From 744ac0c82d866d40e4e9f6ac2b8cd3afaf9b3380 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 15:33:27 +0800 Subject: [PATCH 01/69] [kafka] Use the DDL table contract in Metadata Integrate the DDL mapping prerequisite and use its resolver for Metadata discovery. Omit unsupported tables from all-topic queries and return per-topic mapping errors for named queries. Preserve metadata for compatible tables in mixed requests. Validation: Java 11, mvn -o -pl fluss-kafka clean verify (62 unit tests and 2 integration tests); Checkstyle, Spotless and RAT passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 742/742 AI-Contributed/UT: 571/571 --- .../fluss/kafka/KafkaProtocolPlugin.java | 3 +- .../fluss/kafka/KafkaRequestHandler.java | 22 +- .../kafka/api/metadata/MetadataHandler.java | 190 ++++++ .../metadata/GatewayKafkaMetadataBackend.java | 322 ++++++++++ .../metadata/KafkaClusterMetadata.java | 210 ++++++ .../metadata/KafkaMetadataBackend.java | 30 + .../backend/metadata/KafkaMetadataQuery.java | 102 +++ .../fluss/kafka/KafkaMetadataHandlerTest.java | 602 ++++++++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 8 +- 9 files changed, 1482 insertions(+), 7 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 9a9e2e9ff6e..fd185fdef12 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -66,6 +66,7 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " + service.getClass().getSimpleName()); } - return new KafkaRequestHandler(); + TabletServerGateway gateway = (TabletServerGateway) service; + return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index b3b555a8f9a..d6fb1a0430e 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,14 +17,20 @@ package org.apache.fluss.kafka; +import org.apache.fluss.kafka.api.metadata.MetadataHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import java.util.Collections; +import java.util.Arrays; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { @@ -32,11 +38,19 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaApiRegistry registry; private final KafkaRequestDispatcher dispatcher; - /** Creates a Kafka request handler with the implemented server capabilities. */ - public KafkaRequestHandler() { + /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ + public KafkaRequestHandler( + RpcGatewayService service, TabletServerGateway gateway, String kafkaDatabase) { + checkNotNull(service); + checkNotNull(gateway); + checkNotNull(kafkaDatabase); this.registry = new KafkaApiRegistry( - Collections.singletonList(new ApiVersionsHandler(this::apiRegistry))); + Arrays.asList( + new ApiVersionsHandler(this::apiRegistry), + new MetadataHandler( + new GatewayKafkaMetadataBackend( + service, gateway, kafkaDatabase)))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java new file mode 100644 index 00000000000..ee8bfe7a5a8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataBackend; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements Kafka Metadata versions 0 through 11 using a narrow Fluss metadata backend. */ +@Internal +public final class MetadataHandler implements KafkaApiHandler { + + private static final short MAX_SUPPORTED_VERSION = 11; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.METADATA, + ApiKeys.METADATA.oldestVersion(), + (short) Math.min(ApiKeys.METADATA.latestVersion(), MAX_SUPPORTED_VERSION), + true); + + private final KafkaMetadataBackend backend; + + /** Creates a Metadata handler. */ + public MetadataHandler(KafkaMetadataBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, MetadataRequest request) { + List validTopics = new ArrayList<>(); + List invalidTopics = new ArrayList<>(); + if (!request.isAllTopics()) { + for (MetadataRequestTopic topic : request.data().topics()) { + if (topic.name() == null) { + throw new InvalidRequestException( + "Topic name must be set because topic ID lookup is not supported by " + + "Metadata versions 10 and 11."); + } else if (!Topic.isValid(topic.name())) { + invalidTopics.add( + new KafkaClusterMetadata.Topic( + topic.name(), + topic.topicId(), + TopicError.INVALID_TOPIC, + Collections.emptyList())); + } else { + // Kafka added the topic ID fields in v10, but ID-based Metadata lookup was not + // implemented until v12. This handler intentionally stops at v11. + validTopics.add(new TopicReference(topic.name(), Uuid.ZERO_UUID)); + } + } + } + + KafkaMetadataQuery query = + new KafkaMetadataQuery( + request.isAllTopics(), + validTopics, + context.listenerName(), + clientAddress(context.remoteAddress())); + return backend.getMetadata(query) + .thenApply( + metadata -> { + List topics = + new ArrayList<>(metadata.topics()); + topics.addAll(invalidTopics); + return toResponse( + request.version(), + new KafkaClusterMetadata(metadata.brokers(), topics)); + }); + } + + private static MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { + MetadataResponseData data = + new MetadataResponseData() + .setThrottleTimeMs(0) + .setControllerId(MetadataResponse.NO_CONTROLLER_ID) + .setClusterAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Broker broker : metadata.brokers()) { + MetadataResponseData.MetadataResponseBroker responseBroker = + new MetadataResponseData.MetadataResponseBroker() + .setNodeId(broker.id()) + .setHost(broker.host()) + .setPort(broker.port()); + if (broker.rack() != null) { + responseBroker.setRack(broker.rack()); + } + data.brokers().add(responseBroker); + } + for (KafkaClusterMetadata.Topic topic : metadata.topics()) { + MetadataResponseData.MetadataResponseTopic responseTopic = + new MetadataResponseData.MetadataResponseTopic() + .setName(topic.name()) + .setTopicId(topic.topicId()) + .setErrorCode(toKafkaError(topic.error()).code()) + .setIsInternal(topic.name() != null && Topic.isInternal(topic.name())) + .setTopicAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Partition partition : topic.partitions()) { + responseTopic + .partitions() + .add( + new MetadataResponseData.MetadataResponsePartition() + .setErrorCode( + partition.leaderAvailable() + ? Errors.NONE.code() + : Errors.LEADER_NOT_AVAILABLE.code()) + .setPartitionIndex(partition.partitionId()) + .setLeaderId(partition.leaderId()) + .setLeaderEpoch(partition.leaderEpoch()) + .setReplicaNodes(partition.replicas()) + .setIsrNodes(partition.isr()) + .setOfflineReplicas(partition.offlineReplicas())); + } + data.topics().add(responseTopic); + } + return new MetadataResponse(data, version); + } + + private static Errors toKafkaError(TopicError error) { + switch (error) { + case NONE: + return Errors.NONE; + case UNKNOWN_TOPIC_OR_PARTITION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case UNKNOWN_TOPIC_ID: + return Errors.UNKNOWN_TOPIC_ID; + case INVALID_TOPIC: + return Errors.INVALID_TOPIC_EXCEPTION; + default: + throw new IllegalArgumentException("Unsupported metadata error " + error); + } + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java new file mode 100644 index 00000000000..5905624654f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.MetadataResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.Uuid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */ +@Internal +public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { + + private static final Logger LOG = LoggerFactory.getLogger(GatewayKafkaMetadataBackend.class); + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final String databaseName; + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + + /** Creates a metadata backend backed by the local TabletServer gateway. */ + public GatewayKafkaMetadataBackend( + RpcGatewayService service, TabletServerGateway gateway, String databaseName) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.databaseName = checkNotNull(databaseName); + this.topicMapper = new KafkaTopicMapper(databaseName); + } + + @Override + public CompletableFuture getMetadata(KafkaMetadataQuery query) { + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenCompose( + response -> + requestFlussMetadata( + query, + new LinkedHashSet<>(response.getTableNamesList()))); + } + + Set topicNames = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + topicNames.add(topic.topicName()); + } + } + return requestFlussMetadata(query, topicNames); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames) { + return requestFlussMetadata(query, topicNames, true); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { + MetadataRequest request = new MetadataRequest(); + for (String topicName : topicNames) { + TablePath tablePath = TablePath.of(databaseName, topicName); + if (!topicMapper.isMappedTable(tablePath)) { + continue; + } + tablePath = topicMapper.toTablePath(topicName); + request.addAllTablePaths( + Collections.singletonList( + new PbTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()))); + } + setCurrentSession(query); + try { + return gateway.metadata(request) + .handle( + (response, failure) -> + failure == null + ? CompletableFuture.completedFuture( + toKafkaMetadata(query, response)) + : recoverMetadataFailure( + query, failure, refreshAndRetry)) + .thenCompose(future -> future); + } catch (Throwable failure) { + return recoverMetadataFailure(query, failure, refreshAndRetry); + } + } + + private CompletableFuture recoverMetadataFailure( + KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) { + if (refreshAndRetry) { + return currentTopicNames(query) + .thenCompose(currentNames -> requestFlussMetadata(query, currentNames, false)); + } + LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure)); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(unwrap(failure)); + return failed; + } + + private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenApply( + response -> { + Set currentNames = + new LinkedHashSet<>(response.getTableNamesList()); + if (!query.allTopics() && !containsTopicId(query.topics())) { + Set requestedNames = new HashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + requestedNames.add(topic.topicName()); + } + } + currentNames.retainAll(requestedNames); + } + return currentNames; + }); + } + + private KafkaClusterMetadata toKafkaMetadata( + KafkaMetadataQuery query, MetadataResponse response) { + List brokers = new ArrayList<>(); + Set aliveBrokerIds = new HashSet<>(); + for (PbServerNode server : response.getTabletServersList()) { + brokers.add( + new Broker( + server.getNodeId(), + server.getHost(), + server.getPort(), + server.hasRack() ? server.getRack() : null)); + aliveBrokerIds.add(server.getNodeId()); + } + Collections.sort(brokers, Comparator.comparingInt(Broker::id)); + + Map topicsByName = new HashMap<>(); + Map topicsById = new HashMap<>(); + for (PbTableMetadata table : response.getTableMetadatasList()) { + TablePath tablePath = + TablePath.of( + table.getTablePath().getDatabaseName(), + table.getTablePath().getTableName()); + if (!topicMapper.isMappedTable(tablePath)) { + continue; + } + Topic topic = toKafkaTopic(table, aliveBrokerIds); + topicsByName.put(topic.name(), topic); + topicsById.put(topic.topicId(), topic); + } + + List topics = new ArrayList<>(); + if (query.allTopics()) { + for (Topic topic : topicsByName.values()) { + if (topic.error() == TopicError.NONE) { + topics.add(topic); + } + } + Collections.sort(topics, Comparator.comparing(Topic::name)); + } else { + for (TopicReference reference : query.topics()) { + Topic topic = + reference.hasTopicId() + ? topicsById.get(reference.topicId()) + : topicsByName.get(reference.topicName()); + if (topic != null && matches(reference, topic)) { + topics.add(topic); + } else { + topics.add(missingTopic(reference)); + } + } + } + return new KafkaClusterMetadata(brokers, topics); + } + + private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { + try { + schemaResolver.resolve(TableDescriptor.fromJsonBytes(table.getTableJson())); + } catch (IllegalArgumentException e) { + LOG.debug( + "Table {} does not define a supported Kafka mapping: {}", + table.getTablePath().getTableName(), + e.getMessage()); + return new Topic( + table.getTablePath().getTableName(), + topicMapper.toTopicId(table.getTableId()), + TopicError.INVALID_TOPIC, + Collections.emptyList()); + } + List partitions = new ArrayList<>(); + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + boolean leaderAvailable = + bucket.hasLeaderId() && aliveBrokerIds.contains(bucket.getLeaderId()); + List replicas = new ArrayList<>(); + List isr = new ArrayList<>(); + List offlineReplicas = new ArrayList<>(); + Set isrIds = new HashSet<>(); + if (bucket.hasBucketEpoch()) { + for (int isrId : bucket.getIsrs()) { + isrIds.add(isrId); + } + } else if (leaderAvailable) { + // Legacy metadata has no authoritative ISR. Report only the available leader + // as a conservative fallback; live followers may still be out of sync. + isrIds.add(bucket.getLeaderId()); + } + for (int replicaId : bucket.getReplicaIds()) { + replicas.add(replicaId); + if (isrIds.contains(replicaId)) { + isr.add(replicaId); + } + if (!aliveBrokerIds.contains(replicaId)) { + offlineReplicas.add(replicaId); + } + } + partitions.add( + new Partition( + bucket.getBucketId(), + leaderAvailable ? bucket.getLeaderId() : -1, + bucket.hasLeaderEpoch() ? bucket.getLeaderEpoch() : -1, + replicas, + isr, + offlineReplicas, + leaderAvailable)); + } + Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); + return new Topic( + table.getTablePath().getTableName(), + topicMapper.toTopicId(table.getTableId()), + TopicError.NONE, + partitions); + } + + private static Topic missingTopic(TopicReference reference) { + TopicError error = + reference.hasTopicId() + ? TopicError.UNKNOWN_TOPIC_ID + : TopicError.UNKNOWN_TOPIC_OR_PARTITION; + return new Topic( + reference.topicName(), reference.topicId(), error, Collections.emptyList()); + } + + private static boolean matches(TopicReference reference, Topic topic) { + return (reference.topicName() == null || reference.topicName().equals(topic.name())) + && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId())); + } + + private void setCurrentSession(KafkaMetadataQuery query) { + service.setCurrentSession( + new Session( + (short) 0, + query.listenerName(), + false, + query.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static boolean containsTopicId(List topics) { + for (TopicReference topic : topics) { + if (topic.hasTopicId()) { + return true; + } + } + return false; + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java new file mode 100644 index 00000000000..bbc634af6f0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Kafka-domain cluster metadata returned by a Fluss metadata backend. */ +@Internal +public final class KafkaClusterMetadata { + + private final List brokers; + private final List topics; + + /** Creates cluster metadata. */ + public KafkaClusterMetadata(List brokers, List topics) { + this.brokers = immutableCopy(brokers); + this.topics = immutableCopy(topics); + } + + /** Returns Kafka-reachable brokers. */ + public List brokers() { + return brokers; + } + + /** Returns topic metadata and topic-level errors. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Kafka-reachable broker information. */ + @Internal + public static final class Broker { + + private final int id; + private final String host; + private final int port; + private final @Nullable String rack; + + /** Creates broker information. */ + public Broker(int id, String host, int port, @Nullable String rack) { + this.id = id; + this.host = checkNotNull(host); + this.port = port; + this.rack = rack; + } + + /** Returns the Kafka broker ID. */ + public int id() { + return id; + } + + /** Returns the Kafka listener host. */ + public String host() { + return host; + } + + /** Returns the Kafka listener port. */ + public int port() { + return port; + } + + /** Returns the broker rack, if configured. */ + public @Nullable String rack() { + return rack; + } + } + + /** Topic-level error independent of a Kafka response schema version. */ + @Internal + public enum TopicError { + NONE, + UNKNOWN_TOPIC_OR_PARTITION, + UNKNOWN_TOPIC_ID, + INVALID_TOPIC + } + + /** Metadata for one Kafka topic. */ + @Internal + public static final class Topic { + + private final @Nullable String name; + private final Uuid topicId; + private final TopicError error; + private final List partitions; + + /** Creates topic metadata. */ + public Topic( + @Nullable String name, Uuid topicId, TopicError error, List partitions) { + this.name = name; + this.topicId = checkNotNull(topicId); + this.error = checkNotNull(error); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name, if known. */ + public @Nullable String name() { + return name; + } + + /** Returns the stable Kafka topic ID. */ + public Uuid topicId() { + return topicId; + } + + /** Returns the topic-level domain error. */ + public TopicError error() { + return error; + } + + /** Returns the topic partitions. */ + public List partitions() { + return partitions; + } + } + + /** Metadata for one Kafka partition backed by a Fluss bucket. */ + @Internal + public static final class Partition { + + private final int partitionId; + private final int leaderId; + private final int leaderEpoch; + private final List replicas; + private final List isr; + private final List offlineReplicas; + private final boolean leaderAvailable; + + /** Creates partition metadata. */ + public Partition( + int partitionId, + int leaderId, + int leaderEpoch, + List replicas, + List isr, + List offlineReplicas, + boolean leaderAvailable) { + this.partitionId = partitionId; + this.leaderId = leaderId; + this.leaderEpoch = leaderEpoch; + this.replicas = immutableCopy(replicas); + this.isr = immutableCopy(isr); + this.offlineReplicas = immutableCopy(offlineReplicas); + this.leaderAvailable = leaderAvailable; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the current leader ID, or {@code -1} when unavailable. */ + public int leaderId() { + return leaderId; + } + + /** Returns the leader epoch, or {@code -1} when unavailable. */ + public int leaderEpoch() { + return leaderEpoch; + } + + /** Returns assigned replica IDs. */ + public List replicas() { + return replicas; + } + + /** Returns replica IDs currently visible as in-sync. */ + public List isr() { + return isr; + } + + /** Returns assigned replicas whose TabletServers are unavailable. */ + public List offlineReplicas() { + return offlineReplicas; + } + + /** Returns whether the partition has a reachable leader. */ + public boolean leaderAvailable() { + return leaderAvailable; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java new file mode 100644 index 00000000000..abb02a920ed --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Metadata API. */ +@Internal +public interface KafkaMetadataBackend { + + /** Resolves Kafka-domain metadata asynchronously. */ + CompletableFuture getMetadata(KafkaMetadataQuery query); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java new file mode 100644 index 00000000000..01e21fa4440 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Domain query used by the Metadata API to access the Fluss adapter layer. */ +@Internal +public final class KafkaMetadataQuery { + + private final boolean allTopics; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a metadata query. */ + public KafkaMetadataQuery( + boolean allTopics, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.allTopics = allTopics; + this.topics = Collections.unmodifiableList(new ArrayList<>(checkNotNull(topics))); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns whether all Kafka topics should be returned. */ + public boolean allTopics() { + return allTopics; + } + + /** Returns the explicitly requested topic identities. */ + public List topics() { + return topics; + } + + /** Returns the Kafka listener used by the client connection. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client address when it is available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + /** Kafka topic name and ID supplied by a Metadata request. */ + @Internal + public static final class TopicReference { + + private final @Nullable String topicName; + private final Uuid topicId; + + /** Creates a topic reference. */ + public TopicReference(@Nullable String topicName, Uuid topicId) { + this.topicName = topicName; + this.topicId = checkNotNull(topicId); + } + + /** Returns the requested topic name, if present. */ + public @Nullable String topicName() { + return topicName; + } + + /** Returns the requested topic ID, or {@link Uuid#ZERO_UUID} when absent. */ + public Uuid topicId() { + return topicId; + } + + /** Returns whether this reference identifies a topic by ID. */ + public boolean hasTopicId() { + return !Uuid.ZERO_UUID.equals(topicId); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java new file mode 100644 index 00000000000..2614b2df3fd --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -0,0 +1,602 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol compatibility tests for the Kafka Metadata API. */ +public class KafkaMetadataHandlerTest { + + private static final Uuid TOPIC_ID = new Uuid(0x466c757373000000L, 123L); + + @Test + public void testNamedTopicForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Collections.singletonList("topic"))), + version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(100, new byte[] {1, 2, 3})); + request.data() + .topics() + .get(0) + .unknownTaggedFields() + .add(new RawTaggedField(101, new byte[] {4, 5, 6})); + } + MetadataResponse response = handle(service, request, version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.controller()).isNull(); + Node broker = response.brokers().iterator().next(); + assertThat(broker.host()).isEqualTo("broker-1"); + assertThat(broker.port()).isEqualTo(9092); + assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.partitionIndex()).isZero(); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).isEmpty(); + } + assertThat(service.lastListenerName).isEqualTo("KAFKA"); + } + + @Test + public void testAllTopicsForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = allTopicsRequest(version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(102, new byte[] {7, 8, 9})); + } + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("other", "topic"); + } + } + + @Test + public void testAllTopicsAndMissingAndInvalidTopic() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse allTopics = + handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); + assertThat(allTopics.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactlyInAnyOrder("other", "topic"); + + MetadataRequest requestedTopics = + new MetadataRequest.Builder(Arrays.asList("missing", "invalid topic"), false) + .build((short) 9); + MetadataResponse errors = handle(service, requestedTopics, (short) 9); + assertThat(errors.errors()) + .containsEntry("missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("invalid topic", Errors.INVALID_TOPIC_EXCEPTION); + } + + @Test + public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName("topic") + .setTopicId( + new Uuid( + 0x466c757373000000L, + 999L)))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(response.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + } + } + + @Test + public void testV10AndV11RejectTopicIdOnlyLookup() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Uuid requestedTopicId = new Uuid(0x466c757373000000L, 999L); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(null) + .setTopicId(requestedTopicId))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()).hasSize(1); + MetadataResponseTopic responseTopic = response.data().topics().iterator().next(); + assertThat(responseTopic.name()).isEmpty(); + assertThat(responseTopic.topicId()).isEqualTo(requestedTopicId); + assertThat(responseTopic.errorCode()).isEqualTo(Errors.INVALID_REQUEST.code()); + } + } + + @Test + public void testTopicIdentityAcrossDeleteAndRecreate() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse initial = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(initial.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + + service.removeTable("topic"); + MetadataResponse deleted = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(deleted.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + + service.putTable("topic", 223L); + Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); + MetadataResponse recreatedByName = + handle( + service, + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11), + (short) 11); + assertThat(recreatedByName.data().topics().find("topic").topicId()) + .isEqualTo(recreatedTopicId) + .isNotEqualTo(TOPIC_ID); + } + + @Test + public void testDeleteRaceBecomesUnknownTopicResult() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.removeTable("topic"); + service.failNextMetadataAsMissing = true; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + } + + @Test + public void testUnavailableLeaderUsesPartitionError() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicLeaderAvailable = false; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.leaderId()).isEqualTo(-1); + assertThat(partition.replicaNodes()).containsExactly(1, 2, 3); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).containsExactly(3); + } + + @Test + public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicIsr = new int[] {1}; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataUsesLeaderOnlyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + service.topicLeaderAvailable = false; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.isrNodes()).isEmpty(); + } + + @Test + public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { + for (int bucketEpoch : new int[] {7, -1}) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = bucketEpoch; + service.topicIsr = new int[0]; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.isrNodes()).isEmpty(); + } + } + + @Test + public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failMetadata = true; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 1)); + assertThat(response.brokers()).isEmpty(); + } + + @Test + public void testMetadataUsesDdlContractForEveryVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable( + "no_mapping", + 125L, + TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(2) + .build()); + service.putTable( + "bad_format", + 126L, + TableDescriptor.builder(defaultDescriptor()) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") + .build()); + service.putTable( + "primary_key", + 127L, + TableDescriptor.builder(defaultDescriptor()) + .schema( + Schema.newBuilder() + .column("body", DataTypes.BYTES().copy(false)) + .primaryKey("body") + .build()) + .build()); + service.putTable( + "partitioned", + 128L, + TableDescriptor.builder(defaultDescriptor()).partitionedBy("body").build()); + service.putTable( + "indexed", + 129L, + TableDescriptor.builder(defaultDescriptor()).logFormat(LogFormat.INDEXED).build()); + service.putTable("invalid topic", 130L); + List invalidMappings = + Arrays.asList("no_mapping", "bad_format", "primary_key", "partitioned", "indexed"); + for (short version = 0; version <= 11; version++) { + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("other", "topic"); + List requested = new ArrayList<>(invalidMappings); + requested.add("topic"); + requested.add("missing"); + MetadataResponse named = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + requested)), + version), + version); + for (String name : invalidMappings) { + MetadataResponseTopic topic = named.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(topic.partitions()).isEmpty(); + } + assertThat(named.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(named.data().topics().find("missing").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + } + + @Test + public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable( + "topic", + 123L, + TableDescriptor.builder(defaultDescriptor()) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .build()); + MetadataResponse invalid = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(invalid.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + service.putTable("topic", 123L); + MetadataResponse valid = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(valid.data().topics().find("topic").errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + } + + private static TableDescriptor defaultDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .build(); + } + + private static MetadataRequest namedTopicRequest(String topicName) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(topicName) + .setTopicId(Uuid.ZERO_UUID))), + (short) 11); + } + + private static MetadataRequest allTopicsRequest(short version) { + MetadataRequestData data = new MetadataRequestData(); + data.setTopics(version == 0 ? Collections.emptyList() : null); + return new MetadataRequest(data, version); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { + KafkaRequestHandler handler = new KafkaRequestHandler(service, service, "kafka"); + ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); + KafkaRequest request; + try { + request = + new KafkaRequest( + ApiKeys.METADATA, + version, + new RequestHeader(ApiKeys.METADATA, version, "client-id", 1), + requestBody, + "KAFKA", + requestBuffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + } finally { + // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest. + requestBuffer.release(); + } + handler.processRequest(request); + ByteBuf responseBuffer = request.responseBuffer(); + try { + return (MetadataResponse) + AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + assertThat(requestBuffer.refCnt()).isZero(); + } + } + + private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + + private final Map tables = new LinkedHashMap<>(); + private final Map descriptors = new LinkedHashMap<>(); + private String lastListenerName; + private boolean topicLeaderAvailable = true; + private int[] topicIsr = new int[] {1, 2}; + private Integer topicBucketEpoch = 7; + private boolean failMetadata; + private boolean failNextMetadataAsMissing; + + private TestingMetadataGatewayService() { + putTable("topic", 123L); + putTable("other", 124L); + } + + @Override + public CompletableFuture listTables(ListTablesRequest request) { + assertThat(request.getDatabaseName()).isEqualTo("kafka"); + return CompletableFuture.completedFuture( + new ListTablesResponse().addAllTableNames(new ArrayList<>(tables.keySet()))); + } + + @Override + public CompletableFuture metadata( + org.apache.fluss.rpc.messages.MetadataRequest request) { + lastListenerName = currentListenerName(); + if (failMetadata) { + CompletableFuture failure = + new CompletableFuture<>(); + failure.completeExceptionally(new IllegalStateException("metadata unavailable")); + return failure; + } + if (failNextMetadataAsMissing) { + failNextMetadataAsMissing = false; + throw new TableNotExistException("table was deleted"); + } + List topics = new ArrayList<>(); + for (PbTablePath tablePath : request.getTablePathsList()) { + Long tableId = tables.get(tablePath.getTableName()); + if (tableId != null) { + topics.add( + tableMetadata( + tablePath.getTableName(), + tableId, + !"topic".equals(tablePath.getTableName()) + || topicLeaderAvailable, + "topic".equals(tablePath.getTableName()) + ? topicIsr + : new int[] {1, 2}, + "topic".equals(tablePath.getTableName()) + ? topicBucketEpoch + : Integer.valueOf(7)) + .setTableJson( + descriptors + .get(tablePath.getTableName()) + .toJsonBytes())); + } + } + return CompletableFuture.completedFuture( + new org.apache.fluss.rpc.messages.MetadataResponse() + .addAllTabletServers( + Arrays.asList( + new PbServerNode() + .setNodeId(1) + .setHost("broker-1") + .setPort(9092) + .setRack("rack-a"), + new PbServerNode() + .setNodeId(2) + .setHost("broker-2") + .setPort(9093))) + .addAllTableMetadatas(topics)); + } + + private void putTable(String topic, long tableId) { + putTable(topic, tableId, defaultDescriptor()); + } + + private void putTable(String topic, long tableId, TableDescriptor descriptor) { + tables.put(topic, tableId); + descriptors.put(topic, descriptor); + } + + private void removeTable(String topic) { + tables.remove(topic); + descriptors.remove(topic); + } + + private static PbTableMetadata tableMetadata( + String topic, + long tableId, + boolean leaderAvailable, + int[] isr, + Integer bucketEpoch) { + PbTableMetadata table = + new PbTableMetadata() + .setTablePath( + new PbTablePath().setDatabaseName("kafka").setTableName(topic)) + .setTableId(tableId) + .addAllBucketMetadatas( + Arrays.asList( + new PbBucketMetadata() + .setBucketId(0) + .setLeaderId(leaderAvailable ? 1 : 3) + .setLeaderEpoch(5) + .setReplicaIds( + leaderAvailable + ? new int[] {1, 2} + : new int[] {1, 2, 3}) + .setIsrs( + bucketEpoch == null ? new int[0] : isr), + new PbBucketMetadata() + .setBucketId(1) + .setLeaderId(2) + .setLeaderEpoch(6) + .setReplicaIds(new int[] {1, 2}) + .setIsrs( + bucketEpoch == null + ? new int[0] + : new int[] {1, 2}))); + if (bucketEpoch != null) { + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + bucket.setBucketEpoch(bucketEpoch); + } + } + return table; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 40fdab04cc4..d00c436defe 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.kafka; +import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; @@ -145,7 +146,9 @@ private static void assertSuccessfulResponseDefaults(ApiVersionsResponse respons private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) - .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); + .containsExactly( + tuple(ApiKeys.METADATA.id, (short) 0, (short) 11), + tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } @ParameterizedTest @@ -292,6 +295,7 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + return new KafkaRequestHandler(service, service, "kafka"); } } From c0c3816397dff885b84cb82d8af833a83e0913da Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 14:39:43 +0800 Subject: [PATCH 02/69] [kafka] Discover qualified topics and isolate missing databases Route Metadata queries through the database.table mapping from PR03 and enumerate compatible topics across databases. Preserve brokers and valid topics when a requested database is absent or disappears during discovery; propagate other gateway failures. Cover qualified names, same-named tables across databases, synchronous and asynchronous lookup failures, and Metadata v0-v11 over a real listener. Validated on Java 11: 90 unit tests and 4 integration tests passed, along with Checkstyle, Spotless, RAT, and git diff --check. Five additional interleaved asynchronous review probes passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 129/129 AI-Contributed/UT: 514/514 --- .../fluss/kafka/KafkaProtocolPlugin.java | 2 +- .../fluss/kafka/KafkaRequestHandler.java | 7 +- .../kafka/api/metadata/MetadataHandler.java | 3 +- .../metadata/GatewayKafkaMetadataBackend.java | 116 +++++-- .../fluss/kafka/KafkaMetadataHandlerTest.java | 312 ++++++++++++++---- .../fluss/kafka/KafkaMetadataITCase.java | 200 +++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 2 +- 7 files changed, 531 insertions(+), 111 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index fd185fdef12..dddfff1f9eb 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -67,6 +67,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { + service.getClass().getSimpleName()); } TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE)); + return new KafkaRequestHandler(service, gateway); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index d6fb1a0430e..3278d8328a2 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -39,18 +39,15 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ - public KafkaRequestHandler( - RpcGatewayService service, TabletServerGateway gateway, String kafkaDatabase) { + public KafkaRequestHandler(RpcGatewayService service, TabletServerGateway gateway) { checkNotNull(service); checkNotNull(gateway); - checkNotNull(kafkaDatabase); this.registry = new KafkaApiRegistry( Arrays.asList( new ApiVersionsHandler(this::apiRegistry), new MetadataHandler( - new GatewayKafkaMetadataBackend( - service, gateway, kafkaDatabase)))); + new GatewayKafkaMetadataBackend(service, gateway)))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java index ee8bfe7a5a8..2ceca11fbe1 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -28,6 +28,7 @@ import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InvalidRequestException; @@ -85,7 +86,7 @@ public CompletableFuture handle( throw new InvalidRequestException( "Topic name must be set because topic ID lookup is not supported by " + "Metadata versions 10 and 11."); - } else if (!Topic.isValid(topic.name())) { + } else if (!KafkaTopicMapper.isValidTopic(topic.name())) { invalidTopics.add( new KafkaClusterMetadata.Topic( topic.name(), diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java index 5905624654f..202658b9f0f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.backend.metadata; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; @@ -29,7 +30,9 @@ import org.apache.fluss.metadata.TablePath; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; import org.apache.fluss.rpc.messages.MetadataRequest; import org.apache.fluss.rpc.messages.MetadataResponse; import org.apache.fluss.rpc.messages.PbBucketMetadata; @@ -38,6 +41,7 @@ import org.apache.fluss.rpc.messages.PbTablePath; import org.apache.fluss.rpc.netty.server.Session; import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.common.Uuid; import org.slf4j.Logger; @@ -65,29 +69,20 @@ public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { private final RpcGatewayService service; private final TabletServerGateway gateway; - private final String databaseName; - private final KafkaTopicMapper topicMapper; + private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); /** Creates a metadata backend backed by the local TabletServer gateway. */ - public GatewayKafkaMetadataBackend( - RpcGatewayService service, TabletServerGateway gateway, String databaseName) { + public GatewayKafkaMetadataBackend(RpcGatewayService service, TabletServerGateway gateway) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); - this.databaseName = checkNotNull(databaseName); - this.topicMapper = new KafkaTopicMapper(databaseName); } @Override public CompletableFuture getMetadata(KafkaMetadataQuery query) { if (query.allTopics() || containsTopicId(query.topics())) { - setCurrentSession(query); - return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) - .thenCompose( - response -> - requestFlussMetadata( - query, - new LinkedHashSet<>(response.getTableNamesList()))); + return currentTopicNames(query) + .thenCompose(topicNames -> requestFlussMetadata(query, topicNames)); } Set topicNames = new LinkedHashSet<>(); @@ -108,11 +103,7 @@ private CompletableFuture requestFlussMetadata( KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { MetadataRequest request = new MetadataRequest(); for (String topicName : topicNames) { - TablePath tablePath = TablePath.of(databaseName, topicName); - if (!topicMapper.isMappedTable(tablePath)) { - continue; - } - tablePath = topicMapper.toTablePath(topicName); + TablePath tablePath = topicMapper.toTablePath(topicName); request.addAllTablePaths( Collections.singletonList( new PbTablePath() @@ -148,12 +139,27 @@ private CompletableFuture recoverMetadataFailure( } private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { - setCurrentSession(query); - return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + CompletableFuture> databasesFuture; + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + databasesFuture = + gateway.listDatabases(new ListDatabasesRequest()) + .thenApply( + response -> + new LinkedHashSet<>(response.getDatabaseNamesList())); + } else { + Set databases = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + databases.add(topicMapper.toTablePath(topic.topicName()).getDatabaseName()); + } + } + databasesFuture = CompletableFuture.completedFuture(databases); + } + return databasesFuture + .thenCompose(databases -> listTopicNames(query, databases)) .thenApply( - response -> { - Set currentNames = - new LinkedHashSet<>(response.getTableNamesList()); + currentNames -> { if (!query.allTopics() && !containsTopicId(query.topics())) { Set requestedNames = new HashSet<>(); for (TopicReference topic : query.topics()) { @@ -167,6 +173,56 @@ private CompletableFuture> currentTopicNames(KafkaMetadataQuery quer }); } + private CompletableFuture> listTopicNames( + KafkaMetadataQuery query, Set databases) { + CompletableFuture> topicsFuture = + CompletableFuture.completedFuture(new LinkedHashSet<>()); + for (String database : databases) { + topicsFuture = + topicsFuture.thenCompose( + topicNames -> + listTables(query, database) + .thenApply( + response -> { + for (String tableName : + response.getTableNamesList()) { + TablePath tablePath = + TablePath.of( + database, tableName); + if (topicMapper.isMappedTable( + tablePath)) { + topicNames.add( + topicMapper.toTopicName( + tablePath)); + } + } + return topicNames; + })); + } + return topicsFuture; + } + + private CompletableFuture listTables( + KafkaMetadataQuery query, String database) { + setCurrentSession(query); + CompletableFuture tablesFuture; + try { + tablesFuture = gateway.listTables(new ListTablesRequest().setDatabaseName(database)); + } catch (Exception failure) { + tablesFuture = FutureUtils.completedExceptionally(failure); + } + return tablesFuture.exceptionally( + failure -> { + Throwable cause = unwrap(failure); + if (cause instanceof DatabaseNotExistException) { + // A database may be missing or deleted after listDatabases. Keep loading + // the remaining topics and brokers through the normal metadata request. + return new ListTablesResponse(); + } + throw new CompletionException(cause); + }); + } + private KafkaClusterMetadata toKafkaMetadata( KafkaMetadataQuery query, MetadataResponse response) { List brokers = new ArrayList<>(); @@ -192,7 +248,7 @@ private KafkaClusterMetadata toKafkaMetadata( if (!topicMapper.isMappedTable(tablePath)) { continue; } - Topic topic = toKafkaTopic(table, aliveBrokerIds); + Topic topic = toKafkaTopic(table, topicMapper.toTopicName(tablePath), aliveBrokerIds); topicsByName.put(topic.name(), topic); topicsById.put(topic.topicId(), topic); } @@ -221,16 +277,17 @@ private KafkaClusterMetadata toKafkaMetadata( return new KafkaClusterMetadata(brokers, topics); } - private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { + private Topic toKafkaTopic( + PbTableMetadata table, String topicName, Set aliveBrokerIds) { try { schemaResolver.resolve(TableDescriptor.fromJsonBytes(table.getTableJson())); } catch (IllegalArgumentException e) { LOG.debug( "Table {} does not define a supported Kafka mapping: {}", - table.getTablePath().getTableName(), + topicName, e.getMessage()); return new Topic( - table.getTablePath().getTableName(), + topicName, topicMapper.toTopicId(table.getTableId()), TopicError.INVALID_TOPIC, Collections.emptyList()); @@ -273,10 +330,7 @@ private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { } Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); return new Topic( - table.getTablePath().getTableName(), - topicMapper.toTopicId(table.getTableId()), - TopicError.NONE, - partitions); + topicName, topicMapper.toTopicId(table.getTableId()), TopicError.NONE, partitions); } private static Topic missingTopic(TopicReference reference) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 2614b2df3fd..4c33a51635a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -17,12 +17,15 @@ package org.apache.fluss.kafka; +import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; +import org.apache.fluss.rpc.messages.ListDatabasesResponse; import org.apache.fluss.rpc.messages.ListTablesRequest; import org.apache.fluss.rpc.messages.ListTablesResponse; import org.apache.fluss.rpc.messages.PbBucketMetadata; @@ -32,6 +35,7 @@ import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.common.Node; import org.apache.kafka.common.Uuid; @@ -47,14 +51,20 @@ import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; @@ -72,7 +82,7 @@ public void testNamedTopicForEverySupportedVersion() { new MetadataRequestData() .setTopics( MetadataRequest.convertToMetadataRequestTopic( - Collections.singletonList("topic"))), + Collections.singletonList("kafka.topic"))), version); if (version >= 9) { request.data() @@ -92,7 +102,7 @@ public void testNamedTopicForEverySupportedVersion() { assertThat(broker.host()).isEqualTo("broker-1"); assertThat(broker.port()).isEqualTo(9092); assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); - MetadataResponseTopic topic = response.data().topics().find("topic"); + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); assertThat(topic.partitions()).hasSize(2); assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); @@ -121,7 +131,7 @@ public void testAllTopicsForEverySupportedVersion() { assertThat(response.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactly("other", "topic"); + .containsExactly("kafka.other", "kafka.topic"); } } @@ -133,15 +143,16 @@ public void testAllTopicsAndMissingAndInvalidTopic() { handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); assertThat(allTopics.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactlyInAnyOrder("other", "topic"); + .containsExactlyInAnyOrder("kafka.other", "kafka.topic"); MetadataRequest requestedTopics = - new MetadataRequest.Builder(Arrays.asList("missing", "invalid topic"), false) + new MetadataRequest.Builder( + Arrays.asList("kafka.missing", "kafka.invalid topic"), false) .build((short) 9); MetadataResponse errors = handle(service, requestedTopics, (short) 9); assertThat(errors.errors()) - .containsEntry("missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) - .containsEntry("invalid topic", Errors.INVALID_TOPIC_EXCEPTION); + .containsEntry("kafka.missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("kafka.invalid topic", Errors.INVALID_TOPIC_EXCEPTION); } @Test @@ -154,7 +165,7 @@ public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { .setTopics( Collections.singletonList( new MetadataRequestTopic() - .setName("topic") + .setName("kafka.topic") .setTopicId( new Uuid( 0x466c757373000000L, @@ -164,7 +175,7 @@ public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { MetadataResponse response = handle(service, request, version); assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); - assertThat(response.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + assertThat(response.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); } } @@ -197,24 +208,24 @@ public void testV10AndV11RejectTopicIdOnlyLookup() { public void testTopicIdentityAcrossDeleteAndRecreate() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - MetadataResponse initial = handle(service, namedTopicRequest("topic"), (short) 11); - assertThat(initial.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + MetadataResponse initial = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + assertThat(initial.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); - service.removeTable("topic"); - MetadataResponse deleted = handle(service, namedTopicRequest("topic"), (short) 11); + service.removeTable("kafka.topic"); + MetadataResponse deleted = handle(service, namedTopicRequest("kafka.topic"), (short) 11); assertThat(deleted.errorCounts()) .containsExactlyEntriesOf( Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); - service.putTable("topic", 223L); + service.putTable("kafka.topic", 223L); Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); MetadataResponse recreatedByName = handle( service, - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11), (short) 11); - assertThat(recreatedByName.data().topics().find("topic").topicId()) + assertThat(recreatedByName.data().topics().find("kafka.topic").topicId()) .isEqualTo(recreatedTopicId) .isNotEqualTo(TOPIC_ID); } @@ -222,10 +233,10 @@ public void testTopicIdentityAcrossDeleteAndRecreate() { @Test public void testDeleteRaceBecomesUnknownTopicResult() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.removeTable("topic"); + service.removeTable("kafka.topic"); service.failNextMetadataAsMissing = true; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); assertThat(response.errorCounts()) .containsExactlyEntriesOf( @@ -237,12 +248,12 @@ public void testUnavailableLeaderUsesPartitionError() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicLeaderAvailable = false; MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11); MetadataResponse response = handle(service, request, (short) 11); - MetadataResponseTopic topic = response.data().topics().find("topic"); + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); MetadataResponsePartition partition = topic.partitions().get(0); assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); @@ -257,13 +268,13 @@ public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicIsr = new int[] {1}; MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11); MetadataResponse response = handle(service, request, (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.replicaNodes()).containsExactly(1, 2); assertThat(partition.isrNodes()).containsExactly(1); assertThat(partition.offlineReplicas()).isEmpty(); @@ -274,10 +285,10 @@ public void testLegacyMetadataUsesLeaderOnlyIsr() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicBucketEpoch = null; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.leaderId()).isEqualTo(1); assertThat(partition.replicaNodes()).containsExactly(1, 2); assertThat(partition.isrNodes()).containsExactly(1); @@ -290,10 +301,10 @@ public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() { service.topicBucketEpoch = null; service.topicLeaderAvailable = false; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); assertThat(partition.isrNodes()).isEmpty(); } @@ -305,10 +316,11 @@ public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { service.topicBucketEpoch = bucketEpoch; service.topicIsr = new int[0]; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = + handle(service, namedTopicRequest("kafka.topic"), (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.isrNodes()).isEmpty(); } } @@ -318,7 +330,7 @@ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.failMetadata = true; MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11); MetadataResponse response = handle(service, request, (short) 11); @@ -332,20 +344,20 @@ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { public void testMetadataUsesDdlContractForEveryVersion() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.putTable( - "no_mapping", + "kafka.no_mapping", 125L, TableDescriptor.builder() .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) .distributedBy(2) .build()); service.putTable( - "bad_format", + "kafka.bad_format", 126L, TableDescriptor.builder(defaultDescriptor()) .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") .build()); service.putTable( - "primary_key", + "kafka.primary_key", 127L, TableDescriptor.builder(defaultDescriptor()) .schema( @@ -355,24 +367,29 @@ public void testMetadataUsesDdlContractForEveryVersion() { .build()) .build()); service.putTable( - "partitioned", + "kafka.partitioned", 128L, TableDescriptor.builder(defaultDescriptor()).partitionedBy("body").build()); service.putTable( - "indexed", + "kafka.indexed", 129L, TableDescriptor.builder(defaultDescriptor()).logFormat(LogFormat.INDEXED).build()); - service.putTable("invalid topic", 130L); + service.putTable("kafka.invalid topic", 130L); List invalidMappings = - Arrays.asList("no_mapping", "bad_format", "primary_key", "partitioned", "indexed"); + Arrays.asList( + "kafka.no_mapping", + "kafka.bad_format", + "kafka.primary_key", + "kafka.partitioned", + "kafka.indexed"); for (short version = 0; version <= 11; version++) { MetadataResponse all = handle(service, allTopicsRequest(version), version); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactly("other", "topic"); + .containsExactly("kafka.other", "kafka.topic"); List requested = new ArrayList<>(invalidMappings); - requested.add("topic"); - requested.add("missing"); + requested.add("kafka.topic"); + requested.add("kafka.missing"); MetadataResponse named = handle( service, @@ -388,9 +405,9 @@ public void testMetadataUsesDdlContractForEveryVersion() { assertThat(topic.errorCode()).isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); assertThat(topic.partitions()).isEmpty(); } - assertThat(named.data().topics().find("topic").errorCode()) + assertThat(named.data().topics().find("kafka.topic").errorCode()) .isEqualTo(Errors.NONE.code()); - assertThat(named.data().topics().find("missing").errorCode()) + assertThat(named.data().topics().find("kafka.missing").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); } } @@ -399,18 +416,135 @@ public void testMetadataUsesDdlContractForEveryVersion() { public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.putTable( - "topic", + "kafka.topic", 123L, TableDescriptor.builder(defaultDescriptor()) .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") .build()); - MetadataResponse invalid = handle(service, namedTopicRequest("topic"), (short) 11); - assertThat(invalid.data().topics().find("topic").errorCode()) + MetadataResponse invalid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + assertThat(invalid.data().topics().find("kafka.topic").errorCode()) .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); - service.putTable("topic", 123L); - MetadataResponse valid = handle(service, namedTopicRequest("topic"), (short) 11); - assertThat(valid.data().topics().find("topic").errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(valid.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + service.putTable("kafka.topic", 123L); + MetadataResponse valid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + assertThat(valid.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(valid.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); + } + + @Test + public void testSameTableNameInDifferentDatabasesForEveryVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("sales.topic", 223L); + for (short version = 0; version <= 11; version++) { + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic", "sales.topic"); + MetadataResponse named = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList( + "kafka.topic", "sales.topic"))), + version), + version); + assertThat(named.data().topics()).hasSize(2); + assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); + if (version >= 10) { + assertThat(named.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); + assertThat(named.data().topics().find("sales.topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 223L)); + } + } + } + + @Test + public void testRejectUnqualifiedAndMalformedNamesWithoutHidingValidTopics() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + List invalidNames = Arrays.asList("topic", ".topic", "kafka.", "kafka.topic.extra"); + List names = new ArrayList<>(invalidNames); + names.add("kafka.topic"); + MetadataResponse response = + handle( + service, + new MetadataRequest.Builder(names, false).build((short) 11), + (short) 11); + for (String name : invalidNames) { + assertThat(response.data().topics().find(name).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + } + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMissingDatabaseRetainsBrokersAndValidTopics(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failListTablesAsFuture = failedFuture; + for (short version = 0; version <= 11; version++) { + for (List names : + Arrays.asList( + Collections.singletonList("missing_db.topic"), + Arrays.asList("missing_db.topic", "kafka.topic"))) { + MetadataResponse response = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest + .convertToMetadataRequestTopic( + names)), + version), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(names.size()); + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + if (names.contains("kafka.topic")) { + MetadataResponseTopic valid = response.data().topics().find("kafka.topic"); + assertThat(valid.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.partitions()).hasSize(2); + } + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testDatabaseDeletedDuringDiscoveryRetainsOtherTopics(boolean failedFuture) { + for (short version = 0; version <= 11; version++) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("deleted_db.topic", 223L); + service.deleteDatabaseBeforeListing = "deleted_db"; + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, allTopicsRequest(version), version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic"); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testUnexpectedListTablesFailureIsNotTreatedAsMissingDatabase(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failNextMetadataAsMissing = true; + service.listTablesFailure = new IllegalStateException("metadata unavailable"); + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_SERVER_ERROR.code()); } private static TableDescriptor defaultDescriptor() { @@ -441,7 +575,7 @@ private static MetadataRequest allTopicsRequest(short version) { private static MetadataResponse handle( TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { - KafkaRequestHandler handler = new KafkaRequestHandler(service, service, "kafka"); + KafkaRequestHandler handler = new KafkaRequestHandler(service, service); ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); KafkaRequest request; try { @@ -472,6 +606,7 @@ private static MetadataResponse handle( private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + private final Set databases = new LinkedHashSet<>(); private final Map tables = new LinkedHashMap<>(); private final Map descriptors = new LinkedHashMap<>(); private String lastListenerName; @@ -480,17 +615,49 @@ private static final class TestingMetadataGatewayService extends TestingTabletGa private Integer topicBucketEpoch = 7; private boolean failMetadata; private boolean failNextMetadataAsMissing; + private boolean failListTablesAsFuture; + private String deleteDatabaseBeforeListing; + private RuntimeException listTablesFailure; private TestingMetadataGatewayService() { - putTable("topic", 123L); - putTable("other", 124L); + putTable("kafka.topic", 123L); + putTable("kafka.other", 124L); + } + + @Override + public CompletableFuture listDatabases( + ListDatabasesRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + return CompletableFuture.completedFuture( + new ListDatabasesResponse().addAllDatabaseNames(databases)); } @Override public CompletableFuture listTables(ListTablesRequest request) { - assertThat(request.getDatabaseName()).isEqualTo("kafka"); + assertThat(currentListenerName()).isEqualTo("KAFKA"); + String database = request.getDatabaseName(); + if (database.equals(deleteDatabaseBeforeListing)) { + databases.remove(database); + tables.keySet().removeIf(name -> name.startsWith(database + ".")); + } + RuntimeException failure = listTablesFailure; + if (failure == null && !databases.contains(database)) { + failure = new DatabaseNotExistException("Database does not exist."); + } + if (failure != null) { + if (failListTablesAsFuture) { + return FutureUtils.completedExceptionally(new CompletionException(failure)); + } + throw failure; + } + String prefix = request.getDatabaseName() + "."; + List names = + tables.keySet().stream() + .filter(name -> name.startsWith(prefix)) + .map(name -> name.substring(prefix.length())) + .collect(Collectors.toList()); return CompletableFuture.completedFuture( - new ListTablesResponse().addAllTableNames(new ArrayList<>(tables.keySet()))); + new ListTablesResponse().addAllTableNames(names)); } @Override @@ -509,25 +676,23 @@ public CompletableFuture metadat } List topics = new ArrayList<>(); for (PbTablePath tablePath : request.getTablePathsList()) { - Long tableId = tables.get(tablePath.getTableName()); - if (tableId != null) { - topics.add( - tableMetadata( - tablePath.getTableName(), - tableId, - !"topic".equals(tablePath.getTableName()) - || topicLeaderAvailable, - "topic".equals(tablePath.getTableName()) - ? topicIsr - : new int[] {1, 2}, - "topic".equals(tablePath.getTableName()) - ? topicBucketEpoch - : Integer.valueOf(7)) - .setTableJson( - descriptors - .get(tablePath.getTableName()) - .toJsonBytes())); + String topicName = tablePath.getDatabaseName() + "." + tablePath.getTableName(); + Long tableId = tables.get(topicName); + if (tableId == null) { + throw new TableNotExistException("Table does not exist: " + topicName); } + topics.add( + tableMetadata( + topicName, + tableId, + !"kafka.topic".equals(topicName) || topicLeaderAvailable, + "kafka.topic".equals(topicName) + ? topicIsr + : new int[] {1, 2}, + "kafka.topic".equals(topicName) + ? topicBucketEpoch + : Integer.valueOf(7)) + .setTableJson(descriptors.get(topicName).toJsonBytes())); } return CompletableFuture.completedFuture( new org.apache.fluss.rpc.messages.MetadataResponse() @@ -550,6 +715,7 @@ private void putTable(String topic, long tableId) { } private void putTable(String topic, long tableId, TableDescriptor descriptor) { + databases.add(topic.substring(0, topic.indexOf('.'))); tables.put(topic, tableId); descriptors.put(topic, descriptor); } @@ -568,7 +734,9 @@ private static PbTableMetadata tableMetadata( PbTableMetadata table = new PbTableMetadata() .setTablePath( - new PbTablePath().setDatabaseName("kafka").setTableName(topic)) + new PbTablePath() + .setDatabaseName(topic.substring(0, topic.indexOf('.'))) + .setTableName(topic.substring(topic.indexOf('.') + 1))) .setTableId(tableId) .addAllBucketMetadatas( Arrays.asList( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java new file mode 100644 index 00000000000..2f18ffe495d --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.dropDatabase; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies qualified Metadata discovery over a real Kafka listener and TabletServer. */ +public class KafkaMetadataITCase { + + @RegisterExtension + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfiguration()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + public void testQualifiedMetadataAcrossDatabases() throws Exception { + TableDescriptor descriptor = topicDescriptor(); + long first = createTable(CLUSTER, TablePath.of("first_db", "events"), descriptor); + long second = createTable(CLUSTER, TablePath.of("second_db", "events"), descriptor); + CLUSTER.waitUntilTableReady(first); + CLUSTER.waitUntilTableReady(second); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + query( + node, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList( + "first_db.events", + "second_db.events", + "first_db.missing"))), + version)); + assertThat(named.data().topics().find("first_db.events").partitions()).hasSize(2); + assertThat(named.data().topics().find("second_db.events").partitions()).hasSize(2); + assertThat(named.data().topics().find("first_db.missing").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(named.data().topics().find("first_db.events").topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373000000L, first) : Uuid.ZERO_UUID); + assertThat(named.data().topics().find("second_db.events").topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373000000L, second) : Uuid.ZERO_UUID); + assertThat(named.brokers()).hasSize(1); + assertThat(named.brokers().iterator().next().port()).isEqualTo(node.port()); + + MetadataRequestData allTopics = new MetadataRequestData(); + allTopics.setTopics(version == 0 ? Collections.emptyList() : null); + MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("first_db.events", "second_db.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @Test + public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exception { + TableDescriptor descriptor = topicDescriptor(); + long existing = createTable(CLUSTER, TablePath.of("existing_db", "events"), descriptor); + long deleted = createTable(CLUSTER, TablePath.of("deleted_db", "events"), descriptor); + CLUSTER.waitUntilTableReady(existing); + CLUSTER.waitUntilTableReady(deleted); + dropDatabase(CLUSTER, "deleted_db"); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + + for (short version = 0; version <= 11; version++) { + MetadataResponse missing = + query( + node, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Collections.singletonList( + "deleted_db.events"))), + version)); + assertThat(missing.brokers()).hasSize(1); + assertThat(missing.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + + MetadataResponse mixed = + query( + node, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList( + "deleted_db.events", + "existing_db.events"))), + version)); + assertThat(mixed.brokers()).hasSize(1); + assertThat(mixed.brokers().iterator().next().port()).isEqualTo(node.port()); + assertThat(mixed.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(mixed.data().topics().find("existing_db.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(mixed.data().topics().find("existing_db.events").partitions()).hasSize(2); + + MetadataRequestData allTopics = new MetadataRequestData(); + allTopics.setTopics(version == 0 ? Collections.emptyList() : null); + MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); + assertThat(all.brokers()).hasSize(1); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("existing_db.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + private static TableDescriptor topicDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty("kafka.value.format", "raw") + .build(); + } + + private static Configuration clusterConfiguration() { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.KAFKA_ENABLED, true); + conf.set(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS, 3); + return conf; + } + + private static MetadataResponse query(ServerNode node, MetadataRequest request) + throws Exception { + RequestHeader header = + new RequestHeader(ApiKeys.METADATA, request.version(), "metadata-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] requestBytes = new byte[encoded.remaining()]; + encoded.get(requestBytes); + try (Socket socket = new Socket(node.host(), node.port())) { + socket.setSoTimeout(10000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + byte[] responseBytes = new byte[input.readInt()]; + input.readFully(responseBytes); + return (MetadataResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(responseBytes), header); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index d00c436defe..89b855f7538 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -296,6 +296,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { private static KafkaRequestHandler createKafkaRequestHandler() { TestingTabletGatewayService service = new TestingTabletGatewayService(); - return new KafkaRequestHandler(service, service, "kafka"); + return new KafkaRequestHandler(service, service); } } From 1ed86e1328ad0cb18bdf6a2c14980ebcf639b309 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 18:14:14 +0800 Subject: [PATCH 03/69] [kafka] Preserve Metadata regressions on the updated PR03 base Keep the ApiVersions wire fallback assertion limited to its retry range, while successful negotiation verifies both registered API ranges. Cover Metadata v0-v11 handling of native gateway visibility filtering in named queries, all-topic discovery, and missing-database retries. Verify Metadata v12 is rejected before invoking the gateway. Retain the existing qualified-topic discovery contract on prerequisite c508de6a8df974727c30539080d5e58f56de930a. New Core-dependent aliases and persisted table enablement remain deferred. Validation: Java 11 reactor verify passed 105 unit tests and 5 integration tests, with Spotless, Checkstyle, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 71/71 --- .../fluss/kafka/KafkaMetadataHandlerTest.java | 66 +++++++++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 5 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 4c33a51635a..53dcbeb9e48 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -480,6 +480,62 @@ public void testRejectUnqualifiedAndMalformedNamesWithoutHidingValidTopics() { .isEqualTo(Errors.NONE.code()); } + @Test + public void testUnauthorizedTopicsRemainHiddenForEveryVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.hiddenTopics.add("kafka.topic"); + for (short version = 0; version <= 11; version++) { + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other"); + for (List names : + Arrays.asList( + Arrays.asList("kafka.topic", "kafka.other"), + Arrays.asList("kafka.topic", "kafka.other", "missing_db.topic"))) { + MetadataResponse named = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest + .convertToMetadataRequestTopic( + names)), + version), + version); + assertThat(named.brokers()).hasSize(2); + MetadataResponseTopic hidden = named.data().topics().find("kafka.topic"); + assertThat(hidden.errorCode()).isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(hidden.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(hidden.partitions()).isEmpty(); + MetadataResponseTopic visible = named.data().topics().find("kafka.other"); + assertThat(visible.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(visible.partitions()).hasSize(2); + assertThat(named.data().topics()).hasSize(names.size()); + if (names.contains("missing_db.topic")) { + assertThat(named.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + } + } + } + + @Test + public void testMetadataV12IsRejectedBeforeCallingGateway() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) + .build((short) 12); + + MetadataResponse response = handle(service, request, (short) 12); + + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + assertThat(service.gatewayCalls).isZero(); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) public void testMissingDatabaseRetainsBrokersAndValidTopics(boolean failedFuture) { @@ -607,8 +663,10 @@ private static MetadataResponse handle( private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { private final Set databases = new LinkedHashSet<>(); + private final Set hiddenTopics = new LinkedHashSet<>(); private final Map tables = new LinkedHashMap<>(); private final Map descriptors = new LinkedHashMap<>(); + private int gatewayCalls; private String lastListenerName; private boolean topicLeaderAvailable = true; private int[] topicIsr = new int[] {1, 2}; @@ -627,6 +685,7 @@ private TestingMetadataGatewayService() { @Override public CompletableFuture listDatabases( ListDatabasesRequest request) { + gatewayCalls++; assertThat(currentListenerName()).isEqualTo("KAFKA"); return CompletableFuture.completedFuture( new ListDatabasesResponse().addAllDatabaseNames(databases)); @@ -634,6 +693,7 @@ public CompletableFuture listDatabases( @Override public CompletableFuture listTables(ListTablesRequest request) { + gatewayCalls++; assertThat(currentListenerName()).isEqualTo("KAFKA"); String database = request.getDatabaseName(); if (database.equals(deleteDatabaseBeforeListing)) { @@ -653,6 +713,7 @@ public CompletableFuture listTables(ListTablesRequest reques String prefix = request.getDatabaseName() + "."; List names = tables.keySet().stream() + .filter(name -> !hiddenTopics.contains(name)) .filter(name -> name.startsWith(prefix)) .map(name -> name.substring(prefix.length())) .collect(Collectors.toList()); @@ -663,6 +724,7 @@ public CompletableFuture listTables(ListTablesRequest reques @Override public CompletableFuture metadata( org.apache.fluss.rpc.messages.MetadataRequest request) { + gatewayCalls++; lastListenerName = currentListenerName(); if (failMetadata) { CompletableFuture failure = @@ -677,6 +739,10 @@ public CompletableFuture metadat List topics = new ArrayList<>(); for (PbTablePath tablePath : request.getTablePathsList()) { String topicName = tablePath.getDatabaseName() + "." + tablePath.getTableName(); + // Native Metadata and listTables omit tables without DESCRIBE permission. + if (hiddenTopics.contains(topicName)) { + continue; + } Long tableId = tables.get(topicName); if (tableId == null) { throw new TableNotExistException("Table does not exist: " + topicName); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 89b855f7538..0d06962e301 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -77,7 +77,10 @@ public void testUnsupportedWireVersionFallsBackAndAllowsRenegotiation() { ApiVersionsResponse unsupportedResponse = parseWireResponse(channel, (short) 0, 41); assertThat(unsupportedResponse.data().errorCode()) .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); - assertBrokerCapabilities(unsupportedResponse); + // The version error only advertises the range needed to retry ApiVersions. + assertThat(unsupportedResponse.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); ApiVersionsRequestData data = new ApiVersionsRequestData() From 2c7e03dda2a935300bf053b542611133fc5881d0 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 11:14:01 +0800 Subject: [PATCH 04/69] [test] Wait for coordinator metadata before snapshot lease retry Wait for all server gateways to advertise the restarted coordinator before acquire, release and drop lease checks. Assert that the client still holds its cached address so each operation exercises metadata refresh and retry. Rebased PR04 onto PR03 at 86d2735ba6c4f61eddf9d86ad9b4ef3732714972. Validation: restart case passed in the full Admin suite and focused rerun; Kafka 110 tests, RPC retry 6 tests, Spotless and Checkstyle passed. The full Admin suite passed 46/47; the unchanged rolling-upgrade health test first observed RED instead of GREEN and passed the subsequent focused rerun. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 11/11 --- .../apache/fluss/client/admin/FlussAdminITCase.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index b39739c6b01..8938993fa5a 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -19,8 +19,10 @@ import org.apache.fluss.client.Connection; import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.FlussConnection; import org.apache.fluss.client.metadata.KvSnapshotMetadata; import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.MetadataUpdater; import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.writer.AppendWriter; @@ -1211,19 +1213,24 @@ void testKvSnapshotLeaseAfterCoordinatorServerRestart() throws Exception { assertThat(zkClient.getKvSnapshotLeaseMetadata(lease.leaseId())).isPresent(); restartCoordinatorServer(zkClient); - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); - lease.dropLease().get(); assertThat(zkClient.getKvSnapshotLeaseMetadata(lease.leaseId())).isNotPresent(); } private void restartCoordinatorServer(ZooKeeperClient zkClient) throws Exception { + MetadataUpdater metadataUpdater = ((FlussConnection) conn).getMetadataUpdater(); + ServerNode cachedCoordinator = metadataUpdater.getCoordinatorServer(); FLUSS_CLUSTER_EXTENSION.stopCoordinatorServer(); waitUntil( () -> !zkClient.getCoordinatorLeaderAddress().isPresent(), Duration.ofMinutes(1), "Coordinator server node still exists in ZooKeeper"); FLUSS_CLUSTER_EXTENSION.startCoordinatorServer(); + // Election completes before the new coordinator address reaches every tablet server. + // Wait for server metadata to converge so that the client's retry can discover it. + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + // Keep the client cache stale to exercise the lease's metadata refresh and retry. + assertThat(metadataUpdater.getCoordinatorServer()).isEqualTo(cachedCoordinator); } @Test From a23937ddf5c7154242327464e62fb7d7f281f811 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 17:40:33 +0800 Subject: [PATCH 05/69] [kafka] Apply persisted topic enablement to Metadata Use the actual server default database and the PR03 shared mapper/schema contract. Resolve aliases by TablePath, preserve requested names, and retain valid aliases when a missing database triggers metadata discovery retry. Expose only tables with persisted table.kafka.enabled. Return unknown-topic for disabled or missing tables and invalid-topic for damaged enabled contracts in named and all-topic requests. Discovery emits canonical names without duplicate aliases and retains native authorization and routing. Verify configuration injection, no fallback lookup or database creation, enablement/reset changes, invalid historical contracts and v0-v11 behavior through real Kafka listeners. Preserve the coordinator snapshot lease fix. Validation: Java 11 clean reactor verify passed 160 distinct cases: 118 Kafka (including 6 Metadata integration cases), 31 shared schema, 3 atomic DDL, 6 RPC retry and 2 snapshot lease cases. Spotless, Checkstyle and RAT passed. The four existing PR04 commits were rebased unchanged onto PR03 4a9aa6b007053ba7af5e8c0a0222a106c477a9f2 and checked with range-diff. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 155/155 AI-Contributed/UT: 571/571 --- .../fluss/kafka/KafkaProtocolPlugin.java | 2 +- .../fluss/kafka/KafkaRequestHandler.java | 10 +- .../kafka/api/metadata/MetadataHandler.java | 18 +- .../metadata/GatewayKafkaMetadataBackend.java | 124 +++---- .../fluss/kafka/KafkaMetadataHandlerTest.java | 265 ++++++++++++++- .../fluss/kafka/KafkaMetadataITCase.java | 303 +++++++++++++++++- .../fluss/kafka/KafkaRequestHandlerTest.java | 3 +- 7 files changed, 642 insertions(+), 83 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index dddfff1f9eb..d0363e3f6f4 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -67,6 +67,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { + service.getClass().getSimpleName()); } TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(service, gateway); + return new KafkaRequestHandler(conf, service, gateway); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 3278d8328a2..46dc00355bf 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,12 +17,14 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; import org.apache.fluss.kafka.api.metadata.MetadataHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; @@ -39,15 +41,19 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ - public KafkaRequestHandler(RpcGatewayService service, TabletServerGateway gateway) { + public KafkaRequestHandler( + Configuration conf, RpcGatewayService service, TabletServerGateway gateway) { checkNotNull(service); checkNotNull(gateway); + KafkaTopicMapper topicMapper = new KafkaTopicMapper(checkNotNull(conf)); this.registry = new KafkaApiRegistry( Arrays.asList( new ApiVersionsHandler(this::apiRegistry), new MetadataHandler( - new GatewayKafkaMetadataBackend(service, gateway)))); + new GatewayKafkaMetadataBackend( + service, gateway, topicMapper), + topicMapper))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java index 2ceca11fbe1..2787cf83124 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -32,6 +32,7 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.internals.Topic; import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; import org.apache.kafka.common.message.MetadataResponseData; @@ -64,10 +65,12 @@ public final class MetadataHandler implements KafkaApiHandler { true); private final KafkaMetadataBackend backend; + private final KafkaTopicMapper topicMapper; /** Creates a Metadata handler. */ - public MetadataHandler(KafkaMetadataBackend backend) { + public MetadataHandler(KafkaMetadataBackend backend, KafkaTopicMapper topicMapper) { this.backend = checkNotNull(backend); + this.topicMapper = checkNotNull(topicMapper); } @Override @@ -86,7 +89,7 @@ public CompletableFuture handle( throw new InvalidRequestException( "Topic name must be set because topic ID lookup is not supported by " + "Metadata versions 10 and 11."); - } else if (!KafkaTopicMapper.isValidTopic(topic.name())) { + } else if (!isValidTopic(topic.name())) { invalidTopics.add( new KafkaClusterMetadata.Topic( topic.name(), @@ -119,6 +122,17 @@ public CompletableFuture handle( }); } + private boolean isValidTopic(String name) { + try { + // Validate the resolved path too: a bare name's canonical form includes the configured + // default database and must fit the same Kafka name limit as an explicit name. + topicMapper.toTablePath(name); + return true; + } catch (InvalidTopicException e) { + return false; + } + } + private static MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { MetadataResponseData data = new MetadataResponseData() diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java index 202658b9f0f..0d4ed5a78c3 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -44,9 +44,12 @@ import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -69,41 +72,42 @@ public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { private final RpcGatewayService service; private final TabletServerGateway gateway; - private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); + private final KafkaTopicMapper topicMapper; private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); /** Creates a metadata backend backed by the local TabletServer gateway. */ - public GatewayKafkaMetadataBackend(RpcGatewayService service, TabletServerGateway gateway) { + public GatewayKafkaMetadataBackend( + RpcGatewayService service, TabletServerGateway gateway, KafkaTopicMapper topicMapper) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); + this.topicMapper = checkNotNull(topicMapper); } @Override public CompletableFuture getMetadata(KafkaMetadataQuery query) { if (query.allTopics() || containsTopicId(query.topics())) { - return currentTopicNames(query) - .thenCompose(topicNames -> requestFlussMetadata(query, topicNames)); + return currentTablePaths(query) + .thenCompose(tablePaths -> requestFlussMetadata(query, tablePaths)); } - Set topicNames = new LinkedHashSet<>(); + Set tablePaths = new LinkedHashSet<>(); for (TopicReference topic : query.topics()) { if (topic.topicName() != null) { - topicNames.add(topic.topicName()); + tablePaths.add(topicMapper.toTablePath(topic.topicName())); } } - return requestFlussMetadata(query, topicNames); + return requestFlussMetadata(query, tablePaths); } private CompletableFuture requestFlussMetadata( - KafkaMetadataQuery query, Set topicNames) { - return requestFlussMetadata(query, topicNames, true); + KafkaMetadataQuery query, Set tablePaths) { + return requestFlussMetadata(query, tablePaths, true); } private CompletableFuture requestFlussMetadata( - KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { + KafkaMetadataQuery query, Set tablePaths, boolean refreshAndRetry) { MetadataRequest request = new MetadataRequest(); - for (String topicName : topicNames) { - TablePath tablePath = topicMapper.toTablePath(topicName); + for (TablePath tablePath : tablePaths) { request.addAllTablePaths( Collections.singletonList( new PbTablePath() @@ -129,8 +133,8 @@ private CompletableFuture requestFlussMetadata( private CompletableFuture recoverMetadataFailure( KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) { if (refreshAndRetry) { - return currentTopicNames(query) - .thenCompose(currentNames -> requestFlussMetadata(query, currentNames, false)); + return currentTablePaths(query) + .thenCompose(currentPaths -> requestFlussMetadata(query, currentPaths, false)); } LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure)); CompletableFuture failed = new CompletableFuture<>(); @@ -138,7 +142,7 @@ private CompletableFuture recoverMetadataFailure( return failed; } - private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { + private CompletableFuture> currentTablePaths(KafkaMetadataQuery query) { CompletableFuture> databasesFuture; if (query.allTopics() || containsTopicId(query.topics())) { setCurrentSession(query); @@ -157,49 +161,46 @@ private CompletableFuture> currentTopicNames(KafkaMetadataQuery quer databasesFuture = CompletableFuture.completedFuture(databases); } return databasesFuture - .thenCompose(databases -> listTopicNames(query, databases)) + .thenCompose(databases -> listTablePaths(query, databases)) .thenApply( - currentNames -> { + currentPaths -> { if (!query.allTopics() && !containsTopicId(query.topics())) { - Set requestedNames = new HashSet<>(); + Set requestedTables = new HashSet<>(); for (TopicReference topic : query.topics()) { if (topic.topicName() != null) { - requestedNames.add(topic.topicName()); + requestedTables.add( + topicMapper.toTablePath(topic.topicName())); } } - currentNames.retainAll(requestedNames); + // Canonical names and aliases share one table identity, including + // when a missing database triggers this discovery retry. + currentPaths.retainAll(requestedTables); } - return currentNames; + return currentPaths; }); } - private CompletableFuture> listTopicNames( + private CompletableFuture> listTablePaths( KafkaMetadataQuery query, Set databases) { - CompletableFuture> topicsFuture = + CompletableFuture> tablesFuture = CompletableFuture.completedFuture(new LinkedHashSet<>()); for (String database : databases) { - topicsFuture = - topicsFuture.thenCompose( - topicNames -> + tablesFuture = + tablesFuture.thenCompose( + tablePaths -> listTables(query, database) .thenApply( response -> { for (String tableName : response.getTableNamesList()) { - TablePath tablePath = + tablePaths.add( TablePath.of( - database, tableName); - if (topicMapper.isMappedTable( - tablePath)) { - topicNames.add( - topicMapper.toTopicName( - tablePath)); - } + database, tableName)); } - return topicNames; + return tablePaths; })); } - return topicsFuture; + return tablesFuture; } private CompletableFuture listTables( @@ -238,37 +239,40 @@ private KafkaClusterMetadata toKafkaMetadata( } Collections.sort(brokers, Comparator.comparingInt(Broker::id)); - Map topicsByName = new HashMap<>(); + Map topicsByTable = new HashMap<>(); Map topicsById = new HashMap<>(); for (PbTableMetadata table : response.getTableMetadatasList()) { TablePath tablePath = TablePath.of( table.getTablePath().getDatabaseName(), table.getTablePath().getTableName()); - if (!topicMapper.isMappedTable(tablePath)) { - continue; + Topic topic = toKafkaTopic(table, tablePath, aliveBrokerIds); + if (topic != null) { + topicsByTable.put(tablePath, topic); + topicsById.put(topic.topicId(), topic); } - Topic topic = toKafkaTopic(table, topicMapper.toTopicName(tablePath), aliveBrokerIds); - topicsByName.put(topic.name(), topic); - topicsById.put(topic.topicId(), topic); } List topics = new ArrayList<>(); if (query.allTopics()) { - for (Topic topic : topicsByName.values()) { - if (topic.error() == TopicError.NONE) { - topics.add(topic); - } - } + // Invalid enabled tables remain visible as errors. Only disabled tables are omitted. + topics.addAll(topicsByTable.values()); Collections.sort(topics, Comparator.comparing(Topic::name)); } else { for (TopicReference reference : query.topics()) { Topic topic = reference.hasTopicId() ? topicsById.get(reference.topicId()) - : topicsByName.get(reference.topicName()); + : topicsByTable.get(topicMapper.toTablePath(reference.topicName())); if (topic != null && matches(reference, topic)) { - topics.add(topic); + topics.add( + new Topic( + reference.topicName() == null + ? topic.name() + : reference.topicName(), + topic.topicId(), + topic.error(), + topic.partitions())); } else { topics.add(missingTopic(reference)); } @@ -277,13 +281,20 @@ private KafkaClusterMetadata toKafkaMetadata( return new KafkaClusterMetadata(brokers, topics); } + @Nullable private Topic toKafkaTopic( - PbTableMetadata table, String topicName, Set aliveBrokerIds) { + PbTableMetadata table, TablePath tablePath, Set aliveBrokerIds) { + String topicName = tablePath.toString(); try { - schemaResolver.resolve(TableDescriptor.fromJsonBytes(table.getTableJson())); - } catch (IllegalArgumentException e) { + TableDescriptor descriptor = TableDescriptor.fromJsonBytes(table.getTableJson()); + if (!schemaResolver.isEnabled(descriptor)) { + return null; + } + topicMapper.toTopicName(tablePath); + schemaResolver.resolve(descriptor); + } catch (IllegalArgumentException | InvalidTopicException e) { LOG.debug( - "Table {} does not define a supported Kafka mapping: {}", + "Enabled table {} has an invalid Kafka contract: {}", topicName, e.getMessage()); return new Topic( @@ -342,8 +353,11 @@ private static Topic missingTopic(TopicReference reference) { reference.topicName(), reference.topicId(), error, Collections.emptyList()); } - private static boolean matches(TopicReference reference, Topic topic) { - return (reference.topicName() == null || reference.topicName().equals(topic.name())) + private boolean matches(TopicReference reference, Topic topic) { + return (reference.topicName() == null + || topicMapper + .toTablePath(reference.topicName()) + .equals(topicMapper.toTablePath(topic.name()))) && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId())); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 53dcbeb9e48..bd06a8cb78e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -17,9 +17,10 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.exception.TableNotExistException; -import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; @@ -347,14 +348,17 @@ public void testMetadataUsesDdlContractForEveryVersion() { "kafka.no_mapping", 125L, TableDescriptor.builder() - .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) .distributedBy(2) .build()); service.putTable( "kafka.bad_format", 126L, TableDescriptor.builder(defaultDescriptor()) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "unimplemented") .build()); service.putTable( "kafka.primary_key", @@ -362,14 +366,14 @@ public void testMetadataUsesDdlContractForEveryVersion() { TableDescriptor.builder(defaultDescriptor()) .schema( Schema.newBuilder() - .column("body", DataTypes.BYTES().copy(false)) - .primaryKey("body") + .column("_kafka_value", DataTypes.BYTES().copy(false)) + .primaryKey("_kafka_value") .build()) .build()); service.putTable( "kafka.partitioned", 128L, - TableDescriptor.builder(defaultDescriptor()).partitionedBy("body").build()); + TableDescriptor.builder(defaultDescriptor()).partitionedBy("_kafka_value").build()); service.putTable( "kafka.indexed", 129L, @@ -377,17 +381,24 @@ public void testMetadataUsesDdlContractForEveryVersion() { service.putTable("kafka.invalid topic", 130L); List invalidMappings = Arrays.asList( - "kafka.no_mapping", "kafka.bad_format", "kafka.primary_key", "kafka.partitioned", - "kafka.indexed"); + "kafka.indexed", + "kafka.invalid topic"); for (short version = 0; version <= 11; version++) { MetadataResponse all = handle(service, allTopicsRequest(version), version); + List exposed = new ArrayList<>(invalidMappings); + exposed.addAll(Arrays.asList("kafka.other", "kafka.topic")); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactly("kafka.other", "kafka.topic"); + .containsExactlyInAnyOrderElementsOf(exposed); + for (String name : invalidMappings) { + assertThat(all.data().topics().find(name).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + } List requested = new ArrayList<>(invalidMappings); + requested.add("kafka.no_mapping"); requested.add("kafka.topic"); requested.add("kafka.missing"); MetadataResponse named = @@ -409,6 +420,8 @@ public void testMetadataUsesDdlContractForEveryVersion() { .isEqualTo(Errors.NONE.code()); assertThat(named.data().topics().find("kafka.missing").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(named.data().topics().find("kafka.no_mapping").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); } } @@ -419,7 +432,7 @@ public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { "kafka.topic", 123L, TableDescriptor.builder(defaultDescriptor()) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "string") .build()); MetadataResponse invalid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); assertThat(invalid.data().topics().find("kafka.topic").errorCode()) @@ -462,9 +475,9 @@ public void testSameTableNameInDifferentDatabasesForEveryVersion() { } @Test - public void testRejectUnqualifiedAndMalformedNamesWithoutHidingValidTopics() { + public void testRejectMalformedNamesWithoutHidingValidTopics() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - List invalidNames = Arrays.asList("topic", ".topic", "kafka.", "kafka.topic.extra"); + List invalidNames = Arrays.asList(".topic", "kafka.", "kafka.topic.extra"); List names = new ArrayList<>(invalidNames); names.add("kafka.topic"); MetadataResponse response = @@ -603,12 +616,220 @@ public void testUnexpectedListTablesFailureIsNotTreatedAsMissingDatabase(boolean .isEqualTo(Errors.UNKNOWN_SERVER_ERROR.code()); } + @ParameterizedTest + @ValueSource(strings = {"fluss", "custom_db"}) + public void testAliasesPreserveRequestedNamesAndShareIdentity(String defaultDatabase) { + Configuration config = new Configuration(); + if (!"fluss".equals(defaultDatabase)) { + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, defaultDatabase); + } + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + String canonical = defaultDatabase + ".topic"; + service.putTable(canonical, 777L); + service.putTable("sales.topic", 778L); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle( + service, + topicsRequest(version, "topic", canonical, "sales.topic"), + version, + config); + assertThat(named.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("topic", canonical, "sales.topic"); + assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = named.data().topics().find("topic"); + MetadataResponseTopic qualified = named.data().topics().find(canonical); + assertThat(alias.partitions()).isEqualTo(qualified.partitions()); + assertThat(alias.topicId()).isEqualTo(qualified.topicId()); + if (version >= 10) { + assertThat(alias.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)) + .isNotEqualTo(named.data().topics().find("sales.topic").topicId()); + } + MetadataResponse all = handle(service, allTopicsRequest(version), version, config); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .contains(canonical, "sales.topic") + .doesNotContain("topic") + .doesNotHaveDuplicates(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testAliasSurvivesMissingDatabaseRetry(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + service.failListTablesAsFuture = failedFuture; + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + handle( + service, + topicsRequest( + version, + "topic", + "missing_db.topic", + "fluss.topic", + "kafka.topic"), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(4); + for (String name : Arrays.asList("topic", "fluss.topic", "kafka.topic")) { + assertThat(response.data().topics().find(name).errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(response.data().topics().find(name).partitions()).hasSize(2); + } + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("topic").topicId()) + .isEqualTo(response.data().topics().find("fluss.topic").topicId()); + } + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.listedDatabases) + .containsExactlyInAnyOrder("fluss", "kafka", "missing_db"); + } + + @Test + public void testMissingDefaultDatabaseNeverSearchesOtherDatabases() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + MetadataResponse response = + handle( + service, + topicsRequest((short) 11, "topic", "kafka.topic"), + (short) 11, + config); + assertThat(response.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.requestedTables) + .containsExactlyInAnyOrder("missing_default.topic", "kafka.topic"); + assertThat(service.listedDatabases).containsExactlyInAnyOrder("missing_default", "kafka"); + } + + @Test + public void testAliasAuthorizationDoesNotFallbackToSameNamedVisibleTable() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + service.hiddenTopics.add("fluss.topic"); + service.putTable("sales.topic", 778L); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + handle( + service, + topicsRequest( + version, + "topic", + "fluss.topic", + "sales.topic", + "missing_db.topic"), + version); + for (String hidden : Arrays.asList("topic", "fluss.topic")) { + MetadataResponseTopic topic = response.data().topics().find(hidden); + assertThat(topic.errorCode()).isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + } + assertThat(response.data().topics().find("sales.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .contains("sales.topic") + .doesNotContain("topic", "fluss.topic"); + } + assertThat(service.requestedTables).doesNotContain("sales.other"); + } + + @Test + public void testPersistedEnablementControlsBothAliasesAndDiscovery() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + TableDescriptor missingFlag = + TableDescriptor.builder() + .schema(defaultDescriptor().getSchema()) + .distributedBy(2) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + TableDescriptor disabledInvalid = + TableDescriptor.builder(defaultDescriptor()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "unimplemented") + .build(); + for (TableDescriptor disabled : Arrays.asList(missingFlag, disabledInvalid)) { + service.putTable("fluss.topic", 777L, disabled); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + assertThat(named.errors()) + .containsEntry("topic", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("fluss.topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(named.data().topics()) + .allSatisfy( + topic -> { + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + }); + assertThat(handle(service, allTopicsRequest(version), version).data().topics()) + .extracting(MetadataResponseTopic::name) + .doesNotContain("topic", "fluss.topic"); + } + } + service.putTable("fluss.topic", 777L); + MetadataResponse enabled = + handle(service, topicsRequest((short) 11, "topic", "fluss.topic"), (short) 11); + assertThat(enabled.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(enabled.data().topics().find("topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)); + service.putTable("fluss.topic", 777L, disabledInvalid); + assertThat(handle(service, namedTopicRequest("topic"), (short) 11).errors()) + .containsEntry("topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + + @Test + public void testResolvedAliasNameLimitDoesNotHideValidTopic() { + String database = String.join("", Collections.nCopies(125, "d")); + String table = String.join("", Collections.nCopies(125, "t")); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, database); + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + MetadataResponse response = + handle( + service, + topicsRequest((short) 11, table, "kafka.topic"), + (short) 11, + config); + assertThat(response.errors()).containsEntry(table, Errors.INVALID_TOPIC_EXCEPTION); + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(service.requestedTables).containsExactly("kafka.topic"); + + // A historical enabled table with an unrepresentable canonical name must be visible as + // an error during discovery rather than silently classified as disabled. + service.putTable(database + "." + table, 777L); + assertThat(handle(service, allTopicsRequest((short) 11), (short) 11, config).errors()) + .containsEntry(database + "." + table, Errors.INVALID_TOPIC_EXCEPTION); + } + + private static MetadataRequest topicsRequest(short version, String... names) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList(names))), + version); + } + private static TableDescriptor defaultDescriptor() { return TableDescriptor.builder() - .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) .distributedBy(2) .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") .build(); } @@ -631,7 +852,15 @@ private static MetadataRequest allTopicsRequest(short version) { private static MetadataResponse handle( TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { - KafkaRequestHandler handler = new KafkaRequestHandler(service, service); + return handle(service, requestBody, version, new Configuration()); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, + MetadataRequest requestBody, + short version, + Configuration config) { + KafkaRequestHandler handler = new KafkaRequestHandler(config, service, service); ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); KafkaRequest request; try { @@ -667,6 +896,9 @@ private static final class TestingMetadataGatewayService extends TestingTabletGa private final Map tables = new LinkedHashMap<>(); private final Map descriptors = new LinkedHashMap<>(); private int gatewayCalls; + private int listDatabasesCalls; + private final Set requestedTables = new LinkedHashSet<>(); + private final Set listedDatabases = new LinkedHashSet<>(); private String lastListenerName; private boolean topicLeaderAvailable = true; private int[] topicIsr = new int[] {1, 2}; @@ -686,6 +918,7 @@ private TestingMetadataGatewayService() { public CompletableFuture listDatabases( ListDatabasesRequest request) { gatewayCalls++; + listDatabasesCalls++; assertThat(currentListenerName()).isEqualTo("KAFKA"); return CompletableFuture.completedFuture( new ListDatabasesResponse().addAllDatabaseNames(databases)); @@ -696,6 +929,7 @@ public CompletableFuture listTables(ListTablesRequest reques gatewayCalls++; assertThat(currentListenerName()).isEqualTo("KAFKA"); String database = request.getDatabaseName(); + listedDatabases.add(database); if (database.equals(deleteDatabaseBeforeListing)) { databases.remove(database); tables.keySet().removeIf(name -> name.startsWith(database + ".")); @@ -739,6 +973,7 @@ public CompletableFuture metadat List topics = new ArrayList<>(); for (PbTablePath tablePath : request.getTablePathsList()) { String topicName = tablePath.getDatabaseName() + "." + tablePath.getTableName(); + requestedTables.add(topicName); // Native Metadata and listTables omit tables without DESCRIBE permission. if (hiddenTopics.contains(topicName)) { continue; diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java index 2f18ffe495d..d162ed837a4 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -25,6 +25,7 @@ import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.data.TableRegistration; import org.apache.fluss.types.DataTypes; import org.apache.kafka.common.Uuid; @@ -43,11 +44,17 @@ import java.io.DataOutputStream; import java.net.Socket; import java.nio.ByteBuffer; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.dropDatabase; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newDatabaseExistsRequest; +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.assertj.core.api.Assertions.assertThat; /** Verifies qualified Metadata discovery over a real Kafka listener and TabletServer. */ @@ -56,7 +63,7 @@ public class KafkaMetadataITCase { @RegisterExtension public static final FlussClusterExtension CLUSTER = FlussClusterExtension.builder() - .setNumOfTabletServers(1) + .setNumOfTabletServers(2) .setClusterConf(clusterConfiguration()) .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") .build(); @@ -93,7 +100,7 @@ public void testQualifiedMetadataAcrossDatabases() throws Exception { assertThat(named.data().topics().find("second_db.events").topicId()) .isEqualTo( version >= 10 ? new Uuid(0x466c757373000000L, second) : Uuid.ZERO_UUID); - assertThat(named.brokers()).hasSize(1); + assertThat(named.brokers()).hasSize(2); assertThat(named.brokers().iterator().next().port()).isEqualTo(node.port()); MetadataRequestData allTopics = new MetadataRequestData(); @@ -127,7 +134,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio Collections.singletonList( "deleted_db.events"))), version)); - assertThat(missing.brokers()).hasSize(1); + assertThat(missing.brokers()).hasSize(2); assertThat(missing.data().topics().find("deleted_db.events").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); @@ -142,7 +149,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio "deleted_db.events", "existing_db.events"))), version)); - assertThat(mixed.brokers()).hasSize(1); + assertThat(mixed.brokers()).hasSize(2); assertThat(mixed.brokers().iterator().next().port()).isEqualTo(node.port()); assertThat(mixed.data().topics().find("deleted_db.events").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); @@ -153,7 +160,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio MetadataRequestData allTopics = new MetadataRequestData(); allTopics.setTopics(version == 0 ? Collections.emptyList() : null); MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); - assertThat(all.brokers()).hasSize(1); + assertThat(all.brokers()).hasSize(2); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) .containsExactly("existing_db.events"); @@ -161,18 +168,300 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio } } + @Test + public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception { + long other = createTable(CLUSTER, TablePath.of("sales", "events"), topicDescriptor()); + CLUSTER.waitUntilTableReady(other); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + for (short version = 0; version <= 11; version++) { + MetadataResponse before = query(node, named(version, "events", "sales.events")); + assertThat(before.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(before.data().topics().find("sales.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + long tableId = createTable(CLUSTER, TablePath.of("fluss", "events"), topicDescriptor()); + CLUSTER.waitUntilTableReady(tableId); + for (short version = 0; version <= 11; version++) { + for (ServerNode broker : CLUSTER.getTabletServerNodes("KAFKA")) { + // The missing database forces the same discovery retry that used to discard + // aliases. + MetadataResponse response = + query( + broker, + named( + version, + "events", + "fluss.events", + "sales.events", + "missing_db.events")); + MetadataResponseTopic alias = response.data().topics().find("events"); + MetadataResponseTopic canonical = response.data().topics().find("fluss.events"); + assertThat(alias.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(alias.partitions()).isEqualTo(canonical.partitions()).hasSize(2); + assertThat(alias.partitions()) + .allSatisfy( + partition -> { + assertThat(partition.leaderId()).isIn(0, 1); + assertThat(partition.replicaNodes()) + .containsExactlyInAnyOrder(0, 1); + assertThat(partition.isrNodes()) + .containsExactlyInAnyOrder(0, 1); + }); + assertThat(alias.topicId()).isEqualTo(canonical.topicId()); + if (version >= 10) { + assertThat(alias.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, tableId)) + .isNotEqualTo(response.data().topics().find("sales.events").topicId()); + } + assertThat(response.data().topics().find("missing_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(query(broker, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events", "sales.events"); + } + } + } + + @Test + public void testPersistedEnablementChangesAreVisibleThroughBothNames() throws Exception { + TablePath path = TablePath.of("fluss", "events"); + TableDescriptor disabled = + TableDescriptor.builder(topicDescriptor()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build(); + TableDescriptor unset = + TableDescriptor.builder() + .schema(topicDescriptor().getSchema()) + .distributedBy(2) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + long tableId = createTable(CLUSTER, path, disabled); + long unsetId = createTable(CLUSTER, TablePath.of("sales", "unset"), unset); + CLUSTER.waitUntilTableReady(tableId); + CLUSTER.waitUntilTableReady(unsetId); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + assertExposure(node, false, tableId); + setEnabled(CLUSTER, path, true); + assertExposure(node, true, tableId); + setEnabled(CLUSTER, path, false); + assertExposure(node, false, tableId); + setEnabled(CLUSTER, path, true); + assertExposure(node, true, tableId); + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList(ConfigOptions.TABLE_KAFKA_ENABLED.key()), + Collections.emptyList(), + false)) + .get(); + assertExposure(node, false, tableId); + } + + @Test + public void testInvalidEnabledContractRemainsVisibleInNamedAndAllTopics() throws Exception { + TablePath path = TablePath.of("fluss", "events"); + long tableId = createTable(CLUSTER, path, topicDescriptor()); + long validId = createTable(CLUSTER, TablePath.of("sales", "valid"), topicDescriptor()); + CLUSTER.waitUntilTableReady(tableId); + CLUSTER.waitUntilTableReady(validId); + TableRegistration registration = CLUSTER.getZooKeeperClient().getTable(path).get(); + Map invalidProperties = new HashMap<>(registration.properties); + invalidProperties.put(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "string"); + // Bypass current DDL validation to model historical persisted metadata with a damaged + // contract. Metadata must expose the enabled table's error, not hide it as disabled. + CLUSTER.getZooKeeperClient() + .updateTable( + path, + registration.newProperties( + invalidProperties, registration.customProperties)); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + query(node, named((short) 11, "events")) + .data() + .topics() + .find("events") + .errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code())); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + query(node, named(version, "events", "fluss.events", "sales.valid")); + for (String topic : Arrays.asList("events", "fluss.events")) { + assertThat(named.data().topics().find(topic).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(named.data().topics().find(topic).partitions()).isEmpty(); + } + MetadataResponse all = query(node, allTopics(version)); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events", "sales.valid"); + assertThat(all.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(all.data().topics().find("sales.valid").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + } + + @Test + public void testCustomDefaultDatabaseIsInjectedAndNeverCreatedByMetadata() throws Exception { + Configuration config = clusterConfiguration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "custom_default"); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + FlussClusterExtension cluster = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(config) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + cluster.start(); + try { + long nativeDefaultId = + createTable(cluster, TablePath.of("fluss", "events"), topicDescriptor()); + cluster.waitUntilTableReady(nativeDefaultId); + ServerNode node = cluster.getTabletServerInfos().get(0).node("KAFKA"); + assertThat( + cluster.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("custom_default")) + .get() + .isExists()) + .isFalse(); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = query(node, named(version, "events", "fluss.events")); + assertThat(response.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + long customId = + createTable( + cluster, TablePath.of("custom_default", "events"), topicDescriptor()); + cluster.waitUntilTableReady(customId); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + query( + node, + named(version, "events", "custom_default.events", "fluss.events")); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = response.data().topics().find("events"); + assertThat(alias.partitions()) + .isEqualTo( + response.data() + .topics() + .find("custom_default.events") + .partitions()); + if (version >= 10) { + assertThat(alias.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, customId)) + .isNotEqualTo(response.data().topics().find("fluss.events").topicId()); + } + assertThat(query(node, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("custom_default.events", "fluss.events"); + } + dropDatabase(cluster, "custom_default"); + MetadataResponse deleted = query(node, named((short) 11, "events", "fluss.events")); + assertThat(deleted.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(deleted.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat( + cluster.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("custom_default")) + .get() + .isExists()) + .isFalse(); + } finally { + cluster.close(); + } + } + + private static void assertExposure(ServerNode node, boolean enabled, long tableId) + throws Exception { + short expected = enabled ? Errors.NONE.code() : Errors.UNKNOWN_TOPIC_OR_PARTITION.code(); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + query(node, named((short) 11, "events")) + .data() + .topics() + .find("events") + .errorCode()) + .isEqualTo(expected)); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + query(node, named(version, "events", "fluss.events", "sales.unset")); + for (String name : Arrays.asList("events", "fluss.events")) { + MetadataResponseTopic topic = response.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(expected); + assertThat(topic.topicId()) + .isEqualTo( + enabled && version >= 10 + ? new Uuid(0x466c757373000000L, tableId) + : Uuid.ZERO_UUID); + } + assertThat(response.data().topics().find("sales.unset").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + if (enabled) { + assertThat(query(node, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events"); + } else { + assertThat(query(node, allTopics(version)).data().topics()).isEmpty(); + } + } + } + + private static void setEnabled(FlussClusterExtension cluster, TablePath path, boolean enabled) + throws Exception { + cluster.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled)), + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private static MetadataRequest named(short version, String... names) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList(names))), + version); + } + + private static MetadataRequest allTopics(short version) { + return new MetadataRequest( + new MetadataRequestData().setTopics(version == 0 ? Collections.emptyList() : null), + version); + } + private static TableDescriptor topicDescriptor() { return TableDescriptor.builder() - .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) .distributedBy(2) .logFormat(LogFormat.ARROW) - .customProperty("kafka.value.format", "raw") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") .build(); } private static Configuration clusterConfiguration() { Configuration conf = new Configuration(); conf.set(ConfigOptions.KAFKA_ENABLED, true); + conf.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 2); conf.set(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS, 3); return conf; } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 0d06962e301..f55b1832c0b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; @@ -299,6 +300,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { private static KafkaRequestHandler createKafkaRequestHandler() { TestingTabletGatewayService service = new TestingTabletGatewayService(); - return new KafkaRequestHandler(service, service); + return new KafkaRequestHandler(new Configuration(), service, service); } } From 993d234e7be1239218aa8f9804904951fcd28676 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 18:44:54 +0800 Subject: [PATCH 06/69] [kafka] Give topic aliases distinct stable wire IDs Keep canonical topic IDs unchanged and encode alias wire IDs in a separate namespace using the same Fluss table ID. Kafka clients can then cache both names and detect table recreation even when the new leader epoch is lower. Cover Kafka Metadata v10/v11 full and partial updates, recreation from epoch 5 to 0 without a missing-topic response, and native Metadata responses. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 31/31 AI-Contributed/UT: 135/135 --- .../kafka/api/metadata/MetadataHandler.java | 12 ++- .../fluss/kafka/mapping/KafkaTopicMapper.java | 19 +++- .../fluss/kafka/KafkaMetadataHandlerTest.java | 96 ++++++++++++++++++- .../fluss/kafka/KafkaMetadataITCase.java | 23 ++++- .../kafka/mapping/KafkaTopicMapperTest.java | 16 ++++ 5 files changed, 155 insertions(+), 11 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java index 2787cf83124..bffc28d6de2 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -133,7 +133,7 @@ private boolean isValidTopic(String name) { } } - private static MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { + private MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { MetadataResponseData data = new MetadataResponseData() .setThrottleTimeMs(0) @@ -155,7 +155,15 @@ private static MetadataResponse toResponse(short version, KafkaClusterMetadata m MetadataResponseData.MetadataResponseTopic responseTopic = new MetadataResponseData.MetadataResponseTopic() .setName(topic.name()) - .setTopicId(topic.topicId()) + // The backend retains the shared table identity. Distinguish names + // only on the wire so clients can cache both and detect recreation. + .setTopicId( + topic.name() != null + && topic.name().indexOf('.') < 0 + && topicMapper.isMappedTopicId(topic.topicId()) + ? topicMapper.toAliasTopicId( + topicMapper.toTableId(topic.topicId())) + : topic.topicId()) .setErrorCode(toKafkaError(topic.error()).code()) .setIsInternal(topic.name() != null && Topic.isInternal(topic.name())) .setTopicAuthorizedOperations( diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index aa31d618986..8098917cc65 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -35,6 +35,8 @@ public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + // ASCII "FlussA": Kafka clients require distinct IDs for distinct wire topic names. + private static final long ALIAS_TOPIC_ID_NAMESPACE = 0x466c757373410000L; private final String defaultDatabase; @@ -104,16 +106,29 @@ public static boolean isValidTopic(String topicName) { topicName.substring(0, separator), topicName.substring(separator + 1))); } - /** Maps a Fluss table ID to a stable Kafka topic ID. */ + /** Maps a Fluss table ID to the canonical name's stable Kafka topic ID. */ public Uuid toTopicId(long tableId) { checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); return new Uuid(TOPIC_ID_NAMESPACE, tableId); } + /** + * Maps a Fluss table ID to the default-database alias's stable Kafka topic ID. + * + *

The wire ID must differ from the canonical name's ID because Kafka clients maintain a + * one-to-one ID-to-name map. It must also change on recreation so a client can accept a new + * table's lower leader epoch without first observing a missing-topic response. + */ + public Uuid toAliasTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(ALIAS_TOPIC_ID_NAMESPACE, tableId); + } + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ public boolean isMappedTopicId(Uuid topicId) { return topicId != null - && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && (topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + || topicId.getMostSignificantBits() == ALIAS_TOPIC_ID_NAMESPACE) && topicId.getLeastSignificantBits() >= 0L; } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index bd06a8cb78e..73801d678c5 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -38,8 +38,11 @@ import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.concurrent.FutureUtils; +import org.apache.kafka.clients.Metadata; import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.message.MetadataRequestData; import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; @@ -51,6 +54,7 @@ import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.utils.LogContext; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -641,10 +645,13 @@ public void testAliasesPreserveRequestedNamesAndShareIdentity(String defaultData MetadataResponseTopic alias = named.data().topics().find("topic"); MetadataResponseTopic qualified = named.data().topics().find(canonical); assertThat(alias.partitions()).isEqualTo(qualified.partitions()); - assertThat(alias.topicId()).isEqualTo(qualified.topicId()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373410000L, 777L) : Uuid.ZERO_UUID); if (version >= 10) { - assertThat(alias.topicId()) + assertThat(qualified.topicId()) .isEqualTo(new Uuid(0x466c757373000000L, 777L)) + .isNotEqualTo(alias.topicId()) .isNotEqualTo(named.data().topics().find("sales.topic").topicId()); } MetadataResponse all = handle(service, allTopicsRequest(version), version, config); @@ -683,7 +690,12 @@ public void testAliasSurvivesMissingDatabaseRetry(boolean failedFuture) { assertThat(response.data().topics().find("missing_db.topic").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); assertThat(response.data().topics().find("topic").topicId()) - .isEqualTo(response.data().topics().find("fluss.topic").topicId()); + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373410000L, 777L) : Uuid.ZERO_UUID); + if (version >= 10) { + assertThat(response.data().topics().find("fluss.topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)); + } } assertThat(service.listDatabasesCalls).isZero(); assertThat(service.listedDatabases) @@ -783,6 +795,8 @@ public void testPersistedEnablementControlsBothAliasesAndDiscovery() { handle(service, topicsRequest((short) 11, "topic", "fluss.topic"), (short) 11); assertThat(enabled.errorCounts()).containsOnlyKeys(Errors.NONE); assertThat(enabled.data().topics().find("topic").topicId()) + .isEqualTo(new Uuid(0x466c757373410000L, 777L)); + assertThat(enabled.data().topics().find("fluss.topic").topicId()) .isEqualTo(new Uuid(0x466c757373000000L, 777L)); service.putTable("fluss.topic", 777L, disabledInvalid); assertThat(handle(service, namedTopicRequest("topic"), (short) 11).errors()) @@ -814,6 +828,82 @@ public void testResolvedAliasNameLimitDoesNotHideValidTopic() { .containsEntry(database + "." + table, Errors.INVALID_TOPIC_EXCEPTION); } + @ParameterizedTest + @ValueSource(shorts = {10, 11}) + public void testKafkaClientCanCacheCanonicalNameAndAliasTogether(short version) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + Metadata client = + new Metadata(0L, 0L, 60000L, new LogContext(), new ClusterResourceListeners()); + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "fluss.topic"), version), false, 1L); + // A producer discovers the alias after already caching the canonical name. Nonzero + // duplicate IDs would make Kafka's MetadataSnapshot throw while building topicNames. + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "topic"), version), true, 2L); + assertThat(client.fetch().partitionsForTopic("topic")).hasSize(2); + assertThat(client.fetch().partitionsForTopic("fluss.topic")).hasSize(2); + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "topic", "fluss.topic"), version), + false, + 3L); + assertThat(client.fetch().partitionsForTopic("topic")).hasSize(2); + assertThat(client.fetch().partitionsForTopic("fluss.topic")).hasSize(2); + assertThat(client.topicIds()).containsOnlyKeys("topic", "fluss.topic"); + assertThat(client.topicNames()) + .containsEntry(new Uuid(0x466c757373000000L, 777L), "fluss.topic") + .containsEntry(new Uuid(0x466c757373410000L, 777L), "topic"); + client.close(); + } + + @ParameterizedTest + @ValueSource(shorts = {10, 11}) + public void testKafkaClientRefreshesRecreatedAliasWithLowerLeaderEpoch(short version) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + Metadata client = + new Metadata(0L, 0L, 60000L, new LogContext(), new ClusterResourceListeners()); + MetadataResponse original = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + original.data() + .topics() + .forEach( + topic -> + topic.partitions() + .forEach(p -> p.setLeaderId(1).setLeaderEpoch(5))); + client.updateWithCurrentRequestVersion(original, false, 1L); + for (String name : Arrays.asList("topic", "fluss.topic")) { + assertThat(client.fetch().leaderFor(new TopicPartition(name, 0)).id()).isEqualTo(1); + assertThat(client.lastSeenLeaderEpoch(new TopicPartition(name, 0))).contains(5); + } + + // The client never observes the missing table between deletion and recreation. + service.removeTable("fluss.topic"); + service.putTable("fluss.topic", 778L); + MetadataResponse recreated = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + recreated + .data() + .topics() + .forEach( + topic -> + topic.partitions() + .forEach(p -> p.setLeaderId(2).setLeaderEpoch(0))); + client.updateWithCurrentRequestVersion(recreated, false, 2L); + for (String name : Arrays.asList("fluss.topic", "topic")) { + assertThat(client.fetch().leaderFor(new TopicPartition(name, 0)).id()) + .as("Leader after recreating %s", name) + .isEqualTo(2); + assertThat(client.lastSeenLeaderEpoch(new TopicPartition(name, 0))).contains(0); + } + assertThat(client.topicNames()) + .containsEntry(new Uuid(0x466c757373000000L, 778L), "fluss.topic") + .containsEntry(new Uuid(0x466c757373410000L, 778L), "topic") + .doesNotContainKeys( + new Uuid(0x466c757373000000L, 777L), new Uuid(0x466c757373410000L, 777L)); + client.close(); + } + private static MetadataRequest topicsRequest(short version, String... names) { return new MetadataRequest( new MetadataRequestData() diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java index d162ed837a4..94b5c35f14f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -208,10 +208,15 @@ public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception assertThat(partition.isrNodes()) .containsExactlyInAnyOrder(0, 1); }); - assertThat(alias.topicId()).isEqualTo(canonical.topicId()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, tableId) + : Uuid.ZERO_UUID); if (version >= 10) { - assertThat(alias.topicId()) + assertThat(canonical.topicId()) .isEqualTo(new Uuid(0x466c757373000000L, tableId)) + .isNotEqualTo(alias.topicId()) .isNotEqualTo(response.data().topics().find("sales.events").topicId()); } assertThat(response.data().topics().find("missing_db.events").errorCode()) @@ -355,9 +360,15 @@ public void testCustomDefaultDatabaseIsInjectedAndNeverCreatedByMetadata() throw .topics() .find("custom_default.events") .partitions()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, customId) + : Uuid.ZERO_UUID); if (version >= 10) { - assertThat(alias.topicId()) + assertThat(response.data().topics().find("custom_default.events").topicId()) .isEqualTo(new Uuid(0x466c757373000000L, customId)) + .isNotEqualTo(alias.topicId()) .isNotEqualTo(response.data().topics().find("fluss.events").topicId()); } assertThat(query(node, allTopics(version)).data().topics()) @@ -403,7 +414,11 @@ private static void assertExposure(ServerNode node, boolean enabled, long tableI assertThat(topic.topicId()) .isEqualTo( enabled && version >= 10 - ? new Uuid(0x466c757373000000L, tableId) + ? new Uuid( + name.indexOf('.') >= 0 + ? 0x466c757373000000L + : 0x466c757373410000L, + tableId) : Uuid.ZERO_UUID); } assertThat(response.data().topics().find("sales.unset").errorCode()) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index c2aa5206640..5638c9a0666 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -162,6 +162,22 @@ public void testTopicIdMapping() { assertThat(mapper.toTableId(firstTableTopicId)).isZero(); } + @Test + public void testAliasTopicIdsKeepTableIdentityAndDetectRecreation() { + for (long tableId : new long[] {0L, 123L, Long.MAX_VALUE}) { + Uuid canonical = mapper.toTopicId(tableId); + Uuid alias = mapper.toAliasTopicId(tableId); + assertThat(alias) + .isNotIn(canonical, Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID) + .isEqualTo(new KafkaTopicMapper().toAliasTopicId(tableId)); + assertThat(mapper.isMappedTopicId(alias)).isTrue(); + assertThat(mapper.toTableId(alias)).isEqualTo(mapper.toTableId(canonical)); + } + assertThat(mapper.toAliasTopicId(124L)).isNotEqualTo(mapper.toAliasTopicId(123L)); + assertThatThrownBy(() -> mapper.toAliasTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + } + @Test public void testRejectsInvalidTopicIds() { assertThatThrownBy(() -> mapper.toTopicId(-1L)) From dcec63346dd3786696de4e4ab42565b3a0788c7c Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 14:30:55 +0800 Subject: [PATCH 07/69] [kafka] Validate Metadata against the updated PR03 contract Adapt the decoder regression to the configured request handler. Cover optional three-field metadata columns and legacy offset rejection across Metadata v0-v11, including native DDL and both real TabletServer listeners. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 109/109 --- .../fluss/kafka/KafkaCommandDecoderTest.java | 5 +- .../fluss/kafka/KafkaMetadataHandlerTest.java | 59 +++++++++++++++++++ .../fluss/kafka/KafkaMetadataITCase.java | 45 ++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index dda2e52bde5..147b39dd834 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -17,6 +17,8 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; @@ -70,7 +72,8 @@ public void testUnsupportedApiVersionsRequestReturnsVersionZeroErrorResponse() { assertThat(parsedRequest.hasUnsupportedRequestVersion()).isTrue(); assertThat(parsedRequest.version()).isZero(); - new KafkaRequestHandler().processRequest(request); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + new KafkaRequestHandler(new Configuration(), service, service).processRequest(request); channel.runPendingTasks(); responseBuffer = channel.readOutbound(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 73801d678c5..528932410a1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -36,6 +36,7 @@ import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.clients.Metadata; @@ -429,6 +430,64 @@ public void testMetadataUsesDdlContractForEveryVersion() { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMetadataColumnContractForEveryVersion(boolean legacyOffset) { + RowType metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + if (legacyOffset) { + metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT())); + } + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + // Also model an invalid historical descriptor that native DDL would now reject. + service.putTable( + "fluss.metadata_events", + 777L, + TableDescriptor.builder(defaultDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_metadata", metadataType) + .build()) + .build()); + short expected = legacyOffset ? Errors.INVALID_TOPIC_EXCEPTION.code() : Errors.NONE.code(); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle( + service, + topicsRequest( + version, + "metadata_events", + "fluss.metadata_events", + "kafka.topic"), + version); + for (String name : Arrays.asList("metadata_events", "fluss.metadata_events")) { + MetadataResponseTopic topic = named.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(expected); + assertThat(topic.partitions()).hasSize(legacyOffset ? 0 : 2); + } + assertThat(named.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics().find("fluss.metadata_events").errorCode()) + .isEqualTo(expected); + assertThat(all.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .doesNotContain("metadata_events"); + } + } + @Test public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java index 94b5c35f14f..0ac5279c401 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -113,6 +113,51 @@ public void testQualifiedMetadataAcrossDatabases() throws Exception { } } + @Test + public void testOptionalMetadataColumnOverRealListeners() throws Exception { + TableDescriptor descriptor = + TableDescriptor.builder(topicDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()))) + .build()) + .build(); + long tableId = createTable(CLUSTER, TablePath.of("fluss", "metadata_events"), descriptor); + CLUSTER.waitUntilTableReady(tableId); + for (int serverId = 0; serverId < 2; serverId++) { + ServerNode node = CLUSTER.getTabletServerInfos().get(serverId).node("KAFKA"); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + query(node, named(version, "metadata_events", "fluss.metadata_events")); + assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = named.data().topics().find("metadata_events"); + MetadataResponseTopic canonical = + named.data().topics().find("fluss.metadata_events"); + assertThat(alias.partitions()).hasSize(2).isEqualTo(canonical.partitions()); + if (version >= 10) { + assertThat(canonical.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, tableId)); + assertThat(alias.topicId()).isEqualTo(new Uuid(0x466c757373410000L, tableId)); + } + MetadataResponse all = query(node, allTopics(version)); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.metadata_events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + } + @Test public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exception { TableDescriptor descriptor = topicDescriptor(); From 5275f903c134aea16a932da78a1095a71eb832e4 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 20:24:12 +0800 Subject: [PATCH 08/69] [kafka] Simplify Metadata wiring and regression tests Pass one RPC service through the request handler and validate its TabletServerGateway capability inside the Metadata backend. Consolidate overlapping Metadata tests while retaining protocol-version, alias identity, client cache, authorization and discovery regressions. Align the custom default database test with PR03 startup creation and verify that Metadata does not recreate a deleted default database. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 35/35 AI-Contributed/UT: 546/546 --- .../fluss/kafka/KafkaProtocolPlugin.java | 9 +- .../fluss/kafka/KafkaRequestHandler.java | 9 +- .../metadata/GatewayKafkaMetadataBackend.java | 17 +- .../fluss/kafka/KafkaCommandDecoderTest.java | 2 +- .../fluss/kafka/KafkaMetadataHandlerTest.java | 375 +++--------------- .../fluss/kafka/KafkaMetadataITCase.java | 167 +++----- .../fluss/kafka/KafkaRequestHandlerTest.java | 2 +- 7 files changed, 123 insertions(+), 458 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index d0363e3f6f4..2d0fa9220e3 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -20,7 +20,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.rpc.RpcGatewayService; -import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; @@ -61,12 +60,6 @@ public ChannelHandler createChannelHandler( @Override public RequestHandler createRequestHandler(RpcGatewayService service) { - if (!(service instanceof TabletServerGateway)) { - throw new IllegalArgumentException( - "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " - + service.getClass().getSimpleName()); - } - TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(conf, service, gateway); + return new KafkaRequestHandler(conf, service); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 46dc00355bf..31da9143823 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -26,7 +26,6 @@ import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.rpc.RpcGatewayService; -import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; @@ -41,18 +40,14 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ - public KafkaRequestHandler( - Configuration conf, RpcGatewayService service, TabletServerGateway gateway) { - checkNotNull(service); - checkNotNull(gateway); + public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { KafkaTopicMapper topicMapper = new KafkaTopicMapper(checkNotNull(conf)); this.registry = new KafkaApiRegistry( Arrays.asList( new ApiVersionsHandler(this::apiRegistry), new MetadataHandler( - new GatewayKafkaMetadataBackend( - service, gateway, topicMapper), + new GatewayKafkaMetadataBackend(service, topicMapper), topicMapper))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java index 0d4ed5a78c3..2c2ff42e117 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -62,6 +62,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */ @@ -75,11 +76,19 @@ public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { private final KafkaTopicMapper topicMapper; private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); - /** Creates a metadata backend backed by the local TabletServer gateway. */ - public GatewayKafkaMetadataBackend( - RpcGatewayService service, TabletServerGateway gateway, KafkaTopicMapper topicMapper) { + /** + * Creates a metadata backend backed by the local TabletServer gateway. + * + * @throws IllegalArgumentException if the service does not implement {@link + * TabletServerGateway} + */ + public GatewayKafkaMetadataBackend(RpcGatewayService service, KafkaTopicMapper topicMapper) { this.service = checkNotNull(service); - this.gateway = checkNotNull(gateway); + checkArgument( + service instanceof TabletServerGateway, + "Kafka protocol endpoints can only be enabled on TabletServers, but the service is %s", + service.getClass().getSimpleName()); + this.gateway = (TabletServerGateway) service; this.topicMapper = checkNotNull(topicMapper); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index 147b39dd834..f745fe7fe8b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -73,7 +73,7 @@ public void testUnsupportedApiVersionsRequestReturnsVersionZeroErrorResponse() { assertThat(parsedRequest.version()).isZero(); TestingTabletGatewayService service = new TestingTabletGatewayService(); - new KafkaRequestHandler(new Configuration(), service, service).processRequest(request); + new KafkaRequestHandler(new Configuration(), service).processRequest(request); channel.runPendingTasks(); responseBuffer = channel.readOutbound(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 528932410a1..b1359f1c705 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -83,13 +83,7 @@ public class KafkaMetadataHandlerTest { public void testNamedTopicForEverySupportedVersion() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { - MetadataRequest request = - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Collections.singletonList("kafka.topic"))), - version); + MetadataRequest request = topicsRequest(version, "kafka.topic"); if (version >= 9) { request.data() .unknownTaggedFields() @@ -141,26 +135,6 @@ public void testAllTopicsForEverySupportedVersion() { } } - @Test - public void testAllTopicsAndMissingAndInvalidTopic() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - - MetadataResponse allTopics = - handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); - assertThat(allTopics.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactlyInAnyOrder("kafka.other", "kafka.topic"); - - MetadataRequest requestedTopics = - new MetadataRequest.Builder( - Arrays.asList("kafka.missing", "kafka.invalid topic"), false) - .build((short) 9); - MetadataResponse errors = handle(service, requestedTopics, (short) 9); - assertThat(errors.errors()) - .containsEntry("kafka.missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) - .containsEntry("kafka.invalid topic", Errors.INVALID_TOPIC_EXCEPTION); - } - @Test public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -210,32 +184,6 @@ public void testV10AndV11RejectTopicIdOnlyLookup() { } } - @Test - public void testTopicIdentityAcrossDeleteAndRecreate() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - - MetadataResponse initial = handle(service, namedTopicRequest("kafka.topic"), (short) 11); - assertThat(initial.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); - - service.removeTable("kafka.topic"); - MetadataResponse deleted = handle(service, namedTopicRequest("kafka.topic"), (short) 11); - assertThat(deleted.errorCounts()) - .containsExactlyEntriesOf( - Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); - - service.putTable("kafka.topic", 223L); - Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); - MetadataResponse recreatedByName = - handle( - service, - new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) - .build((short) 11), - (short) 11); - assertThat(recreatedByName.data().topics().find("kafka.topic").topicId()) - .isEqualTo(recreatedTopicId) - .isNotEqualTo(TOPIC_ID); - } - @Test public void testDeleteRaceBecomesUnknownTopicResult() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -253,9 +201,7 @@ public void testDeleteRaceBecomesUnknownTopicResult() { public void testUnavailableLeaderUsesPartitionError() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicLeaderAvailable = false; - MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) - .build((short) 11); + MetadataRequest request = namedTopicRequest("kafka.topic"); MetadataResponse response = handle(service, request, (short) 11); @@ -273,9 +219,7 @@ public void testUnavailableLeaderUsesPartitionError() { public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicIsr = new int[] {1}; - MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) - .build((short) 11); + MetadataRequest request = namedTopicRequest("kafka.topic"); MetadataResponse response = handle(service, request, (short) 11); @@ -335,9 +279,7 @@ public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.failMetadata = true; - MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) - .build((short) 11); + MetadataRequest request = namedTopicRequest("kafka.topic"); MetadataResponse response = handle(service, request, (short) 11); @@ -346,90 +288,6 @@ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { assertThat(response.brokers()).isEmpty(); } - @Test - public void testMetadataUsesDdlContractForEveryVersion() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.putTable( - "kafka.no_mapping", - 125L, - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .distributedBy(2) - .build()); - service.putTable( - "kafka.bad_format", - 126L, - TableDescriptor.builder(defaultDescriptor()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "unimplemented") - .build()); - service.putTable( - "kafka.primary_key", - 127L, - TableDescriptor.builder(defaultDescriptor()) - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES().copy(false)) - .primaryKey("_kafka_value") - .build()) - .build()); - service.putTable( - "kafka.partitioned", - 128L, - TableDescriptor.builder(defaultDescriptor()).partitionedBy("_kafka_value").build()); - service.putTable( - "kafka.indexed", - 129L, - TableDescriptor.builder(defaultDescriptor()).logFormat(LogFormat.INDEXED).build()); - service.putTable("kafka.invalid topic", 130L); - List invalidMappings = - Arrays.asList( - "kafka.bad_format", - "kafka.primary_key", - "kafka.partitioned", - "kafka.indexed", - "kafka.invalid topic"); - for (short version = 0; version <= 11; version++) { - MetadataResponse all = handle(service, allTopicsRequest(version), version); - List exposed = new ArrayList<>(invalidMappings); - exposed.addAll(Arrays.asList("kafka.other", "kafka.topic")); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactlyInAnyOrderElementsOf(exposed); - for (String name : invalidMappings) { - assertThat(all.data().topics().find(name).errorCode()) - .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); - } - List requested = new ArrayList<>(invalidMappings); - requested.add("kafka.no_mapping"); - requested.add("kafka.topic"); - requested.add("kafka.missing"); - MetadataResponse named = - handle( - service, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - requested)), - version), - version); - for (String name : invalidMappings) { - MetadataResponseTopic topic = named.data().topics().find(name); - assertThat(topic.errorCode()).isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); - assertThat(topic.partitions()).isEmpty(); - } - assertThat(named.data().topics().find("kafka.topic").errorCode()) - .isEqualTo(Errors.NONE.code()); - assertThat(named.data().topics().find("kafka.missing").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(named.data().topics().find("kafka.no_mapping").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - } - } - @ParameterizedTest @ValueSource(booleans = {false, true}) public void testMetadataColumnContractForEveryVersion(boolean legacyOffset) { @@ -488,55 +346,6 @@ public void testMetadataColumnContractForEveryVersion(boolean legacyOffset) { } } - @Test - public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.putTable( - "kafka.topic", - 123L, - TableDescriptor.builder(defaultDescriptor()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "string") - .build()); - MetadataResponse invalid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); - assertThat(invalid.data().topics().find("kafka.topic").errorCode()) - .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); - service.putTable("kafka.topic", 123L); - MetadataResponse valid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); - assertThat(valid.data().topics().find("kafka.topic").errorCode()) - .isEqualTo(Errors.NONE.code()); - assertThat(valid.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); - } - - @Test - public void testSameTableNameInDifferentDatabasesForEveryVersion() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.putTable("sales.topic", 223L); - for (short version = 0; version <= 11; version++) { - MetadataResponse all = handle(service, allTopicsRequest(version), version); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactly("kafka.other", "kafka.topic", "sales.topic"); - MetadataResponse named = - handle( - service, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Arrays.asList( - "kafka.topic", "sales.topic"))), - version), - version); - assertThat(named.data().topics()).hasSize(2); - assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); - if (version >= 10) { - assertThat(named.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); - assertThat(named.data().topics().find("sales.topic").topicId()) - .isEqualTo(new Uuid(0x466c757373000000L, 223L)); - } - } - } - @Test public void testRejectMalformedNamesWithoutHidingValidTopics() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -556,48 +365,6 @@ public void testRejectMalformedNamesWithoutHidingValidTopics() { .isEqualTo(Errors.NONE.code()); } - @Test - public void testUnauthorizedTopicsRemainHiddenForEveryVersion() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.hiddenTopics.add("kafka.topic"); - for (short version = 0; version <= 11; version++) { - MetadataResponse all = handle(service, allTopicsRequest(version), version); - assertThat(all.brokers()).hasSize(2); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactly("kafka.other"); - for (List names : - Arrays.asList( - Arrays.asList("kafka.topic", "kafka.other"), - Arrays.asList("kafka.topic", "kafka.other", "missing_db.topic"))) { - MetadataResponse named = - handle( - service, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest - .convertToMetadataRequestTopic( - names)), - version), - version); - assertThat(named.brokers()).hasSize(2); - MetadataResponseTopic hidden = named.data().topics().find("kafka.topic"); - assertThat(hidden.errorCode()).isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(hidden.topicId()).isEqualTo(Uuid.ZERO_UUID); - assertThat(hidden.partitions()).isEmpty(); - MetadataResponseTopic visible = named.data().topics().find("kafka.other"); - assertThat(visible.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(visible.partitions()).hasSize(2); - assertThat(named.data().topics()).hasSize(names.size()); - if (names.contains("missing_db.topic")) { - assertThat(named.data().topics().find("missing_db.topic").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - } - } - } - } - @Test public void testMetadataV12IsRejectedBeforeCallingGateway() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -614,36 +381,47 @@ public void testMetadataV12IsRejectedBeforeCallingGateway() { @ParameterizedTest @ValueSource(booleans = {false, true}) - public void testMissingDatabaseRetainsBrokersAndValidTopics(boolean failedFuture) { + public void testMissingDatabaseRetainsBrokersAndAliases(boolean failedFuture) { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); service.failListTablesAsFuture = failedFuture; for (short version = 0; version <= 11; version++) { for (List names : Arrays.asList( Collections.singletonList("missing_db.topic"), - Arrays.asList("missing_db.topic", "kafka.topic"))) { + Arrays.asList( + "missing_db.topic", "topic", "fluss.topic", "kafka.topic"))) { MetadataResponse response = handle( service, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest - .convertToMetadataRequestTopic( - names)), - version), + topicsRequest(version, names.toArray(new String[0])), version); assertThat(response.brokers()).hasSize(2); assertThat(response.data().topics()).hasSize(names.size()); assertThat(response.data().topics().find("missing_db.topic").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - if (names.contains("kafka.topic")) { - MetadataResponseTopic valid = response.data().topics().find("kafka.topic"); - assertThat(valid.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(valid.partitions()).hasSize(2); + if (names.contains("topic")) { + for (String name : Arrays.asList("topic", "fluss.topic", "kafka.topic")) { + MetadataResponseTopic valid = response.data().topics().find(name); + assertThat(valid.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.partitions()).hasSize(2); + } + assertThat(response.data().topics().find("topic").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, 777L) + : Uuid.ZERO_UUID); + assertThat(response.data().topics().find("fluss.topic").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373000000L, 777L) + : Uuid.ZERO_UUID); } } } + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.listedDatabases) + .containsExactlyInAnyOrder("fluss", "kafka", "missing_db"); } @ParameterizedTest @@ -722,45 +500,6 @@ public void testAliasesPreserveRequestedNamesAndShareIdentity(String defaultData } } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - public void testAliasSurvivesMissingDatabaseRetry(boolean failedFuture) { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.putTable("fluss.topic", 777L); - service.failListTablesAsFuture = failedFuture; - for (short version = 0; version <= 11; version++) { - MetadataResponse response = - handle( - service, - topicsRequest( - version, - "topic", - "missing_db.topic", - "fluss.topic", - "kafka.topic"), - version); - assertThat(response.brokers()).hasSize(2); - assertThat(response.data().topics()).hasSize(4); - for (String name : Arrays.asList("topic", "fluss.topic", "kafka.topic")) { - assertThat(response.data().topics().find(name).errorCode()) - .isEqualTo(Errors.NONE.code()); - assertThat(response.data().topics().find(name).partitions()).hasSize(2); - } - assertThat(response.data().topics().find("missing_db.topic").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(response.data().topics().find("topic").topicId()) - .isEqualTo( - version >= 10 ? new Uuid(0x466c757373410000L, 777L) : Uuid.ZERO_UUID); - if (version >= 10) { - assertThat(response.data().topics().find("fluss.topic").topicId()) - .isEqualTo(new Uuid(0x466c757373000000L, 777L)); - } - } - assertThat(service.listDatabasesCalls).isZero(); - assertThat(service.listedDatabases) - .containsExactlyInAnyOrder("fluss", "kafka", "missing_db"); - } - @Test public void testMissingDefaultDatabaseNeverSearchesOtherDatabases() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -783,37 +522,46 @@ public void testMissingDefaultDatabaseNeverSearchesOtherDatabases() { } @Test - public void testAliasAuthorizationDoesNotFallbackToSameNamedVisibleTable() { + public void testUnauthorizedAliasesRemainHiddenWithAndWithoutRetry() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.putTable("fluss.topic", 777L); service.hiddenTopics.add("fluss.topic"); service.putTable("sales.topic", 778L); for (short version = 0; version <= 11; version++) { - MetadataResponse response = - handle( - service, - topicsRequest( - version, - "topic", - "fluss.topic", - "sales.topic", - "missing_db.topic"), - version); - for (String hidden : Arrays.asList("topic", "fluss.topic")) { - MetadataResponseTopic topic = response.data().topics().find(hidden); - assertThat(topic.errorCode()).isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); - assertThat(topic.partitions()).isEmpty(); + for (List names : + Arrays.asList( + Arrays.asList("topic", "fluss.topic", "sales.topic"), + Arrays.asList( + "topic", "fluss.topic", "sales.topic", "missing_db.topic"))) { + MetadataResponse response = + handle( + service, + topicsRequest(version, names.toArray(new String[0])), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(names.size()); + for (String hidden : Arrays.asList("topic", "fluss.topic")) { + MetadataResponseTopic topic = response.data().topics().find(hidden); + assertThat(topic.errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + } + MetadataResponseTopic visible = response.data().topics().find("sales.topic"); + assertThat(visible.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(visible.partitions()).hasSize(2); + if (names.contains("missing_db.topic")) { + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } } - assertThat(response.data().topics().find("sales.topic").errorCode()) - .isEqualTo(Errors.NONE.code()); MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.brokers()).hasSize(2); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) .contains("sales.topic") .doesNotContain("topic", "fluss.topic"); } - assertThat(service.requestedTables).doesNotContain("sales.other"); } @Test @@ -983,14 +731,7 @@ private static TableDescriptor defaultDescriptor() { } private static MetadataRequest namedTopicRequest(String topicName) { - return new MetadataRequest( - new MetadataRequestData() - .setTopics( - Collections.singletonList( - new MetadataRequestTopic() - .setName(topicName) - .setTopicId(Uuid.ZERO_UUID))), - (short) 11); + return topicsRequest((short) 11, topicName); } private static MetadataRequest allTopicsRequest(short version) { @@ -1009,7 +750,7 @@ private static MetadataResponse handle( MetadataRequest requestBody, short version, Configuration config) { - KafkaRequestHandler handler = new KafkaRequestHandler(config, service, service); + KafkaRequestHandler handler = new KafkaRequestHandler(config, service); ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); KafkaRequest request; try { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java index 0ac5279c401..3a5e582ee71 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -28,6 +28,7 @@ import org.apache.fluss.server.zk.data.TableRegistration; import org.apache.fluss.types.DataTypes; +import org.apache.kafka.common.Node; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.message.MetadataRequestData; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; @@ -68,96 +69,6 @@ public class KafkaMetadataITCase { .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") .build(); - @Test - public void testQualifiedMetadataAcrossDatabases() throws Exception { - TableDescriptor descriptor = topicDescriptor(); - long first = createTable(CLUSTER, TablePath.of("first_db", "events"), descriptor); - long second = createTable(CLUSTER, TablePath.of("second_db", "events"), descriptor); - CLUSTER.waitUntilTableReady(first); - CLUSTER.waitUntilTableReady(second); - ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); - - for (short version = 0; version <= 11; version++) { - MetadataResponse named = - query( - node, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Arrays.asList( - "first_db.events", - "second_db.events", - "first_db.missing"))), - version)); - assertThat(named.data().topics().find("first_db.events").partitions()).hasSize(2); - assertThat(named.data().topics().find("second_db.events").partitions()).hasSize(2); - assertThat(named.data().topics().find("first_db.missing").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(named.data().topics().find("first_db.events").topicId()) - .isEqualTo( - version >= 10 ? new Uuid(0x466c757373000000L, first) : Uuid.ZERO_UUID); - assertThat(named.data().topics().find("second_db.events").topicId()) - .isEqualTo( - version >= 10 ? new Uuid(0x466c757373000000L, second) : Uuid.ZERO_UUID); - assertThat(named.brokers()).hasSize(2); - assertThat(named.brokers().iterator().next().port()).isEqualTo(node.port()); - - MetadataRequestData allTopics = new MetadataRequestData(); - allTopics.setTopics(version == 0 ? Collections.emptyList() : null); - MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactly("first_db.events", "second_db.events"); - assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); - } - } - - @Test - public void testOptionalMetadataColumnOverRealListeners() throws Exception { - TableDescriptor descriptor = - TableDescriptor.builder(topicDescriptor()) - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column( - "_kafka_metadata", - DataTypes.ROW( - DataTypes.FIELD( - "timestamp", - DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD( - "timestamp_type", DataTypes.INT()), - DataTypes.FIELD( - "partition", DataTypes.INT()))) - .build()) - .build(); - long tableId = createTable(CLUSTER, TablePath.of("fluss", "metadata_events"), descriptor); - CLUSTER.waitUntilTableReady(tableId); - for (int serverId = 0; serverId < 2; serverId++) { - ServerNode node = CLUSTER.getTabletServerInfos().get(serverId).node("KAFKA"); - for (short version = 0; version <= 11; version++) { - MetadataResponse named = - query(node, named(version, "metadata_events", "fluss.metadata_events")); - assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); - MetadataResponseTopic alias = named.data().topics().find("metadata_events"); - MetadataResponseTopic canonical = - named.data().topics().find("fluss.metadata_events"); - assertThat(alias.partitions()).hasSize(2).isEqualTo(canonical.partitions()); - if (version >= 10) { - assertThat(canonical.topicId()) - .isEqualTo(new Uuid(0x466c757373000000L, tableId)); - assertThat(alias.topicId()).isEqualTo(new Uuid(0x466c757373410000L, tableId)); - } - MetadataResponse all = query(node, allTopics(version)); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactly("fluss.metadata_events"); - assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); - } - } - } - @Test public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exception { TableDescriptor descriptor = topicDescriptor(); @@ -169,31 +80,13 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); for (short version = 0; version <= 11; version++) { - MetadataResponse missing = - query( - node, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Collections.singletonList( - "deleted_db.events"))), - version)); + MetadataResponse missing = query(node, named(version, "deleted_db.events")); assertThat(missing.brokers()).hasSize(2); assertThat(missing.data().topics().find("deleted_db.events").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); MetadataResponse mixed = - query( - node, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Arrays.asList( - "deleted_db.events", - "existing_db.events"))), - version)); + query(node, named(version, "deleted_db.events", "existing_db.events")); assertThat(mixed.brokers()).hasSize(2); assertThat(mixed.brokers().iterator().next().port()).isEqualTo(node.port()); assertThat(mixed.data().topics().find("deleted_db.events").errorCode()) @@ -202,9 +95,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio .isEqualTo(Errors.NONE.code()); assertThat(mixed.data().topics().find("existing_db.events").partitions()).hasSize(2); - MetadataRequestData allTopics = new MetadataRequestData(); - allTopics.setTopics(version == 0 ? Collections.emptyList() : null); - MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); + MetadataResponse all = query(node, allTopics(version)); assertThat(all.brokers()).hasSize(2); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) @@ -214,7 +105,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio } @Test - public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception { + public void testAliasesAndOptionalMetadataAcrossDatabases() throws Exception { long other = createTable(CLUSTER, TablePath.of("sales", "events"), topicDescriptor()); CLUSTER.waitUntilTableReady(other); ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); @@ -225,7 +116,25 @@ public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception assertThat(before.data().topics().find("sales.events").errorCode()) .isEqualTo(Errors.NONE.code()); } - long tableId = createTable(CLUSTER, TablePath.of("fluss", "events"), topicDescriptor()); + // Exercise PR03's optional metadata column through the same alias and discovery paths. + TableDescriptor withMetadata = + TableDescriptor.builder(topicDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()))) + .build()) + .build(); + long tableId = createTable(CLUSTER, TablePath.of("fluss", "events"), withMetadata); CLUSTER.waitUntilTableReady(tableId); for (short version = 0; version <= 11; version++) { for (ServerNode broker : CLUSTER.getTabletServerNodes("KAFKA")) { @@ -239,7 +148,20 @@ public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception "events", "fluss.events", "sales.events", + "fluss.missing", "missing_db.events")); + assertThat(response.brokers()).hasSize(2); + assertThat(response.brokers()).extracting(Node::port).contains(broker.port()); + for (String name : Arrays.asList("fluss.events", "sales.events")) { + MetadataResponseTopic topic = response.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + } + assertThat(response.data().topics().find("sales.events").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373000000L, other) + : Uuid.ZERO_UUID); MetadataResponseTopic alias = response.data().topics().find("events"); MetadataResponseTopic canonical = response.data().topics().find("fluss.events"); assertThat(alias.errorCode()).isEqualTo(Errors.NONE.code()); @@ -264,11 +186,15 @@ public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception .isNotEqualTo(alias.topicId()) .isNotEqualTo(response.data().topics().find("sales.events").topicId()); } - assertThat(response.data().topics().find("missing_db.events").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(query(broker, allTopics(version)).data().topics()) + for (String missing : Arrays.asList("fluss.missing", "missing_db.events")) { + assertThat(response.data().topics().find(missing).errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + MetadataResponse all = query(broker, allTopics(version)); + assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) .containsExactly("fluss.events", "sales.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); } } } @@ -359,7 +285,7 @@ public void testInvalidEnabledContractRemainsVisibleInNamedAndAllTopics() throws } @Test - public void testCustomDefaultDatabaseIsInjectedAndNeverCreatedByMetadata() throws Exception { + public void testCustomDefaultDatabaseAliasAndNoRecreationByMetadata() throws Exception { Configuration config = clusterConfiguration(); config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "custom_default"); config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); @@ -375,12 +301,13 @@ public void testCustomDefaultDatabaseIsInjectedAndNeverCreatedByMetadata() throw createTable(cluster, TablePath.of("fluss", "events"), topicDescriptor()); cluster.waitUntilTableReady(nativeDefaultId); ServerNode node = cluster.getTabletServerInfos().get(0).node("KAFKA"); + // PR03 creates the configured database at startup, before any Metadata request. assertThat( cluster.newCoordinatorClient() .databaseExists(newDatabaseExistsRequest("custom_default")) .get() .isExists()) - .isFalse(); + .isTrue(); for (short version = 0; version <= 11; version++) { MetadataResponse response = query(node, named(version, "events", "fluss.events")); assertThat(response.data().topics().find("events").errorCode()) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index f55b1832c0b..3c1bfe6128b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -300,6 +300,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { private static KafkaRequestHandler createKafkaRequestHandler() { TestingTabletGatewayService service = new TestingTabletGatewayService(); - return new KafkaRequestHandler(new Configuration(), service, service); + return new KafkaRequestHandler(new Configuration(), service); } } From ffdefa3f20775d3724a6ffafdd9d02b51655cbe5 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 10:33:17 +0800 Subject: [PATCH 09/69] [kafka] Adapt Metadata tests to merged DDL configuration Use KafkaValueConversionType for typed table properties after rebasing onto main. Preserve the raw invalid value in the disabled-table regression to keep verifying that Metadata filters it before schema resolution. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 21/21 --- .../apache/fluss/kafka/KafkaMetadataHandlerTest.java | 12 +++++++++--- .../org/apache/fluss/kafka/KafkaMetadataITCase.java | 9 +++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index b1359f1c705..f41134c5e47 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.metadata.LogFormat; @@ -571,12 +572,15 @@ public void testPersistedEnablementControlsBothAliasesAndDiscovery() { TableDescriptor.builder() .schema(defaultDescriptor().getSchema()) .distributedBy(2) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) .build(); TableDescriptor disabledInvalid = TableDescriptor.builder(defaultDescriptor()) .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "unimplemented") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "unimplemented") .build(); for (TableDescriptor disabled : Arrays.asList(missingFlag, disabledInvalid)) { service.putTable("fluss.topic", 777L, disabled); @@ -726,7 +730,9 @@ private static TableDescriptor defaultDescriptor() { .distributedBy(2) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) .build(); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java index 3a5e582ee71..b281349a68f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -20,6 +20,7 @@ import org.apache.fluss.cluster.ServerNode; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; @@ -210,7 +211,9 @@ public void testPersistedEnablementChangesAreVisibleThroughBothNames() throws Ex TableDescriptor.builder() .schema(topicDescriptor().getSchema()) .distributedBy(2) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) .build(); long tableId = createTable(CLUSTER, path, disabled); long unsetId = createTable(CLUSTER, TablePath.of("sales", "unset"), unset); @@ -441,7 +444,9 @@ private static TableDescriptor topicDescriptor() { .distributedBy(2) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) .build(); } From 4c70505287f44007b47d21374b8aed71ee3e5a74 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 11:41:24 +0800 Subject: [PATCH 10/69] [kafka] Rename default database topic ID namespace Clarify the namespace name and comment for default-database aliases. Keep the UUID namespace value and mapping behavior unchanged. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 8/8 AI-Contributed/UT: 0/0 --- .../org/apache/fluss/kafka/mapping/KafkaTopicMapper.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index 8098917cc65..e0f0d8d23fb 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -35,8 +35,8 @@ public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; - // ASCII "FlussA": Kafka clients require distinct IDs for distinct wire topic names. - private static final long ALIAS_TOPIC_ID_NAMESPACE = 0x466c757373410000L; + // ASCII "FlussA": distinguishes default-database aliases from canonical topic names. + private static final long DEFAULT_DATABASE_TOPIC_ID_NAMESPACE = 0x466c757373410000L; private final String defaultDatabase; @@ -121,14 +121,14 @@ public Uuid toTopicId(long tableId) { */ public Uuid toAliasTopicId(long tableId) { checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); - return new Uuid(ALIAS_TOPIC_ID_NAMESPACE, tableId); + return new Uuid(DEFAULT_DATABASE_TOPIC_ID_NAMESPACE, tableId); } /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ public boolean isMappedTopicId(Uuid topicId) { return topicId != null && (topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE - || topicId.getMostSignificantBits() == ALIAS_TOPIC_ID_NAMESPACE) + || topicId.getMostSignificantBits() == DEFAULT_DATABASE_TOPIC_ID_NAMESPACE) && topicId.getLeastSignificantBits() >= 0L; } From d20cf7ccc0b44f5b141117338c6a76df93582dff Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:18:03 +0800 Subject: [PATCH 11/69] [kafka] Add independently testable Produce protocol handling Validate non-idempotent Produce v3-v11 and isolate invalid partitions before invoking a protocol-independent backend. Preserve partition order, acknowledgements and owned record data. Leave production registration to the append integration PR. Validation: Java 11, mvn -o -pl fluss-kafka spotless:apply clean verify; 69 unit tests and 2 integration tests passed, including Checkstyle, Spotless and RAT. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 621/621 AI-Contributed/UT: 366/366 --- .../kafka/api/produce/ProduceHandler.java | 284 ++++++++++++++ .../backend/produce/KafkaProduceBackend.java | 29 ++ .../backend/produce/KafkaProduceCommand.java | 197 ++++++++++ .../backend/produce/KafkaProduceResult.java | 111 ++++++ .../fluss/kafka/KafkaProduceProtocolTest.java | 366 ++++++++++++++++++ 5 files changed, 987 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java new file mode 100644 index 00000000000..0eb14b26ff2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidRequiredAcksException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.BaseRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements non-idempotent Kafka Produce versions 3 through 11. */ +@Internal +public final class ProduceHandler implements KafkaApiHandler { + + private static final short MIN_SUPPORTED_VERSION = 3; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec(ApiKeys.PRODUCE, MIN_SUPPORTED_VERSION, (short) 11, true); + + private final KafkaProduceBackend backend; + + /** Creates a non-idempotent Produce handler. */ + public ProduceHandler(KafkaProduceBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ProduceRequest request) { + validateRequest(request); + List topics = new ArrayList<>(); + Map failures = new HashMap<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + try { + if (!Topic.isValid(topic.name())) { + throw new InvalidTopicException("Invalid Kafka topic name " + topic.name()); + } + if (partition.index() < 0) { + throw new InvalidRequestException("Negative Kafka partition ID."); + } + partitions.add( + new PartitionWrite( + partition.index(), + copyRecords(request.version(), partition.records()))); + } catch (RuntimeException failure) { + failures.put( + new TopicPartition(topic.name(), partition.index()), + failedPartition(partition.index(), failure)); + } + } + if (!partitions.isEmpty()) { + topics.add(new TopicWrite(topic.name(), partitions)); + } + } + KafkaProduceCommand command = + new KafkaProduceCommand( + request.acks(), + request.timeout(), + topics, + context.listenerName(), + clientAddress(context.remoteAddress())); + CompletableFuture result; + try { + result = + topics.isEmpty() + ? CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList())) + : checkNotNull(backend.write(command)); + } catch (RuntimeException failure) { + result = new CompletableFuture<>(); + result.completeExceptionally(failure); + } + return result.handle( + (written, failure) -> { + Map results = new HashMap<>(failures); + if (failure == null && written != null) { + for (TopicResult topic : written.topics()) { + for (PartitionResult partition : topic.partitions()) { + results.putIfAbsent( + new TopicPartition( + topic.topicName(), partition.partitionId()), + partition); + } + } + } + List ordered = new ArrayList<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + TopicPartition key = + new TopicPartition(topic.name(), partition.index()); + PartitionResult value = results.get(key); + if (value == null) { + value = + failedPartition( + partition.index(), + failure == null + ? new IllegalStateException( + "Produce backend omitted this partition.") + : failure); + } + partitions.add(value); + } + ordered.add(new TopicResult(topic.name(), partitions)); + } + return toResponse(new KafkaProduceResult(ordered)); + }); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + while (failure instanceof CompletionException && failure.getCause() != null) { + failure = failure.getCause(); + } + return new PartitionResult( + partitionId, Errors.forException(failure), -1L, failure.getMessage()); + } + + private static void validateRequest(ProduceRequest request) { + Set names = new HashSet<>(); + for (TopicProduceData topic : request.data().topicData()) { + if (!names.add(topic.name())) { + throw new InvalidRequestException("Duplicate Kafka topic in Produce request."); + } + Set partitions = new HashSet<>(); + for (PartitionProduceData partition : topic.partitionData()) { + if (!partitions.add(partition.index())) { + throw new InvalidRequestException( + "Duplicate Kafka partition in Produce request."); + } + } + } + if (request.transactionalId() != null) { + throw new InvalidRequestException( + "Transactional Produce is not supported by the Fluss Kafka compatibility layer."); + } + if (request.acks() != -1 && request.acks() != 0 && request.acks() != 1) { + throw new InvalidRequiredAcksException("Invalid required acks " + request.acks()); + } + } + + private static List copyRecords( + short version, BaseRecords baseRecords) { + if (!(baseRecords instanceof Records)) { + throw new InvalidRequestException("Unsupported Kafka records representation."); + } + ProduceRequest.validateRecords(version, baseRecords); + Records records = (Records) baseRecords; + if (records.sizeInBytes() == 0) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + List copied = new ArrayList<>(); + int validBytes = 0; + for (RecordBatch batch : records.batches()) { + validBytes += batch.sizeInBytes(); + batch.ensureValid(); + if (batch.hasProducerId() || batch.isTransactional() || batch.isControlBatch()) { + throw new InvalidRequestException( + "Idempotent, transactional, and control record batches are not supported."); + } + for (org.apache.kafka.common.record.Record record : batch) { + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } + } + if (copied.isEmpty() || validBytes != records.sizeInBytes()) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + return copied; + } + + private static ProduceResponse toResponse(KafkaProduceResult result) { + ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0); + for (TopicResult topic : result.topics()) { + ProduceResponseData.TopicProduceResponse topicResponse = + new ProduceResponseData.TopicProduceResponse().setName(topic.topicName()); + for (PartitionResult partition : topic.partitions()) { + topicResponse + .partitionResponses() + .add( + new ProduceResponseData.PartitionProduceResponse() + .setIndex(partition.partitionId()) + .setErrorCode(partition.error().code()) + .setBaseOffset(partition.baseOffset()) + .setLogAppendTimeMs(-1L) + .setLogStartOffset(-1L) + .setErrorMessage(partition.errorMessage())); + } + data.responses().add(topicResponse); + } + return new ProduceResponse(data); + } + + private static List copyHeaders(Header[] headers) { + List copied = new ArrayList<>(headers.length); + for (Header header : headers) { + copied.add(new RecordHeader(header.key(), header.value())); + } + return copied; + } + + private static byte[] copyBuffer(ByteBuffer buffer) { + if (buffer == null) { + return null; + } + ByteBuffer duplicate = buffer.duplicate(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return bytes; + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java new file mode 100644 index 00000000000..3b8428c5670 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Produce API. */ +@Internal +public interface KafkaProduceBackend { + /** Writes copied Kafka records through the native Fluss write path. */ + CompletableFuture write(KafkaProduceCommand command); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java new file mode 100644 index 00000000000..722f3d3d245 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Protocol-independent write command used by the Kafka Produce backend. */ +@Internal +public final class KafkaProduceCommand { + + private final short acks; + private final int timeoutMs; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a Kafka write command. */ + public KafkaProduceCommand( + short acks, + int timeoutMs, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.acks = acks; + this.timeoutMs = timeoutMs; + this.topics = immutableCopy(topics); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns Kafka required acknowledgements. */ + public short acks() { + return acks; + } + + /** Returns the Produce timeout in milliseconds. */ + public int timeoutMs() { + return timeoutMs; + } + + /** Returns the topic writes in request order. */ + public List topics() { + return topics; + } + + /** Returns the listener that received the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client network address when available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Records addressed to one Kafka topic. */ + @Internal + public static final class TopicWrite { + private final String topicName; + private final List partitions; + + /** Creates the writes for one topic. */ + public TopicWrite(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns partition writes in request order. */ + public List partitions() { + return partitions; + } + } + + /** Records addressed to one Kafka partition. */ + @Internal + public static final class PartitionWrite { + private final int partitionId; + private final List records; + + /** Creates the writes for one partition. */ + public PartitionWrite(int partitionId, List records) { + this.partitionId = partitionId; + this.records = immutableCopy(records); + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns copied records in append order. */ + public List records() { + return records; + } + } + + /** A copied Kafka record whose lifetime is independent of the network request buffer. */ + @Internal + public static final class Record { + private final long timestamp; + private final @Nullable byte[] key; + private final @Nullable byte[] value; + private final List headers; + + /** Creates a copied Kafka record. */ + public Record( + long timestamp, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { + this.timestamp = timestamp; + this.key = copyNullable(key); + this.value = copyNullable(value); + this.headers = immutableCopy(headers); + } + + /** Returns the Kafka record timestamp. */ + public long timestamp() { + return timestamp; + } + + /** Returns a copy of the nullable Kafka record key. */ + public @Nullable byte[] key() { + return copyNullable(key); + } + + /** Returns a copy of the nullable Kafka record value. */ + public @Nullable byte[] value() { + return copyNullable(value); + } + + /** Returns the copied Kafka headers in record order. */ + public List headers() { + return headers; + } + + private static @Nullable byte[] copyNullable(@Nullable byte[] value) { + return value == null ? null : value.clone(); + } + } + + /** A copied Kafka record header. */ + @Internal + public static final class RecordHeader { + private final String name; + private final @Nullable byte[] value; + + /** Creates a copied Kafka record header. */ + public RecordHeader(String name, @Nullable byte[] value) { + this.name = checkNotNull(name); + this.value = value == null ? null : value.clone(); + } + + /** Returns the header name. */ + public String name() { + return name; + } + + /** Returns a copy of the nullable header value. */ + public @Nullable byte[] value() { + return value == null ? null : value.clone(); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java new file mode 100644 index 00000000000..218c566bde4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Result of a Kafka Produce backend invocation. */ +@Internal +public final class KafkaProduceResult { + private final List topics; + + /** Creates a Produce result. */ + public KafkaProduceResult(List topics) { + this.topics = immutableCopy(topics); + } + + /** Returns topic results in request order. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Results for one topic. */ + @Internal + public static final class TopicResult { + private final String topicName; + private final List partitions; + + /** Creates the result for one topic. */ + public TopicResult(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns the partition results in request order. */ + public List partitions() { + return partitions; + } + } + + /** Result for one partition. */ + @Internal + public static final class PartitionResult { + private final int partitionId; + private final Errors error; + private final long baseOffset; + private final @Nullable String errorMessage; + + /** Creates the result for one partition. */ + public PartitionResult( + int partitionId, Errors error, long baseOffset, @Nullable String errorMessage) { + this.partitionId = partitionId; + this.error = checkNotNull(error); + this.baseOffset = baseOffset; + this.errorMessage = errorMessage; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the Kafka protocol error. */ + public Errors error() { + return error; + } + + /** Returns the first appended offset, or {@code -1} on failure. */ + public long baseOffset() { + return baseOffset; + } + + /** Returns an optional diagnostic error message. */ + public @Nullable String errorMessage() { + return errorMessage; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java new file mode 100644 index 00000000000..bf043823689 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.kafka.api.produce.ProduceHandler; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; + +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol tests independent of the Fluss append and record conversion backends. */ +class KafkaProduceProtocolTest { + + @Test + void testSupportedVersionsCopyRecordsAndForwardContext() { + for (short version = 3; version <= 11; version++) { + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request(version, (short) -1, partition(0, records())), + command -> { + copied.set(command); + return successful(command); + }); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(42L); + KafkaProduceCommand command = copied.get(); + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(4321); + assertThat(command.listenerName()).isEqualTo("KAFKA"); + KafkaProduceCommand.Record record = + command.topics().get(0).partitions().get(0).records().get(0); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()).hasSize(1); + assertThat(record.headers().get(0).value()).isNull(); + byte[] key = record.key(); + key[0] = 9; + assertThat(record.key()).containsExactly((byte) 1); + } + } + + @Test + void testCorruptPartitionDoesNotSuppressValidPartition() { + MemoryRecords corrupt = records(); + corrupt.buffer().put(corrupt.sizeInBytes() - 1, (byte) 99); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, corrupt), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + assertThat(command.topics().get(0).partitions().get(0).partitionId()) + .isEqualTo(1); + return successful(command); + }); + assertErrors(response, Errors.CORRUPT_MESSAGE, Errors.NONE); + } + + @Test + void testInvalidTopicIsIsolated() { + ProduceRequest request = request((short) 11, (short) 1, partition(0, records())); + request.data() + .topicData() + .add( + new TopicProduceData() + .setName("bad/name") + .setPartitionData( + Collections.singletonList(partition(0, records())))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()).hasSize(1); + return successful(command); + }); + assertThat(response.errorCounts()) + .containsEntry(Errors.NONE, 1) + .containsEntry(Errors.INVALID_TOPIC_EXCEPTION, 1); + } + + @Test + void testEmptyNegativeAndIdempotentPartitionsAreRejectedLocally() { + MemoryRecords idempotent = + MemoryRecords.withIdempotentRecords( + Compression.NONE, 10L, (short) 0, 0, new SimpleRecord(new byte[] {1})); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(-1, records()), + partition(2, idempotent), + partition(3, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + return successful(command); + }); + assertErrors( + response, + Errors.INVALID_RECORD, + Errors.INVALID_REQUEST, + Errors.INVALID_REQUEST, + Errors.NONE); + } + + @Test + void testRequestValidationNeverCallsBackend() { + KafkaProduceBackend unused = + command -> { + throw new AssertionError("Backend must not be called"); + }; + assertErrors( + dispatch(request((short) 11, (short) 2, partition(0, records())), unused), + Errors.INVALID_REQUIRED_ACKS); + ProduceRequest transactional = request((short) 11, (short) 1, partition(0, records())); + transactional.data().setTransactionalId("transaction"); + assertErrors( + dispatch(new ProduceRequest(transactional.data(), transactional.version()), unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch( + request( + (short) 11, + (short) 1, + partition(0, records()), + partition(0, records())), + unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch(request((short) 2, (short) 1, partition(0, records())), unused), + Errors.UNSUPPORTED_VERSION); + } + + @Test + void testBackendFailuresAndMissingResultsPreserveValidationErrors() { + for (boolean synchronous : Arrays.asList(true, false)) { + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(1, records())), + command -> { + if (synchronous) { + throw new TimeoutException("append timed out"); + } + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally( + new TimeoutException("append timed out")); + return failed; + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.REQUEST_TIMED_OUT); + } + assertErrors( + dispatch( + request((short) 11, (short) 1, partition(0, records())), + command -> + CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList()))), + Errors.UNKNOWN_SERVER_ERROR); + } + + @Test + void testAcksZeroSuccessAndFailureReleaseBufferWithoutSendingResponse() { + for (boolean failure : Arrays.asList(false, true)) { + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + ProduceRequest body = request((short) 11, (short) 0, partition(0, records())); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, (short) 11, "producer", 1); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + CompletableFuture pending = new CompletableFuture<>(); + try { + channel.writeInbound(buffer); + KafkaRequest parsed = (KafkaRequest) requests.pollRequest(1000); + dispatcher(command -> pending) + .dispatch(parsed) + .whenComplete( + (response, error) -> { + if (error == null) { + parsed.complete(response); + } else { + parsed.fail(error); + } + }); + assertThat(parsed.future()).isNotDone(); + if (failure) { + pending.completeExceptionally(new TimeoutException("failure")); + } else { + pending.complete( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, Errors.NONE, 42L, null)))))); + } + channel.runPendingTasks(); + assertThat(parsed.future()).isDone(); + assertThat((Object) channel.readOutbound()).isNull(); + assertThat(buffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + } + + private static ProduceResponse dispatch(ProduceRequest body, KafkaProduceBackend backend) { + ByteBuf buffer = Unpooled.buffer(1); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, body.version(), "producer", 1); + KafkaRequest request = + new KafkaRequest( + ApiKeys.PRODUCE, + body.version(), + header, + body, + "KAFKA", + buffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + buffer.release(); + request.complete(dispatcher(backend).dispatch(request).join()); + ByteBuf response = request.responseBuffer(); + try { + ByteBuffer bytes = response.nioBuffer(); + ResponseHeader.parse(bytes, header.toResponseHeader().headerVersion()); + return (ProduceResponse) + AbstractResponse.parseResponse(ApiKeys.PRODUCE, bytes, body.version()); + } finally { + response.release(); + } + } + + private static KafkaRequestDispatcher dispatcher(KafkaProduceBackend backend) { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ProduceHandler(backend)); + registry.freeze(); + return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); + } + + private static CompletableFuture successful(KafkaProduceCommand command) { + List topics = new ArrayList<>(); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + List partitions = new ArrayList<>(); + for (KafkaProduceCommand.PartitionWrite partition : topic.partitions()) { + partitions.add( + new PartitionResult(partition.partitionId(), Errors.NONE, 42L, null)); + } + topics.add(new TopicResult(topic.topicName(), partitions)); + } + return CompletableFuture.completedFuture(new KafkaProduceResult(topics)); + } + + private static ProduceRequest request( + short version, short acks, PartitionProduceData... partitions) { + TopicProduceData topic = + new TopicProduceData().setName("topic").setPartitionData(Arrays.asList(partitions)); + return new ProduceRequest( + new ProduceRequestData() + .setAcks(acks) + .setTimeoutMs(4321) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topic).iterator())), + version); + } + + private static PartitionProduceData partition(int index, MemoryRecords records) { + return new PartitionProduceData().setIndex(index).setRecords(records); + } + + private static MemoryRecords records() { + return MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.NONE, + new SimpleRecord( + 123L, + new byte[] {1}, + new byte[] {2}, + new Header[] {new RecordHeader("header", null)})); + } + + private static void assertErrors(ProduceResponse response, Errors... errors) { + assertThat(response.data().responses().find("topic").partitionResponses()) + .extracting(PartitionProduceResponse::errorCode) + .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); + } +} From c568cd59eb9288c379340b431d95982e54f0d61a Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 17:04:41 +0800 Subject: [PATCH 12/69] [kafka] Validate Produce batch offsets and stream compressed records Read compressed records through a closeable streaming iterator to avoid preallocating a record list from the declared batch count. Reject incoming Kafka batches whose offset range disagrees with their record count before copying records or invoking the backend. Keep Fluss-assigned offsets independent of this input validation and preserve partition error isolation. Cover all five codecs, owned record copies, malformed counts and offset ranges, and backend-assigned offsets. Java 11 validation passed: 117 unit tests and 4 integration tests, plus Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 42/42 AI-Contributed/UT: 168/168 --- .../kafka/api/produce/ProduceHandler.java | 42 ++++- .../fluss/kafka/KafkaProduceProtocolTest.java | 168 ++++++++++++++++++ 2 files changed, 202 insertions(+), 8 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java index 0eb14b26ff2..a598c0d6cfa 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -30,6 +30,7 @@ import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.kafka.common.InvalidRecordException; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.InvalidRequiredAcksException; @@ -47,6 +48,8 @@ import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ProduceRequest; import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.CloseableIterator; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -219,14 +222,37 @@ private static List copyRecords( throw new InvalidRequestException( "Idempotent, transactional, and control record batches are not supported."); } - for (org.apache.kafka.common.record.Record record : batch) { - record.ensureValid(); - copied.add( - new KafkaProduceCommand.Record( - record.timestamp(), - copyBuffer(record.hasKey() ? record.key() : null), - copyBuffer(record.hasValue() ? record.value() : null), - copyHeaders(record.headers()))); + // Validate the incoming Kafka batch before copying away its offset metadata. Fluss + // assigns its own storage offsets later, independently of this input validation. + long countFromOffsets = batch.lastOffset() - batch.baseOffset() + 1; + Integer recordCount = batch.countOrNull(); + if (countFromOffsets <= 0 + || recordCount == null + || recordCount <= 0 + || countFromOffsets != recordCount) { + throw new InvalidRecordException( + "Invalid Kafka record batch offset range [" + + batch.baseOffset() + + ", " + + batch.lastOffset() + + "] for record count " + + recordCount + + "."); + } + // The ordinary compressed iterator preallocates a list using the untrusted record + // count. Stream records instead and close the decompressor on success and failure. + try (CloseableIterator iterator = + batch.streamingIterator(BufferSupplier.NO_CACHING)) { + while (iterator.hasNext()) { + org.apache.kafka.common.record.Record record = iterator.next(); + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } } } if (copied.isEmpty() || validBytes != records.sizeInBytes()) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java index bf043823689..c2160805510 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -41,6 +41,8 @@ import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.DefaultRecordBatch; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.SimpleRecord; @@ -50,7 +52,10 @@ import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.requests.ResponseHeader; +import org.apache.kafka.common.utils.Crc32C; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -102,6 +107,161 @@ void testSupportedVersionsCopyRecordsAndForwardContext() { } } + @ParameterizedTest + @EnumSource(CompressionType.class) + void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType) { + byte[] value = new byte[32 * 1024]; + Arrays.fill(value, (byte) 7); + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord( + 123L, + new byte[] {1}, + value, + new Header[] { + new RecordHeader("header", new byte[] {2, 3}), + new RecordHeader("header", null) + }), + new SimpleRecord(456L, null, new byte[0])); + AtomicReference copied = new AtomicReference<>(); + + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + copied.set(command); + return successful(command); + }); + assertErrors(response, Errors.NONE); + ByteBuffer input = records.buffer().duplicate(); + while (input.hasRemaining()) { + input.put((byte) 0); + } + + List result = + copied.get().topics().get(0).partitions().get(0).records(); + assertThat(result).hasSize(2); + assertThat(result.get(0).timestamp()).isEqualTo(123L); + assertThat(result.get(0).key()).containsExactly((byte) 1); + assertThat(result.get(0).value()).containsExactly(value); + assertThat(result.get(0).headers()) + .extracting(KafkaProduceCommand.RecordHeader::name) + .containsExactly("header", "header"); + assertThat(result.get(0).headers().get(0).value()).containsExactly((byte) 2, (byte) 3); + assertThat(result.get(0).headers().get(1).value()).isNull(); + assertThat(result.get(1).timestamp()).isEqualTo(456L); + assertThat(result.get(1).key()).isNull(); + assertThat(result.get(1).value()).isEmpty(); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testIncorrectRecordCountDoesNotSuppressValidPartition(CompressionType compressionType) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + ByteBuffer bytes = malformed.buffer(); + bytes.putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, 32); + bytes.putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, 31); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testInvalidBatchOffsetRangeDoesNotSuppressValidPartition(CompressionType compressionType) { + for (int[] invalidHeader : + new int[][] {{1, -1}, {1, 9}, {1, Integer.MAX_VALUE}, {0, 0}, {-1, 0}}) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + int recordCount = invalidHeader[0]; + int lastOffsetDelta = invalidHeader[1]; + malformed.buffer().putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, recordCount); + malformed.buffer().putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, lastOffsetDelta); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testValidBatchUsesOffsetAssignedByBackend(CompressionType compressionType) { + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1}), + new SimpleRecord(new byte[] {2}), + new SimpleRecord(new byte[] {3})); + long backendBaseOffset = 1L << 40; + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + assertThat(command.topics().get(0).partitions().get(0).records()) + .hasSize(3); + return CompletableFuture.completedFuture( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, + Errors.NONE, + backendBaseOffset, + null)))))); + }); + assertErrors(response, Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(backendBaseOffset); + } + @Test void testCorruptPartitionDoesNotSuppressValidPartition() { MemoryRecords corrupt = records(); @@ -363,4 +523,12 @@ private static void assertErrors(ProduceResponse response, Errors... errors) { .extracting(PartitionProduceResponse::errorCode) .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); } + + private static void updateBatchChecksum(MemoryRecords records) { + ByteBuffer bytes = records.buffer(); + int checksumStart = DefaultRecordBatch.CRC_OFFSET + Integer.BYTES; + bytes.putInt( + DefaultRecordBatch.CRC_OFFSET, + (int) Crc32C.compute(bytes, checksumStart, bytes.limit() - checksumStart)); + } } From 91ef63cde565e0c303615222474fb3ffc1af47b5 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 18:25:10 +0800 Subject: [PATCH 13/69] [kafka] Preserve batch timestamp types in Produce commands Carry timestamp provenance through a protocol-independent enum on each copied record. Read the type from its batch for compressed and uncompressed records; keep timestamp presence separate. Retain the existing constructor with an explicit UNKNOWN type instead of guessing provenance from the numeric value. Cover mixed batch types across partitions, absent timestamps, owned data and legacy constructors. Keep partition validation and acks=0 behavior, and check that ApiVersions advertises the registered Produce range. Metadata row storage and production append registration remain outside this protocol-only change. Validated with Java 11 and this worktree's clean reactor: 152 unit tests and 5 integration tests passed, including ApiVersions and Metadata regressions. Spotless, Checkstyle, RAT and git diff --check passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 64/64 AI-Contributed/UT: 326/326 --- .../kafka/api/produce/ProduceHandler.java | 16 ++ .../backend/produce/KafkaProduceCommand.java | 48 ++++- .../fluss/kafka/KafkaProduceProtocolTest.java | 159 ++++++++++++++++- .../api/versions/ApiVersionsHandlerTest.java | 7 + .../produce/KafkaProduceCommandTest.java | 164 ++++++++++++++++++ 5 files changed, 388 insertions(+), 6 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java index a598c0d6cfa..a679636a2c9 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -23,6 +23,7 @@ import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; @@ -239,6 +240,7 @@ private static List copyRecords( + recordCount + "."); } + TimestampType timestampType = timestampType(batch); // The ordinary compressed iterator preallocates a list using the untrusted record // count. Stream records instead and close the decompressor on success and failure. try (CloseableIterator iterator = @@ -249,6 +251,7 @@ private static List copyRecords( copied.add( new KafkaProduceCommand.Record( record.timestamp(), + timestampType, copyBuffer(record.hasKey() ? record.key() : null), copyBuffer(record.hasValue() ? record.value() : null), copyHeaders(record.headers()))); @@ -261,6 +264,19 @@ private static List copyRecords( return copied; } + private static TimestampType timestampType(RecordBatch batch) { + switch (batch.timestampType()) { + case CREATE_TIME: + return TimestampType.CREATE_TIME; + case LOG_APPEND_TIME: + return TimestampType.LOG_APPEND_TIME; + case NO_TIMESTAMP_TYPE: + return TimestampType.UNKNOWN; + default: + throw new InvalidRecordException("Unknown batch timestamp type."); + } + } + private static ProduceResponse toResponse(KafkaProduceResult result) { ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0); for (TopicResult topic : result.topics()) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index 722f3d3d245..9a4f035ad19 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -127,31 +127,75 @@ public List records() { } } + /** + * Timestamp provenance supplied by a batch, independent of whether a record has a timestamp. + */ + @Internal + public enum TimestampType { + /** No batch timestamp type was supplied, including by the legacy record constructor. */ + UNKNOWN, + /** The timestamp was assigned when the record was created. */ + CREATE_TIME, + /** The timestamp was assigned when the batch was appended to a log. */ + LOG_APPEND_TIME + } + /** A copied Kafka record whose lifetime is independent of the network request buffer. */ @Internal public static final class Record { + /** Sentinel used when the input record has no timestamp. */ + public static final long NO_TIMESTAMP = -1L; + private final long timestamp; + private final TimestampType timestampType; private final @Nullable byte[] key; private final @Nullable byte[] value; private final List headers; - /** Creates a copied Kafka record. */ + /** + * Creates a copied record without batch timestamp provenance. + * + *

The type is {@link TimestampType#UNKNOWN} even when a timestamp is present. Callers + * that know the batch type should use the explicit constructor instead of inferring it from + * the timestamp value. + */ public Record( long timestamp, @Nullable byte[] key, @Nullable byte[] value, List headers) { + this(timestamp, TimestampType.UNKNOWN, key, value, headers); + } + + /** Creates a copied record with the timestamp type supplied by its source batch. */ + public Record( + long timestamp, + TimestampType timestampType, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { this.timestamp = timestamp; + this.timestampType = checkNotNull(timestampType); this.key = copyNullable(key); this.value = copyNullable(value); this.headers = immutableCopy(headers); } - /** Returns the Kafka record timestamp. */ + /** Returns the record timestamp in milliseconds, or {@link #NO_TIMESTAMP} when absent. */ public long timestamp() { return timestamp; } + /** Returns whether the input record has a timestamp, independently of its batch type. */ + public boolean hasTimestamp() { + return timestamp != NO_TIMESTAMP; + } + + /** Returns the source batch's timestamp type, never inferred from the timestamp value. */ + public TimestampType timestampType() { + return timestampType; + } + /** Returns a copy of the nullable Kafka record key. */ public @Nullable byte[] key() { return copyNullable(key); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java index c2160805510..63d9f8b7c6d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -20,6 +20,7 @@ import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; @@ -44,6 +45,7 @@ import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.DefaultRecordBatch; import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.MemoryRecordsBuilder; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.SimpleRecord; import org.apache.kafka.common.requests.AbstractResponse; @@ -97,6 +99,8 @@ void testSupportedVersionsCopyRecordsAndForwardContext() { KafkaProduceCommand.Record record = command.topics().get(0).partitions().get(0).records().get(0); assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(record.hasTimestamp()).isTrue(); assertThat(record.key()).containsExactly((byte) 1); assertThat(record.value()).containsExactly((byte) 2); assertThat(record.headers()).hasSize(1); @@ -107,6 +111,120 @@ void testSupportedVersionsCopyRecordsAndForwardContext() { } } + @ParameterizedTest + @EnumSource(CompressionType.class) + void testTimestampTypesRemainLocalToTheirBatches(CompressionType compressionType) { + MemoryRecords createTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + 123L, + 456L); + MemoryRecords appendTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + 123L, + 789L, + 987L); + MemoryRecords missingCreateTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP); + MemoryRecords missingAppendTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP); + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, createTime), + partition(1, appendTime), + partition(2, missingCreateTime), + partition(3, missingAppendTime)), + command -> { + copied.set(command); + return successful(command); + }); + assertErrors(response, Errors.NONE, Errors.NONE, Errors.NONE, Errors.NONE); + for (MemoryRecords input : + Arrays.asList(createTime, appendTime, missingCreateTime, missingAppendTime)) { + ByteBuffer buffer = input.buffer().duplicate(); + while (buffer.hasRemaining()) { + buffer.put((byte) 0); + } + } + + List partitions = + copied.get().topics().get(0).partitions(); + assertThat(partitions).hasSize(4); + assertThat(partitions.get(0).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(123L, 456L); + // The same numeric timestamp can have a different batch type. + assertThat(partitions.get(1).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(123L, 123L); + for (int i = 0; i < partitions.size(); i++) { + TimestampType expectedType = + i % 2 == 0 ? TimestampType.CREATE_TIME : TimestampType.LOG_APPEND_TIME; + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::timestampType) + .containsExactly(expectedType, expectedType); + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::hasTimestamp) + .containsExactly(i < 2, i < 2); + if (i >= 2) { + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(RecordBatch.NO_TIMESTAMP, RecordBatch.NO_TIMESTAMP); + } + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testMultipleBatchesPerPartitionRemainInvalid(CompressionType compressionType) { + MemoryRecords first = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + 123L); + MemoryRecords second = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + 456L, + 123L); + ByteBuffer combined = ByteBuffer.allocate(first.sizeInBytes() + second.sizeInBytes()); + combined.put(first.buffer().duplicate()).put(second.buffer().duplicate()).flip(); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.readableRecords(combined)), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + @ParameterizedTest @EnumSource(CompressionType.class) void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType) { @@ -145,6 +263,7 @@ void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType copied.get().topics().get(0).partitions().get(0).records(); assertThat(result).hasSize(2); assertThat(result.get(0).timestamp()).isEqualTo(123L); + assertThat(result.get(0).timestampType()).isEqualTo(TimestampType.CREATE_TIME); assertThat(result.get(0).key()).containsExactly((byte) 1); assertThat(result.get(0).value()).containsExactly(value); assertThat(result.get(0).headers()) @@ -153,6 +272,7 @@ void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType assertThat(result.get(0).headers().get(0).value()).containsExactly((byte) 2, (byte) 3); assertThat(result.get(0).headers().get(1).value()).isNull(); assertThat(result.get(1).timestamp()).isEqualTo(456L); + assertThat(result.get(1).timestampType()).isEqualTo(TimestampType.CREATE_TIME); assertThat(result.get(1).key()).isNull(); assertThat(result.get(1).value()).isEmpty(); } @@ -409,7 +529,19 @@ void testAcksZeroSuccessAndFailureReleaseBufferWithoutSendingResponse() { try { channel.writeInbound(buffer); KafkaRequest parsed = (KafkaRequest) requests.pollRequest(1000); - dispatcher(command -> pending) + dispatcher( + command -> { + assertThat( + command.topics() + .get(0) + .partitions() + .get(0) + .records() + .get(0) + .timestampType()) + .isEqualTo(TimestampType.CREATE_TIME); + return pending; + }) .dispatch(parsed) .whenComplete( (response, error) -> { @@ -469,9 +601,8 @@ private static ProduceResponse dispatch(ProduceRequest body, KafkaProduceBackend } private static KafkaRequestDispatcher dispatcher(KafkaProduceBackend backend) { - KafkaApiRegistry registry = new KafkaApiRegistry(); - registry.register(new ProduceHandler(backend)); - registry.freeze(); + KafkaApiRegistry registry = + new KafkaApiRegistry(Collections.singletonList(new ProduceHandler(backend))); return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @@ -518,6 +649,26 @@ private static MemoryRecords records() { new Header[] {new RecordHeader("header", null)})); } + private static MemoryRecords timestampedRecords( + CompressionType compressionType, + org.apache.kafka.common.record.TimestampType timestampType, + long logAppendTime, + long... timestamps) { + try (MemoryRecordsBuilder builder = + MemoryRecords.builder( + ByteBuffer.allocate(1024), + RecordBatch.MAGIC_VALUE_V2, + Compression.of(compressionType).build(), + timestampType, + 0L, + logAppendTime)) { + for (long timestamp : timestamps) { + builder.append(new SimpleRecord(timestamp, new byte[] {1}, new byte[] {2})); + } + return builder.build(); + } + } + private static void assertErrors(ProduceResponse response, Errors... errors) { assertThat(response.data().responses().find("topic").partitionResponses()) .extracting(PartitionProduceResponse::errorCode) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java index 0492ee3572f..9d7912b5b42 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.api.versions; import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; @@ -59,6 +60,11 @@ void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { new KafkaApiRegistry( Arrays.asList( handler, + new ProduceHandler( + command -> { + throw new AssertionError( + "Capability advertisement must not invoke the Produce backend."); + }), metadataHandler, new TestingApiHandler( new KafkaApiSpec( @@ -78,6 +84,7 @@ void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, (short) 11), tuple(ApiKeys.METADATA.id, (short) 2, (short) 7), tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java new file mode 100644 index 00000000000..01fa6825b28 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests timestamp provenance and ownership of protocol-independent Produce commands. */ +class KafkaProduceCommandTest { + + @ParameterizedTest + @ValueSource(longs = {-1L, 0L, 123L}) + void testLegacyConstructorDoesNotInferBatchType(long timestamp) { + Record record = new Record(timestamp, null, null, Collections.emptyList()); + + assertThat(record.timestamp()).isEqualTo(timestamp); + assertThat(record.hasTimestamp()).isEqualTo(timestamp != Record.NO_TIMESTAMP); + assertThat(record.timestampType()).isEqualTo(TimestampType.UNKNOWN); + assertThat(record.key()).isNull(); + assertThat(record.value()).isNull(); + } + + @ParameterizedTest + @EnumSource(TimestampType.class) + void testTimestampPresenceDoesNotChangeExplicitBatchType(TimestampType timestampType) { + for (long timestamp : new long[] {Record.NO_TIMESTAMP, 0L, 123L}) { + Record record = + new Record(timestamp, timestampType, null, null, Collections.emptyList()); + + assertThat(record.timestampType()).isEqualTo(timestampType); + assertThat(record.timestamp()).isEqualTo(timestamp); + assertThat(record.hasTimestamp()).isEqualTo(timestamp != Record.NO_TIMESTAMP); + } + } + + @Test + void testExplicitBatchTypeMustNotBeNull() { + assertThatThrownBy(() -> new Record(123L, null, null, null, Collections.emptyList())) + .isInstanceOf(NullPointerException.class); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testBothConstructorsOwnTheirRecordData(boolean explicitTimestampType) { + byte[] key = {1}; + byte[] value = {2}; + byte[] headerValue = {3}; + List headers = + new ArrayList<>( + Arrays.asList( + new RecordHeader("header", headerValue), + new RecordHeader("header", null))); + Record record = + explicitTimestampType + ? new Record(123L, TimestampType.LOG_APPEND_TIME, key, value, headers) + : new Record(123L, key, value, headers); + key[0] = 4; + value[0] = 5; + headerValue[0] = 6; + headers.clear(); + record.key()[0] = 7; + record.value()[0] = 8; + record.headers().get(0).value()[0] = 9; + + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()) + .extracting(RecordHeader::name) + .containsExactly("header", "header"); + assertThat(record.headers().get(0).value()).containsExactly((byte) 3); + assertThat(record.headers().get(1).value()).isNull(); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.timestampType()) + .isEqualTo( + explicitTimestampType + ? TimestampType.LOG_APPEND_TIME + : TimestampType.UNKNOWN); + assertThatThrownBy(() -> record.headers().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { + List records = + new ArrayList<>( + Arrays.asList( + new Record( + 123L, + TimestampType.CREATE_TIME, + null, + null, + Collections.emptyList()), + new Record( + 123L, + TimestampType.LOG_APPEND_TIME, + null, + null, + Collections.emptyList()), + new Record( + Record.NO_TIMESTAMP, null, null, Collections.emptyList()))); + List partitions = + new ArrayList<>(Collections.singletonList(new PartitionWrite(2, records))); + List topics = + new ArrayList<>(Collections.singletonList(new TopicWrite("db.topic", partitions))); + KafkaProduceCommand command = + new KafkaProduceCommand((short) -1, 1000, topics, "KAFKA", null); + records.clear(); + partitions.clear(); + topics.clear(); + + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(1000); + assertThat(command.topics()).extracting(TopicWrite::topicName).containsExactly("db.topic"); + assertThat(command.topics().get(0).partitions()) + .extracting(PartitionWrite::partitionId) + .containsExactly(2); + List copied = command.topics().get(0).partitions().get(0).records(); + assertThat(copied) + .extracting(Record::timestampType) + .containsExactly( + TimestampType.CREATE_TIME, + TimestampType.LOG_APPEND_TIME, + TimestampType.UNKNOWN); + assertThat(copied) + .extracting(Record::timestamp) + .containsExactly(123L, 123L, Record.NO_TIMESTAMP); + assertThatThrownBy(() -> command.topics().clear()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> command.topics().get(0).partitions().clear()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(copied::clear).isInstanceOf(UnsupportedOperationException.class); + } +} From a565b8aa0e62194863e056458f7dc495e792493f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 17:50:34 +0800 Subject: [PATCH 14/69] [kafka] Verify Produce topic aliases preserve request names Document that the protocol backend receives original topic names and returns results keyed by those names, leaving database resolution to the adapter. Cover bare and qualified names together across Produce v3-v11, reordered backend results, local validation failures, backend errors and invalid acks. Keep null and empty record values distinct in the owned protocol command. Validated on Java 11 with this worktree's clean reactor: 162 Kafka unit tests, 23 integration tests, and 34 Common/Server schema and DDL tests passed. Spotless, Checkstyle, RAT and git diff --check passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 11/11 AI-Contributed/UT: 150/150 --- .../kafka/api/produce/ProduceHandler.java | 7 +- .../backend/produce/KafkaProduceCommand.java | 2 +- .../backend/produce/KafkaProduceResult.java | 2 +- .../fluss/kafka/KafkaProduceProtocolTest.java | 150 +++++++++++++++++- 4 files changed, 155 insertions(+), 6 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java index a679636a2c9..c485e618152 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -68,7 +68,12 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Implements non-idempotent Kafka Produce versions 3 through 11. */ +/** + * Implements non-idempotent Kafka Produce versions 3 through 11. + * + *

Topic names cross the backend boundary unchanged. The backend resolves bare and qualified + * names and returns results keyed by the original request names. + */ @Internal public final class ProduceHandler implements KafkaApiHandler { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index 9a4f035ad19..aadaa36f737 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -93,7 +93,7 @@ public TopicWrite(String topicName, List partitions) { this.partitions = immutableCopy(partitions); } - /** Returns the Kafka topic name. */ + /** Returns the original request topic name, including any default-database alias. */ public String topicName() { return topicName; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java index 218c566bde4..8f41864f85b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java @@ -60,7 +60,7 @@ public TopicResult(String topicName, List partitions) { this.partitions = immutableCopy(partitions); } - /** Returns the Kafka topic name. */ + /** Returns the original request topic name, rather than a resolved table path. */ public String topicName() { return topicName; } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java index 63d9f8b7c6d..1a801cd55ed 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -40,6 +40,7 @@ import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.message.ProduceResponseData.TopicProduceResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.CompressionType; @@ -58,6 +59,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -68,6 +70,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; /** Protocol tests independent of the Fluss append and record conversion backends. */ class KafkaProduceProtocolTest { @@ -111,6 +114,134 @@ void testSupportedVersionsCopyRecordsAndForwardContext() { } } + @ParameterizedTest + @ValueSource(shorts = {3, 4, 5, 6, 7, 8, 9, 10, 11}) + void testBareAndQualifiedTopicNamesRemainUnchanged(short version) { + MemoryRecords nullableValues = + MemoryRecords.withRecords( + Compression.NONE, + new SimpleRecord(123L, new byte[] {1}, null), + new SimpleRecord(456L, null, new byte[0])); + ProduceRequest request = + requestForTopics( + version, + (short) 1, + topic("orders", partition(0, nullableValues)), + topic("fluss.orders", partition(0, records())), + topic("sales.orders", partition(0, records()))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()) + .extracting(KafkaProduceCommand.TopicWrite::topicName) + .containsExactly("orders", "fluss.orders", "sales.orders"); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + assertThat(topic.partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(0); + } + List copied = + command.topics().get(0).partitions().get(0).records(); + assertThat(copied).hasSize(2); + // The protocol command preserves null and empty values for the adapter. + assertThat(copied.get(0).value()).isNull(); + assertThat(copied.get(1).value()).isEmpty(); + List results = + new ArrayList<>(successful(command).join().topics()); + Collections.reverse(results); + return CompletableFuture.completedFuture( + new KafkaProduceResult(results)); + }); + + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactly("orders", "fluss.orders", "sales.orders"); + for (TopicProduceResponse topic : response.data().responses()) { + assertThat(topic.partitionResponses()) + .extracting( + PartitionProduceResponse::index, + PartitionProduceResponse::errorCode, + PartitionProduceResponse::baseOffset) + .containsExactly(tuple(0, Errors.NONE.code(), 42L)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"orders", "fluss.orders"}) + void testLocalAndBackendErrorsAreIsolatedByRequestTopicName(String invalidTopic) { + String validTopic = invalidTopic.equals("orders") ? "fluss.orders" : "orders"; + ProduceResponse response = + dispatch( + requestForTopics( + (short) 11, + (short) 1, + topic(invalidTopic, partition(0, MemoryRecords.EMPTY)), + topic(validTopic, partition(0, records())), + topic("sales.orders", partition(0, records()))), + command -> { + assertThat(command.topics()) + .extracting(KafkaProduceCommand.TopicWrite::topicName) + .containsExactly(validTopic, "sales.orders"); + return CompletableFuture.completedFuture( + new KafkaProduceResult( + Arrays.asList( + new TopicResult( + "sales.orders", + Collections.singletonList( + new PartitionResult( + 0, + Errors + .TOPIC_AUTHORIZATION_FAILED, + -1L, + null))), + new TopicResult( + validTopic, + Collections.singletonList( + new PartitionResult( + 0, + Errors.NONE, + 42L, + null)))))); + }); + + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactly(invalidTopic, validTopic, "sales.orders"); + assertTopicError(response, invalidTopic, Errors.INVALID_RECORD); + assertTopicError(response, validTopic, Errors.NONE); + assertTopicError(response, "sales.orders", Errors.TOPIC_AUTHORIZATION_FAILED); + } + + @ParameterizedTest + @ValueSource(shorts = {3, 4, 5, 6, 7, 8, 9, 10, 11}) + void testRequestAndBackendFailuresPreserveTopicNames(short version) { + for (boolean invalidAcks : Arrays.asList(false, true)) { + ProduceResponse response = + dispatch( + requestForTopics( + version, + invalidAcks ? (short) 2 : (short) 1, + topic("orders", partition(0, records())), + topic("fluss.orders", partition(0, records())), + topic("sales.orders", partition(0, records()))), + command -> { + assertThat(invalidAcks).isFalse(); + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally(new TimeoutException("append failed")); + return failed; + }); + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactlyInAnyOrder("orders", "fluss.orders", "sales.orders"); + Errors error = invalidAcks ? Errors.INVALID_REQUIRED_ACKS : Errors.REQUEST_TIMED_OUT; + for (String name : Arrays.asList("orders", "fluss.orders", "sales.orders")) { + assertTopicError(response, name, error); + } + } + } + @ParameterizedTest @EnumSource(CompressionType.class) void testTimestampTypesRemainLocalToTheirBatches(CompressionType compressionType) { @@ -621,18 +752,25 @@ private static CompletableFuture successful(KafkaProduceComm private static ProduceRequest request( short version, short acks, PartitionProduceData... partitions) { - TopicProduceData topic = - new TopicProduceData().setName("topic").setPartitionData(Arrays.asList(partitions)); + return requestForTopics(version, acks, topic("topic", partitions)); + } + + private static ProduceRequest requestForTopics( + short version, short acks, TopicProduceData... topics) { return new ProduceRequest( new ProduceRequestData() .setAcks(acks) .setTimeoutMs(4321) .setTopicData( new ProduceRequestData.TopicProduceDataCollection( - Collections.singletonList(topic).iterator())), + Arrays.asList(topics).iterator())), version); } + private static TopicProduceData topic(String name, PartitionProduceData... partitions) { + return new TopicProduceData().setName(name).setPartitionData(Arrays.asList(partitions)); + } + private static PartitionProduceData partition(int index, MemoryRecords records) { return new PartitionProduceData().setIndex(index).setRecords(records); } @@ -675,6 +813,12 @@ private static void assertErrors(ProduceResponse response, Errors... errors) { .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); } + private static void assertTopicError(ProduceResponse response, String topic, Errors error) { + assertThat(response.data().responses().find(topic).partitionResponses()) + .extracting(PartitionProduceResponse::index, PartitionProduceResponse::errorCode) + .containsExactly(tuple(0, error.code())); + } + private static void updateBatchChecksum(MemoryRecords records) { ByteBuffer bytes = records.buffer(); int checksumStart = DefaultRecordBatch.CRC_OFFSET + Integer.BYTES; From 28e43efe34b59d7308a410b438aa35eab46afdb9 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:21:35 +0800 Subject: [PATCH 15/69] [kafka] Transcode mapped raw and string records to Arrow Reuse the DDL contract to assemble key, value, timestamp and ordered headers into physical rows. Decode strings with strict UTF-8 validation and preserve nullable values. Return owned heap log bytes and recycle Arrow writers on success and failure. Validation: Java 11, mvn -o -pl fluss-kafka spotless:apply clean verify; 75 unit tests and 2 integration tests passed, including Checkstyle, Spotless and RAT. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 314/314 AI-Contributed/UT: 326/326 --- .../transcode/ArrowKafkaRecordTranscoder.java | 87 +++++ .../transcode/FlussArrowRecordEncoder.java | 71 ++++ .../KafkaRecordEncodingException.java | 35 ++ .../transcode/KafkaRecordTranscoder.java | 32 ++ .../kafka/transcode/KafkaRowAssembler.java | 89 +++++ .../ArrowKafkaRecordTranscoderTest.java | 326 ++++++++++++++++++ 6 files changed, 640 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java new file mode 100644 index 00000000000..717f5cd33ff --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; + +import javax.annotation.Nullable; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Converts raw/string Kafka records using the DDL mapping into owned Fluss Arrow log bytes. */ +@Internal +public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); + + @Override + public BytesView transcode(List records, TableInfo tableInfo) throws Exception { + checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); + KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaRowAssembler assembler = new KafkaRowAssembler(schema); + List rows = new ArrayList<>(records.size()); + for (Record record : records) { + Object[] key = + schema.keyFormat() == null + ? new Object[0] + : new Object[] {decode(schema.keyFormat(), record.key())}; + rows.add( + assembler.assemble( + key, + new Object[] {decode(schema.valueFormat(), record.value())}, + record.timestamp(), + record.headers())); + } + return arrowRecordEncoder.encode(rows, tableInfo); + } + + private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) { + if (bytes == null || format == KafkaDataFormat.RAW) { + return bytes; + } + if (format != KafkaDataFormat.STRING) { + throw new KafkaRecordEncodingException("Unsupported Kafka data format: " + format); + } + try { + return BinaryString.fromString( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString()); + } catch (CharacterCodingException e) { + throw new KafkaRecordEncodingException("Kafka string field is not valid UTF-8.", e); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java new file mode 100644 index 00000000000..f74680ccaa2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.memory.UnmanagedPagedOutputView; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.MemoryLogRecordsArrowBuilder; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.arrow.ArrowWriter; +import org.apache.fluss.row.arrow.ArrowWriterPool; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator; + +import java.util.List; + +/** Encodes assembled physical Fluss rows into one native Arrow log batch. */ +@Internal +public final class FlussArrowRecordEncoder { + + private static final int INITIAL_PAGE_SIZE = 4096; + + /** Encodes all rows using the table's current schema ID and Arrow compression settings. */ + public BytesView encode(List rows, TableInfo tableInfo) throws Exception { + try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + ArrowWriterPool provider = new ArrowWriterPool(allocator)) { + ArrowWriter writer = + provider.getOrCreateWriter( + tableInfo.getTableId(), + tableInfo.getSchemaId(), + Integer.MAX_VALUE, + tableInfo.getRowType(), + tableInfo.getTableConfig().getArrowCompressionInfo()); + long epoch = writer.getEpoch(); + try { + MemoryLogRecordsArrowBuilder builder = + MemoryLogRecordsArrowBuilder.builder( + tableInfo.getSchemaId(), + writer, + new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), + true, + null); + for (GenericRow row : rows) { + builder.append(ChangeType.APPEND_ONLY, row); + } + // The output view owns heap pages. The result remains valid after the Arrow + // writer and allocator close, including across asynchronous append completion. + return builder.build(); + } finally { + writer.recycle(epoch); + } + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java new file mode 100644 index 00000000000..b48dd9d894f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that Kafka record bytes cannot be decoded using the configured data format. */ +@Internal +public final class KafkaRecordEncodingException extends IllegalArgumentException { + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message) { + super(message); + } + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java new file mode 100644 index 00000000000..eeff2f3d14f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; + +import java.util.List; + +/** Converts copied Kafka records into the native Fluss log representation. */ +@Internal +public interface KafkaRecordTranscoder { + /** Transcodes records according to the target Fluss table schema and log format. */ + BytesView transcode(List records, TableInfo tableInfo) throws Exception; +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java new file mode 100644 index 00000000000..4ebd506dc3c --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.schema.KafkaFieldProjection; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; + +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Assembles decoded Kafka key, value, and metadata into a physical Fluss row. */ +@Internal +public final class KafkaRowAssembler { + + private final KafkaTopicSchema topicSchema; + + /** Creates an assembler for the resolved topic schema. */ + public KafkaRowAssembler(KafkaTopicSchema topicSchema) { + this.topicSchema = checkNotNull(topicSchema); + } + + /** Assembles one Fluss row. */ + public GenericRow assemble( + Object[] keyValues, Object[] valueValues, long timestamp, List headers) { + GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); + setProjectedFields(row, topicSchema.keyProjection(), keyValues); + setProjectedFields(row, topicSchema.valueProjection(), valueValues); + if (topicSchema.timestampPosition() >= 0) { + row.setField(topicSchema.timestampPosition(), TimestampLtz.fromEpochMillis(timestamp)); + } + if (topicSchema.headersPosition() >= 0) { + row.setField(topicSchema.headersPosition(), toHeaders(headers)); + } + return row; + } + + private static void setProjectedFields( + GenericRow row, KafkaFieldProjection projection, Object[] values) { + if (values.length != projection.size()) { + throw new IllegalArgumentException( + "Kafka decoder returned " + + values.length + + " fields for a projection of " + + projection.size() + + "."); + } + for (int i = 0; i < values.length; i++) { + Object value = values[i]; + if (value == null && !projection.dataTypeAt(i).isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka record cannot populate NOT NULL field '" + + projection.nameAt(i) + + "' with null."); + } + row.setField(projection.positionAt(i), value); + } + } + + private static GenericArray toHeaders(List headers) { + Object[] rows = new Object[headers.size()]; + for (int i = 0; i < headers.size(); i++) { + RecordHeader header = headers.get(i); + rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.value()); + } + return new GenericArray(rows); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java new file mode 100644 index 00000000000..6936b5c8d61 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.CloseableIterator; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; + +/** Verifies mapped Arrow records after transcoding resources have closed. */ +class ArrowKafkaRecordTranscoderTest { + private final ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + + @Test + void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { + for (boolean string : new boolean[] {false, true}) { + TableInfo table = envelope(string); + Record record = + new Record( + 123L, + bytes("键"), + bytes("message"), + Arrays.asList( + new RecordHeader("duplicate", bytes("first")), + new RecordHeader("duplicate", null))); + BytesView encoded = transcoder.transcode(Arrays.asList(record, record), table); + read( + encoded, + table, + 2, + row -> { + if (string) { + assertThat(row.getString(0).toString()).isEqualTo("message"); + assertThat(row.getString(3).toString()).isEqualTo("键"); + } else { + assertThat(row.getBytes(0)).isEqualTo(bytes("message")); + assertThat(row.getBytes(3)).isEqualTo(bytes("键")); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + InternalArray headers = row.getArray(2); + assertThat(headers.size()).isEqualTo(2); + assertThat(headers.getRow(0, 2).getString(0).toString()) + .isEqualTo("duplicate"); + assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("first")); + assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); + }); + } + } + + @Test + void testNullsAndEmptyBytesRemainDistinct() throws Exception { + TableInfo table = envelope(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, null, null, Collections.emptyList())), + table), + table, + 1, + row -> { + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.isNullAt(3)).isTrue(); + assertThat(row.getArray(2).size()).isZero(); + }); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, new byte[0], new byte[0], Collections.emptyList())), + table), + table, + 1, + row -> { + assertThat(row.getBytes(0)).isEmpty(); + assertThat(row.getBytes(3)).isEmpty(); + }); + } + + @Test + void testUnmappedKeyIsIgnoredAndMixedFormatsWork() throws Exception { + TableInfo valueOnly = valueTable(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + new byte[] {(byte) 0xff}, + bytes("value"), + Collections.emptyList())), + valueOnly), + valueOnly, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("value")); + Schema schema = + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("id", DataTypes.STRING()) + .build(); + TableInfo mixed = + table( + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .build()); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + bytes("key"), + new byte[] {(byte) 0xff}, + Collections.emptyList())), + mixed), + mixed, + 1, + row -> { + assertThat(row.getString(1).toString()).isEqualTo("key"); + assertThat(row.getBytes(0)).containsExactly((byte) 0xff); + }); + } + + @Test + void testRejectsMalformedUtf8AndNotNullViolationsThenRemainsUsable() throws Exception { + TableInfo table = valueTable(false); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + new byte[] {(byte) 0xc3, 0x28}, + Collections.emptyList())), + table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("UTF-8"); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, null, null, Collections.emptyList())), + valueTable(true))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL"); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, null, bytes("ok"), Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("ok")); + } + + @Test + void testRejectsInvalidMappingAndEmptyPartition() { + TableInfo invalid = + table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING()) + .build()) + .distributedBy(1) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + bytes("value"), + Collections.emptyList())), + invalid)) + .isInstanceOf(KafkaTopicSchemaException.class); + assertThatThrownBy(() -> transcoder.transcode(Collections.emptyList(), valueTable(false))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("empty"); + } + + @Test + void testArrowWriterFailureClosesAllocatorWithoutLeaking() { + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encode( + Collections.singletonList(GenericRow.of(123)), + valueTable(false))); + assertThat(failure).isInstanceOf(ClassCastException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + + private static TableInfo valueTable(boolean notNull) { + return table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING().copy(!notNull)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .build()); + } + + private static TableInfo envelope(boolean string) { + Schema schema = + Schema.newBuilder() + .column("body", string ? DataTypes.STRING() : DataTypes.BYTES()) + .column("time", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "attrs", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES())))) + .column("id", string ? DataTypes.STRING() : DataTypes.BYTES()) + .build(); + return table( + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty( + KafkaDataFormat.VALUE_FORMAT_CONFIG, string ? "string" : "raw") + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, string ? "string" : "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "time") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attrs") + .build()); + } + + private static TableInfo table(TableDescriptor descriptor) { + return TableInfo.of(TablePath.of("kafka", "topic"), 42L, 3, descriptor, null, 1L, 1L); + } + + private static void read( + BytesView encoded, TableInfo table, int count, Consumer verify) + throws Exception { + ByteBuf buffer = encoded.getByteBuf(); + try { + LogRecordBatch batch = + MemoryLogRecords.pointToByteBuffer(buffer.nioBuffer()) + .batches() + .iterator() + .next(); + batch.ensureValid(); + assertThat(batch.schemaId()).isEqualTo((short) table.getSchemaId()); + assertThat(batch.getRecordCount()).isEqualTo(count); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + table.getRowType(), + table.getSchemaId(), + new TestingSchemaGetter( + new SchemaInfo( + table.getSchema(), table.getSchemaId()))); + CloseableIterator records = batch.records(context)) { + for (int i = 0; i < count; i++) { + assertThat(records.hasNext()).isTrue(); + verify.accept(records.next().getRow()); + } + assertThat(records.hasNext()).isFalse(); + } + } finally { + buffer.release(); + } + } + + private static byte[] bytes(String text) { + return text.getBytes(StandardCharsets.UTF_8); + } +} From 3762f51b909956de1ab698902e0f3c2caa095a78 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 17:19:52 +0800 Subject: [PATCH 16/69] [kafka] Stream record transcoding and reuse owned payloads Decode and append one record at a time instead of retaining an entire row list. Borrow command-owned key, value and header bytes during synchronous conversion while preserving defensive-copy accessors. Cover row and payload reuse, batches beyond initial Arrow capacity, and resource cleanup after partial conversion failures. Validation on Java 11: reactor clean verify passed with 130 unit tests and 4 integration tests, plus Checkstyle, Spotless, RAT and git diff --check. An additional 25 boundary scenarios passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 101/101 AI-Contributed/UT: 224/224 --- .../backend/produce/KafkaProduceCommand.java | 30 ++++ .../transcode/ArrowKafkaRecordTranscoder.java | 36 +++-- .../transcode/FlussArrowRecordEncoder.java | 26 +++- .../kafka/transcode/KafkaRowAssembler.java | 9 +- .../produce/KafkaProduceCommandTest.java | 47 ++++++ .../ArrowKafkaRecordTranscoderTest.java | 144 ++++++++++++++++++ 6 files changed, 271 insertions(+), 21 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index aadaa36f737..db13f63b314 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -201,11 +201,31 @@ public TimestampType timestampType() { return copyNullable(key); } + /** + * Borrows the nullable key owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #key()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedKey() { + return key; + } + /** Returns a copy of the nullable Kafka record value. */ public @Nullable byte[] value() { return copyNullable(value); } + /** + * Borrows the nullable value owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } + /** Returns the copied Kafka headers in record order. */ public List headers() { return headers; @@ -237,5 +257,15 @@ public String name() { public @Nullable byte[] value() { return value == null ? null : value.clone(); } + + /** + * Borrows the nullable header value for synchronous conversion. + * + *

The returned array is owned by this header and must not be modified or retained after + * conversion. Use {@link #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 717f5cd33ff..9fcaaaabf31 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -25,7 +25,6 @@ import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.BinaryString; -import org.apache.fluss.row.GenericRow; import javax.annotation.Nullable; @@ -33,7 +32,6 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -49,20 +47,26 @@ public BytesView transcode(List records, TableInfo tableInfo) throws Exc checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); KafkaRowAssembler assembler = new KafkaRowAssembler(schema); - List rows = new ArrayList<>(records.size()); - for (Record record : records) { - Object[] key = - schema.keyFormat() == null - ? new Object[0] - : new Object[] {decode(schema.keyFormat(), record.key())}; - rows.add( - assembler.assemble( - key, - new Object[] {decode(schema.valueFormat(), record.value())}, - record.timestamp(), - record.headers())); - } - return arrowRecordEncoder.encode(rows, tableInfo); + return arrowRecordEncoder.encodeStreaming( + consumer -> { + for (Record record : records) { + Object[] key = + schema.keyFormat() == null + ? new Object[0] + : new Object[] { + decode(schema.keyFormat(), record.borrowedKey()) + }; + consumer.append( + assembler.assemble( + key, + new Object[] { + decode(schema.valueFormat(), record.borrowedValue()) + }, + record.timestamp(), + record.headers())); + } + }, + tableInfo); } private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java index f74680ccaa2..c84dd20d690 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java @@ -39,6 +39,16 @@ public final class FlussArrowRecordEncoder { /** Encodes all rows using the table's current schema ID and Arrow compression settings. */ public BytesView encode(List rows, TableInfo tableInfo) throws Exception { + return encodeStreaming( + consumer -> { + for (GenericRow row : rows) { + consumer.append(row); + } + }, + tableInfo); + } + + BytesView encodeStreaming(RowSource rowSource, TableInfo tableInfo) throws Exception { try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); ArrowWriterPool provider = new ArrowWriterPool(allocator)) { ArrowWriter writer = @@ -57,9 +67,7 @@ public BytesView encode(List rows, TableInfo tableInfo) throws Excep new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), true, null); - for (GenericRow row : rows) { - builder.append(ChangeType.APPEND_ONLY, row); - } + rowSource.produce(row -> builder.append(ChangeType.APPEND_ONLY, row)); // The output view owns heap pages. The result remains valid after the Arrow // writer and allocator close, including across asynchronous append completion. return builder.build(); @@ -68,4 +76,16 @@ public BytesView encode(List rows, TableInfo tableInfo) throws Excep } } } + + @FunctionalInterface + interface RowSource { + /** Produces rows synchronously and never retains the supplied consumer. */ + void produce(RowConsumer consumer) throws Exception; + } + + @FunctionalInterface + interface RowConsumer { + /** Serializes one row before returning and never retains the row. */ + void append(GenericRow row) throws Exception; + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index 4ebd506dc3c..cac8ac3e841 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -41,7 +41,12 @@ public KafkaRowAssembler(KafkaTopicSchema topicSchema) { this.topicSchema = checkNotNull(topicSchema); } - /** Assembles one Fluss row. */ + /** + * Assembles one Fluss row for immediate synchronous encoding. + * + *

The row borrows decoded values and header payloads; it must not modify them or retain them + * after conversion. + */ public GenericRow assemble( Object[] keyValues, Object[] valueValues, long timestamp, List headers) { GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); @@ -82,7 +87,7 @@ private static GenericArray toHeaders(List headers) { Object[] rows = new Object[headers.size()]; for (int i = 0; i < headers.size(); i++) { RecordHeader header = headers.get(i); - rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.value()); + rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.borrowedValue()); } return new GenericArray(rows); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java index 01fa6825b28..3b9c8f15ec0 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -161,4 +161,51 @@ void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { .isInstanceOf(UnsupportedOperationException.class); assertThatThrownBy(copied::clear).isInstanceOf(UnsupportedOperationException.class); } + + @Test + void testBorrowedPayloadsReuseOwnedArraysWhileCopyAccessorsRemainDefensive() { + byte[] key = {1}; + byte[] value = {2}; + byte[] headerValue = {3}; + RecordHeader header = new RecordHeader("header", headerValue); + List headers = new ArrayList<>(); + headers.add(header); + Record record = new Record(1L, key, value, headers); + + assertThat(record.borrowedKey()).isNotSameAs(key); + assertThat(record.borrowedValue()).isNotSameAs(value); + assertThat(header.borrowedValue()).isNotSameAs(headerValue); + key[0] = 10; + value[0] = 20; + headerValue[0] = 30; + headers.clear(); + + assertThat(record.borrowedKey()).containsExactly((byte) 1); + assertThat(record.borrowedValue()).containsExactly((byte) 2); + assertThat(header.borrowedValue()).containsExactly((byte) 3); + assertThat(record.headers()).containsExactly(header); + assertThat(record.borrowedKey()).isSameAs(record.borrowedKey()); + assertThat(record.borrowedValue()).isSameAs(record.borrowedValue()); + assertThat(header.borrowedValue()).isSameAs(header.borrowedValue()); + + record.key()[0] = 40; + record.value()[0] = 50; + header.value()[0] = 60; + assertThat(record.key()).containsExactly((byte) 1).isNotSameAs(record.borrowedKey()); + assertThat(record.value()).containsExactly((byte) 2).isNotSameAs(record.borrowedValue()); + assertThat(header.value()).containsExactly((byte) 3).isNotSameAs(header.borrowedValue()); + } + + @Test + void testBorrowedPayloadsPreserveNullAndEmptyBytes() { + Record nullRecord = new Record(1L, null, null, Collections.emptyList()); + assertThat(nullRecord.borrowedKey()).isNull(); + assertThat(nullRecord.borrowedValue()).isNull(); + assertThat(new RecordHeader("header", null).borrowedValue()).isNull(); + + Record emptyRecord = new Record(1L, new byte[0], new byte[0], Collections.emptyList()); + assertThat(emptyRecord.borrowedKey()).isEmpty(); + assertThat(emptyRecord.borrowedValue()).isEmpty(); + assertThat(new RecordHeader("header", new byte[0]).borrowedValue()).isEmpty(); + } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 6936b5c8d61..f1df39504d2 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -33,18 +33,27 @@ import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.record.TestingSchemaGetter; import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.InternalArray; import org.apache.fluss.row.InternalRow; +import org.apache.fluss.row.TimestampLtz; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.CloseableIterator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; @@ -242,6 +251,141 @@ void testArrowWriterFailureClosesAllocatorWithoutLeaking() { assertThat(failure.getSuppressed()).isEmpty(); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean string) + throws Exception { + TableInfo table = envelope(string); + List records = new ArrayList<>(); + for (int i = 0; i < 2050; i++) { + records.add( + new Record( + i, + bytes("key-" + i), + bytes("消息-" + i), + Arrays.asList( + new RecordHeader("duplicate", bytes("header-" + i)), + new RecordHeader("duplicate", null)))); + } + BytesView encoded = transcoder.transcode(records, table); + AtomicInteger nextRecord = new AtomicInteger(); + read( + encoded, + table, + records.size(), + row -> { + int i = nextRecord.getAndIncrement(); + if (string) { + assertThat(row.getString(0).toString()).isEqualTo("消息-" + i); + assertThat(row.getString(3).toString()).isEqualTo("key-" + i); + } else { + assertThat(row.getBytes(0)).isEqualTo(bytes("消息-" + i)); + assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + InternalArray headers = row.getArray(2); + assertThat(headers.size()).isEqualTo(2); + assertThat(headers.getRow(0, 2).getString(0).toString()).isEqualTo("duplicate"); + assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("header-" + i)); + assertThat(headers.getRow(1, 2).getString(0).toString()).isEqualTo("duplicate"); + assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); + }); + } + + @Test + void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exception { + TableInfo table = envelope(false); + byte[] key = new byte[1]; + byte[] value = new byte[1]; + byte[] headerValue = new byte[1]; + GenericRow row = + GenericRow.of( + value, + TimestampLtz.fromEpochMillis(1L), + new GenericArray( + new Object[] { + GenericRow.of(BinaryString.fromString("header"), headerValue) + }), + key); + BytesView encoded = + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + for (int i = 0; i < 3; i++) { + key[0] = (byte) i; + value[0] = (byte) (i + 10); + headerValue[0] = (byte) (i + 20); + row.setField(1, TimestampLtz.fromEpochMillis(i)); + consumer.append(row); + } + Arrays.fill(key, (byte) -1); + Arrays.fill(value, (byte) -1); + Arrays.fill(headerValue, (byte) -1); + row.setField(0, null); + }, + table); + AtomicInteger nextRecord = new AtomicInteger(); + read( + encoded, + table, + 3, + actual -> { + int i = nextRecord.getAndIncrement(); + assertThat(actual.getBytes(3)).containsExactly((byte) i); + assertThat(actual.getBytes(0)).containsExactly((byte) (i + 10)); + assertThat(actual.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + assertThat(actual.getArray(2).getRow(0, 2).getBytes(1)) + .containsExactly((byte) (i + 20)); + }); + } + + @Test + void testDecodeFailureAfterValidRecordClosesAllocatorAndRemainsUsable() throws Exception { + TableInfo table = valueTable(true); + Record valid = new Record(1L, null, bytes("valid"), Collections.emptyList()); + for (byte[] invalidValue : new byte[][] {null, {(byte) 0xc3, 0x28}}) { + Throwable failure = + catchThrowable( + () -> + transcoder.transcode( + Arrays.asList( + valid, + new Record( + 2L, + null, + invalidValue, + Collections.emptyList())), + table)); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + read( + transcoder.transcode(Collections.singletonList(valid), table), + table, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); + } + + @Test + void testStreamingSourceFailureClosesAllocator() { + IOException expected = new IOException("source failed after the first row"); + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + consumer.append( + GenericRow.of( + BinaryString.fromString( + "valid"))); + throw expected; + }, + valueTable(false))); + assertThat(failure).isSameAs(expected); + assertThat(failure.getSuppressed()).isEmpty(); + } + private static TableInfo valueTable(boolean notNull) { return table( TableDescriptor.builder() From ea5dc34a229ff563d68b06c2fd8e1a96a59e151c Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:31:14 +0800 Subject: [PATCH 17/69] [kafka] Encode timestamp metadata as UTC TIMESTAMP values Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 13/13 AI-Contributed/UT: 75/75 --- .../kafka/transcode/KafkaRowAssembler.java | 13 +++- .../ArrowKafkaRecordTranscoderTest.java | 75 +++++++++++++++++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index cac8ac3e841..01886849d2b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -24,7 +24,7 @@ import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.GenericArray; import org.apache.fluss.row.GenericRow; -import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; import java.util.List; @@ -53,7 +53,16 @@ public GenericRow assemble( setProjectedFields(row, topicSchema.keyProjection(), keyValues); setProjectedFields(row, topicSchema.valueProjection(), valueValues); if (topicSchema.timestampPosition() >= 0) { - row.setField(topicSchema.timestampPosition(), TimestampLtz.fromEpochMillis(timestamp)); + int position = topicSchema.timestampPosition(); + if (timestamp == -1L) { + if (!topicSchema.rowType().getTypeAt(position).isNullable()) { + throw new KafkaRecordEncodingException( + "Missing Kafka timestamp cannot populate a NOT NULL timestamp column."); + } + } else { + // Epoch milliseconds define UTC calendar fields, independently of the JVM zone. + row.setField(position, TimestampNtz.fromMillis(timestamp)); + } } if (topicSchema.headersPosition() >= 0) { row.setField(topicSchema.headersPosition(), toHeaders(headers)); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index f1df39504d2..1b91bcdbc53 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -38,7 +38,7 @@ import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.InternalArray; import org.apache.fluss.row.InternalRow; -import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.CloseableIterator; @@ -49,10 +49,14 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.TimeZone; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -89,7 +93,7 @@ void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { assertThat(row.getBytes(0)).isEqualTo(bytes("message")); assertThat(row.getBytes(3)).isEqualTo(bytes("键")); } - assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(123L); InternalArray headers = row.getArray(2); assertThat(headers.size()).isEqualTo(2); assertThat(headers.getRow(0, 2).getString(0).toString()) @@ -100,6 +104,59 @@ void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { } } + @Test + void testTimestampCalendarValuesAreIndependentOfDefaultTimeZone() throws Exception { + TimeZone original = TimeZone.getDefault(); + long millis = 1700000000123L; + try { + for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/Los_Angeles"}) { + TimeZone.setDefault(TimeZone.getTimeZone(zone)); + TableInfo table = envelope(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + millis, + null, + bytes("value"), + Collections.emptyList())), + table), + table, + 1, + row -> + assertThat(row.getTimestampNtz(1, 3).toLocalDateTime()) + .isEqualTo( + LocalDateTime.ofInstant( + Instant.ofEpochMilli(millis), + ZoneOffset.UTC))); + } + } finally { + TimeZone.setDefault(original); + } + } + + @Test + void testMissingTimestampRespectsColumnNullability() throws Exception { + List records = + Collections.singletonList( + new Record(-1L, null, bytes("value"), Collections.emptyList())); + TableInfo nullable = envelope(false, true); + read( + transcoder.transcode(records, nullable), + nullable, + 1, + row -> assertThat(row.isNullAt(1)).isTrue()); + assertThatThrownBy(() -> transcoder.transcode(records, envelope(false))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("Missing Kafka timestamp"); + TableInfo unmapped = valueTable(false); + read( + transcoder.transcode(records, unmapped), + unmapped, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("value")); + } + @Test void testNullsAndEmptyBytesRemainDistinct() throws Exception { TableInfo table = envelope(false); @@ -282,7 +339,7 @@ void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean str assertThat(row.getBytes(0)).isEqualTo(bytes("消息-" + i)); assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); } - assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(i); InternalArray headers = row.getArray(2); assertThat(headers.size()).isEqualTo(2); assertThat(headers.getRow(0, 2).getString(0).toString()).isEqualTo("duplicate"); @@ -301,7 +358,7 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce GenericRow row = GenericRow.of( value, - TimestampLtz.fromEpochMillis(1L), + TimestampNtz.fromMillis(1L), new GenericArray( new Object[] { GenericRow.of(BinaryString.fromString("header"), headerValue) @@ -315,7 +372,7 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce key[0] = (byte) i; value[0] = (byte) (i + 10); headerValue[0] = (byte) (i + 20); - row.setField(1, TimestampLtz.fromEpochMillis(i)); + row.setField(1, TimestampNtz.fromMillis(i)); consumer.append(row); } Arrays.fill(key, (byte) -1); @@ -333,7 +390,7 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce int i = nextRecord.getAndIncrement(); assertThat(actual.getBytes(3)).containsExactly((byte) i); assertThat(actual.getBytes(0)).containsExactly((byte) (i + 10)); - assertThat(actual.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + assertThat(actual.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(i); assertThat(actual.getArray(2).getRow(0, 2).getBytes(1)) .containsExactly((byte) (i + 20)); }); @@ -400,10 +457,14 @@ private static TableInfo valueTable(boolean notNull) { } private static TableInfo envelope(boolean string) { + return envelope(string, false); + } + + private static TableInfo envelope(boolean string, boolean nullableTimestamp) { Schema schema = Schema.newBuilder() .column("body", string ? DataTypes.STRING() : DataTypes.BYTES()) - .column("time", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("time", DataTypes.TIMESTAMP(3).copy(nullableTimestamp)) .column( "attrs", DataTypes.ARRAY( From 8b0a9c6b24804be2ee8a29687fbdbc01566b6a22 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 18:05:19 +0800 Subject: [PATCH 18/69] [kafka] Transcode binary and string into fixed system columns Use the shared KafkaTableSchema contract for the fixed value column, raw optional keys, and last-value-wins header maps. Validate final map and key nullability, keep optional rescue values null, and remove prototype field projections and timestamp-column mapping. Reject metadata until native per-row offsets are available; JSON decoding remains a later stage. Preserve synchronous streaming, borrowed payloads, owned Arrow output and failure-path cleanup. Cover all optional system-column combinations, raw bytes, strict UTF-8, header deduplication, multi-page output and partial conversion failures without changing null-value filtering. Validation: Java 11 current-worktree clean reactor, 277 unit tests and 27 integration tests passed, including cumulative Kafka, shared schema, DDL and Arrow coverage. The final 23 transcoder cases passed again after strengthening the multi-page test. Spotless, Checkstyle, RAT and diff checks passed. All SNAPSHOT test dependencies came from this worktree. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 95/95 AI-Contributed/UT: 741/741 --- .../transcode/ArrowKafkaRecordTranscoder.java | 30 +- .../kafka/transcode/KafkaRowAssembler.java | 65 +- .../ArrowKafkaRecordTranscoderTest.java | 741 +++++++++++------- 3 files changed, 530 insertions(+), 306 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 9fcaaaabf31..88711f42323 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -19,9 +19,9 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.kafka.schema.KafkaTopicSchema; import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.BinaryString; @@ -36,7 +36,7 @@ import static org.apache.fluss.utils.Preconditions.checkArgument; -/** Converts raw/string Kafka records using the DDL mapping into owned Fluss Arrow log bytes. */ +/** Converts binary/string Kafka values and fixed system columns into owned Arrow log bytes. */ @Internal public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); @@ -45,37 +45,31 @@ public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { @Override public BytesView transcode(List records, TableInfo tableInfo) throws Exception { checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); - KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaTableSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + if (schema.valueType() != ValueType.BINARY && schema.valueType() != ValueType.STRING) { + throw new KafkaRecordEncodingException( + "Unsupported Kafka value conversion type: " + schema.valueType()); + } KafkaRowAssembler assembler = new KafkaRowAssembler(schema); return arrowRecordEncoder.encodeStreaming( consumer -> { for (Record record : records) { - Object[] key = - schema.keyFormat() == null - ? new Object[0] - : new Object[] { - decode(schema.keyFormat(), record.borrowedKey()) - }; consumer.append( assembler.assemble( - key, new Object[] { - decode(schema.valueFormat(), record.borrowedValue()) + decode(schema.valueType(), record.borrowedValue()) }, - record.timestamp(), + record.borrowedKey(), record.headers())); } }, tableInfo); } - private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) { - if (bytes == null || format == KafkaDataFormat.RAW) { + private static @Nullable Object decode(ValueType format, @Nullable byte[] bytes) { + if (bytes == null || format == ValueType.BINARY) { return bytes; } - if (format != KafkaDataFormat.STRING) { - throw new KafkaRecordEncodingException("Unsupported Kafka data format: " + format); - } try { return BinaryString.fromString( StandardCharsets.UTF_8 diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index 01886849d2b..00747192122 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -19,25 +19,29 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; -import org.apache.fluss.kafka.schema.KafkaFieldProjection; -import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; -import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; -import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.types.MapType; +import javax.annotation.Nullable; + +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Assembles decoded Kafka key, value, and metadata into a physical Fluss row. */ +/** Assembles decoded values, raw keys and headers using the shared fixed-column contract. */ @Internal public final class KafkaRowAssembler { - private final KafkaTopicSchema topicSchema; + private final KafkaTableSchema topicSchema; /** Creates an assembler for the resolved topic schema. */ - public KafkaRowAssembler(KafkaTopicSchema topicSchema) { + public KafkaRowAssembler(KafkaTableSchema topicSchema) { this.topicSchema = checkNotNull(topicSchema); } @@ -48,25 +52,24 @@ public KafkaRowAssembler(KafkaTopicSchema topicSchema) { * after conversion. */ public GenericRow assemble( - Object[] keyValues, Object[] valueValues, long timestamp, List headers) { + Object[] valueValues, @Nullable byte[] key, List headers) { GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); - setProjectedFields(row, topicSchema.keyProjection(), keyValues); setProjectedFields(row, topicSchema.valueProjection(), valueValues); - if (topicSchema.timestampPosition() >= 0) { - int position = topicSchema.timestampPosition(); - if (timestamp == -1L) { - if (!topicSchema.rowType().getTypeAt(position).isNullable()) { - throw new KafkaRecordEncodingException( - "Missing Kafka timestamp cannot populate a NOT NULL timestamp column."); - } - } else { - // Epoch milliseconds define UTC calendar fields, independently of the JVM zone. - row.setField(position, TimestampNtz.fromMillis(timestamp)); + if (topicSchema.keyPosition() >= 0) { + int position = topicSchema.keyPosition(); + if (key == null && !topicSchema.rowType().getTypeAt(position).isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka key cannot populate NOT NULL _kafka_key."); } + row.setField(position, key); } if (topicSchema.headersPosition() >= 0) { - row.setField(topicSchema.headersPosition(), toHeaders(headers)); + int position = topicSchema.headersPosition(); + row.setField( + position, + toHeaders(headers, (MapType) topicSchema.rowType().getTypeAt(position))); } + // A declared _kafka_rescue stays null for binary/string; omitted columns do not exist. return row; } @@ -92,12 +95,22 @@ private static void setProjectedFields( } } - private static GenericArray toHeaders(List headers) { - Object[] rows = new Object[headers.size()]; - for (int i = 0; i < headers.size(); i++) { - RecordHeader header = headers.get(i); - rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.borrowedValue()); + private static GenericMap toHeaders(List headers, MapType type) { + Map values = new LinkedHashMap<>(); + for (RecordHeader header : headers) { + values.put(BinaryString.fromString(header.name()), header.borrowedValue()); + } + // Validate the final map: an overwritten null must not reject a non-null last value. + for (Map.Entry entry : values.entrySet()) { + if (entry.getKey() == null && !type.getKeyType().isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka header name cannot populate NOT NULL _kafka_headers map keys."); + } + if (entry.getValue() == null && !type.getValueType().isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka header value cannot populate NOT NULL _kafka_headers map values."); + } } - return new GenericArray(rows); + return new GenericMap(values); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 1b91bcdbc53..ee01c5e4e60 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -17,10 +17,12 @@ package org.apache.fluss.kafka.transcode; +import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; -import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.SchemaInfo; @@ -34,12 +36,12 @@ import org.apache.fluss.record.TestingSchemaGetter; import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.BinaryString; -import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; -import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalMap; import org.apache.fluss.row.InternalRow; -import org.apache.fluss.row.TimestampNtz; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.CloseableIterator; @@ -49,270 +51,405 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; -import java.util.TimeZone; +import java.util.Map; +import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; -/** Verifies mapped Arrow records after transcoding resources have closed. */ +/** Verifies fixed-column Arrow records after transcoding resources have closed. */ class ArrowKafkaRecordTranscoderTest { private final ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); - @Test - void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { - for (boolean string : new boolean[] {false, true}) { - TableInfo table = envelope(string); - Record record = - new Record( - 123L, - bytes("键"), - bytes("message"), - Arrays.asList( - new RecordHeader("duplicate", bytes("first")), - new RecordHeader("duplicate", null))); - BytesView encoded = transcoder.transcode(Arrays.asList(record, record), table); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) throws Exception { + // The value looks like JSON but binary/string must not interpret or rescue its members. + byte[] value = bytes("{\"_kafka_rescue\":\"消息\",\"_kafka_key\":123}"); + byte[] key = {(byte) 0xff, 0, (byte) 0xc3}; + Record record = + new Record( + 123L, + TimestampType.LOG_APPEND_TIME, + key, + value, + Arrays.asList( + new RecordHeader("duplicate", bytes("first")), + new RecordHeader("nullable", bytes("discarded")), + new RecordHeader("duplicate", bytes("last")), + new RecordHeader("nullable", null))); + for (int mask = 0; mask < 8; mask++) { + TableInfo table = envelope(string, mask); + TableDescriptor before = table.toTableDescriptor(); + KafkaTableSchema contract = KafkaTableSchema.resolve(before); read( - encoded, + transcoder.transcode(Collections.singletonList(record), table), table, - 2, + 1, row -> { - if (string) { - assertThat(row.getString(0).toString()).isEqualTo("message"); - assertThat(row.getString(3).toString()).isEqualTo("键"); - } else { - assertThat(row.getBytes(0)).isEqualTo(bytes("message")); - assertThat(row.getBytes(3)).isEqualTo(bytes("键")); + assertThat(row.getFieldCount()) + .isEqualTo(table.getRowType().getFieldCount()); + assertValue(row, contract.valueProjection().positionAt(0), string, value); + if (contract.keyPosition() >= 0) { + assertThat(row.getBytes(contract.keyPosition())).isEqualTo(key); + } + if (contract.headersPosition() >= 0) { + Map headers = + headers(row.getMap(contract.headersPosition())); + assertThat(headers).hasSize(2).containsKey("nullable"); + assertThat(headers.get("duplicate")).isEqualTo(bytes("last")); + assertThat(headers.get("nullable")).isNull(); + } + if (contract.rescuePosition() >= 0) { + assertThat(row.isNullAt(contract.rescuePosition())).isTrue(); } - assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(123L); - InternalArray headers = row.getArray(2); - assertThat(headers.size()).isEqualTo(2); - assertThat(headers.getRow(0, 2).getString(0).toString()) - .isEqualTo("duplicate"); - assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("first")); - assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); }); + assertThat(table.toTableDescriptor()).isEqualTo(before); + assertThat(before.getProperties()).doesNotContainKey("table.kafka.convert.key.type"); + assertThat(before.getCustomProperties()).isEmpty(); } + assertThat(record.timestampType()).isEqualTo(TimestampType.LOG_APPEND_TIME); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.borrowedKey()).isEqualTo(key); + assertThat(record.borrowedValue()).isEqualTo(value); } - @Test - void testTimestampCalendarValuesAreIndependentOfDefaultTimeZone() throws Exception { - TimeZone original = TimeZone.getDefault(); - long millis = 1700000000123L; - try { - for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/Los_Angeles"}) { - TimeZone.setDefault(TimeZone.getTimeZone(zone)); - TableInfo table = envelope(false); - read( - transcoder.transcode( - Collections.singletonList( - new Record( - millis, - null, - bytes("value"), - Collections.emptyList())), - table), - table, - 1, - row -> - assertThat(row.getTimestampNtz(1, 3).toLocalDateTime()) - .isEqualTo( - LocalDateTime.ofInstant( - Instant.ofEpochMilli(millis), - ZoneOffset.UTC))); - } - } finally { - TimeZone.setDefault(original); - } - } - - @Test - void testMissingTimestampRespectsColumnNullability() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testNullAndEmptyPayloadsRemainDistinct(boolean string) throws Exception { + TableInfo table = envelope(string, 7); + // This converter retains its existing nullability checks; ingress filtering is separate. List records = - Collections.singletonList( - new Record(-1L, null, bytes("value"), Collections.emptyList())); - TableInfo nullable = envelope(false, true); + Arrays.asList( + new Record(Record.NO_TIMESTAMP, null, null, Collections.emptyList()), + new Record(0L, new byte[0], new byte[0], Collections.emptyList())); + AtomicInteger next = new AtomicInteger(); read( - transcoder.transcode(records, nullable), - nullable, - 1, - row -> assertThat(row.isNullAt(1)).isTrue()); - assertThatThrownBy(() -> transcoder.transcode(records, envelope(false))) + transcoder.transcode(records, table), + table, + 2, + row -> { + int value = table.getRowType().getFieldIndex("_kafka_value"); + int key = table.getRowType().getFieldIndex("_kafka_key"); + if (next.getAndIncrement() == 0) { + assertThat(row.isNullAt(value)).isTrue(); + assertThat(row.isNullAt(key)).isTrue(); + } else { + assertValue(row, value, string, new byte[0]); + assertThat(row.getBytes(key)).isEmpty(); + } + assertThat( + row.getMap(table.getRowType().getFieldIndex("_kafka_headers")) + .size()) + .isZero(); + assertThat(row.isNullAt(table.getRowType().getFieldIndex("_kafka_rescue"))) + .isTrue(); + }); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES().copy(false)) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + bytes("valid"), + Collections.emptyList())), + table)) .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("Missing Kafka timestamp"); - TableInfo unmapped = valueTable(false); + .hasMessageContaining("NOT NULL _kafka_key"); + byte[] invalidUtf8Key = {(byte) 0xff}; read( - transcoder.transcode(records, unmapped), - unmapped, + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + invalidUtf8Key, + bytes("valid"), + Collections.emptyList())), + table), + table, 1, - row -> assertThat(row.getString(0).toString()).isEqualTo("value")); + row -> assertThat(row.getBytes(0)).isEqualTo(invalidUtf8Key)); } - @Test - void testNullsAndEmptyBytesRemainDistinct() throws Exception { - TableInfo table = envelope(false); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING() + .copy(false), + DataTypes.BYTES() + .copy( + nullableValue)) + .copy(false)) + .column("_kafka_value", DataTypes.STRING()) + .build(), + true) + .build()); + List overwrittenNull = + Arrays.asList( + new RecordHeader("same", null), + new RecordHeader("", new byte[0]), + new RecordHeader("same", bytes("last"))); read( transcoder.transcode( Collections.singletonList( - new Record(1L, null, null, Collections.emptyList())), + new Record(1L, null, bytes("v"), overwrittenNull)), table), table, 1, row -> { - assertThat(row.isNullAt(0)).isTrue(); - assertThat(row.isNullAt(3)).isTrue(); - assertThat(row.getArray(2).size()).isZero(); + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(2); + assertThat(map.get("same")).isEqualTo(bytes("last")); + assertThat(map.get("")).isEmpty(); }); + Record finalNull = + new Record( + 1L, + null, + bytes("v"), + Arrays.asList( + new RecordHeader("same", bytes("first")), + new RecordHeader("same", null))); + if (nullableValue) { + read( + transcoder.transcode(Collections.singletonList(finalNull), table), + table, + 1, + row -> + assertThat(headers(row.getMap(0))) + .containsOnlyKeys("same") + .containsEntry("same", null)); + } else { + assertThatThrownBy( + () -> transcoder.transcode(Collections.singletonList(finalNull), table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL _kafka_headers map values"); + } read( transcoder.transcode( Collections.singletonList( - new Record(1L, new byte[0], new byte[0], Collections.emptyList())), + new Record(1L, null, bytes("v"), Collections.emptyList())), table), table, 1, + row -> assertThat(row.getMap(0).size()).isZero()); + } + + @Test + void testOmittedSystemColumnsIgnoreTimestampKeyAndHeaders() throws Exception { + TableInfo table = valueTable(false); + List records = new ArrayList<>(); + for (TimestampType type : TimestampType.values()) { + for (long timestamp : new long[] {Record.NO_TIMESTAMP, 0L, Long.MAX_VALUE}) { + records.add( + new Record( + timestamp, + type, + new byte[] {(byte) 0xff}, + bytes("v"), + Collections.singletonList(new RecordHeader("ignored", null)))); + } + } + read( + transcoder.transcode(records, table), + table, + records.size(), row -> { - assertThat(row.getBytes(0)).isEmpty(); - assertThat(row.getBytes(3)).isEmpty(); + assertThat(row.getFieldCount()).isEqualTo(1); + assertThat(row.getString(0).toString()).isEqualTo("v"); }); } @Test - void testUnmappedKeyIsIgnoredAndMixedFormatsWork() throws Exception { - TableInfo valueOnly = valueTable(false); - read( + void testBinaryValuePreservesArbitraryBytesAcrossOutputPages() throws Exception { + byte[] value = new byte[32768]; + new Random(42L).nextBytes(value); + value[0] = (byte) 0xff; + TableInfo table = envelope(false, 4); + BytesView encoded = transcoder.transcode( Collections.singletonList( - new Record( - 1L, - new byte[] {(byte) 0xff}, - bytes("value"), - Collections.emptyList())), - valueOnly), - valueOnly, - 1, - row -> assertThat(row.getString(0).toString()).isEqualTo("value")); - Schema schema = - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("id", DataTypes.STRING()) - .build(); - TableInfo mixed = - table( - TableDescriptor.builder() - .schema(schema) - .distributedBy(1) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty( - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .build()); + new Record(1L, null, value, Collections.emptyList())), + table); + assertThat(encoded.getBytesLength()).isGreaterThan(4096); read( - transcoder.transcode( - Collections.singletonList( - new Record( - 1L, - bytes("key"), - new byte[] {(byte) 0xff}, - Collections.emptyList())), - mixed), - mixed, + encoded, + table, 1, row -> { - assertThat(row.getString(1).toString()).isEqualTo("key"); - assertThat(row.getBytes(0)).containsExactly((byte) 0xff); + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.getBytes(1)).isEqualTo(value); }); } - @Test - void testRejectsMalformedUtf8AndNotNullViolationsThenRemainsUsable() throws Exception { - TableInfo table = valueTable(false); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testValueNotNullConstraint(boolean string) { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_value", + valueType(string).copy(false)) + .build(), + string) + .build()); assertThatThrownBy( () -> transcoder.transcode( Collections.singletonList( new Record( - 1L, - null, - new byte[] {(byte) 0xc3, 0x28}, - Collections.emptyList())), + 1L, null, null, Collections.emptyList())), table)) .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("UTF-8"); + .hasMessageContaining("NOT NULL field '_kafka_value'"); + } + + @Test + void testRejectsMetadataUntilActualOffsetsCanBePersisted() { + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))) + .build(); assertThatThrownBy( () -> transcoder.transcode( Collections.singletonList( new Record( - 1L, null, null, Collections.emptyList())), - valueTable(true))) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("NOT NULL"); - read( - transcoder.transcode( - Collections.singletonList( - new Record(1L, null, bytes("ok"), Collections.emptyList())), - table), - table, - 1, - row -> assertThat(row.getString(0).toString()).isEqualTo("ok")); + 123L, + TimestampType.CREATE_TIME, + null, + bytes("v"), + Collections.emptyList())), + table(descriptor(schema, true).build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("actual per-row offsets"); } @Test - void testRejectsInvalidMappingAndEmptyPartition() { - TableInfo invalid = - table( - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("body", DataTypes.STRING()) - .build()) - .distributedBy(1) - .build()); + void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { + List records = + Collections.singletonList( + new Record(1L, null, bytes("v"), Collections.emptyList())); + Schema fixed = Schema.newBuilder().column("_kafka_value", DataTypes.STRING()).build(); assertThatThrownBy( () -> transcoder.transcode( - Collections.singletonList( - new Record( - 1L, - null, - bytes("value"), - Collections.emptyList())), - invalid)) - .isInstanceOf(KafkaTopicSchemaException.class); + records, + table( + descriptor(fixed, true) + .property( + ConfigOptions.TABLE_KAFKA_ENABLED, + false) + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); + assertThatThrownBy( + () -> + transcoder.transcode( + records, + table( + descriptor( + Schema.newBuilder() + .column( + "body", + DataTypes.STRING()) + .build(), + true) + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value"); + assertThatThrownBy( + () -> + transcoder.transcode( + records, + table( + descriptor(fixed, true) + .customProperty( + "kafka.value.format", "string") + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("custom property"); assertThatThrownBy(() -> transcoder.transcode(Collections.emptyList(), valueTable(false))) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("empty"); } @Test - void testArrowWriterFailureClosesAllocatorWithoutLeaking() { - Throwable failure = - catchThrowable( - () -> - new FlussArrowRecordEncoder() - .encode( - Collections.singletonList(GenericRow.of(123)), - valueTable(false))); - assertThat(failure).isInstanceOf(ClassCastException.class); - assertThat(failure.getSuppressed()).isEmpty(); + void testJsonIsNotActivatedEvenForNullValues() { + TableInfo table = + table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build()); + for (byte[] value : new byte[][] {null, bytes("{}")}) { + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + value, + Collections.emptyList())), + table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("PARSE_JSON"); + } } @ParameterizedTest @ValueSource(booleans = {false, true}) void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean string) throws Exception { - TableInfo table = envelope(string); + TableInfo table = envelope(string, 7); List records = new ArrayList<>(); for (int i = 0; i < 2050; i++) { records.add( @@ -321,48 +458,57 @@ void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean str bytes("key-" + i), bytes("消息-" + i), Arrays.asList( - new RecordHeader("duplicate", bytes("header-" + i)), - new RecordHeader("duplicate", null)))); + new RecordHeader("duplicate", null), + new RecordHeader("duplicate", bytes("header-" + i))))); } - BytesView encoded = transcoder.transcode(records, table); - AtomicInteger nextRecord = new AtomicInteger(); + AtomicInteger next = new AtomicInteger(); read( - encoded, + transcoder.transcode(records, table), table, records.size(), row -> { - int i = nextRecord.getAndIncrement(); - if (string) { - assertThat(row.getString(0).toString()).isEqualTo("消息-" + i); - assertThat(row.getString(3).toString()).isEqualTo("key-" + i); - } else { - assertThat(row.getBytes(0)).isEqualTo(bytes("消息-" + i)); - assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); + int i = next.getAndIncrement(); + assertValue(row, 2, string, bytes("消息-" + i)); + assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(1); + assertThat(map.get("duplicate")).isEqualTo(bytes("header-" + i)); + assertThat(row.isNullAt(1)).isTrue(); + }); + } + + @Test + void testOneRecordWithManyDistinctHeadersGrowsArrowMapVectors() throws Exception { + TableInfo table = envelope(false, 2); + List input = new ArrayList<>(); + for (int i = 0; i < 2050; i++) { + input.add(new RecordHeader("header-" + i, bytes("value-" + i))); + } + read( + transcoder.transcode( + Collections.singletonList(new Record(1L, null, bytes("v"), input)), table), + table, + 1, + row -> { + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(2050); + for (int i = 0; i < 2050; i++) { + assertThat(map.get("header-" + i)).isEqualTo(bytes("value-" + i)); } - assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(i); - InternalArray headers = row.getArray(2); - assertThat(headers.size()).isEqualTo(2); - assertThat(headers.getRow(0, 2).getString(0).toString()).isEqualTo("duplicate"); - assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("header-" + i)); - assertThat(headers.getRow(1, 2).getString(0).toString()).isEqualTo("duplicate"); - assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); }); } @Test void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exception { - TableInfo table = envelope(false); + TableInfo table = envelope(false, 7); byte[] key = new byte[1]; byte[] value = new byte[1]; byte[] headerValue = new byte[1]; GenericRow row = GenericRow.of( + GenericMap.of(BinaryString.fromString("header"), headerValue), + null, value, - TimestampNtz.fromMillis(1L), - new GenericArray( - new Object[] { - GenericRow.of(BinaryString.fromString("header"), headerValue) - }), key); BytesView encoded = new FlussArrowRecordEncoder() @@ -372,49 +518,50 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce key[0] = (byte) i; value[0] = (byte) (i + 10); headerValue[0] = (byte) (i + 20); - row.setField(1, TimestampNtz.fromMillis(i)); consumer.append(row); } Arrays.fill(key, (byte) -1); Arrays.fill(value, (byte) -1); Arrays.fill(headerValue, (byte) -1); - row.setField(0, null); + row.setField(2, null); }, table); - AtomicInteger nextRecord = new AtomicInteger(); + AtomicInteger next = new AtomicInteger(); read( encoded, table, 3, actual -> { - int i = nextRecord.getAndIncrement(); + int i = next.getAndIncrement(); assertThat(actual.getBytes(3)).containsExactly((byte) i); - assertThat(actual.getBytes(0)).containsExactly((byte) (i + 10)); - assertThat(actual.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(i); - assertThat(actual.getArray(2).getRow(0, 2).getBytes(1)) + assertThat(actual.getBytes(2)).containsExactly((byte) (i + 10)); + assertThat(headers(actual.getMap(0)).get("header")) .containsExactly((byte) (i + 20)); }); } @Test - void testDecodeFailureAfterValidRecordClosesAllocatorAndRemainsUsable() throws Exception { + void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { TableInfo table = valueTable(true); Record valid = new Record(1L, null, bytes("valid"), Collections.emptyList()); - for (byte[] invalidValue : new byte[][] {null, {(byte) 0xc3, 0x28}}) { + for (byte[] invalid : new byte[][] {null, {(byte) 0xc3, 0x28}, {(byte) 0xff}}) { + AtomicReference output = new AtomicReference<>(); Throwable failure = catchThrowable( () -> - transcoder.transcode( - Arrays.asList( - valid, - new Record( - 2L, - null, - invalidValue, - Collections.emptyList())), - table)); + output.set( + transcoder.transcode( + Arrays.asList( + valid, + new Record( + 2L, + null, + invalid, + Collections.emptyList())), + table))); assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); } read( transcoder.transcode(Collections.singletonList(valid), table), @@ -423,6 +570,63 @@ void testDecodeFailureAfterValidRecordClosesAllocatorAndRemainsUsable() throws E row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); } + @Test + void testHeaderFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.BYTES().copy(false))) + .build(), + true) + .build()); + Record valid = + new Record( + 1L, + null, + bytes("v"), + Collections.singletonList(new RecordHeader("h", bytes("v")))); + Record invalid = + new Record( + 1L, + null, + bytes("v"), + Collections.singletonList(new RecordHeader("h", null))); + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + Arrays.asList(valid, invalid), table))); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + read( + transcoder.transcode(Collections.singletonList(valid), table), + table, + 1, + row -> assertThat(headers(row.getMap(1)).get("h")).isEqualTo(bytes("v"))); + } + + @Test + void testArrowWriterFailureClosesAllocatorWithoutLeaking() { + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encode( + Collections.singletonList(GenericRow.of(123)), + valueTable(false))); + assertThat(failure).isInstanceOf(ClassCastException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + @Test void testStreamingSourceFailureClosesAllocator() { IOException expected = new IOException("source failed after the first row"); @@ -445,52 +649,65 @@ void testStreamingSourceFailureClosesAllocator() { private static TableInfo valueTable(boolean notNull) { return table( - TableDescriptor.builder() - .schema( + descriptor( Schema.newBuilder() - .column("body", DataTypes.STRING().copy(!notNull)) - .build()) - .distributedBy(1) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .column("_kafka_value", DataTypes.STRING().copy(!notNull)) + .build(), + true) .build()); } - private static TableInfo envelope(boolean string) { - return envelope(string, false); + private static TableInfo envelope(boolean string, int mask) { + Schema.Builder schema = Schema.newBuilder(); + if ((mask & 2) != 0) { + schema.column("_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + } + if ((mask & 4) != 0) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + schema.column("_kafka_value", valueType(string)); + if ((mask & 1) != 0) { + schema.column("_kafka_key", DataTypes.BYTES()); + } + return table(descriptor(schema.build(), string).build()); + } + + private static DataType valueType(boolean string) { + return string ? DataTypes.STRING() : DataTypes.BYTES(); } - private static TableInfo envelope(boolean string, boolean nullableTimestamp) { - Schema schema = - Schema.newBuilder() - .column("body", string ? DataTypes.STRING() : DataTypes.BYTES()) - .column("time", DataTypes.TIMESTAMP(3).copy(nullableTimestamp)) - .column( - "attrs", - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES())))) - .column("id", string ? DataTypes.STRING() : DataTypes.BYTES()) - .build(); - return table( - TableDescriptor.builder() - .schema(schema) - .distributedBy(1) - .logFormat(LogFormat.ARROW) - .customProperty( - KafkaDataFormat.VALUE_FORMAT_CONFIG, string ? "string" : "raw") - .customProperty( - KafkaDataFormat.KEY_FORMAT_CONFIG, string ? "string" : "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "time") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attrs") - .build()); + private static TableDescriptor.Builder descriptor(Schema schema, boolean string) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, string ? "string" : "binary"); } private static TableInfo table(TableDescriptor descriptor) { - return TableInfo.of(TablePath.of("kafka", "topic"), 42L, 3, descriptor, null, 1L, 1L); + return TableInfo.of(TablePath.of("fluss", "topic"), 42L, 3, descriptor, null, 1L, 1L); + } + + private static Map headers(InternalMap map) { + Map result = new LinkedHashMap<>(); + for (int i = 0; i < map.size(); i++) { + assertThat(map.keyArray().isNullAt(i)).isFalse(); + String key = map.keyArray().getString(i).toString(); + assertThat(result).doesNotContainKey(key); + result.put(key, map.valueArray().isNullAt(i) ? null : map.valueArray().getBytes(i)); + } + return result; + } + + private static void assertValue(InternalRow row, int position, boolean string, byte[] value) { + if (string) { + assertThat(row.getString(position).toString()) + .isEqualTo(new String(value, StandardCharsets.UTF_8)); + } else { + assertThat(row.getBytes(position)).isEqualTo(value); + } } private static void read( From 706ea4d370a6fecce604170e4f599da72596dc0d Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 17:02:56 +0800 Subject: [PATCH 19/69] [kafka] Preserve metadata rejection on the updated PR05 base The shared schema now accepts three-field Kafka metadata without offsets. Reject it explicitly during row assembler construction so binary/string conversion cannot silently emit null metadata. Keep the existing streaming and borrowed-payload paths unchanged. Cover nullable and non-null metadata in both formats, and update the legacy offset-field regression to the current schema validation message. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 5/5 AI-Contributed/UT: 41/41 --- .../kafka/transcode/KafkaRowAssembler.java | 5 +++ .../ArrowKafkaRecordTranscoderTest.java | 41 ++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index 00747192122..b3ab60dbf82 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -19,6 +19,7 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.metadata.KafkaFieldProjection; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; @@ -43,6 +44,10 @@ public final class KafkaRowAssembler { /** Creates an assembler for the resolved topic schema. */ public KafkaRowAssembler(KafkaTableSchema topicSchema) { this.topicSchema = checkNotNull(topicSchema); + if (topicSchema.metadataPosition() >= 0) { + throw new KafkaTopicSchemaException( + "_kafka_metadata conversion is not supported by this transcoder yet."); + } } /** diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index ee01c5e4e60..ab343b831a1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -337,8 +337,43 @@ void testValueNotNullConstraint(boolean string) { .hasMessageContaining("NOT NULL field '_kafka_value'"); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRejectsMetadataUntilTranscodingIsSupported(boolean string) { + DataType metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + for (boolean nullable : new boolean[] {false, true}) { + TableDescriptor descriptor = + descriptor( + Schema.newBuilder() + .column("_kafka_value", valueType(string)) + .column("_kafka_metadata", metadataType.copy(nullable)) + .build(), + string) + .build(); + // DDL accepts the new contract, but encoding must not silently drop its metadata. + assertThat(KafkaTableSchema.resolve(descriptor).metadataPosition()).isEqualTo(1); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 123L, + TimestampType.CREATE_TIME, + null, + bytes("v"), + Collections.emptyList())), + table(descriptor))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_metadata conversion is not supported"); + } + } + @Test - void testRejectsMetadataUntilActualOffsetsCanBePersisted() { + void testRejectsLegacyMetadataOffsetField() { Schema schema = Schema.newBuilder() .column("_kafka_value", DataTypes.STRING()) @@ -362,7 +397,9 @@ void testRejectsMetadataUntilActualOffsetsCanBePersisted() { Collections.emptyList())), table(descriptor(schema, true).build()))) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("actual per-row offsets"); + .hasMessageContaining( + "_kafka_metadata must be ROW"); } @Test From dfa57463b0885b04e7bfa19f6e897e23cf3a7077 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 23:09:59 +0800 Subject: [PATCH 20/69] [kafka] Transcode three-field metadata from Produce records Populate TIMESTAMP_LTZ(3), Kafka batch timestamp type and the requested partition before Arrow encoding. Preserve timestamp provenance independently of timestamp presence, and enforce nested timestamp NOT NULL constraints. Keep synchronous streaming and borrowed key/value/header payloads. Cover all 16 optional-column combinations, timestamp boundaries, partition isolation, Arrow vector growth and failure atomicity for binary and string values. Validation: Java 11 offline clean verify; 348 tests passed (317 unit and 31 integration), with Spotless, Checkstyle and RAT checks passing. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 74/74 AI-Contributed/UT: 233/233 --- .../transcode/ArrowKafkaRecordTranscoder.java | 8 +- .../transcode/KafkaRecordTranscoder.java | 8 +- .../kafka/transcode/KafkaRowAssembler.java | 58 +++-- .../ArrowKafkaRecordTranscoderTest.java | 233 ++++++++++++++---- 4 files changed, 244 insertions(+), 63 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 88711f42323..dcbc0143c89 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -43,7 +43,9 @@ public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); @Override - public BytesView transcode(List records, TableInfo tableInfo) throws Exception { + public BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception { + checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTableSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); if (schema.valueType() != ValueType.BINARY && schema.valueType() != ValueType.STRING) { @@ -59,8 +61,8 @@ public BytesView transcode(List records, TableInfo tableInfo) throws Exc new Object[] { decode(schema.valueType(), record.borrowedValue()) }, - record.borrowedKey(), - record.headers())); + record, + partitionId)); } }, tableInfo); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java index eeff2f3d14f..9bcc465b2a2 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -27,6 +27,10 @@ /** Converts copied Kafka records into the native Fluss log representation. */ @Internal public interface KafkaRecordTranscoder { - /** Transcodes records according to the target Fluss table schema and log format. */ - BytesView transcode(List records, TableInfo tableInfo) throws Exception; + /** + * Transcodes records from the requested Kafka partition according to the target Fluss table + * schema and log format. + */ + BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index b3ab60dbf82..b5cb170b483 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -18,16 +18,17 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; -import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; import org.apache.fluss.metadata.KafkaFieldProjection; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; import org.apache.fluss.types.MapType; - -import javax.annotation.Nullable; +import org.apache.fluss.types.RowType; import java.util.LinkedHashMap; import java.util.List; @@ -35,7 +36,7 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Assembles decoded values, raw keys and headers using the shared fixed-column contract. */ +/** Assembles decoded values and system columns using the shared fixed-column contract. */ @Internal public final class KafkaRowAssembler { @@ -44,24 +45,21 @@ public final class KafkaRowAssembler { /** Creates an assembler for the resolved topic schema. */ public KafkaRowAssembler(KafkaTableSchema topicSchema) { this.topicSchema = checkNotNull(topicSchema); - if (topicSchema.metadataPosition() >= 0) { - throw new KafkaTopicSchemaException( - "_kafka_metadata conversion is not supported by this transcoder yet."); - } } /** - * Assembles one Fluss row for immediate synchronous encoding. + * Assembles one Fluss row from a record and its requested partition for immediate synchronous + * encoding. * - *

The row borrows decoded values and header payloads; it must not modify them or retain them - * after conversion. + *

The row borrows decoded values, keys and header payloads; it must not modify them or + * retain them after conversion. */ - public GenericRow assemble( - Object[] valueValues, @Nullable byte[] key, List headers) { + public GenericRow assemble(Object[] valueValues, Record record, int partitionId) { GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); setProjectedFields(row, topicSchema.valueProjection(), valueValues); if (topicSchema.keyPosition() >= 0) { int position = topicSchema.keyPosition(); + byte[] key = record.borrowedKey(); if (key == null && !topicSchema.rowType().getTypeAt(position).isNullable()) { throw new KafkaRecordEncodingException( "Null Kafka key cannot populate NOT NULL _kafka_key."); @@ -72,12 +70,44 @@ public GenericRow assemble( int position = topicSchema.headersPosition(); row.setField( position, - toHeaders(headers, (MapType) topicSchema.rowType().getTypeAt(position))); + toHeaders( + record.headers(), (MapType) topicSchema.rowType().getTypeAt(position))); + } + if (topicSchema.metadataPosition() >= 0) { + int position = topicSchema.metadataPosition(); + RowType type = (RowType) topicSchema.rowType().getTypeAt(position); + if (!record.hasTimestamp() && !type.getTypeAt(0).isNullable()) { + throw new KafkaRecordEncodingException( + "Missing Kafka timestamp cannot populate NOT NULL _kafka_metadata.timestamp."); + } + row.setField( + position, + GenericRow.of( + record.hasTimestamp() + ? TimestampLtz.fromEpochMillis(record.timestamp()) + : null, + timestampTypeId(record.timestampType()), + partitionId)); } // A declared _kafka_rescue stays null for binary/string; omitted columns do not exist. return row; } + private static int timestampTypeId(TimestampType type) { + // Preserve batch provenance even when a record's timestamp is absent. The legacy record + // constructor supplies UNKNOWN, so a present timestamp alone cannot imply CreateTime. + switch (type) { + case UNKNOWN: + return -1; + case CREATE_TIME: + return 0; + case LOG_APPEND_TIME: + return 1; + default: + throw new KafkaRecordEncodingException("Unsupported Kafka timestamp type: " + type); + } + } + private static void setProjectedFields( GenericRow row, KafkaFieldProjection projection, Object[] values) { if (values.length != projection.size()) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index ab343b831a1..2ffa89093c3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -87,12 +87,12 @@ void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) thro new RecordHeader("nullable", bytes("discarded")), new RecordHeader("duplicate", bytes("last")), new RecordHeader("nullable", null))); - for (int mask = 0; mask < 8; mask++) { + for (int mask = 0; mask < 16; mask++) { TableInfo table = envelope(string, mask); TableDescriptor before = table.toTableDescriptor(); KafkaTableSchema contract = KafkaTableSchema.resolve(before); read( - transcoder.transcode(Collections.singletonList(record), table), + transcoder.transcode(7, Collections.singletonList(record), table), table, 1, row -> { @@ -109,6 +109,9 @@ void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) thro assertThat(headers.get("duplicate")).isEqualTo(bytes("last")); assertThat(headers.get("nullable")).isNull(); } + if (contract.metadataPosition() >= 0) { + assertMetadata(row, contract.metadataPosition(), 123L, 1, 7); + } if (contract.rescuePosition() >= 0) { assertThat(row.isNullAt(contract.rescuePosition())).isTrue(); } @@ -134,7 +137,7 @@ void testNullAndEmptyPayloadsRemainDistinct(boolean string) throws Exception { new Record(0L, new byte[0], new byte[0], Collections.emptyList())); AtomicInteger next = new AtomicInteger(); read( - transcoder.transcode(records, table), + transcoder.transcode(0, records, table), table, 2, row -> { @@ -171,6 +174,7 @@ void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { assertThatThrownBy( () -> transcoder.transcode( + 0, Collections.singletonList( new Record( 1L, @@ -183,6 +187,7 @@ void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { byte[] invalidUtf8Key = {(byte) 0xff}; read( transcoder.transcode( + 0, Collections.singletonList( new Record( 1L, @@ -222,6 +227,7 @@ void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) thr new RecordHeader("same", bytes("last"))); read( transcoder.transcode( + 0, Collections.singletonList( new Record(1L, null, bytes("v"), overwrittenNull)), table), @@ -243,7 +249,7 @@ void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) thr new RecordHeader("same", null))); if (nullableValue) { read( - transcoder.transcode(Collections.singletonList(finalNull), table), + transcoder.transcode(0, Collections.singletonList(finalNull), table), table, 1, row -> @@ -252,12 +258,15 @@ void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) thr .containsEntry("same", null)); } else { assertThatThrownBy( - () -> transcoder.transcode(Collections.singletonList(finalNull), table)) + () -> + transcoder.transcode( + 0, Collections.singletonList(finalNull), table)) .isInstanceOf(KafkaRecordEncodingException.class) .hasMessageContaining("NOT NULL _kafka_headers map values"); } read( transcoder.transcode( + 0, Collections.singletonList( new Record(1L, null, bytes("v"), Collections.emptyList())), table), @@ -282,7 +291,7 @@ void testOmittedSystemColumnsIgnoreTimestampKeyAndHeaders() throws Exception { } } read( - transcoder.transcode(records, table), + transcoder.transcode(0, records, table), table, records.size(), row -> { @@ -299,6 +308,7 @@ void testBinaryValuePreservesArbitraryBytesAcrossOutputPages() throws Exception TableInfo table = envelope(false, 4); BytesView encoded = transcoder.transcode( + 0, Collections.singletonList( new Record(1L, null, value, Collections.emptyList())), table); @@ -329,6 +339,7 @@ void testValueNotNullConstraint(boolean string) { assertThatThrownBy( () -> transcoder.transcode( + 0, Collections.singletonList( new Record( 1L, null, null, Collections.emptyList())), @@ -339,39 +350,117 @@ void testValueNotNullConstraint(boolean string) { @ParameterizedTest @ValueSource(booleans = {false, true}) - void testRejectsMetadataUntilTranscodingIsSupported(boolean string) { - DataType metadataType = - DataTypes.ROW( - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), - DataTypes.FIELD("partition", DataTypes.INT())); + void testMetadataPreservesInputTimestampBatchTypeAndRequestedPartition(boolean string) + throws Exception { + long[] timestamps = {Record.NO_TIMESTAMP, 0L, 123L, 1728912345678L, Long.MAX_VALUE}; + TimestampType[] types = { + TimestampType.UNKNOWN, TimestampType.CREATE_TIME, TimestampType.LOG_APPEND_TIME + }; + int[] typeIds = {-1, 0, 1}; + List records = new ArrayList<>(); + for (TimestampType type : types) { + for (long timestamp : timestamps) { + records.add(new Record(timestamp, type, null, bytes("v"), Collections.emptyList())); + } + } for (boolean nullable : new boolean[] {false, true}) { - TableDescriptor descriptor = - descriptor( - Schema.newBuilder() - .column("_kafka_value", valueType(string)) - .column("_kafka_metadata", metadataType.copy(nullable)) - .build(), - string) - .build(); - // DDL accepts the new contract, but encoding must not silently drop its metadata. - assertThat(KafkaTableSchema.resolve(descriptor).metadataPosition()).isEqualTo(1); - assertThatThrownBy( - () -> - transcoder.transcode( - Collections.singletonList( - new Record( - 123L, - TimestampType.CREATE_TIME, - null, - bytes("v"), - Collections.emptyList())), - table(descriptor))) - .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("_kafka_metadata conversion is not supported"); + TableInfo table = metadataTable(string, nullable, true); + TableDescriptor before = table.toTableDescriptor(); + // Reuse the same converter and records across requests without retaining partition + // state. + for (int partition : new int[] {7, 0, 3}) { + AtomicInteger next = new AtomicInteger(); + read( + transcoder.transcode(partition, records, table), + table, + records.size(), + row -> { + int i = next.getAndIncrement(); + assertMetadata( + row, + 0, + timestamps[i % timestamps.length], + typeIds[i / timestamps.length], + partition); + assertValue(row, 1, string, bytes("v")); + }); + } + assertThat(table.toTableDescriptor()).isEqualTo(before); } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMetadataTimestampNotNullFailureHasNoPartialOutputAndRemainsUsable(boolean string) + throws Exception { + Record valid = + new Record( + 0L, TimestampType.CREATE_TIME, null, bytes("v"), Collections.emptyList()); + for (boolean nullable : new boolean[] {false, true}) { + TableInfo table = metadataTable(string, nullable, false); + for (TimestampType type : TimestampType.values()) { + Record invalid = + new Record( + Record.NO_TIMESTAMP, + type, + null, + bytes("v"), + Collections.emptyList()); + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + 7, Arrays.asList(valid, invalid), table))); + assertThat(failure) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL _kafka_metadata.timestamp"); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + read( + transcoder.transcode(3, Collections.singletonList(valid), table), + table, + 1, + row -> { + assertMetadata(row, 0, 0L, 0, 3); + assertValue(row, 1, string, bytes("v")); + }); + } + } + } + + @Test + void testLegacyRecordDoesNotInferBatchTimestampType() throws Exception { + TableInfo table = metadataTable(true, false, false); + read( + transcoder.transcode( + 7, + Collections.singletonList( + new Record(123L, null, bytes("v"), Collections.emptyList())), + table), + table, + 1, + row -> assertMetadata(row, 0, 123L, -1, 7)); + } + + @Test + void testRejectsNegativePartitionId() { + assertThatThrownBy( + () -> + transcoder.transcode( + -1, + Collections.singletonList( + new Record( + Record.NO_TIMESTAMP, + null, + bytes("v"), + Collections.emptyList())), + envelope(false, 8))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Kafka partition ID must be non-negative"); + } + @Test void testRejectsLegacyMetadataOffsetField() { Schema schema = @@ -388,6 +477,7 @@ void testRejectsLegacyMetadataOffsetField() { assertThatThrownBy( () -> transcoder.transcode( + 0, Collections.singletonList( new Record( 123L, @@ -411,6 +501,7 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { assertThatThrownBy( () -> transcoder.transcode( + 0, records, table( descriptor(fixed, true) @@ -423,6 +514,7 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { assertThatThrownBy( () -> transcoder.transcode( + 0, records, table( descriptor( @@ -438,6 +530,7 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { assertThatThrownBy( () -> transcoder.transcode( + 0, records, table( descriptor(fixed, true) @@ -446,7 +539,8 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { .build()))) .isInstanceOf(KafkaTopicSchemaException.class) .hasMessageContaining("custom property"); - assertThatThrownBy(() -> transcoder.transcode(Collections.emptyList(), valueTable(false))) + assertThatThrownBy( + () -> transcoder.transcode(0, Collections.emptyList(), valueTable(false))) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("empty"); } @@ -470,6 +564,7 @@ void testJsonIsNotActivatedEvenForNullValues() { assertThatThrownBy( () -> transcoder.transcode( + 0, Collections.singletonList( new Record( 1L, @@ -486,12 +581,17 @@ void testJsonIsNotActivatedEvenForNullValues() { @ValueSource(booleans = {false, true}) void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean string) throws Exception { - TableInfo table = envelope(string, 7); + TableInfo table = envelope(string, 15); List records = new ArrayList<>(); for (int i = 0; i < 2050; i++) { records.add( new Record( - i, + i % 3 == 0 ? Record.NO_TIMESTAMP : i, + i % 3 == 0 + ? TimestampType.UNKNOWN + : (i % 3 == 1 + ? TimestampType.CREATE_TIME + : TimestampType.LOG_APPEND_TIME), bytes("key-" + i), bytes("消息-" + i), Arrays.asList( @@ -500,7 +600,7 @@ void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean str } AtomicInteger next = new AtomicInteger(); read( - transcoder.transcode(records, table), + transcoder.transcode(7, records, table), table, records.size(), row -> { @@ -511,6 +611,7 @@ void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean str assertThat(map).hasSize(1); assertThat(map.get("duplicate")).isEqualTo(bytes("header-" + i)); assertThat(row.isNullAt(1)).isTrue(); + assertMetadata(row, 4, i % 3 == 0 ? Record.NO_TIMESTAMP : i, i % 3 - 1, 7); }); } @@ -523,7 +624,9 @@ void testOneRecordWithManyDistinctHeadersGrowsArrowMapVectors() throws Exception } read( transcoder.transcode( - Collections.singletonList(new Record(1L, null, bytes("v"), input)), table), + 0, + Collections.singletonList(new Record(1L, null, bytes("v"), input)), + table), table, 1, row -> { @@ -588,6 +691,7 @@ void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw () -> output.set( transcoder.transcode( + 0, Arrays.asList( valid, new Record( @@ -601,7 +705,7 @@ void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw assertThat(output).hasNullValue(); } read( - transcoder.transcode(Collections.singletonList(valid), table), + transcoder.transcode(0, Collections.singletonList(valid), table), table, 1, row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); @@ -640,12 +744,12 @@ void testHeaderFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw () -> output.set( transcoder.transcode( - Arrays.asList(valid, invalid), table))); + 0, Arrays.asList(valid, invalid), table))); assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); assertThat(failure.getSuppressed()).isEmpty(); assertThat(output).hasNullValue(); read( - transcoder.transcode(Collections.singletonList(valid), table), + transcoder.transcode(0, Collections.singletonList(valid), table), table, 1, row -> assertThat(headers(row.getMap(1)).get("h")).isEqualTo(bytes("v"))); @@ -706,9 +810,33 @@ private static TableInfo envelope(boolean string, int mask) { if ((mask & 1) != 0) { schema.column("_kafka_key", DataTypes.BYTES()); } + if ((mask & 8) != 0) { + schema.column("_kafka_metadata", metadataType(true)); + } return table(descriptor(schema.build(), string).build()); } + private static DataType metadataType(boolean nullableTimestamp) { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3).copy(nullableTimestamp)), + DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))); + } + + private static TableInfo metadataTable( + boolean string, boolean nullableRow, boolean nullableTimestamp) { + return table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_metadata", + metadataType(nullableTimestamp).copy(nullableRow)) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + } + private static DataType valueType(boolean string) { return string ? DataTypes.STRING() : DataTypes.BYTES(); } @@ -716,7 +844,7 @@ private static DataType valueType(boolean string) { private static TableDescriptor.Builder descriptor(Schema schema, boolean string) { return TableDescriptor.builder() .schema(schema) - .distributedBy(1) + .distributedBy(8) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) .property( @@ -747,6 +875,23 @@ private static void assertValue(InternalRow row, int position, boolean string, b } } + private static void assertMetadata( + InternalRow row, int position, long timestamp, int timestampType, int partition) { + assertThat(row.isNullAt(position)).isFalse(); + InternalRow metadata = row.getRow(position, 3); + assertThat(metadata.getFieldCount()).isEqualTo(3); + if (timestamp == Record.NO_TIMESTAMP) { + assertThat(metadata.isNullAt(0)).isTrue(); + } else { + assertThat(metadata.isNullAt(0)).isFalse(); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(timestamp); + } + assertThat(metadata.isNullAt(1)).isFalse(); + assertThat(metadata.getInt(1)).isEqualTo(timestampType); + assertThat(metadata.isNullAt(2)).isFalse(); + assertThat(metadata.getInt(2)).isEqualTo(partition); + } + private static void read( BytesView encoded, TableInfo table, int count, Consumer verify) throws Exception { From 6985a0d689d647f217ac888f7ae91703a6a6738d Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 12:00:00 +0800 Subject: [PATCH 21/69] [kafka] Align Produce transcoding with current schema contract Use the typed conversion API from the current metadata contract and align transcoder tests with schema-time nullability validation. Co-Authored-By: Codex AI-Model: gpt-6.1-sol AI-Contributed/Feature: 16/16 AI-Contributed/UT: 237/237 --- .../transcode/ArrowKafkaRecordTranscoder.java | 16 +- .../ArrowKafkaRecordTranscoderTest.java | 237 ++++++++++-------- 2 files changed, 140 insertions(+), 113 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index dcbc0143c89..f441eb96c6b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -18,10 +18,10 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.metadata.KafkaTableSchema; -import org.apache.fluss.metadata.KafkaTableSchema.ValueType; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.BinaryString; @@ -48,9 +48,10 @@ public BytesView transcode(int partitionId, List records, TableInfo tabl checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTableSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); - if (schema.valueType() != ValueType.BINARY && schema.valueType() != ValueType.STRING) { + if (schema.valueConversionType() != KafkaValueConversionType.BINARY + && schema.valueConversionType() != KafkaValueConversionType.STRING) { throw new KafkaRecordEncodingException( - "Unsupported Kafka value conversion type: " + schema.valueType()); + "Unsupported Kafka value conversion type: " + schema.valueConversionType()); } KafkaRowAssembler assembler = new KafkaRowAssembler(schema); return arrowRecordEncoder.encodeStreaming( @@ -59,7 +60,9 @@ public BytesView transcode(int partitionId, List records, TableInfo tabl consumer.append( assembler.assemble( new Object[] { - decode(schema.valueType(), record.borrowedValue()) + decode( + schema.valueConversionType(), + record.borrowedValue()) }, record, partitionId)); @@ -68,8 +71,9 @@ public BytesView transcode(int partitionId, List records, TableInfo tabl tableInfo); } - private static @Nullable Object decode(ValueType format, @Nullable byte[] bytes) { - if (bytes == null || format == ValueType.BINARY) { + private static @Nullable Object decode( + KafkaValueConversionType format, @Nullable byte[] bytes) { + if (bytes == null || format == KafkaValueConversionType.BINARY) { return bytes; } try { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 2ffa89093c3..f039715fb10 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; @@ -162,7 +163,7 @@ void testNullAndEmptyPayloadsRemainDistinct(boolean string) throws Exception { @ParameterizedTest @ValueSource(booleans = {false, true}) void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { - TableInfo table = + TableInfo nonNullableKeyTable = table( descriptor( Schema.newBuilder() @@ -181,9 +182,27 @@ void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { null, bytes("valid"), Collections.emptyList())), - table)) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("NOT NULL _kafka_key"); + nonNullableKeyTable)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_key' is NOT NULL"); + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES()) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record(1L, null, bytes("valid"), Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.isNullAt(0)).isTrue()); byte[] invalidUtf8Key = {(byte) 0xff}; read( transcoder.transcode( @@ -215,7 +234,7 @@ void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) thr DataTypes.BYTES() .copy( nullableValue)) - .copy(false)) + .copy(true)) .column("_kafka_value", DataTypes.STRING()) .build(), true) @@ -225,6 +244,19 @@ void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) thr new RecordHeader("same", null), new RecordHeader("", new byte[0]), new RecordHeader("same", bytes("last"))); + if (!nullableValue) { + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, null, bytes("v"), overwrittenNull)), + table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_headers"); + return; + } read( transcoder.transcode( 0, @@ -247,23 +279,14 @@ void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) thr Arrays.asList( new RecordHeader("same", bytes("first")), new RecordHeader("same", null))); - if (nullableValue) { - read( - transcoder.transcode(0, Collections.singletonList(finalNull), table), - table, - 1, - row -> - assertThat(headers(row.getMap(0))) - .containsOnlyKeys("same") - .containsEntry("same", null)); - } else { - assertThatThrownBy( - () -> - transcoder.transcode( - 0, Collections.singletonList(finalNull), table)) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("NOT NULL _kafka_headers map values"); - } + read( + transcoder.transcode(0, Collections.singletonList(finalNull), table), + table, + 1, + row -> + assertThat(headers(row.getMap(0))) + .containsOnlyKeys("same") + .containsEntry("same", null)); read( transcoder.transcode( 0, @@ -344,8 +367,8 @@ void testValueNotNullConstraint(boolean string) { new Record( 1L, null, null, Collections.emptyList())), table)) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("NOT NULL field '_kafka_value'"); + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value' is NOT NULL"); } @ParameterizedTest @@ -363,76 +386,52 @@ void testMetadataPreservesInputTimestampBatchTypeAndRequestedPartition(boolean s records.add(new Record(timestamp, type, null, bytes("v"), Collections.emptyList())); } } - for (boolean nullable : new boolean[] {false, true}) { - TableInfo table = metadataTable(string, nullable, true); - TableDescriptor before = table.toTableDescriptor(); - // Reuse the same converter and records across requests without retaining partition - // state. - for (int partition : new int[] {7, 0, 3}) { - AtomicInteger next = new AtomicInteger(); - read( - transcoder.transcode(partition, records, table), - table, - records.size(), - row -> { - int i = next.getAndIncrement(); - assertMetadata( - row, - 0, - timestamps[i % timestamps.length], - typeIds[i / timestamps.length], - partition); - assertValue(row, 1, string, bytes("v")); - }); - } - assertThat(table.toTableDescriptor()).isEqualTo(before); + TableInfo table = metadataTable(string, true, true); + TableDescriptor before = table.toTableDescriptor(); + // Reuse the same converter and records across requests without retaining partition state. + for (int partition : new int[] {7, 0, 3}) { + AtomicInteger next = new AtomicInteger(); + read( + transcoder.transcode(partition, records, table), + table, + records.size(), + row -> { + int i = next.getAndIncrement(); + assertMetadata( + row, + 0, + timestamps[i % timestamps.length], + typeIds[i / timestamps.length], + partition); + assertValue(row, 1, string, bytes("v")); + }); } + assertThat(table.toTableDescriptor()).isEqualTo(before); } @ParameterizedTest @ValueSource(booleans = {false, true}) - void testMetadataTimestampNotNullFailureHasNoPartialOutputAndRemainsUsable(boolean string) - throws Exception { - Record valid = - new Record( - 0L, TimestampType.CREATE_TIME, null, bytes("v"), Collections.emptyList()); - for (boolean nullable : new boolean[] {false, true}) { - TableInfo table = metadataTable(string, nullable, false); - for (TimestampType type : TimestampType.values()) { - Record invalid = - new Record( - Record.NO_TIMESTAMP, - type, - null, - bytes("v"), - Collections.emptyList()); - AtomicReference output = new AtomicReference<>(); - Throwable failure = - catchThrowable( - () -> - output.set( - transcoder.transcode( - 7, Arrays.asList(valid, invalid), table))); - assertThat(failure) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("NOT NULL _kafka_metadata.timestamp"); - assertThat(failure.getSuppressed()).isEmpty(); - assertThat(output).hasNullValue(); - read( - transcoder.transcode(3, Collections.singletonList(valid), table), - table, - 1, - row -> { - assertMetadata(row, 0, 0L, 0, 3); - assertValue(row, 1, string, bytes("v")); - }); - } - } + void testMetadataTimestampNotNullIsRejectedAtSchemaResolution(boolean string) { + TableInfo table = metadataTable(string, true, false); + assertThatThrownBy( + () -> + transcoder.transcode( + 7, + Collections.singletonList( + new Record( + Record.NO_TIMESTAMP, + TimestampType.UNKNOWN, + null, + bytes("v"), + Collections.emptyList())), + table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_metadata.timestamp' is NOT NULL"); } @Test void testLegacyRecordDoesNotInferBatchTimestampType() throws Exception { - TableInfo table = metadataTable(true, false, false); + TableInfo table = metadataTable(true, true, true); read( transcoder.transcode( 7, @@ -493,7 +492,7 @@ void testRejectsLegacyMetadataOffsetField() { } @Test - void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { + void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() throws Exception { List records = Collections.singletonList( new Record(1L, null, bytes("v"), Collections.emptyList())); @@ -527,18 +526,16 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { .build()))) .isInstanceOf(KafkaTopicSchemaException.class) .hasMessageContaining("_kafka_value"); - assertThatThrownBy( - () -> - transcoder.transcode( - 0, - records, - table( - descriptor(fixed, true) - .customProperty( - "kafka.value.format", "string") - .build()))) - .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("custom property"); + TableInfo unrelatedCustomProperty = + table( + descriptor(fixed, true) + .customProperty("kafka.value.format", "string") + .build()); + read( + transcoder.transcode(0, records, unrelatedCustomProperty), + unrelatedCustomProperty, + records.size(), + row -> assertThat(row.getString(0).toString()).isEqualTo("v")); assertThatThrownBy( () -> transcoder.transcode(0, Collections.emptyList(), valueTable(false))) .isInstanceOf(IllegalArgumentException.class) @@ -558,7 +555,8 @@ void testJsonIsNotActivatedEvenForNullValues() { .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) .property( - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.PARSE_JSON) .build()); for (byte[] value : new byte[][] {null, bytes("{}")}) { assertThatThrownBy( @@ -682,9 +680,9 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce @Test void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { - TableInfo table = valueTable(true); + TableInfo table = valueTable(false); Record valid = new Record(1L, null, bytes("valid"), Collections.emptyList()); - for (byte[] invalid : new byte[][] {null, {(byte) 0xc3, 0x28}, {(byte) 0xff}}) { + for (byte[] invalid : new byte[][] {{(byte) 0xc3, 0x28}, {(byte) 0xff}}) { AtomicReference output = new AtomicReference<>(); Throwable failure = catchThrowable( @@ -709,11 +707,20 @@ void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw table, 1, row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record(2L, null, null, Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.isNullAt(0)).isTrue()); } @Test void testHeaderFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { - TableInfo table = + TableInfo invalidTable = table( descriptor( Schema.newBuilder() @@ -744,10 +751,25 @@ void testHeaderFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw () -> output.set( transcoder.transcode( - 0, Arrays.asList(valid, invalid), table))); - assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + 0, Arrays.asList(valid, invalid), invalidTable))); + assertThat(failure) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_headers"); assertThat(failure.getSuppressed()).isEmpty(); assertThat(output).hasNullValue(); + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.BYTES())) + .build(), + true) + .build()); read( transcoder.transcode(0, Collections.singletonList(valid), table), table, @@ -819,8 +841,8 @@ private static TableInfo envelope(boolean string, int mask) { private static DataType metadataType(boolean nullableTimestamp) { return DataTypes.ROW( DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3).copy(nullableTimestamp)), - DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), - DataTypes.FIELD("partition", DataTypes.INT().copy(false))); + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); } private static TableInfo metadataTable( @@ -848,7 +870,8 @@ private static TableDescriptor.Builder descriptor(Schema schema, boolean string) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) .property( - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, string ? "string" : "binary"); + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + string ? KafkaValueConversionType.STRING : KafkaValueConversionType.BINARY); } private static TableInfo table(TableDescriptor descriptor) { From 7c76aa2b80ba4d2d008177a6ee9e2bccf7403b6f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:31:15 +0800 Subject: [PATCH 22/69] [kafka] Connect Produce to native Fluss append Register Produce v3-v11 and adapt the local TabletServer gateway using the shared qualified-topic DDL contract and raw/string transcoder. Preserve the resolved TablePath across metadata lookup and conversion, isolate table/partition failures, and map invalid topic names correctly. Drain deferred actions immediately after the synchronous append invocation so acks=all does not wait for its own delayed replica fetches. Base on PR06 04a9093b3 and independent core PR #3412 at 969f985ab. Validation: Java 11 reactor verify; 139 Kafka unit tests, 7 Kafka integration tests, and 10 targeted RPC/server tests passed. Three-replica tests cover raw/string, gzip, acks=0/1/all, offsets, timestamp/headers, native readback, and independent writes to same-named tables in different databases. Checkstyle, Spotless, RAT, and git diff --check passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 319/319 AI-Contributed/UT: 606/606 --- fluss-kafka/pom.xml | 6 + .../fluss/kafka/KafkaRequestHandler.java | 11 +- .../produce/GatewayKafkaProduceBackend.java | 306 +++++++++++++++ .../fluss/kafka/KafkaProduceAppendITCase.java | 240 ++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 1 + .../GatewayKafkaProduceBackendTest.java | 365 ++++++++++++++++++ 6 files changed, 928 insertions(+), 1 deletion(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index b77695848ca..e38487db564 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,6 +64,12 @@ + + ${project.groupId} + fluss-client + ${project.version} + test + org.apache.curator curator-test diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 31da9143823..9e985dbea7f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -19,13 +19,17 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.kafka.api.metadata.MetadataHandler; +import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; +import org.apache.fluss.kafka.backend.produce.GatewayKafkaProduceBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; @@ -48,7 +52,12 @@ public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { new ApiVersionsHandler(this::apiRegistry), new MetadataHandler( new GatewayKafkaMetadataBackend(service, topicMapper), - topicMapper))); + topicMapper), + new ProduceHandler( + new GatewayKafkaProduceBackend( + service, + (TabletServerGateway) service, + new ArrowKafkaRecordTranscoder())))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java new file mode 100644 index 00000000000..dd07df63733 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.protocol.Errors; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the local TabletServer write gateway to the Kafka Produce backend contract. */ +@Internal +public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final KafkaRecordTranscoder transcoder; + + /** Creates a Produce backend backed by the local TabletServer gateway. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + KafkaRecordTranscoder transcoder) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.transcoder = checkNotNull(transcoder); + } + + @Override + public CompletableFuture write(KafkaProduceCommand command) { + List> futures = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + futures.add(writeTopic(command, topic)); + } + CompletableFuture all = + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + return all.thenApply( + ignored -> { + List results = new ArrayList<>(); + for (CompletableFuture future : futures) { + results.add(future.join()); + } + return new KafkaProduceResult(results); + }); + } + + private CompletableFuture writeTopic( + KafkaProduceCommand command, TopicWrite topic) { + try { + TablePath path = topicMapper.toTablePath(topic.topicName()); + setCurrentSession(command); + GetTableInfoRequest request = new GetTableInfoRequest(); + request.setTablePath() + .setDatabaseName(path.getDatabaseName()) + .setTableName(path.getTableName()); + return gateway.getTableInfo(request) + .thenCompose( + response -> produceTopic(command, topic, toTableInfo(path, response))) + .exceptionally(failure -> failedTopic(topic, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedTopic(topic, failure)); + } + } + + private CompletableFuture produceTopic( + KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { + // Admission must match Metadata, including when the schema changes between requests. + schemaResolver.resolve(tableInfo.toTableDescriptor()); + ProduceLogRequest request = + new ProduceLogRequest() + .setTableId(tableInfo.getTableId()) + .setAcks(command.acks()) + .setTimeoutMs(command.timeoutMs()); + Map failures = new HashMap<>(); + for (PartitionWrite partition : topic.partitions()) { + if (partition.partitionId() < 0 + || partition.partitionId() >= tableInfo.getNumBuckets()) { + failures.put( + partition.partitionId(), + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_TOPIC_OR_PARTITION, + -1L, + "Partition is outside the table bucket range.")); + continue; + } + try { + BytesView records = transcoder.transcode(partition.records(), tableInfo); + request.addBucketsReq() + .setBucketId(partition.partitionId()) + .setRecordsBytesView(records); + } catch (Exception failure) { + failures.put( + partition.partitionId(), failedPartition(partition.partitionId(), failure)); + } + } + if (request.getBucketsReqsCount() == 0) { + return CompletableFuture.completedFuture( + toTopicResult(topic, new ProduceLogResponse(), failures)); + } + try { + setCurrentSession(command); + CompletableFuture appended; + try { + appended = gateway.produceLog(request); + } finally { + // produceLog appends synchronously, but acks=-1 can finish later. Wake delayed + // fetches now so replicas can fetch the append needed to complete that response. + service.tryCompleteActions(); + } + return appended.handle( + (response, failure) -> + failure == null + ? toTopicResult(topic, response, failures) + : failedAppend(topic, failures, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedAppend(topic, failures, failure)); + } + } + + private static TopicResult failedAppend( + TopicWrite topic, Map failures, Throwable failure) { + List results = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult local = failures.get(partition.partitionId()); + results.add(local == null ? failedPartition(partition.partitionId(), failure) : local); + } + return new TopicResult(topic.topicName(), results); + } + + private static TableInfo toTableInfo(TablePath path, GetTableInfoResponse response) { + return TableInfo.of( + path, + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null, + response.getCreatedTime(), + response.getModifiedTime()); + } + + private static TopicResult toTopicResult( + TopicWrite topic, ProduceLogResponse response, Map failures) { + Map responses = new HashMap<>(); + for (PbProduceLogRespForBucket bucket : response.getBucketsRespsList()) { + responses.put(bucket.getBucketId(), bucket); + } + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult localFailure = failures.get(partition.partitionId()); + if (localFailure != null) { + partitions.add(localFailure); + continue; + } + PbProduceLogRespForBucket bucket = responses.get(partition.partitionId()); + if (bucket == null) { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_SERVER_ERROR, + -1L, + "Fluss Produce response omitted this bucket.")); + } else if (bucket.hasErrorCode() && bucket.getErrorCode() != 0) { + partitions.add( + new PartitionResult( + partition.partitionId(), + toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forCode( + bucket.getErrorCode())), + -1L, + bucket.hasErrorMessage() ? bucket.getErrorMessage() : null)); + } else { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.NONE, + bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, + null)); + } + } + return new TopicResult(topic.topicName(), partitions); + } + + private static TopicResult failedTopic(TopicWrite topic, Throwable failure) { + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + partitions.add(failedPartition(partition.partitionId(), failure)); + } + return new TopicResult(topic.topicName(), partitions); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + Throwable cause = unwrap(failure); + Errors kafkaError = + cause instanceof KafkaTopicSchemaException || cause instanceof InvalidTopicException + ? Errors.INVALID_TOPIC_EXCEPTION + : cause instanceof KafkaRecordEncodingException + ? Errors.CORRUPT_MESSAGE + : cause instanceof IllegalArgumentException + ? Errors.INVALID_REQUEST + : toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forException( + cause)); + return new PartitionResult(partitionId, kafkaError, -1L, cause.getMessage()); + } + + private static Errors toKafkaError(org.apache.fluss.rpc.protocol.Errors error) { + switch (error) { + case NONE: + return Errors.NONE; + case TABLE_NOT_EXIST: + case UNKNOWN_TABLE_OR_BUCKET_EXCEPTION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case NOT_LEADER_OR_FOLLOWER: + return Errors.NOT_LEADER_OR_FOLLOWER; + case LEADER_NOT_AVAILABLE_EXCEPTION: + return Errors.LEADER_NOT_AVAILABLE; + case RECORD_TOO_LARGE_EXCEPTION: + return Errors.MESSAGE_TOO_LARGE; + case CORRUPT_MESSAGE: + case CORRUPT_RECORD_EXCEPTION: + return Errors.CORRUPT_MESSAGE; + case INVALID_REQUIRED_ACKS: + return Errors.INVALID_REQUIRED_ACKS; + case REQUEST_TIME_OUT: + return Errors.REQUEST_TIMED_OUT; + case NOT_ENOUGH_REPLICAS_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS; + case NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND; + case AUTHORIZATION_EXCEPTION: + return Errors.TOPIC_AUTHORIZATION_FAILED; + case LOG_STORAGE_EXCEPTION: + case STORAGE_EXCEPTION: + case DISK_WRITE_LOCKED: + return Errors.KAFKA_STORAGE_ERROR; + default: + return Errors.UNKNOWN_SERVER_ERROR; + } + } + + private void setCurrentSession(KafkaProduceCommand command) { + service.setCurrentSession( + new Session( + (short) 0, + command.listenerName(), + false, + command.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java new file mode 100644 index 00000000000..8f6fab3a2e5 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Exercises native DDL, Kafka discovery and Produce, replication, and native Fluss readback. */ +class KafkaProduceAppendITCase { + private static final String DATABASE = "kafka"; + private static final byte[] KEY = "key".getBytes(StandardCharsets.UTF_8); + private static final byte[] VALUE = "value".getBytes(StandardCharsets.UTF_8); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @ParameterizedTest + @ValueSource(strings = {"raw", "string"}) + void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception { + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, "produce_" + format + "_" + acks); + String topic = path.toString(); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer(acks)) { + assertThat(producer.partitionsFor(topic)).hasSize(1); + ProducerRecord record = + new ProducerRecord<>( + topic, + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + RecordMetadata first = producer.send(record).get(30, TimeUnit.SECONDS); + RecordMetadata second = producer.send(record).get(30, TimeUnit.SECONDS); + if (!acks.equals("0")) { + assertThat(first.offset()).isZero(); + assertThat(second.offset()).isEqualTo(1L); + } + producer.flush(); + assertReadback(connection, path, format, VALUE); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testSameNamedTablesAcrossDatabases() throws Exception { + Map tables = new LinkedHashMap<>(); + tables.put(TablePath.of("produce_first", "events"), VALUE); + tables.put( + TablePath.of("produce_second", "events"), + "second database".getBytes(StandardCharsets.UTF_8)); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + try { + for (TablePath path : tables.keySet()) { + admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true) + .get(); + admin.createTable(path, descriptor("raw"), false).get(); + } + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all")) { + for (Map.Entry entry : tables.entrySet()) { + String topic = entry.getKey().toString(); + assertThat(producer.partitionsFor(topic)).hasSize(1); + ProducerRecord record = + new ProducerRecord<>( + topic, + 0, + 123L, + KEY, + entry.getValue(), + Arrays.asList( + new RecordHeader("source", entry.getValue()), + new RecordHeader("source", null))); + assertThat(producer.send(record).get(30, TimeUnit.SECONDS).offset()) + .isZero(); + assertThat(producer.send(record).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(1L); + } + for (Map.Entry entry : tables.entrySet()) { + assertReadback(connection, entry.getKey(), "raw", entry.getValue()); + } + } + } finally { + for (TablePath path : tables.keySet()) { + admin.dropTable(path, true).get(); + } + } + } + } + + private static void assertReadback( + Connection connection, TablePath path, String format, byte[] expectedValue) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < 2 && System.nanoTime() < deadline) { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + InternalRow row = record.getRow(); + if (format.equals("raw")) { + assertThat(row.getBytes(0)).isEqualTo(expectedValue); + assertThat(row.getBytes(3)).isEqualTo(KEY); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo(new String(expectedValue, StandardCharsets.UTF_8)); + assertThat(row.getString(3).toString()).isEqualTo("key"); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(row.getArray(2).size()).isEqualTo(2); + assertThat(row.getArray(2).getRow(0, 2).getString(0).toString()) + .isEqualTo("source"); + assertThat(row.getArray(2).getRow(1, 2).isNullAt(1)).isTrue(); + count++; + } + } + assertThat(count).isEqualTo(2); + } + } + + private static TableDescriptor descriptor(String format) { + boolean raw = format.equals("raw"); + return TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", raw ? DataTypes.BYTES() : DataTypes.STRING()) + .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "attributes", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD( + "value", DataTypes.BYTES())))) + .column("message_key", raw ? DataTypes.BYTES() : DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format) + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format) + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .build(); + } + + private static KafkaProducer producer(String acks) { + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 3c1bfe6128b..9642ed9bfb1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -151,6 +151,7 @@ private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, (short) 11), tuple(ApiKeys.METADATA.id, (short) 0, (short) 11), tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java new file mode 100644 index 00000000000..d556d910db7 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.protocol.Errors; +import org.junit.jupiter.api.Test; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests append admission, error isolation, session propagation and delayed-fetch completion. */ +class GatewayKafkaProduceBackendTest { + + @Test + void testDrainsImmediatelyWhileAcksAllResponseIsPending() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.append.getAcks()).isEqualTo(-1); + assertThat(service.append.getTimeoutMs()).isEqualTo(4321); + assertThat(service.append.getTableId()).isEqualTo(42L); + assertThat(service.drains).isEqualTo(1); + assertThat(result).isNotDone(); + assertThat(service.pendingAppend).isNotDone(); + service.pendingAppend.complete(success(0)); + assertThat(result.join().topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testDrainsAfterSynchronousAppendFailure() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.throwAppend = true; + assertErrors( + backend(service).write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.UNKNOWN_SERVER_ERROR); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception { + TestingProduceService service = new TestingProduceService(); + PartitionWrite malformed = + new PartitionWrite( + 0, + Collections.singletonList( + new Record( + 1L, + null, + new byte[] {(byte) 0xff}, + Collections.emptyList()))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("kafka.topic", malformed, good(1), good(8)))) + .join(); + assertErrors( + result, Errors.CORRUPT_MESSAGE, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testInvalidAndMissingTablesDoNotSuppressOtherTopics() throws Exception { + for (boolean async : new boolean[] {false, true}) { + TestingProduceService service = new TestingProduceService(); + service.asyncMissing = async; + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("kafka.invalid", good(0)), + topic("kafka.missing", good(0)), + topic("kafka.topic", good(0)))) + .join(); + assertThat(result.topics()).hasSize(3); + assertThat(result.topics().get(0).partitions().get(0).error()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION); + assertThat(result.topics().get(1).partitions().get(0).error()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(result.topics().get(2).partitions().get(0).error()).isEqualTo(Errors.NONE); + assertThat(service.drains).isEqualTo(1); + } + } + + @Test + void testNoAppendOrDrainWhenAllPartitionsFailConversion() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service).write(command((short) 1, topic("kafka.topic", good(8)))).join(); + assertErrors(result, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append).isNull(); + assertThat(service.drains).isZero(); + } + + @Test + void testMapsReplicationFailuresAndMissingResponsesPerPartition() throws Exception { + org.apache.fluss.rpc.protocol.Errors[] flussErrors = { + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.REQUEST_TIME_OUT, + org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER + }; + Errors[] kafkaErrors = { + Errors.NOT_ENOUGH_REPLICAS, + Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, + Errors.REQUEST_TIMED_OUT, + Errors.NOT_LEADER_OR_FOLLOWER + }; + for (int i = 0; i < flussErrors.length; i++) { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = + CompletableFuture.completedFuture( + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setErrorCode(flussErrors[i].code()), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setErrorCode(0) + .setBaseOffset(17L)))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) -1, + topic("kafka.topic", good(0), good(1), good(2)))) + .join(); + assertErrors(result, kafkaErrors[i], Errors.NONE, Errors.UNKNOWN_SERVER_ERROR); + assertThat(result.topics().get(0).partitions().get(1).baseOffset()).isEqualTo(17L); + } + } + + @Test + void testAsyncMetadataCompletionRestoresSessionAndAcksZero() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) 0, topic("kafka.topic", good(0)))); + CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))).join(); + assertErrors(result.join(), Errors.NONE); + assertThat(service.append.getAcks()).isZero(); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testQualifiedTopicsPreserveDatabaseAndTranscoderTablePath() throws Exception { + TestingProduceService service = new TestingProduceService(); + List transcodedPaths = new ArrayList<>(); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + (records, tableInfo) -> { + transcodedPaths.add(tableInfo.getTablePath()); + return transcoder.transcode(records, tableInfo); + }); + + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("kafka.topic", good(0)), + topic("other.topic", good(0)))) + .join(); + + assertThat(service.metadataPaths) + .containsExactly(TablePath.of("kafka", "topic"), TablePath.of("other", "topic")); + assertThat(transcodedPaths).containsExactlyElementsOf(service.metadataPaths); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 43L); + assertThat(result.topics()) + .extracting(KafkaProduceResult.TopicResult::topicName) + .containsExactly("kafka.topic", "other.topic"); + assertThat(result.topics()) + .allSatisfy( + topic -> + assertThat(topic.partitions().get(0).error()) + .isEqualTo(Errors.NONE)); + } + + @Test + void testInvalidQualifiedNamesDoNotReachGatewayOrSuppressValidTopic() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("topic", good(0)), + topic("kafka.topic.extra", good(0)), + topic("kafka.__internal", good(0)), + topic("kafka.topic", good(0)))) + .join(); + + assertThat(result.topics()) + .extracting(topic -> topic.partitions().get(0).error()) + .containsExactly( + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION, + Errors.NONE); + assertThat(service.metadataPaths).containsExactly(TablePath.of("kafka", "topic")); + assertThat(service.appends).hasSize(1); + } + + private static GatewayKafkaProduceBackend backend(TestingProduceService service) { + return new GatewayKafkaProduceBackend(service, service, new ArrowKafkaRecordTranscoder()); + } + + private static KafkaProduceCommand command(short acks, TopicWrite... topics) throws Exception { + return new KafkaProduceCommand( + acks, 4321, Arrays.asList(topics), "KAFKA", InetAddress.getLoopbackAddress()); + } + + private static TopicWrite topic(String name, PartitionWrite... partitions) { + return new TopicWrite(name, Arrays.asList(partitions)); + } + + private static PartitionWrite good(int id) { + return new PartitionWrite( + id, + Collections.singletonList( + new Record(123L, null, new byte[] {65}, Collections.emptyList()))); + } + + private static GetTableInfoResponse metadata(boolean invalid) { + TableDescriptor.Builder descriptor = + TableDescriptor.builder() + .schema(Schema.newBuilder().column("value", DataTypes.STRING()).build()) + .distributedBy(3) + .logFormat(LogFormat.ARROW); + if (!invalid) { + descriptor.customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"); + } + return new GetTableInfoResponse() + .setTableId(42L) + .setSchemaId(1) + .setTableJson(descriptor.build().toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L); + } + + private static ProduceLogResponse success(int id) { + return new ProduceLogResponse() + .addAllBucketsResps( + Collections.singletonList( + new PbProduceLogRespForBucket() + .setBucketId(id) + .setBaseOffset(17L))); + } + + private static void assertErrors(KafkaProduceResult result, Errors... errors) { + List partitions = result.topics().get(0).partitions(); + assertThat(partitions).extracting(PartitionResult::error).containsExactly(errors); + for (PartitionResult partition : partitions) { + if (partition.error() != Errors.NONE) { + assertThat(partition.baseOffset()).isEqualTo(-1L); + } + } + } + + private static class TestingProduceService extends TestingTabletGatewayService { + private final List metadataPaths = new ArrayList<>(); + private final List appends = new ArrayList<>(); + private ProduceLogRequest append; + private CompletableFuture pendingAppend; + private CompletableFuture pendingMetadata; + private boolean throwAppend; + private boolean asyncMissing; + private int drains; + + @Override + public CompletableFuture getTableInfo(GetTableInfoRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + String database = request.getTablePath().getDatabaseName(); + String name = request.getTablePath().getTableName(); + metadataPaths.add(TablePath.of(database, name)); + if (name.equals("missing")) { + if (!asyncMissing) { + throw new TableNotExistException("missing"); + } + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(new TableNotExistException("missing")); + return failed; + } + return pendingMetadata == null + ? CompletableFuture.completedFuture( + GatewayKafkaProduceBackendTest.metadata(name.equals("invalid")) + .setTableId(database.equals("other") ? 43L : 42L)) + : pendingMetadata; + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(currentSession().getInetAddress()) + .isEqualTo(InetAddress.getLoopbackAddress()); + append = request; + appends.add(request); + if (throwAppend) { + throw new IllegalStateException("append failed"); + } + return pendingAppend == null + ? CompletableFuture.completedFuture( + success(request.getBucketsReqsList().get(0).getBucketId())) + : pendingAppend; + } + + @Override + public void tryCompleteActions() { + assertThat(append).isNotNull(); + drains++; + } + } +} From 78b4ac38c8f5d371d5704dca66489ac064113759 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 19:33:20 +0800 Subject: [PATCH 23/69] [kafka] Cache Produce plans and bound conversion execution Reuse parsed table metadata and immutable raw/string write plans while refreshing native metadata and authorization for every topic write. Offload metadata lookup, conversion and native submission to bounded FIFO workers, preserving connection order without waiting for acknowledgements. Close protocol-owned workers after transport and RPC processors stop. Validation: Java 11 reactor verify passed 160 Kafka unit tests, 8 Kafka integration tests and 5 NettyServer tests. Checkstyle, Spotless, RAT and git diff --check passed. Tests cover cache invalidation, authorization refresh, saturation, shutdown, pending acknowledgements and pipelined three-replica append order. Source content is unchanged since validation. ZooKeeper query reduction and byte-based admission remain follow-up work. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 812/812 AI-Contributed/UT: 1047/1047 --- .../fluss/kafka/KafkaProtocolPlugin.java | 40 ++- .../fluss/kafka/KafkaRequestHandler.java | 14 +- .../produce/GatewayKafkaProduceBackend.java | 100 +++++-- .../KafkaProduceConversionExecutor.java | 163 ++++++++++ .../backend/produce/KafkaTableInfoCache.java | 179 +++++++++++ .../transcode/ArrowKafkaRecordTranscoder.java | 22 +- .../KafkaCompiledWritePlanCache.java | 161 ++++++++++ .../transcode/KafkaRecordTranscoder.java | 20 +- .../kafka/transcode/KafkaTopicWritePlan.java | 54 ++++ .../fluss/kafka/KafkaProduceAppendITCase.java | 53 ++++ .../GatewayKafkaProduceBackendTest.java | 141 +++++++++ .../KafkaProduceConversionExecutorTest.java | 184 ++++++++++++ .../produce/KafkaTableInfoCacheTest.java | 276 +++++++++++++++++ .../KafkaCompiledWritePlanCacheTest.java | 278 ++++++++++++++++++ .../fluss/rpc/netty/server/NettyServer.java | 62 ++-- .../rpc/protocol/NetworkProtocolPlugin.java | 6 + .../rpc/netty/server/NettyServerTest.java | 114 +++++++ 17 files changed, 1809 insertions(+), 58 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 2d0fa9220e3..07bdbbfe86b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.backend.produce.KafkaProduceConversionExecutor; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.rpc.netty.server.RequestHandler; @@ -26,11 +27,16 @@ import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkState; /** The Kafka protocol plugin. */ public class KafkaProtocolPlugin implements NetworkProtocolPlugin { private Configuration conf; + private KafkaProduceConversionExecutor conversionExecutor; + private CompletableFuture closeFuture; @Override public String name() { @@ -39,7 +45,7 @@ public String name() { @Override public void setup(Configuration conf) { - this.conf = conf; + this.conf = new Configuration(conf); } @Override @@ -59,7 +65,35 @@ public ChannelHandler createChannelHandler( } @Override - public RequestHandler createRequestHandler(RpcGatewayService service) { - return new KafkaRequestHandler(conf, service); + public synchronized RequestHandler createRequestHandler(RpcGatewayService service) { + checkState(closeFuture == null, "Kafka protocol plugin has already been closed."); + if (conversionExecutor == null) { + // Mirror the RPC worker concurrency, with a separate queue capped at 1024 requests. + int queueCapacity = + Math.max( + 1, + Math.min( + 1024, + conf.get(ConfigOptions.NETTY_SERVER_MAX_QUEUED_REQUESTS))); + int threads = + Math.max( + 1, + Math.min( + queueCapacity, + conf.get(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS))); + conversionExecutor = new KafkaProduceConversionExecutor(threads, queueCapacity); + } + return new KafkaRequestHandler(conf, service, conversionExecutor); + } + + @Override + public synchronized CompletableFuture closeAsync() { + if (closeFuture == null) { + closeFuture = + conversionExecutor == null + ? CompletableFuture.completedFuture(null) + : conversionExecutor.closeAsync(); + } + return closeFuture; } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 9e985dbea7f..2b48e91bbed 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -23,6 +23,7 @@ import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; import org.apache.fluss.kafka.backend.produce.GatewayKafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceConversionExecutor; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; @@ -33,6 +34,8 @@ import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; +import javax.annotation.Nullable; + import java.util.Arrays; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -45,6 +48,14 @@ public class KafkaRequestHandler implements RequestHandler { /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { + this(conf, service, null); + } + + /** Creates a handler with a plugin-owned executor for blocking Produce work. */ + public KafkaRequestHandler( + Configuration conf, + RpcGatewayService service, + @Nullable KafkaProduceConversionExecutor conversionExecutor) { KafkaTopicMapper topicMapper = new KafkaTopicMapper(checkNotNull(conf)); this.registry = new KafkaApiRegistry( @@ -57,7 +68,8 @@ public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { new GatewayKafkaProduceBackend( service, (TabletServerGateway) service, - new ArrowKafkaRecordTranscoder())))); + new ArrowKafkaRecordTranscoder(), + conversionExecutor)))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index dd07df63733..2b3755408b6 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -24,9 +24,9 @@ import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; -import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; @@ -44,12 +44,19 @@ import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.protocol.Errors; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -60,21 +67,48 @@ public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { private final RpcGatewayService service; private final TabletServerGateway gateway; private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); - private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); private final KafkaRecordTranscoder transcoder; + private final KafkaTableInfoCache tableInfoCache = new KafkaTableInfoCache(); + private final @Nullable KafkaProduceConversionExecutor conversionExecutor; /** Creates a Produce backend backed by the local TabletServer gateway. */ public GatewayKafkaProduceBackend( RpcGatewayService service, TabletServerGateway gateway, KafkaRecordTranscoder transcoder) { + this(service, gateway, transcoder, null); + } + + /** Creates a backend that submits work to a caller-owned bounded conversion executor. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + KafkaRecordTranscoder transcoder, + @Nullable KafkaProduceConversionExecutor conversionExecutor) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); this.transcoder = checkNotNull(transcoder); + this.conversionExecutor = conversionExecutor; } @Override public CompletableFuture write(KafkaProduceCommand command) { + if (conversionExecutor != null) { + return conversionExecutor + .submit(Thread.currentThread().getId(), () -> writeTopics(command)) + .exceptionally( + failure -> { + List results = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + results.add(failedTopic(topic, failure)); + } + return new KafkaProduceResult(results); + }); + } + return writeTopics(command); + } + + private CompletableFuture writeTopics(KafkaProduceCommand command) { List> futures = new ArrayList<>(); for (TopicWrite topic : command.topics()) { futures.add(writeTopic(command, topic)); @@ -94,16 +128,27 @@ public CompletableFuture write(KafkaProduceCommand command) private CompletableFuture writeTopic( KafkaProduceCommand command, TopicWrite topic) { try { + checkInterrupted(); TablePath path = topicMapper.toTablePath(topic.topicName()); setCurrentSession(command); GetTableInfoRequest request = new GetTableInfoRequest(); request.setTablePath() .setDatabaseName(path.getDatabaseName()) .setTableName(path.getTableName()); - return gateway.getTableInfo(request) - .thenCompose( + CompletableFuture metadata = gateway.getTableInfo(request); + if (conversionExecutor != null) { + // The local gateway is synchronous. Waiting here also preserves FIFO append + // order if an alternative gateway returns asynchronously; only this worker waits. + GetTableInfoResponse response = + metadata.get(Math.max(0, command.timeoutMs()), TimeUnit.MILLISECONDS); + return produceTopic(command, topic, toTableInfo(path, response)); + } + return metadata.thenCompose( response -> produceTopic(command, topic, toTableInfo(path, response))) .exceptionally(failure -> failedTopic(topic, failure)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + return CompletableFuture.completedFuture(failedTopic(topic, failure)); } catch (Exception failure) { return CompletableFuture.completedFuture(failedTopic(topic, failure)); } @@ -112,7 +157,7 @@ private CompletableFuture writeTopic( private CompletableFuture produceTopic( KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { // Admission must match Metadata, including when the schema changes between requests. - schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaTopicWritePlan writePlan = transcoder.prepare(tableInfo); ProduceLogRequest request = new ProduceLogRequest() .setTableId(tableInfo.getTableId()) @@ -132,7 +177,7 @@ private CompletableFuture produceTopic( continue; } try { - BytesView records = transcoder.transcode(partition.records(), tableInfo); + BytesView records = transcoder.transcode(partition.records(), writePlan); request.addBucketsReq() .setBucketId(partition.partitionId()) .setRecordsBytesView(records); @@ -146,6 +191,7 @@ private CompletableFuture produceTopic( toTopicResult(topic, new ProduceLogResponse(), failures)); } try { + checkInterrupted(); setCurrentSession(command); CompletableFuture appended; try { @@ -175,7 +221,11 @@ private static TopicResult failedAppend( return new TopicResult(topic.topicName(), results); } - private static TableInfo toTableInfo(TablePath path, GetTableInfoResponse response) { + private TableInfo toTableInfo(TablePath path, GetTableInfoResponse response) { + return tableInfoCache.getOrLoad(path, response, () -> parseTableInfo(path, response)); + } + + private static TableInfo parseTableInfo(TablePath path, GetTableInfoResponse response) { return TableInfo.of( path, response.getTableId(), @@ -183,7 +233,8 @@ private static TableInfo toTableInfo(TablePath path, GetTableInfoResponse respon TableDescriptor.fromJsonBytes(response.getTableJson()), response.hasRemoteDataDir() ? response.getRemoteDataDir() : null, response.getCreatedTime(), - response.getModifiedTime()); + response.getModifiedTime(), + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L); } private static TopicResult toTopicResult( @@ -239,15 +290,21 @@ private static TopicResult failedTopic(TopicWrite topic, Throwable failure) { private static PartitionResult failedPartition(int partitionId, Throwable failure) { Throwable cause = unwrap(failure); Errors kafkaError = - cause instanceof KafkaTopicSchemaException || cause instanceof InvalidTopicException - ? Errors.INVALID_TOPIC_EXCEPTION - : cause instanceof KafkaRecordEncodingException - ? Errors.CORRUPT_MESSAGE - : cause instanceof IllegalArgumentException - ? Errors.INVALID_REQUEST - : toKafkaError( - org.apache.fluss.rpc.protocol.Errors.forException( - cause)); + cause instanceof RejectedExecutionException + || cause instanceof TimeoutException + || cause instanceof InterruptedException + || cause instanceof CancellationException + ? Errors.REQUEST_TIMED_OUT + : cause instanceof KafkaTopicSchemaException + || cause instanceof InvalidTopicException + ? Errors.INVALID_TOPIC_EXCEPTION + : cause instanceof KafkaRecordEncodingException + ? Errors.CORRUPT_MESSAGE + : cause instanceof IllegalArgumentException + ? Errors.INVALID_REQUEST + : toKafkaError( + org.apache.fluss.rpc.protocol.Errors + .forException(cause)); return new PartitionResult(partitionId, kafkaError, -1L, cause.getMessage()); } @@ -298,9 +355,16 @@ private void setCurrentSession(KafkaProduceCommand command) { private static Throwable unwrap(Throwable failure) { Throwable current = failure; - while (current instanceof CompletionException && current.getCause() != null) { + while ((current instanceof CompletionException || current instanceof ExecutionException) + && current.getCause() != null) { current = current.getCause(); } return current; } + + private void checkInterrupted() { + if (conversionExecutor != null && Thread.currentThread().isInterrupted()) { + throw new CancellationException("Kafka conversion worker was interrupted."); + } + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java new file mode 100644 index 00000000000..e3953f001e0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; + +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Dedicated, bounded FIFO lanes for Kafka metadata lookup, conversion, and native submission. + * + *

The caller uses its RPC processor identity as the ordering key. The server routes each + * connection to one processor, so its requests remain ordered when moved off that processor. An + * operation must invoke native append before returning, but need not wait for acknowledgements. + */ +@Internal +public final class KafkaProduceConversionExecutor { + private final ThreadPoolExecutor[] workers; + private volatile boolean closed; + private CompletableFuture closeFuture; + + /** Creates fixed single-worker lanes whose combined queue capacity is bounded. */ + public KafkaProduceConversionExecutor(int threads, int queueCapacity) { + checkArgument(threads > 0, "Conversion thread count must be positive."); + checkArgument( + queueCapacity >= threads, "Queue capacity must be at least the thread count."); + workers = new ThreadPoolExecutor[threads]; + for (int i = 0; i < threads; i++) { + int capacity = queueCapacity / threads + (i < queueCapacity % threads ? 1 : 0); + workers[i] = + new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(capacity), + new ExecutorThreadFactory("kafka-produce-conversion-" + i), + new ThreadPoolExecutor.AbortPolicy()); + } + } + + /** Submits ordered work without ever running rejected work on the caller's thread. */ + public CompletableFuture submit( + long orderingKey, Supplier> operation) { + ConversionTask task = new ConversionTask<>(checkNotNull(operation)); + try { + if (closed) { + throw new RejectedExecutionException("Kafka conversion executor is closed."); + } + workers[(int) Math.floorMod(orderingKey, (long) workers.length)].execute(task); + } catch (RejectedExecutionException failure) { + task.cancel(failure); + } + return task.result; + } + + /** + * Stops new work, rejects queued requests, interrupts workers, and awaits their termination. + */ + public synchronized CompletableFuture closeAsync() { + if (closeFuture != null) { + return closeFuture; + } + closed = true; + RejectedExecutionException failure = + new RejectedExecutionException("Kafka conversion executor is shutting down."); + for (ThreadPoolExecutor worker : workers) { + List queued = worker.shutdownNow(); + for (Runnable task : queued) { + ((ConversionTask) task).cancel(failure); + } + } + closeFuture = + CompletableFuture.runAsync( + () -> { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + try { + for (ThreadPoolExecutor worker : workers) { + if (!worker.awaitTermination( + Math.max(0L, deadline - System.nanoTime()), + TimeUnit.NANOSECONDS)) { + throw new IllegalStateException( + "Kafka conversion worker did not terminate."); + } + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted closing Kafka conversion workers.", + interrupted); + } + }); + return closeFuture; + } + + private static final class ConversionTask implements Runnable { + private final AtomicReference>> operation; + private final CompletableFuture result = new CompletableFuture<>(); + private final AtomicBoolean claimed = new AtomicBoolean(); + + private ConversionTask(Supplier> operation) { + this.operation = new AtomicReference<>(operation); + } + + @Override + public void run() { + if (!claimed.compareAndSet(false, true)) { + return; + } + Supplier> action = operation.getAndSet(null); + if (result.isDone()) { + return; + } + try { + checkNotNull(action.get(), "conversion result") + .whenComplete( + (value, failure) -> { + if (failure == null) { + result.complete(value); + } else { + result.completeExceptionally(failure); + } + }); + } catch (Throwable failure) { + result.completeExceptionally(failure); + } + } + + private void cancel(Throwable failure) { + if (claimed.compareAndSet(false, true)) { + operation.set(null); + result.completeExceptionally(failure); + } + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java new file mode 100644 index 00000000000..1d53a5eed8a --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded cache for parsed Kafka topic {@link TableInfo} snapshots. + * + *

The cache never replaces the authoritative {@code GetTableInfo} call. Every Produce request + * still obtains a fresh response (and therefore retains its authorization and freshness semantics). + * This cache only avoids parsing an unchanged descriptor JSON response again. + * + *

An entry is reused only when all response identity fields and the complete descriptor bytes + * match. Comparing the complete descriptor is intentional: a {@code kafka.*} property change may + * alter decoding semantics without changing the schema ID. Cache slots serialize parsing for one + * table, while independent topics remain concurrent. + */ +@Internal +@ThreadSafe +final class KafkaTableInfoCache { + + static final long DEFAULT_MAXIMUM_SIZE = 1_000L; + + private final long maximumSize; + private final ConcurrentMap slots; + private final Object slotCreationLock = new Object(); + + KafkaTableInfoCache() { + this(DEFAULT_MAXIMUM_SIZE); + } + + KafkaTableInfoCache(long maximumSize) { + checkArgument(maximumSize > 0, "Kafka TableInfo cache size must be greater than zero."); + this.maximumSize = maximumSize; + this.slots = new ConcurrentHashMap<>(); + } + + TableInfo getOrLoad( + TablePath tablePath, + GetTableInfoResponse response, + Supplier tableInfoLoader) { + checkNotNull(tablePath); + checkNotNull(response); + checkNotNull(tableInfoLoader); + + while (true) { + CacheSlot slot = getOrCreateSlot(tablePath); + synchronized (slot) { + if (slots.get(tablePath) != slot) { + continue; + } + if (slot.entry != null && slot.entry.matches(response)) { + return slot.entry.tableInfo; + } + TableInfo tableInfo = checkNotNull(tableInfoLoader.get()); + slot.entry = new CacheEntry(response, tableInfo); + return tableInfo; + } + } + } + + void invalidate(TablePath tablePath) { + CacheSlot slot = slots.get(checkNotNull(tablePath)); + if (slot == null) { + return; + } + synchronized (slot) { + slot.entry = null; + slots.remove(tablePath, slot); + } + } + + long estimatedSize() { + return slots.size(); + } + + private CacheSlot getOrCreateSlot(TablePath tablePath) { + CacheSlot slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + synchronized (slotCreationLock) { + slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + if (slots.size() >= maximumSize) { + evictOneSlot(); + } + slot = new CacheSlot(); + slots.put(tablePath, slot); + return slot; + } + } + + private void evictOneSlot() { + for (Map.Entry entry : slots.entrySet()) { + CacheSlot slot = entry.getValue(); + synchronized (slot) { + if (slots.remove(entry.getKey(), slot)) { + return; + } + } + } + } + + private static final class CacheSlot { + private CacheEntry entry; + } + + private static final class CacheEntry { + private final long tableId; + private final int schemaId; + private final byte[] tableJson; + private final long createdTime; + private final long modifiedTime; + private final long bucketCountEpoch; + private final String remoteDataDir; + private final TableInfo tableInfo; + + private CacheEntry(GetTableInfoResponse response, TableInfo tableInfo) { + this.tableId = response.getTableId(); + this.schemaId = response.getSchemaId(); + this.tableJson = response.getTableJson().clone(); + this.createdTime = response.getCreatedTime(); + this.modifiedTime = response.getModifiedTime(); + this.bucketCountEpoch = + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L; + this.remoteDataDir = response.hasRemoteDataDir() ? response.getRemoteDataDir() : null; + this.tableInfo = tableInfo; + } + + private boolean matches(GetTableInfoResponse response) { + return tableId == response.getTableId() + && schemaId == response.getSchemaId() + && createdTime == response.getCreatedTime() + && modifiedTime == response.getModifiedTime() + && bucketCountEpoch + == (response.hasBucketCountEpoch() + ? response.getBucketCountEpoch() + : 0L) + && Objects.equals( + remoteDataDir, + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null) + && Arrays.equals(tableJson, response.getTableJson()); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index f441eb96c6b..e24a38ea652 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -41,19 +41,35 @@ public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); + private final KafkaCompiledWritePlanCache writePlanCache = new KafkaCompiledWritePlanCache(); + + @Override + public KafkaTopicWritePlan prepare(TableInfo tableInfo) { + return writePlanCache.getOrCompile( + tableInfo, + () -> + new KafkaTopicWritePlan( + tableInfo, schemaResolver.resolve(tableInfo.toTableDescriptor()))); + } @Override public BytesView transcode(int partitionId, List records, TableInfo tableInfo) throws Exception { + return transcode(partitionId, records, prepare(tableInfo)); + } + + @Override + public BytesView transcode( + int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); - KafkaTableSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaTableSchema schema = plan.topicSchema(); if (schema.valueConversionType() != KafkaValueConversionType.BINARY && schema.valueConversionType() != KafkaValueConversionType.STRING) { throw new KafkaRecordEncodingException( "Unsupported Kafka value conversion type: " + schema.valueConversionType()); } - KafkaRowAssembler assembler = new KafkaRowAssembler(schema); + KafkaRowAssembler assembler = plan.rowAssembler(); return arrowRecordEncoder.encodeStreaming( consumer -> { for (Record record : records) { @@ -68,7 +84,7 @@ public BytesView transcode(int partitionId, List records, TableInfo tabl partitionId)); } }, - tableInfo); + plan.tableInfo()); } private static @Nullable Object decode( diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java new file mode 100644 index 00000000000..258e8c557b8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded, single-flight cache for immutable Kafka topic write plans. + * + *

The bounded map is keyed by {@link TablePath}, while each slot fences its plan with the exact + * {@link TableInfo} instance from the authoritative metadata cache. This avoids the recursive and + * allocation-heavy {@link TableInfo#hashCode()} on every Produce request. An equivalent but + * separately constructed {@code TableInfo} is deliberately treated as a cache miss. In the + * production path, {@code KafkaTableInfoCache} returns the same instance only when the complete + * {@code GetTableInfo} response is unchanged. + * + *

Consequently a delete/recreate, schema change, or property-only Kafka contract change receives + * a different instance and cannot reuse an older plan. Slots are synchronized individually so a + * concurrent cold start compiles once per table version. + */ +@Internal +@ThreadSafe +final class KafkaCompiledWritePlanCache { + + static final long DEFAULT_MAXIMUM_SIZE = 1_000L; + + private final long maximumSize; + private final ConcurrentMap slots; + private final Object slotCreationLock = new Object(); + + KafkaCompiledWritePlanCache() { + this(DEFAULT_MAXIMUM_SIZE); + } + + KafkaCompiledWritePlanCache(long maximumSize) { + checkArgument( + maximumSize > 0, "Kafka compiled write plan cache size must be greater than zero."); + this.maximumSize = maximumSize; + this.slots = new ConcurrentHashMap<>(); + } + + KafkaTopicWritePlan getIfPresent(TableInfo tableInfo) { + checkNotNull(tableInfo); + CacheSlot slot = slots.get(tableInfo.getTablePath()); + if (slot == null) { + return null; + } + synchronized (slot) { + if (slots.get(tableInfo.getTablePath()) != slot) { + return null; + } + return slot.tableInfo == tableInfo ? slot.plan : null; + } + } + + KafkaTopicWritePlan getOrCompile( + TableInfo tableInfo, Supplier planCompiler) { + checkNotNull(tableInfo); + checkNotNull(planCompiler); + + TablePath tablePath = tableInfo.getTablePath(); + while (true) { + CacheSlot slot = getOrCreateSlot(tablePath); + synchronized (slot) { + if (slots.get(tablePath) != slot) { + continue; + } + if (slot.tableInfo == tableInfo && slot.plan != null) { + return slot.plan; + } + KafkaTopicWritePlan plan = checkNotNull(planCompiler.get()); + slot.tableInfo = tableInfo; + slot.plan = plan; + return plan; + } + } + } + + void invalidate(TableInfo tableInfo) { + checkNotNull(tableInfo); + TablePath tablePath = tableInfo.getTablePath(); + CacheSlot slot = slots.get(tablePath); + if (slot == null) { + return; + } + synchronized (slot) { + if (slot.tableInfo != tableInfo) { + return; + } + slot.tableInfo = null; + slot.plan = null; + slots.remove(tablePath, slot); + } + } + + long estimatedSize() { + return slots.size(); + } + + private CacheSlot getOrCreateSlot(TablePath tablePath) { + CacheSlot slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + synchronized (slotCreationLock) { + slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + if (slots.size() >= maximumSize) { + evictOneSlot(); + } + slot = new CacheSlot(); + slots.put(tablePath, slot); + return slot; + } + } + + private void evictOneSlot() { + for (Map.Entry entry : slots.entrySet()) { + CacheSlot slot = entry.getValue(); + synchronized (slot) { + if (slots.remove(entry.getKey(), slot)) { + return; + } + } + } + } + + private static final class CacheSlot { + private TableInfo tableInfo; + private KafkaTopicWritePlan plan; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java index 9bcc465b2a2..063728b77fa 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -19,6 +19,7 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; @@ -27,10 +28,17 @@ /** Converts copied Kafka records into the native Fluss log representation. */ @Internal public interface KafkaRecordTranscoder { - /** - * Transcodes records from the requested Kafka partition according to the target Fluss table - * schema and log format. - */ - BytesView transcode(int partitionId, List records, TableInfo tableInfo) - throws Exception; + /** Validates the table contract and prepares a shareable plan once per topic write. */ + default KafkaTopicWritePlan prepare(TableInfo tableInfo) { + return new KafkaTopicWritePlan( + tableInfo, new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor())); + } + + /** Transcodes one requested partition using an already prepared plan. */ + default BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { + return transcode(partitionId, records, plan.tableInfo()); + } + + /** Transcodes records from the requested Kafka partition using the target table schema. */ + BytesView transcode(int partitionId, List records, TableInfo tableInfo) throws Exception; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java new file mode 100644 index 00000000000..1f9d744afb5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.TableInfo; + +import javax.annotation.concurrent.ThreadSafe; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Immutable binary/string conversion plan; mutable Arrow resources are owned by each conversion. */ +@Internal +@ThreadSafe +public final class KafkaTopicWritePlan { + private final TableInfo tableInfo; + private final KafkaTableSchema topicSchema; + private final KafkaRowAssembler rowAssembler; + + KafkaTopicWritePlan(TableInfo tableInfo, KafkaTableSchema topicSchema) { + this.tableInfo = checkNotNull(tableInfo); + this.topicSchema = checkNotNull(topicSchema); + this.rowAssembler = new KafkaRowAssembler(topicSchema); + } + + /** Returns the authoritative table snapshot used to compile this plan. */ + public TableInfo tableInfo() { + return tableInfo; + } + + KafkaTableSchema topicSchema() { + return topicSchema; + } + + KafkaRowAssembler rowAssembler() { + return rowAssembler; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 8f6fab3a2e5..fbc35bc0bf8 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -49,10 +49,13 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -156,6 +159,51 @@ void testSameNamedTablesAcrossDatabases() throws Exception { } } + @Test + void testPipelinedProducePreservesAppendOrder() throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_pipelined"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("raw"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + assertThat(producer.partitionsFor(path.toString())).hasSize(1); + List values = new ArrayList<>(); + List> writes = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + byte[] value = new byte[i % 2 == 0 ? 16384 : 1]; + Arrays.fill(value, (byte) i); + values.add(value); + writes.add( + producer.send( + new ProducerRecord<>( + path.toString(), 0, (long) i, KEY, value))); + } + for (int i = 0; i < writes.size(); i++) { + assertThat(writes.get(i).get(30, TimeUnit.SECONDS).offset()).isEqualTo(i); + } + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < values.size() && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.getRow().getBytes(0)).isEqualTo(values.get(count)); + assertThat(record.getRow().getTimestampLtz(1, 3).getEpochMillisecond()) + .isEqualTo(count); + count++; + } + } + assertThat(count).isEqualTo(values.size()); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + private static void assertReadback( Connection connection, TablePath path, String format, byte[] expectedValue) throws Exception { @@ -216,6 +264,10 @@ private static TableDescriptor descriptor(String format) { } private static KafkaProducer producer(String acks) { + return producer(acks, 16384); + } + + private static KafkaProducer producer(String acks, int batchSize) { ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); Map config = new HashMap<>(); config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); @@ -223,6 +275,7 @@ private static KafkaProducer producer(String acks) { config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10000); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index d556d910db7..fb4f999d6e3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -24,10 +24,14 @@ import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.messages.GetTableInfoRequest; import org.apache.fluss.rpc.messages.GetTableInfoResponse; @@ -45,6 +49,8 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -253,6 +259,131 @@ void testInvalidQualifiedNamesDoNotReachGatewayOrSuppressValidTopic() throws Exc assertThat(service.appends).hasSize(1); } + @Test + void testReusesPlansButAlwaysRefreshesMetadataAndValidatesChangedContract() throws Exception { + TestingProduceService service = new TestingProduceService(); + ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); + List prepared = new ArrayList<>(); + List converted = new ArrayList<>(); + KafkaRecordTranscoder transcoder = + new KafkaRecordTranscoder() { + @Override + public KafkaTopicWritePlan prepare(TableInfo tableInfo) { + KafkaTopicWritePlan plan = arrow.prepare(tableInfo); + prepared.add(plan); + return plan; + } + + @Override + public BytesView transcode(List records, TableInfo tableInfo) + throws Exception { + return arrow.transcode(records, tableInfo); + } + + @Override + public BytesView transcode(List records, KafkaTopicWritePlan plan) + throws Exception { + converted.add(plan); + return arrow.transcode(records, plan); + } + }; + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, transcoder); + backend.write(command((short) 1, topic("kafka.topic", good(0), good(1)))).join(); + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(); + assertThat(service.metadataPaths).hasSize(2); + assertThat(prepared).hasSize(2); + assertThat(prepared.get(1)).isSameAs(prepared.get(0)); + assertThat(converted).containsExactly(prepared.get(0), prepared.get(0), prepared.get(0)); + + service.pendingMetadata = CompletableFuture.completedFuture(metadata(true)); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.metadataPaths).hasSize(3); + assertThat(service.appends).hasSize(2); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingMetadata.completeExceptionally( + new org.apache.fluss.exception.AuthorizationException("revoked")); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.metadataPaths).hasSize(4); + assertThat(service.appends).hasSize(2); + } + + @Test + void testOffloadedRequestsAppendInOrderWithoutWaitingForAcknowledgement() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(2, 4); + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingAppend = new CompletableFuture<>(); + service.metadataEntered = new CountDownLatch(1); + service.appended = new CountDownLatch(2); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, new ArrowKafkaRecordTranscoder(), executor); + try { + CompletableFuture first = + backend.write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.metadataEntered.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture second = + backend.write(command((short) -1, topic("kafka.topic", good(1)))); + assertThat(service.metadataThread).isNotSameAs(Thread.currentThread()); + CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))) + .join(); + assertThat(service.appended.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(service.appends) + .extracting(request -> request.getBucketsReqsList().get(0).getBucketId()) + .containsExactly(0, 1); + assertThat(first).isNotDone(); + assertThat(second).isNotDone(); + service.pendingAppend.complete( + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setBaseOffset(17L), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setBaseOffset(18L)))); + assertErrors(first.get(10, TimeUnit.SECONDS), Errors.NONE); + assertErrors(second.get(10, TimeUnit.SECONDS), Errors.NONE); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + service.metadataEntered = new CountDownLatch(1); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, new ArrowKafkaRecordTranscoder(), executor); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); + try { + CompletableFuture running = backend.write(command); + assertThat(service.metadataEntered.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture queued = backend.write(command); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + executor.closeAsync().get(10, TimeUnit.SECONDS); + assertErrors(running.get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + assertErrors(queued.get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + service.pendingMetadata.complete(metadata(false)); + assertThat(service.appends).isEmpty(); + assertThat(service.metadataPaths).hasSize(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + private static GatewayKafkaProduceBackend backend(TestingProduceService service) { return new GatewayKafkaProduceBackend(service, service, new ArrowKafkaRecordTranscoder()); } @@ -318,6 +449,9 @@ private static class TestingProduceService extends TestingTabletGatewayService { private boolean throwAppend; private boolean asyncMissing; private int drains; + private CountDownLatch metadataEntered; + private CountDownLatch appended; + private Thread metadataThread; @Override public CompletableFuture getTableInfo(GetTableInfoRequest request) { @@ -325,6 +459,10 @@ public CompletableFuture getTableInfo(GetTableInfoRequest String database = request.getTablePath().getDatabaseName(); String name = request.getTablePath().getTableName(); metadataPaths.add(TablePath.of(database, name)); + metadataThread = Thread.currentThread(); + if (metadataEntered != null) { + metadataEntered.countDown(); + } if (name.equals("missing")) { if (!asyncMissing) { throw new TableNotExistException("missing"); @@ -360,6 +498,9 @@ public CompletableFuture produceLog(ProduceLogRequest reques public void tryCompleteActions() { assertThat(append).isNotNull(); drains++; + if (appended != null) { + appended.countDown(); + } } } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java new file mode 100644 index 00000000000..88e95f420b6 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests bounded admission, ordering, independent lanes and executor shutdown. */ +class KafkaProduceConversionExecutorTest { + @Test + void testBoundedLanePreservesOrderAndDoesNotBlockAnotherLane() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(2, 2); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + List order = new ArrayList<>(); + AtomicReference worker = new AtomicReference<>(); + AtomicBoolean rejectedRan = new AtomicBoolean(); + try { + CompletableFuture first = + executor.submit( + 0, + () -> { + worker.set(Thread.currentThread()); + started.countDown(); + await(release); + order.add(1); + return CompletableFuture.completedFuture(1); + }); + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture second = + executor.submit( + 0, + () -> { + order.add(2); + return CompletableFuture.completedFuture(2); + }); + CompletableFuture rejected = + executor.submit( + 0, + () -> { + rejectedRan.set(true); + return CompletableFuture.completedFuture(3); + }); + assertThatThrownBy(rejected::join).hasCauseInstanceOf(RejectedExecutionException.class); + assertThat(rejectedRan).isFalse(); + assertThat(worker.get()).isNotSameAs(Thread.currentThread()); + assertThat(second).isNotDone(); + assertThat( + executor.submit(1, () -> CompletableFuture.completedFuture(4)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(4); + release.countDown(); + assertThat(first.get(10, TimeUnit.SECONDS)).isEqualTo(1); + assertThat(second.get(10, TimeUnit.SECONDS)).isEqualTo(2); + assertThat(order).containsExactly(1, 2); + } finally { + release.countDown(); + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testPendingAcknowledgementDoesNotOccupyWorker() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + CompletableFuture acknowledgement = new CompletableFuture<>(); + CountDownLatch submitted = new CountDownLatch(1); + try { + CompletableFuture first = + executor.submit( + 0, + () -> { + submitted.countDown(); + return acknowledgement; + }); + assertThat(submitted.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat( + executor.submit(0, () -> CompletableFuture.completedFuture(2)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(2); + assertThat(first).isNotDone(); + acknowledgement.complete(1); + assertThat(first.get(10, TimeUnit.SECONDS)).isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testCloseInterruptsWorkerCompletesQueuedRequestsAndRejectsNewWork() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + CountDownLatch started = new CountDownLatch(1); + AtomicBoolean queuedRan = new AtomicBoolean(); + CompletableFuture running = + executor.submit( + 0, + () -> { + started.countDown(); + await(new CountDownLatch(1)); + return CompletableFuture.completedFuture(null); + }); + try { + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture queued = + executor.submit( + 0, + () -> { + queuedRan.set(true); + return CompletableFuture.completedFuture(null); + }); + CompletableFuture closed = executor.closeAsync(); + assertThat(executor.closeAsync()).isSameAs(closed); + closed.get(10, TimeUnit.SECONDS); + assertThatThrownBy(running::join).hasRootCauseInstanceOf(InterruptedException.class); + assertThatThrownBy(queued::join).hasCauseInstanceOf(RejectedExecutionException.class); + assertThat(queuedRan).isFalse(); + assertThatThrownBy( + () -> + executor.submit( + 0, + () -> CompletableFuture.completedFuture(null)) + .join()) + .hasCauseInstanceOf(RejectedExecutionException.class); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testFailedOperationDoesNotStopLane() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + try { + CompletableFuture failed = + executor.submit( + 0, + () -> { + throw new IllegalStateException("conversion failed"); + }); + assertThatThrownBy(failed::join).hasCauseInstanceOf(IllegalStateException.class); + assertThat( + executor.submit(0, () -> CompletableFuture.completedFuture(1)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Test worker was not released."); + } + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(failure); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java new file mode 100644 index 00000000000..73c68ec725d --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class KafkaTableInfoCacheTest { + + private static final TablePath TABLE_PATH = TablePath.of("kafka", "orders"); + + @Test + void testReusesUnchangedResponseAndReloadsPropertyOnlyChange() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loadCount = new AtomicInteger(); + GetTableInfoResponse firstResponse = response(1L, 1, descriptor("v1")); + + TableInfo first = + cache.getOrLoad( + TABLE_PATH, + firstResponse, + () -> load(TABLE_PATH, firstResponse, loadCount)); + GetTableInfoResponse equivalentResponse = response(1L, 1, descriptor("v1")); + TableInfo reused = + cache.getOrLoad( + TABLE_PATH, + equivalentResponse, + () -> load(TABLE_PATH, equivalentResponse, loadCount)); + + assertThat(reused).isSameAs(first); + assertThat(loadCount).hasValue(1); + + // A Kafka contract property may change without a schema ID or timestamp change. + GetTableInfoResponse changedResponse = response(1L, 1, descriptor("v2")); + TableInfo changed = + cache.getOrLoad( + TABLE_PATH, + changedResponse, + () -> load(TABLE_PATH, changedResponse, loadCount)); + + assertThat(changed).isNotSameAs(first); + assertThat(loadCount).hasValue(2); + } + + @Test + void testReloadsDeleteAndRecreateAndSupportsExplicitInvalidation() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loadCount = new AtomicInteger(); + GetTableInfoResponse original = response(1L, 1, descriptor("v1")); + TableInfo first = + cache.getOrLoad(TABLE_PATH, original, () -> load(TABLE_PATH, original, loadCount)); + + GetTableInfoResponse recreated = response(2L, 1, descriptor("v1")); + TableInfo second = + cache.getOrLoad( + TABLE_PATH, recreated, () -> load(TABLE_PATH, recreated, loadCount)); + assertThat(second).isNotSameAs(first); + + cache.invalidate(TABLE_PATH); + TableInfo third = + cache.getOrLoad( + TABLE_PATH, recreated, () -> load(TABLE_PATH, recreated, loadCount)); + assertThat(third).isNotSameAs(second); + assertThat(loadCount).hasValue(3); + } + + @Test + void testReloadsWhenOnlyBucketCountEpochChanges() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loads = new AtomicInteger(); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + TableInfo first = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, loads)); + response.setBucketCountEpoch(7L); + TableInfo changed = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, loads)); + assertThat(changed).isNotSameAs(first); + assertThat(changed.getBucketCountEpoch()).isEqualTo(7L); + assertThat(loads).hasValue(2); + } + + @Test + void testLoadFailureIsNotCached() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger attempts = new AtomicInteger(); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + + assertThatThrownBy( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + attempts.incrementAndGet(); + throw new IllegalStateException("broken descriptor"); + })) + .isInstanceOf(IllegalStateException.class) + .hasMessage("broken descriptor"); + + TableInfo loaded = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, attempts)); + assertThat(loaded.getTableId()).isEqualTo(1L); + assertThat(attempts).hasValue(2); + } + + @Test + void testConcurrentColdLoadIsSingleFlight() throws Exception { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + futures.add( + executor.submit( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + loadCount.incrementAndGet(); + loaderStarted.countDown(); + await(releaseLoader); + return toTableInfo(TABLE_PATH, response); + }))); + } + loaderStarted.await(); + releaseLoader.countDown(); + + TableInfo first = futures.get(0).get(); + for (Future future : futures) { + assertThat(future.get()).isSameAs(first); + } + assertThat(loadCount).hasValue(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testInvalidationDoesNotAllowInFlightLoadToResurrect() throws Exception { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future loading = + executor.submit( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + loadCount.incrementAndGet(); + loaderStarted.countDown(); + await(releaseLoader); + return toTableInfo(TABLE_PATH, response); + })); + loaderStarted.await(); + Future invalidating = executor.submit(() -> cache.invalidate(TABLE_PATH)); + releaseLoader.countDown(); + + TableInfo loadedBeforeInvalidation = loading.get(); + invalidating.get(); + TableInfo loadedAfterInvalidation = + cache.getOrLoad( + TABLE_PATH, response, () -> load(TABLE_PATH, response, loadCount)); + + assertThat(loadedAfterInvalidation).isNotSameAs(loadedBeforeInvalidation); + assertThat(loadCount).hasValue(2); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCacheSizeIsBounded() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(2); + for (int i = 0; i < 10; i++) { + TablePath path = TablePath.of("kafka", "topic_" + i); + GetTableInfoResponse response = response(i + 1L, 1, descriptor("v1")); + cache.getOrLoad(path, response, () -> toTableInfo(path, response)); + } + + assertThat(cache.estimatedSize()).isLessThanOrEqualTo(2); + } + + private static TableInfo load( + TablePath tablePath, GetTableInfoResponse response, AtomicInteger loadCount) { + loadCount.incrementAndGet(); + return toTableInfo(tablePath, response); + } + + private static TableInfo toTableInfo(TablePath tablePath, GetTableInfoResponse response) { + return TableInfo.of( + tablePath, + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + null, + response.getCreatedTime(), + response.getModifiedTime(), + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L); + } + + private static GetTableInfoResponse response( + long tableId, int schemaId, TableDescriptor descriptor) { + return new GetTableInfoResponse() + .setTableId(tableId) + .setSchemaId(schemaId) + .setTableJson(descriptor.toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L); + } + + private static TableDescriptor descriptor(String version) { + TableDescriptor.Builder builder = + TableDescriptor.builder() + .schema(Schema.newBuilder().column("value", DataTypes.BYTES()).build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .customProperty("kafka.value.format", "raw"); + if ("v2".equals(version)) { + builder.customProperty("application.version", "changed"); + } + return builder.build(); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for test loader.", e); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java new file mode 100644 index 00000000000..e3026f40685 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class KafkaCompiledWritePlanCacheTest { + + @Test + void testReusesSameIdentityAndSeparatesEquivalentAndChangedInstances() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo original = tableInfo("orders", 1L, 1, "v1"); + + KafkaTopicWritePlan first = + cache.getOrCompile(original, () -> compile(original, compileCount)); + KafkaTopicWritePlan reused = + cache.getOrCompile(original, () -> compile(original, compileCount)); + assertThat(reused).isSameAs(first); + assertThat(cache.getIfPresent(original)).isSameAs(first); + + // The authoritative TableInfo cache owns canonical instances. A separately constructed, + // structurally equivalent object is conservatively treated as a new metadata snapshot. + TableInfo equivalent = tableInfo("orders", 1L, 1, "v1"); + assertThat(cache.getIfPresent(equivalent)).isNull(); + KafkaTopicWritePlan equivalentPlan = + cache.getOrCompile(equivalent, () -> compile(equivalent, compileCount)); + assertThat(equivalentPlan).isNotSameAs(first); + + TableInfo propertyChange = tableInfo("orders", 1L, 1, "v2"); + KafkaTopicWritePlan propertyPlan = + cache.getOrCompile(propertyChange, () -> compile(propertyChange, compileCount)); + TableInfo schemaChange = tableInfo("orders", 1L, 2, "v2"); + KafkaTopicWritePlan schemaPlan = + cache.getOrCompile(schemaChange, () -> compile(schemaChange, compileCount)); + TableInfo recreated = tableInfo("orders", 2L, 2, "v2"); + KafkaTopicWritePlan recreatedPlan = + cache.getOrCompile(recreated, () -> compile(recreated, compileCount)); + + assertThat(propertyPlan).isNotSameAs(first); + assertThat(schemaPlan).isNotSameAs(propertyPlan); + assertThat(recreatedPlan).isNotSameAs(schemaPlan); + assertThat(compileCount).hasValue(5); + } + + @Test + void testEquivalentInstanceCannotInvalidateCurrentIdentity() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo current = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan currentPlan = + cache.getOrCompile(current, () -> compile(current, compileCount)); + + cache.invalidate(tableInfo("orders", 1L, 1, "v1")); + + assertThat(cache.getOrCompile(current, () -> compile(current, compileCount))) + .isSameAs(currentPlan); + assertThat(compileCount).hasValue(1); + } + + @Test + void testInvalidationAndCompileFailureRecovery() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan first = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + + cache.invalidate(tableInfo); + KafkaTopicWritePlan second = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + assertThat(second).isNotSameAs(first); + + TableInfo failingTable = tableInfo("payments", 2L, 1, "v1"); + assertThatThrownBy( + () -> + cache.getOrCompile( + failingTable, + () -> { + compileCount.incrementAndGet(); + throw new IllegalStateException("compile failed"); + })) + .isInstanceOf(IllegalStateException.class) + .hasMessage("compile failed"); + assertThat(cache.getOrCompile(failingTable, () -> compile(failingTable, compileCount))) + .isNotNull(); + assertThat(compileCount).hasValue(4); + } + + @Test + void testConcurrentColdCompileIsSingleFlight() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + AtomicInteger compileCount = new AtomicInteger(); + CountDownLatch compilerStarted = new CountDownLatch(1); + CountDownLatch releaseCompiler = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + futures.add( + executor.submit( + () -> + cache.getOrCompile( + tableInfo, + () -> { + compileCount.incrementAndGet(); + compilerStarted.countDown(); + await(releaseCompiler); + return compile(tableInfo); + }))); + } + compilerStarted.await(); + releaseCompiler.countDown(); + + KafkaTopicWritePlan first = futures.get(0).get(); + for (Future future : futures) { + assertThat(future.get()).isSameAs(first); + } + assertThat(compileCount).hasValue(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testInvalidationDoesNotAllowInFlightCompileToResurrect() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + AtomicInteger compileCount = new AtomicInteger(); + CountDownLatch compilerStarted = new CountDownLatch(1); + CountDownLatch releaseCompiler = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future compiling = + executor.submit( + () -> + cache.getOrCompile( + tableInfo, + () -> { + compileCount.incrementAndGet(); + compilerStarted.countDown(); + await(releaseCompiler); + return compile(tableInfo); + })); + compilerStarted.await(); + Future invalidating = executor.submit(() -> cache.invalidate(tableInfo)); + releaseCompiler.countDown(); + + KafkaTopicWritePlan planBeforeInvalidation = compiling.get(); + invalidating.get(); + KafkaTopicWritePlan planAfterInvalidation = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + + assertThat(planAfterInvalidation).isNotSameAs(planBeforeInvalidation); + assertThat(compileCount).hasValue(2); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCompiledAssemblerIsSafeToShare() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan plan = cache.getOrCompile(tableInfo, () -> compile(tableInfo)); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + final int id = i; + futures.add( + executor.submit( + () -> + plan.rowAssembler() + .assemble( + new Object[0], + new Object[] { + Integer.toString(id) + .getBytes( + StandardCharsets.UTF_8) + }, + id, + new ArrayList<>()) + .getField(0))); + } + for (int i = 0; i < futures.size(); i++) { + assertThat((byte[]) futures.get(i).get()) + .isEqualTo(Integer.toString(i).getBytes(StandardCharsets.UTF_8)); + } + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCacheSizeIsBounded() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(2); + for (int i = 0; i < 10; i++) { + TableInfo tableInfo = tableInfo("topic_" + i, i + 1L, 1, "v1"); + cache.getOrCompile(tableInfo, () -> compile(tableInfo)); + } + + assertThat(cache.estimatedSize()).isLessThanOrEqualTo(2); + } + + private static KafkaTopicWritePlan compile(TableInfo tableInfo, AtomicInteger compileCount) { + compileCount.incrementAndGet(); + return compile(tableInfo); + } + + private static KafkaTopicWritePlan compile(TableInfo tableInfo) { + KafkaTopicSchema schema = + new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor()); + return new KafkaTopicWritePlan(tableInfo, schema); + } + + private static TableInfo tableInfo( + String tableName, long tableId, int schemaId, String propertyValue) { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(Schema.newBuilder().column("payload", DataTypes.BYTES()).build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty("application.version", propertyValue) + .build(); + return TableInfo.of( + TablePath.of("kafka", tableName), tableId, schemaId, descriptor, null, 1L, 1L); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for test compiler.", e); + } + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java index fa0174cb492..37705f2e075 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java @@ -80,6 +80,7 @@ public final class NettyServer implements RpcServer { private EventLoopGroup selectorGroup; private volatile boolean isRunning; + private CompletableFuture closeFuture; public NettyServer( Configuration conf, @@ -104,7 +105,8 @@ public NettyServer( } @Override - public void start() throws IOException { + public synchronized void start() throws IOException { + checkState(closeFuture == null, "Netty server has already been closed."); checkState(bindChannels.isEmpty(), "Netty server has already been initialized."); int numNetworkThreads = conf.getInt(ConfigOptions.NETTY_SERVER_NUM_NETWORK_THREADS); int numWorkerThreads = conf.getInt(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS); @@ -298,34 +300,40 @@ public ScheduledExecutorService getScheduledExecutor() { @Override public CompletableFuture closeAsync() { - if (!isRunning) { - return CompletableFuture.completedFuture(null); + synchronized (this) { + if (closeFuture != null) { + return closeFuture; + } + isRunning = false; + + CompletableFuture transportShutdownFuture = + CompletableFuture.allOf( + shutdownGroup(acceptorGroup), + shutdownGroup(selectorGroup), + FutureUtils.completeAll( + bindChannels.stream() + .map(NettyUtils::shutdownChannel) + .collect(Collectors.toList())), + workerPool.closeAsync()); + closeFuture = closeProtocolsAfter(transportShutdownFuture, protocols); + return closeFuture; } + } - isRunning = false; - - // Stop accepting new connections. - CompletableFuture channelShutdownFuture = - FutureUtils.completeAll( - bindChannels.stream() - .map(NettyUtils::shutdownChannel) - .collect(Collectors.toList())); - // Drain the requests that were already accepted. - CompletableFuture workerShutdownFuture; - if (workerPool != null) { - workerShutdownFuture = workerPool.closeAsync(); - } else { - workerShutdownFuture = CompletableFuture.completedFuture(null); - } - // Shut down the event loops only after both of the above completed. The event loops - // terminate without a quiet period, so they must outlive the tasks still submitted to - // them: the bind channel close listeners, and the responses (and backpressure updates) - // of the requests drained by the worker pool. Shutting the event loops down closes the - // remaining client connections. + static CompletableFuture closeProtocolsAfter( + CompletableFuture shutdownFuture, List protocols) { return FutureUtils.composeAfterwards( - CompletableFuture.allOf(channelShutdownFuture, workerShutdownFuture), - () -> - CompletableFuture.allOf( - shutdownGroup(acceptorGroup), shutdownGroup(selectorGroup))); + shutdownFuture, + () -> { + List> closeFutures = new ArrayList<>(); + for (NetworkProtocolPlugin protocol : protocols) { + try { + closeFutures.add(protocol.closeAsync()); + } catch (Throwable failure) { + closeFutures.add(FutureUtils.completedExceptionally(failure)); + } + } + return FutureUtils.completeAll(closeFutures); + }); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java index f60202228ef..c8921473fad 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java @@ -25,6 +25,7 @@ import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import java.util.List; +import java.util.concurrent.CompletableFuture; /** A network protocol plugin that provides the server side implementation of a network protocol. */ public interface NetworkProtocolPlugin extends Plugin { @@ -52,4 +53,9 @@ public interface NetworkProtocolPlugin extends Plugin { * protocol. */ RequestHandler createRequestHandler(RpcGatewayService service); + + /** Closes protocol-owned resources after transport and request processors have stopped. */ + default CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } } diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java index 6e442187019..528f9fbcef8 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java @@ -25,6 +25,7 @@ import org.apache.fluss.config.MemorySize; import org.apache.fluss.metrics.groups.MetricGroup; import org.apache.fluss.metrics.util.NOPMetricsGroup; +import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.RpcServer; import org.apache.fluss.rpc.TestingGatewayService; import org.apache.fluss.rpc.messages.ApiMessage; @@ -33,6 +34,8 @@ import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; import org.apache.fluss.rpc.netty.client.NettyClient; import org.apache.fluss.rpc.protocol.ApiKeys; +import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; +import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import org.apache.fluss.utils.NetUtils; import org.junit.jupiter.api.Test; @@ -40,11 +43,14 @@ import java.net.ConnectException; import java.net.Socket; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.apache.fluss.utils.NetUtils.getAvailablePort; @@ -54,6 +60,24 @@ /** Test for {@link NettyServer}. */ public class NettyServerTest { + @Test + void testCannotStartAfterClose() { + MetricGroup metricGroup = NOPMetricsGroup.newInstance(); + NettyServer server = + new NettyServer( + new Configuration(), + Endpoint.fromListenersString("FLUSS://localhost:0"), + new TestingGatewayService(), + metricGroup, + RequestsMetrics.createCoordinatorServerRequestMetrics(metricGroup)); + + server.closeAsync().join(); + + assertThatThrownBy(server::start) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already been closed"); + } + @Test void testPortAsZero() throws Exception { List endpoints = Endpoint.fromListenersString("FLUSS://localhost:0"); @@ -210,4 +234,94 @@ public CompletableFuture apiVersions( } } } + + @Test + void testProtocolsCloseOnlyAfterTransportShutdown() { + CompletableFuture transportShutdown = new CompletableFuture<>(); + AtomicInteger closeCalls = new AtomicInteger(); + NetworkProtocolPlugin protocol = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + return CompletableFuture.completedFuture(null); + }); + + CompletableFuture result = + NettyServer.closeProtocolsAfter( + transportShutdown, Collections.singletonList(protocol)); + assertThat(closeCalls).hasValue(0); + assertThat(result).isNotDone(); + + transportShutdown.complete(null); + result.join(); + assertThat(closeCalls).hasValue(1); + } + + @Test + void testAllProtocolsCloseAfterExceptionalTransportShutdown() { + CompletableFuture transportShutdown = new CompletableFuture<>(); + AtomicInteger closeCalls = new AtomicInteger(); + RuntimeException asyncFailure = new RuntimeException("async protocol close failure"); + RuntimeException synchronousFailure = + new RuntimeException("synchronous protocol close failure"); + NetworkProtocolPlugin asyncFailing = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(asyncFailure); + return failed; + }); + NetworkProtocolPlugin synchronouslyFailing = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + throw synchronousFailure; + }); + CompletableFuture result = + NettyServer.closeProtocolsAfter( + transportShutdown, Arrays.asList(asyncFailing, synchronouslyFailing)); + + transportShutdown.completeExceptionally(new RuntimeException("transport failure")); + + assertThatThrownBy(result::join) + .hasStackTraceContaining("transport failure") + .hasStackTraceContaining("async protocol close failure") + .hasStackTraceContaining("synchronous protocol close failure"); + assertThat(closeCalls).hasValue(2); + } + + private static NetworkProtocolPlugin testingProtocol( + Supplier> closeAction) { + return new NetworkProtocolPlugin() { + @Override + public String name() { + return "testing"; + } + + @Override + public void setup(Configuration conf) {} + + @Override + public List listenerNames() { + return Collections.emptyList(); + } + + @Override + public ChannelHandler createChannelHandler( + RequestChannel[] requestChannels, String listenerName) { + return null; + } + + @Override + public RequestHandler createRequestHandler(RpcGatewayService service) { + return null; + } + + @Override + public CompletableFuture closeAsync() { + return closeAction.get(); + } + }; + } } From e0ed833fd9d007828ae985d4abc6b13c6024c1f5 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:36:18 +0800 Subject: [PATCH 24/69] [kafka] Skip null Produce values with write authorization Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 50/50 AI-Contributed/UT: 99/99 --- .../produce/GatewayKafkaProduceBackend.java | 14 ++- .../backend/produce/KafkaProduceCommand.java | 21 +++++ .../fluss/kafka/KafkaProduceAppendITCase.java | 6 +- .../GatewayKafkaProduceBackendTest.java | 91 +++++++++++++++++++ .../apache/fluss/rpc/RpcGatewayService.java | 8 ++ .../fluss/server/tablet/TabletService.java | 7 +- 6 files changed, 141 insertions(+), 6 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index 2b3755408b6..313da86ad7d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -177,7 +177,15 @@ private CompletableFuture produceTopic( continue; } try { - BytesView records = transcoder.transcode(partition.records(), writePlan); + if (partition.hasNullValues() && partition.nonNullRecords().isEmpty()) { + setCurrentSession(command); + service.authorizeTableWrite(tableInfo.getTableId()); + failures.put( + partition.partitionId(), + new PartitionResult(partition.partitionId(), Errors.NONE, -1L, null)); + continue; + } + BytesView records = transcoder.transcode(partition.nonNullRecords(), writePlan); request.addBucketsReq() .setBucketId(partition.partitionId()) .setRecordsBytesView(records); @@ -272,7 +280,9 @@ private static TopicResult toTopicResult( new PartitionResult( partition.partitionId(), Errors.NONE, - bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, + !partition.hasNullValues() && bucket.hasBaseOffset() + ? bucket.getBaseOffset() + : -1L, null)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index db13f63b314..c359c6f8b21 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -109,11 +109,22 @@ public List partitions() { public static final class PartitionWrite { private final int partitionId; private final List records; + private final List nonNullRecords; /** Creates the writes for one partition. */ public PartitionWrite(int partitionId, List records) { this.partitionId = partitionId; this.records = immutableCopy(records); + List retained = new ArrayList<>(); + for (Record record : this.records) { + if (record.borrowedValue() != null) { + retained.add(record); + } + } + this.nonNullRecords = + retained.size() == records.size() + ? this.records + : Collections.unmodifiableList(retained); } /** Returns the Kafka partition ID. */ @@ -121,6 +132,16 @@ public int partitionId() { return partitionId; } + /** Returns records with non-null values, preserving their original order. */ + public List nonNullRecords() { + return nonNullRecords; + } + + /** Whether filtering changed the correspondence between Kafka and native offsets. */ + public boolean hasNullValues() { + return records.size() != nonNullRecords.size(); + } + /** Returns copied records in append order. */ public List records() { return records; diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index fbc35bc0bf8..2fb0fa4c01f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -191,7 +191,7 @@ void testPipelinedProducePreservesAppendOrder() throws Exception { while (count < values.size() && System.nanoTime() < deadline) { for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { assertThat(record.getRow().getBytes(0)).isEqualTo(values.get(count)); - assertThat(record.getRow().getTimestampLtz(1, 3).getEpochMillisecond()) + assertThat(record.getRow().getTimestampNtz(1, 3).getMillisecond()) .isEqualTo(count); count++; } @@ -224,7 +224,7 @@ private static void assertReadback( .isEqualTo(new String(expectedValue, StandardCharsets.UTF_8)); assertThat(row.getString(3).toString()).isEqualTo("key"); } - assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(123L); assertThat(row.getArray(2).size()).isEqualTo(2); assertThat(row.getArray(2).getRow(0, 2).getString(0).toString()) .isEqualTo("source"); @@ -242,7 +242,7 @@ private static TableDescriptor descriptor(String format) { .schema( Schema.newBuilder() .column("body", raw ? DataTypes.BYTES() : DataTypes.STRING()) - .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) .column( "attributes", DataTypes.ARRAY( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index fb4f999d6e3..842a75456b7 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.kafka.backend.produce; +import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; @@ -57,6 +58,84 @@ /** Tests append admission, error isolation, session propagation and delayed-fetch completion. */ class GatewayKafkaProduceBackendTest { + @Test + void testAllNullValuesRequireAuthorizationWithoutAppend() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service) + .write(command((short) 1, topic("kafka.topic", nullValue(0), nullValue(8)))) + .join(); + assertErrors(result, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); + assertThat(service.writeChecks).isEqualTo(1); + assertThat(service.append).isNull(); + assertThat(service.drains).isZero(); + service.denyWrite = true; + assertErrors( + backend(service) + .write(command((short) 1, topic("kafka.topic", nullValue(0)))) + .join(), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.append).isNull(); + } + + @Test + void testNullValuesCannotBypassTableAndSchemaValidation() throws Exception { + TestingProduceService service = new TestingProduceService(); + for (String name : new String[] {"invalid", "missing"}) { + assertErrors( + backend(service) + .write(command((short) 1, topic("kafka." + name, nullValue(0)))) + .join(), + name.equals("invalid") + ? Errors.INVALID_TOPIC_EXCEPTION + : Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + assertThat(service.writeChecks).isZero(); + assertThat(service.append).isNull(); + } + + @Test + void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exception { + TestingProduceService service = new TestingProduceService(); + List seen = new ArrayList<>(); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + (records, table) -> { + seen.addAll(records); + return transcoder.transcode(records, table); + }); + Record empty = new Record(1L, null, new byte[0], Collections.emptyList()); + Record valid = good(0).records().get(0); + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic( + "kafka.topic", + new PartitionWrite( + 0, + Arrays.asList( + nullValue(0).records().get(0), + empty, + valid))))) + .join(); + assertErrors(result, Errors.NONE); + assertThat(seen).containsExactly(empty, valid); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); + assertThat(service.drains).isEqualTo(1); + } + + private static PartitionWrite nullValue(int partition) { + return new PartitionWrite( + partition, + Collections.singletonList( + new Record(-1L, new byte[] {(byte) 0xff}, null, Collections.emptyList()))); + } + @Test void testDrainsImmediatelyWhileAcksAllResponseIsPending() throws Exception { TestingProduceService service = new TestingProduceService(); @@ -449,6 +528,8 @@ private static class TestingProduceService extends TestingTabletGatewayService { private boolean throwAppend; private boolean asyncMissing; private int drains; + private int writeChecks; + private boolean denyWrite; private CountDownLatch metadataEntered; private CountDownLatch appended; private Thread metadataThread; @@ -478,6 +559,16 @@ public CompletableFuture getTableInfo(GetTableInfoRequest : pendingMetadata; } + @Override + public void authorizeTableWrite(long tableId) { + assertThat(tableId).isEqualTo(42L); + assertThat(currentListenerName()).isEqualTo("KAFKA"); + writeChecks++; + if (denyWrite) { + throw new AuthorizationException("Write denied"); + } + } + @Override public CompletableFuture produceLog(ProduceLogRequest request) { assertThat(currentListenerName()).isEqualTo("KAFKA"); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java index 8f2365add70..1dc4a0da4c9 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java @@ -77,6 +77,14 @@ public String currentListenerName() { } } + /** + * Authorizes a table write that may not append any records, using the current request session. + * Services must explicitly support this operation; the default never grants write access. + */ + public void authorizeTableWrite(long tableId) { + throw new UnsupportedOperationException("This service cannot authorize table writes."); + } + /** Returns the provider type of this RPC gateway service. */ public abstract ServerType providerType(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index da73530b13f..a874c516878 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -234,9 +234,14 @@ public String name() { @Override public void shutdown() {} + @Override + public void authorizeTableWrite(long tableId) { + authorizeTable(WRITE, tableId); + } + @Override public CompletableFuture produceLog(ProduceLogRequest request) { - authorizeTable(WRITE, request.getTableId()); + authorizeTableWrite(request.getTableId()); long tableId = request.getTableId(); Map routingErrors = new HashMap<>(); collectRoutingErrors( From 595101108429ba1367903d70495e27341ae183a8 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 18:46:25 +0800 Subject: [PATCH 25/69] [kafka] Verify Produce boundaries on the updated FIP base Cover WRITE revocation after metadata reuse, offloaded metadata identity and schema refresh, and shutdown with an already submitted native append. Preserve Record timestamp provenance and the existing null-value filtering behavior, including native offsets and real Kafka producer readback. The FIP null-value contract and new Core dependencies remain deferred. Validated on PR06 b6f31fc84baf91c030b296701eb319644578056c with Java 11: 206 Kafka unit tests, 10 Kafka integration tests, 8 RPC regression tests and 7 delayed-fetch regression tests passed. Checkstyle, Spotless, RAT and git diff --check passed. This commit changes tests only. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 191/191 --- .../fluss/kafka/KafkaProduceAppendITCase.java | 36 ++++++ .../GatewayKafkaProduceBackendTest.java | 114 +++++++++++++++++- .../produce/KafkaProduceCommandTest.java | 41 +++++++ 3 files changed, 190 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 2fb0fa4c01f..cc41dda5932 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -112,6 +112,42 @@ void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception } } + @Test + void testNullValuesRemainFilteredWithoutAdvancingNativeOffsets() throws Exception { + // Preserve the existing filtering behavior until the FIP defines its null-value contract. + TablePath path = TablePath.of(DATABASE, "produce_filtered_nulls"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("raw"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + ProducerRecord filtered = + new ProducerRecord(path.toString(), 0, 123L, KEY, null); + ProducerRecord retained = + new ProducerRecord<>( + path.toString(), + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + assertThat(producer.send(filtered).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(-1L); + assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()).isZero(); + assertThat(producer.send(filtered).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(-1L); + assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(1L); + assertReadback(connection, path, "raw", VALUE); + } finally { + admin.dropTable(path, true).get(); + } + } + } + @Test void testSameNamedTablesAcrossDatabases() throws Exception { Map tables = new LinkedHashMap<>(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index 842a75456b7..b3f1502396e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -43,6 +43,8 @@ import org.apache.kafka.common.protocol.Errors; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.net.InetAddress; import java.util.ArrayList; @@ -79,6 +81,42 @@ void testAllNullValuesRequireAuthorizationWithoutAppend() throws Exception { assertThat(service.append).isNull(); } + @ParameterizedTest + @ValueSource(strings = {"non-null", "all-null", "mixed"}) + void testCachedMetadataDoesNotBypassRevokedWritePermission(String recordsKind) + throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, new ArrowKafkaRecordTranscoder(), executor); + PartitionWrite partition = + recordsKind.equals("non-null") + ? good(0) + : recordsKind.equals("all-null") + ? nullValue(0) + : new PartitionWrite( + 0, + Arrays.asList( + nullValue(0).records().get(0), + good(0).records().get(0))); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", partition)); + try { + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + int appendedBeforeRevocation = service.appends.size(); + service.denyWrite = true; + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.metadataPaths).hasSize(2); + assertThat(service.writeChecks).isEqualTo(2); + assertThat(service.appends).hasSize(appendedBeforeRevocation); + assertThat(service.metadataThread).isNotSameAs(Thread.currentThread()); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + @Test void testNullValuesCannotBypassTableAndSchemaValidation() throws Exception { TestingProduceService service = new TestingProduceService(); @@ -434,6 +472,77 @@ void testOffloadedRequestsAppendInOrderWithoutWaitingForAcknowledgement() throws } } + @Test + void testOffloadedMetadataRefreshUsesNewTableIdentityAndSchema() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + List snapshots = new ArrayList<>(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + (records, table) -> { + snapshots.add(table); + return transcoder.transcode(records, table); + }, + executor); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); + try { + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableId(84L).setSchemaId(2).setBucketCountEpoch(7L)); + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + assertThat(snapshots).hasSize(3); + assertThat(snapshots.get(1)).isSameAs(snapshots.get(0)); + assertThat(snapshots.get(2)).isNotSameAs(snapshots.get(0)); + assertThat(snapshots.get(2).getTableId()).isEqualTo(84L); + assertThat(snapshots.get(2).getSchemaId()).isEqualTo(2); + assertThat(snapshots.get(2).getBucketCountEpoch()).isEqualTo(7L); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L, 84L); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingMetadata.completeExceptionally(new TableNotExistException("deleted")); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), + Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.metadataPaths).hasSize(4); + assertThat(service.appends).hasSize(3); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testShutdownPreservesAlreadySubmittedAppendResult() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = new CompletableFuture<>(); + service.appended = new CountDownLatch(1); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, new ArrowKafkaRecordTranscoder(), executor); + try { + CompletableFuture result = + backend.write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.appended.await(10, TimeUnit.SECONDS)).isTrue(); + executor.closeAsync().get(10, TimeUnit.SECONDS); + assertThat(result).isNotDone(); + assertThat(service.pendingAppend).isNotCancelled(); + service.pendingAppend.complete(success(0)); + KafkaProduceResult completed = result.get(10, TimeUnit.SECONDS); + assertErrors(completed, Errors.NONE); + assertThat(completed.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.appends).hasSize(1); + assertThat(service.drains).isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + @Test void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Exception { KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); @@ -561,8 +670,10 @@ public CompletableFuture getTableInfo(GetTableInfoRequest @Override public void authorizeTableWrite(long tableId) { - assertThat(tableId).isEqualTo(42L); + assertThat(tableId).isPositive(); assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(currentSession().getInetAddress()) + .isEqualTo(InetAddress.getLoopbackAddress()); writeChecks++; if (denyWrite) { throw new AuthorizationException("Write denied"); @@ -571,6 +682,7 @@ public void authorizeTableWrite(long tableId) { @Override public CompletableFuture produceLog(ProduceLogRequest request) { + authorizeTableWrite(request.getTableId()); assertThat(currentListenerName()).isEqualTo("KAFKA"); assertThat(currentSession().getInetAddress()) .isEqualTo(InetAddress.getLoopbackAddress()); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java index 3b9c8f15ec0..e3624f5535b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -162,6 +162,47 @@ void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { assertThatThrownBy(copied::clear).isInstanceOf(UnsupportedOperationException.class); } + @Test + void testNullFilteringPreservesRecordIdentityAndTimestampProvenance() { + Record filtered = + new Record( + 123L, + TimestampType.CREATE_TIME, + new byte[] {1}, + null, + Collections.emptyList()); + Record retained = + new Record( + 456L, + TimestampType.LOG_APPEND_TIME, + new byte[] {2}, + new byte[] {3}, + Collections.emptyList()); + Record empty = + new Record( + Record.NO_TIMESTAMP, + TimestampType.CREATE_TIME, + null, + new byte[0], + Collections.emptyList()); + List source = new ArrayList<>(Arrays.asList(filtered, retained, empty)); + PartitionWrite partition = new PartitionWrite(0, source); + source.clear(); + + assertThat(partition.records()).containsExactly(filtered, retained, empty); + assertThat(partition.hasNullValues()).isTrue(); + assertThat(partition.nonNullRecords()).containsExactly(retained, empty); + assertThat(partition.nonNullRecords().get(0)).isSameAs(retained); + assertThat(partition.nonNullRecords().get(0).timestampType()) + .isEqualTo(TimestampType.LOG_APPEND_TIME); + assertThat(partition.nonNullRecords().get(0).timestamp()).isEqualTo(456L); + assertThat(partition.nonNullRecords().get(1).timestampType()) + .isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.nonNullRecords().get(1).hasTimestamp()).isFalse(); + assertThatThrownBy(() -> partition.nonNullRecords().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + @Test void testBorrowedPayloadsReuseOwnedArraysWhileCopyAccessorsRemainDefensive() { byte[] key = {1}; From b2c1d53254ea1cc6a4ba0dcc2c4e7b16097fcb93 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 18:45:21 +0800 Subject: [PATCH 26/69] [kafka] Apply persisted table contracts to Produce append Inject the configured topic mapper into Produce, check authoritative table enablement before plan preparation, and use the shared fixed-column schema. Reject conversion failures as INVALID_RECORD and missing databases as UNKNOWN_TOPIC_OR_PARTITION without alias fallback. Cover shared alias plans and offsets, disable/reset and property/schema changes, null filtering, raw keys, last-wins headers and null rescue values. Preserve bounded conversion, authorization and atomic partition conversion. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 34/34 AI-Contributed/UT: 656/656 --- .../fluss/kafka/KafkaRequestHandler.java | 1 + .../produce/GatewayKafkaProduceBackend.java | 22 +- .../transcode/KafkaRecordTranscoder.java | 6 +- .../kafka/transcode/KafkaTopicWritePlan.java | 11 +- .../fluss/kafka/KafkaProduceAppendITCase.java | 301 ++++++++++++-- .../GatewayKafkaProduceBackendTest.java | 366 ++++++++++++++++-- .../produce/KafkaTableInfoCacheTest.java | 10 +- .../KafkaCompiledWritePlanCacheTest.java | 25 +- 8 files changed, 659 insertions(+), 83 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 2b48e91bbed..dc34e6aaaca 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -68,6 +68,7 @@ public KafkaRequestHandler( new GatewayKafkaProduceBackend( service, (TabletServerGateway) service, + topicMapper, new ArrowKafkaRecordTranscoder(), conversionExecutor)))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index 313da86ad7d..fa5daca2e2d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -18,12 +18,14 @@ package org.apache.fluss.kafka.backend.produce; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; @@ -66,7 +68,8 @@ public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { private final RpcGatewayService service; private final TabletServerGateway gateway; - private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); private final KafkaRecordTranscoder transcoder; private final KafkaTableInfoCache tableInfoCache = new KafkaTableInfoCache(); private final @Nullable KafkaProduceConversionExecutor conversionExecutor; @@ -75,18 +78,21 @@ public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { public GatewayKafkaProduceBackend( RpcGatewayService service, TabletServerGateway gateway, + KafkaTopicMapper topicMapper, KafkaRecordTranscoder transcoder) { - this(service, gateway, transcoder, null); + this(service, gateway, topicMapper, transcoder, null); } /** Creates a backend that submits work to a caller-owned bounded conversion executor. */ public GatewayKafkaProduceBackend( RpcGatewayService service, TabletServerGateway gateway, + KafkaTopicMapper topicMapper, KafkaRecordTranscoder transcoder, @Nullable KafkaProduceConversionExecutor conversionExecutor) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); + this.topicMapper = checkNotNull(topicMapper); this.transcoder = checkNotNull(transcoder); this.conversionExecutor = conversionExecutor; } @@ -157,6 +163,11 @@ private CompletableFuture writeTopic( private CompletableFuture produceTopic( KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { // Admission must match Metadata, including when the schema changes between requests. + // Even a cached snapshot is selected only after fetching the authoritative descriptor. + if (!schemaResolver.isEnabled(tableInfo.toTableDescriptor())) { + throw new TableNotExistException( + "Table " + tableInfo.getTablePath() + " is not enabled for Kafka."); + } KafkaTopicWritePlan writePlan = transcoder.prepare(tableInfo); ProduceLogRequest request = new ProduceLogRequest() @@ -185,7 +196,9 @@ private CompletableFuture produceTopic( new PartitionResult(partition.partitionId(), Errors.NONE, -1L, null)); continue; } - BytesView records = transcoder.transcode(partition.nonNullRecords(), writePlan); + BytesView records = + transcoder.transcode( + partition.partitionId(), partition.nonNullRecords(), writePlan); request.addBucketsReq() .setBucketId(partition.partitionId()) .setRecordsBytesView(records); @@ -309,7 +322,7 @@ private static PartitionResult failedPartition(int partitionId, Throwable failur || cause instanceof InvalidTopicException ? Errors.INVALID_TOPIC_EXCEPTION : cause instanceof KafkaRecordEncodingException - ? Errors.CORRUPT_MESSAGE + ? Errors.INVALID_RECORD : cause instanceof IllegalArgumentException ? Errors.INVALID_REQUEST : toKafkaError( @@ -323,6 +336,7 @@ private static Errors toKafkaError(org.apache.fluss.rpc.protocol.Errors error) { case NONE: return Errors.NONE; case TABLE_NOT_EXIST: + case DATABASE_NOT_EXIST: case UNKNOWN_TABLE_OR_BUCKET_EXCEPTION: return Errors.UNKNOWN_TOPIC_OR_PARTITION; case NOT_LEADER_OR_FOLLOWER: diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java index 063728b77fa..6c87d80e273 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -35,10 +35,12 @@ default KafkaTopicWritePlan prepare(TableInfo tableInfo) { } /** Transcodes one requested partition using an already prepared plan. */ - default BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { + default BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { return transcode(partitionId, records, plan.tableInfo()); } /** Transcodes records from the requested Kafka partition using the target table schema. */ - BytesView transcode(int partitionId, List records, TableInfo tableInfo) throws Exception; + BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java index 1f9d744afb5..c68736e4233 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -19,13 +19,16 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; import org.apache.fluss.metadata.TableInfo; import javax.annotation.concurrent.ThreadSafe; import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Immutable binary/string conversion plan; mutable Arrow resources are owned by each conversion. */ +/** + * Immutable binary/string conversion plan; mutable Arrow resources are owned by each conversion. + */ @Internal @ThreadSafe public final class KafkaTopicWritePlan { @@ -36,6 +39,12 @@ public final class KafkaTopicWritePlan { KafkaTopicWritePlan(TableInfo tableInfo, KafkaTableSchema topicSchema) { this.tableInfo = checkNotNull(tableInfo); this.topicSchema = checkNotNull(topicSchema); + // Validate support during preparation as filtered partitions may never call transcode. + if (topicSchema.valueType() != ValueType.BINARY + && topicSchema.valueType() != ValueType.STRING) { + throw new KafkaRecordEncodingException( + "Unsupported Kafka value conversion type: " + topicSchema.valueType()); + } this.rowAssembler = new KafkaRowAssembler(topicSchema); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index cc41dda5932..ddbd1acf806 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -26,7 +26,6 @@ import org.apache.fluss.cluster.ServerNode; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; @@ -40,17 +39,37 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -58,11 +77,13 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; import static org.assertj.core.api.Assertions.assertThat; /** Exercises native DDL, Kafka discovery and Produce, replication, and native Fluss readback. */ class KafkaProduceAppendITCase { - private static final String DATABASE = "kafka"; + private static final String DATABASE = "produce_default"; private static final byte[] KEY = "key".getBytes(StandardCharsets.UTF_8); private static final byte[] VALUE = "value".getBytes(StandardCharsets.UTF_8); @@ -75,7 +96,7 @@ class KafkaProduceAppendITCase { .build(); @ParameterizedTest - @ValueSource(strings = {"raw", "string"}) + @ValueSource(strings = {"binary", "string"}) void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception { try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { @@ -98,7 +119,18 @@ void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception new RecordHeader("source", VALUE), new RecordHeader("source", null))); RecordMetadata first = producer.send(record).get(30, TimeUnit.SECONDS); - RecordMetadata second = producer.send(record).get(30, TimeUnit.SECONDS); + RecordMetadata second = + producer.send( + new ProducerRecord<>( + path.getTableName(), + 0, + 123L, + KEY, + VALUE, + record.headers())) + .get(30, TimeUnit.SECONDS); + assertThat(first.topic()).isEqualTo(topic); + assertThat(second.topic()).isEqualTo(path.getTableName()); if (!acks.equals("0")) { assertThat(first.offset()).isZero(); assertThat(second.offset()).isEqualTo(1L); @@ -119,7 +151,7 @@ void testNullValuesRemainFilteredWithoutAdvancingNativeOffsets() throws Exceptio try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - admin.createTable(path, descriptor("raw"), false).get(); + admin.createTable(path, descriptor("binary"), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); try (KafkaProducer producer = producer("all", 0)) { ProducerRecord filtered = @@ -141,7 +173,7 @@ void testNullValuesRemainFilteredWithoutAdvancingNativeOffsets() throws Exceptio .isEqualTo(-1L); assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()) .isEqualTo(1L); - assertReadback(connection, path, "raw", VALUE); + assertReadback(connection, path, "binary", VALUE); } finally { admin.dropTable(path, true).get(); } @@ -161,7 +193,7 @@ void testSameNamedTablesAcrossDatabases() throws Exception { for (TablePath path : tables.keySet()) { admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true) .get(); - admin.createTable(path, descriptor("raw"), false).get(); + admin.createTable(path, descriptor("binary"), false).get(); } CLUSTER.waitUntilAllGatewayHasSameMetadata(); try (KafkaProducer producer = producer("all")) { @@ -184,7 +216,7 @@ void testSameNamedTablesAcrossDatabases() throws Exception { .isEqualTo(1L); } for (Map.Entry entry : tables.entrySet()) { - assertReadback(connection, entry.getKey(), "raw", entry.getValue()); + assertReadback(connection, entry.getKey(), "binary", entry.getValue()); } } } finally { @@ -201,7 +233,7 @@ void testPipelinedProducePreservesAppendOrder() throws Exception { try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - admin.createTable(path, descriptor("raw"), false).get(); + admin.createTable(path, descriptor("binary"), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); try (KafkaProducer producer = producer("all", 0)) { assertThat(producer.partitionsFor(path.toString())).hasSize(1); @@ -227,8 +259,6 @@ void testPipelinedProducePreservesAppendOrder() throws Exception { while (count < values.size() && System.nanoTime() < deadline) { for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { assertThat(record.getRow().getBytes(0)).isEqualTo(values.get(count)); - assertThat(record.getRow().getTimestampNtz(1, 3).getMillisecond()) - .isEqualTo(count); count++; } } @@ -240,62 +270,260 @@ void testPipelinedProducePreservesAppendOrder() throws Exception { } } + @Test + void testAliasesShareOffsetsAndObservePersistedEnablementOnOneConnection() throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_enablement"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("binary"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.getTableName(), path.toString()}; + assertProduceOffsets(query(socket, names), names, 0L, 1L); + setEnabled(path, false); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setEnabled(path, true); + assertProduceOffsets(query(socket, names), names, 2L, 3L); + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList( + ConfigOptions.TABLE_KAFKA_ENABLED.key()), + Collections.emptyList(), + false)) + .get(); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertReadback(connection, path, "binary", VALUE, 4); + admin.dropTable(path, true).get(); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testDeletedDefaultDatabaseDoesNotFallbackOrRecreateDatabase() throws Exception { + Configuration config = clusterConfig(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "produce_missing_db"); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 1); + FlussClusterExtension cluster = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(config) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + cluster.start(); + try (Connection connection = ConnectionFactory.createConnection(cluster.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + // Startup creates the configured default; Produce must not recreate a deleted one. + assertThat(admin.databaseExists("produce_missing_db").get()).isTrue(); + admin.dropDatabase("produce_missing_db", false, false).get(); + assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); + TablePath existing = TablePath.of("fluss", "events"); + long tableId = createTable(cluster, existing, descriptor("binary")); + cluster.waitUntilTableReady(tableId); + ServerNode node = cluster.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port())) { + String[] missing = { + "events", + "produce_missing_db.events", + "another_missing_db.events", + "fluss.missing" + }; + assertProduceErrors( + query(socket, missing), missing, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertProduceOffsets( + query(socket, existing.toString()), new String[] {existing.toString()}, 0L); + assertProduceOffsets( + query(socket, existing.toString()), new String[] {existing.toString()}, 1L); + assertReadback(connection, existing, "binary", VALUE); + assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); + assertThat(admin.databaseExists("another_missing_db").get()).isFalse(); + } + } finally { + cluster.close(); + } + } + + private static void setEnabled(TablePath path, boolean enabled) throws Exception { + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled)), + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private static ProduceResponse query(Socket socket, String... names) throws Exception { + List topics = new ArrayList<>(); + for (String name : names) { + topics.add( + new TopicProduceData() + .setName(name) + .setPartitionData( + Collections.singletonList( + new PartitionProduceData() + .setIndex(0) + .setRecords( + MemoryRecords.withRecords( + Compression.NONE, + new SimpleRecord( + 123L, + KEY, + VALUE, + new Header[] { + new RecordHeader( + "source", + VALUE), + new RecordHeader( + "source", + null) + })))))); + } + ProduceRequest request = + new ProduceRequest( + new ProduceRequestData() + .setAcks((short) -1) + .setTimeoutMs(10000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + topics.iterator())), + (short) 9); + RequestHeader header = + new RequestHeader(ApiKeys.PRODUCE, request.version(), "produce-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] bytes = new byte[encoded.remaining()]; + encoded.get(bytes); + socket.setSoTimeout(15000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(bytes.length); + output.write(bytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + byte[] response = new byte[input.readInt()]; + input.readFully(response); + return (ProduceResponse) AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static void assertProduceOffsets( + ProduceResponse response, String[] names, long... offsets) { + assertProduceErrors(response, names, Errors.NONE); + for (int i = 0; i < names.length; i++) { + assertThat( + response.data() + .responses() + .find(names[i]) + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(offsets[i]); + } + } + + private static void assertProduceErrors( + ProduceResponse response, String[] names, Errors error) { + assertThat(response.data().responses()) + .extracting(topic -> topic.name()) + .containsExactly(names); + for (String name : names) { + List partitions = + response.data().responses().find(name).partitionResponses(); + assertThat(partitions).hasSize(1); + assertThat(partitions.get(0).index()).isZero(); + assertThat(partitions.get(0).errorCode()).isEqualTo(error.code()); + if (error != Errors.NONE) { + assertThat(partitions.get(0).baseOffset()).isEqualTo(-1L); + } + } + } + private static void assertReadback( Connection connection, TablePath path, String format, byte[] expectedValue) throws Exception { + assertReadback(connection, path, format, expectedValue, 2); + } + + private static void assertReadback( + Connection connection, + TablePath path, + String format, + byte[] expectedValue, + int expectedCount) + throws Exception { try (Table table = connection.getTable(path); LogScanner scanner = table.newScan().createLogScanner()) { scanner.subscribeFromBeginning(0); int count = 0; long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - while (count < 2 && System.nanoTime() < deadline) { + while (count < expectedCount && System.nanoTime() < deadline) { ScanRecords records = scanner.poll(Duration.ofSeconds(1)); for (ScanRecord record : records) { InternalRow row = record.getRow(); - if (format.equals("raw")) { + if (format.equals("binary")) { assertThat(row.getBytes(0)).isEqualTo(expectedValue); - assertThat(row.getBytes(3)).isEqualTo(KEY); } else { assertThat(row.getString(0).toString()) .isEqualTo(new String(expectedValue, StandardCharsets.UTF_8)); - assertThat(row.getString(3).toString()).isEqualTo("key"); } - assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(123L); - assertThat(row.getArray(2).size()).isEqualTo(2); - assertThat(row.getArray(2).getRow(0, 2).getString(0).toString()) + assertThat(row.getBytes(3)).isEqualTo(KEY); + assertThat(row.getMap(1).size()).isEqualTo(1); + assertThat(row.getMap(1).keyArray().getString(0).toString()) .isEqualTo("source"); - assertThat(row.getArray(2).getRow(1, 2).isNullAt(1)).isTrue(); + assertThat(row.getMap(1).valueArray().isNullAt(0)).isTrue(); + assertThat(row.isNullAt(2)).isTrue(); + InternalRow metadata = row.getRow(4, 3); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()) + .isEqualTo(123L); + assertThat(metadata.getInt(1)).isZero(); + assertThat(metadata.getInt(2)).isZero(); count++; } } - assertThat(count).isEqualTo(2); + assertThat(count).isEqualTo(expectedCount); } } private static TableDescriptor descriptor(String format) { - boolean raw = format.equals("raw"); + boolean binary = format.equals("binary"); return TableDescriptor.builder() .schema( Schema.newBuilder() - .column("body", raw ? DataTypes.BYTES() : DataTypes.STRING()) - .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) .column( - "attributes", - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD( - "value", DataTypes.BYTES())))) - .column("message_key", raw ? DataTypes.BYTES() : DataTypes.STRING()) + "_kafka_value", + binary ? DataTypes.BYTES() : DataTypes.STRING()) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))) .build()) .distributedBy(1) .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format) - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format) - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) .build(); } @@ -322,6 +550,7 @@ private static KafkaProducer producer(String acks, int batchSize private static Configuration clusterConfig() { Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, DATABASE); config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); return config; diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index b3f1502396e..9acf78edddf 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -17,13 +17,16 @@ package org.apache.fluss.kafka.backend.produce; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.AuthorizationException; +import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; -import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; @@ -89,7 +92,7 @@ void testCachedMetadataDoesNotBypassRevokedWritePermission(String recordsKind) KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); GatewayKafkaProduceBackend backend = new GatewayKafkaProduceBackend( - service, service, new ArrowKafkaRecordTranscoder(), executor); + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); PartitionWrite partition = recordsKind.equals("non-null") ? good(0) @@ -142,9 +145,10 @@ void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exceptio new GatewayKafkaProduceBackend( service, service, - (records, table) -> { + mapper(), + (partitionId, records, table) -> { seen.addAll(records); - return transcoder.transcode(records, table); + return transcoder.transcode(partitionId, records, table); }); Record empty = new Record(1L, null, new byte[0], Collections.emptyList()); Record valid = good(0).records().get(0); @@ -207,7 +211,8 @@ void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception PartitionWrite malformed = new PartitionWrite( 0, - Collections.singletonList( + Arrays.asList( + good(0).records().get(0), new Record( 1L, null, @@ -220,8 +225,7 @@ void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception (short) 1, topic("kafka.topic", malformed, good(1), good(8)))) .join(); - assertErrors( - result, Errors.CORRUPT_MESSAGE, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertErrors(result, Errors.INVALID_RECORD, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); assertThat(service.append.getBucketsReqsList()).hasSize(1); assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); assertThat(service.drains).isEqualTo(1); @@ -267,13 +271,15 @@ void testMapsReplicationFailuresAndMissingResponsesPerPartition() throws Excepti org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_EXCEPTION, org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION, org.apache.fluss.rpc.protocol.Errors.REQUEST_TIME_OUT, - org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER + org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER, + org.apache.fluss.rpc.protocol.Errors.CORRUPT_RECORD_EXCEPTION }; Errors[] kafkaErrors = { Errors.NOT_ENOUGH_REPLICAS, Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, Errors.REQUEST_TIMED_OUT, - Errors.NOT_LEADER_OR_FOLLOWER + Errors.NOT_LEADER_OR_FOLLOWER, + Errors.CORRUPT_MESSAGE }; for (int i = 0; i < flussErrors.length; i++) { TestingProduceService service = new TestingProduceService(); @@ -322,9 +328,10 @@ void testQualifiedTopicsPreserveDatabaseAndTranscoderTablePath() throws Exceptio new GatewayKafkaProduceBackend( service, service, - (records, tableInfo) -> { + mapper(), + (partitionId, records, tableInfo) -> { transcodedPaths.add(tableInfo.getTablePath()); - return transcoder.transcode(records, tableInfo); + return transcoder.transcode(partitionId, records, tableInfo); }); KafkaProduceResult result = @@ -359,7 +366,7 @@ void testInvalidQualifiedNamesDoNotReachGatewayOrSuppressValidTopic() throws Exc .write( command( (short) 1, - topic("topic", good(0)), + topic(".", good(0)), topic("kafka.topic.extra", good(0)), topic("kafka.__internal", good(0)), topic("kafka.topic", good(0)))) @@ -382,6 +389,7 @@ void testReusesPlansButAlwaysRefreshesMetadataAndValidatesChangedContract() thro ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); List prepared = new ArrayList<>(); List converted = new ArrayList<>(); + List convertedPartitions = new ArrayList<>(); KafkaRecordTranscoder transcoder = new KafkaRecordTranscoder() { @Override @@ -392,26 +400,30 @@ public KafkaTopicWritePlan prepare(TableInfo tableInfo) { } @Override - public BytesView transcode(List records, TableInfo tableInfo) + public BytesView transcode( + int partitionId, List records, TableInfo tableInfo) throws Exception { - return arrow.transcode(records, tableInfo); + return arrow.transcode(partitionId, records, tableInfo); } @Override - public BytesView transcode(List records, KafkaTopicWritePlan plan) + public BytesView transcode( + int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { converted.add(plan); - return arrow.transcode(records, plan); + convertedPartitions.add(partitionId); + return arrow.transcode(partitionId, records, plan); } }; GatewayKafkaProduceBackend backend = - new GatewayKafkaProduceBackend(service, service, transcoder); + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); backend.write(command((short) 1, topic("kafka.topic", good(0), good(1)))).join(); backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(); assertThat(service.metadataPaths).hasSize(2); assertThat(prepared).hasSize(2); assertThat(prepared.get(1)).isSameAs(prepared.get(0)); assertThat(converted).containsExactly(prepared.get(0), prepared.get(0), prepared.get(0)); + assertThat(convertedPartitions).containsExactly(0, 1, 0); service.pendingMetadata = CompletableFuture.completedFuture(metadata(true)); assertErrors( @@ -439,7 +451,7 @@ void testOffloadedRequestsAppendInOrderWithoutWaitingForAcknowledgement() throws service.appended = new CountDownLatch(2); GatewayKafkaProduceBackend backend = new GatewayKafkaProduceBackend( - service, service, new ArrowKafkaRecordTranscoder(), executor); + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); try { CompletableFuture first = backend.write(command((short) -1, topic("kafka.topic", good(0)))); @@ -482,24 +494,41 @@ void testOffloadedMetadataRefreshUsesNewTableIdentityAndSchema() throws Exceptio new GatewayKafkaProduceBackend( service, service, - (records, table) -> { + mapper(), + (partitionId, records, table) -> { snapshots.add(table); - return transcoder.transcode(records, table); + return transcoder.transcode(partitionId, records, table); }, executor); KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); try { assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + TableDescriptor changedSchema = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(3) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); service.pendingMetadata = CompletableFuture.completedFuture( - metadata(false).setTableId(84L).setSchemaId(2).setBucketCountEpoch(7L)); + metadata(false) + .setTableId(84L) + .setSchemaId(2) + .setBucketCountEpoch(7L) + .setTableJson(changedSchema.toJsonBytes())); assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); assertThat(snapshots).hasSize(3); assertThat(snapshots.get(1)).isSameAs(snapshots.get(0)); assertThat(snapshots.get(2)).isNotSameAs(snapshots.get(0)); assertThat(snapshots.get(2).getTableId()).isEqualTo(84L); assertThat(snapshots.get(2).getSchemaId()).isEqualTo(2); + assertThat(snapshots.get(2).getRowType().getTypeAt(0)).isEqualTo(DataTypes.BYTES()); assertThat(snapshots.get(2).getBucketCountEpoch()).isEqualTo(7L); assertThat(service.appends) .extracting(ProduceLogRequest::getTableId) @@ -524,7 +553,7 @@ void testShutdownPreservesAlreadySubmittedAppendResult() throws Exception { KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); GatewayKafkaProduceBackend backend = new GatewayKafkaProduceBackend( - service, service, new ArrowKafkaRecordTranscoder(), executor); + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); try { CompletableFuture result = backend.write(command((short) -1, topic("kafka.topic", good(0)))); @@ -551,7 +580,7 @@ void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Except service.metadataEntered = new CountDownLatch(1); GatewayKafkaProduceBackend backend = new GatewayKafkaProduceBackend( - service, service, new ArrowKafkaRecordTranscoder(), executor); + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); try { CompletableFuture running = backend.write(command); @@ -572,8 +601,283 @@ void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Except } } + @Test + void testAliasesSharePlansAndAuthorizationButRetainRequestNames() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 4); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder, executor); + try { + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("topic", good(0)), + topic("kafka.topic", good(0)))) + .get(10, TimeUnit.SECONDS); + assertThat(result.topics()) + .extracting(KafkaProduceResult.TopicResult::topicName) + .containsExactly("topic", "kafka.topic"); + assertThat(result.topics()) + .allSatisfy( + topic -> + assertThat(topic.partitions().get(0).error()) + .isEqualTo(Errors.NONE)); + assertThat(service.metadataPaths) + .containsExactly( + TablePath.of("kafka", "topic"), TablePath.of("kafka", "topic")); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L); + assertThat(service.writeChecks).isEqualTo(2); + assertThat(transcoder.prepared).hasSize(2); + assertThat(transcoder.prepared.get(0)).isSameAs(transcoder.prepared.get(1)); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDisabledAndResetFlagsRejectBothNamesBeforePrepare(boolean allNull) throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + PartitionWrite partition = allNull ? nullValue(0) : good(0); + assertErrors( + backend.write(command((short) 1, topic("topic", partition))).join(), Errors.NONE); + for (boolean reset : new boolean[] {false, true}) { + TableDescriptor enabled = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + java.util.Map properties = + new java.util.HashMap<>(enabled.getProperties()); + if (reset) { + properties.remove(ConfigOptions.TABLE_KAFKA_ENABLED.key()); + } else { + properties.put(ConfigOptions.TABLE_KAFKA_ENABLED.key(), "false"); + } + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + enabled.withProperties(properties).toJsonBytes())); + int prepared = transcoder.prepared.size(); + int appended = service.appends.size(); + int authorized = service.writeChecks; + for (String name : Arrays.asList("topic", "kafka.topic")) { + assertErrors( + backend.write(command((short) 1, topic(name, partition))).join(), + Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + assertThat(transcoder.prepared).hasSize(prepared); + assertThat(service.appends).hasSize(appended); + assertThat(service.writeChecks).isEqualTo(authorized); + service.pendingMetadata = CompletableFuture.completedFuture(metadata(false)); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", partition))).join(), + Errors.NONE); + assertThat(transcoder.prepared.get(prepared)) + .isNotSameAs(transcoder.prepared.get(prepared - 1)); + } + assertThat(service.metadataPaths).hasSize(7); + } + + @Test + void testPropertyOnlyChangesRejectInvalidContractsAndRecompileRecoveredPlan() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + assertErrors( + backend.write(command((short) 1, topic("topic", good(0)))).join(), Errors.NONE); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + for (TableDescriptor invalid : + Arrays.asList( + TableDescriptor.builder(original) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(), + TableDescriptor.builder(original) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + TableDescriptor.builder(original) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "string") + .build())) { + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(invalid.toJsonBytes())); + for (String name : Arrays.asList("topic", "kafka.topic")) { + assertErrors( + backend.write(command((short) 1, topic(name, nullValue(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + } + } + assertThat(service.appends).hasSize(1); + assertThat(transcoder.prepared).hasSize(1); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE, + "STRING") + .build() + .toJsonBytes())); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.NONE); + assertThat(transcoder.prepared).hasSize(2); + assertThat(transcoder.prepared.get(1)).isNotSameAs(transcoder.prepared.get(0)); + assertThat(transcoder.prepared.get(1).tableInfo().getSchemaId()).isEqualTo(1); + assertThat(service.appends).hasSize(2); + assertThat(service.metadataPaths).hasSize(8); + } + + @Test + void testMissingDefaultAndExplicitDatabasesDoNotFallback() throws Exception { + TestingProduceService service = new TestingProduceService(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_db"); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + new KafkaTopicMapper(config), + new ArrowKafkaRecordTranscoder()); + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("topic", good(0)), + topic("missing_db.topic", good(0)), + topic("other.topic", good(0)))) + .join(); + assertThat(result.topics()) + .extracting(topic -> topic.partitions().get(0).error()) + .containsExactly( + Errors.UNKNOWN_TOPIC_OR_PARTITION, + Errors.UNKNOWN_TOPIC_OR_PARTITION, + Errors.NONE); + assertThat(service.metadataPaths) + .containsExactly( + TablePath.of("missing_db", "topic"), + TablePath.of("missing_db", "topic"), + TablePath.of("other", "topic")); + assertThat(service.appends).extracting(ProduceLogRequest::getTableId).containsExactly(43L); + } + + @Test + void testRecordNullabilityFailureDoesNotAppendValidPrefix() throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.STRING()) + .column( + "_kafka_key", + DataTypes.BYTES() + .copy(false)) + .build()) + .build() + .toJsonBytes())); + Record valid = new Record(1L, new byte[] {1}, new byte[] {65}, Collections.emptyList()); + assertErrors( + backend(service) + .write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 0, + Arrays.asList( + valid, good(0).records().get(0)))))) + .join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + assertThat(service.drains).isZero(); + } + + @Test + void testMetadataOffsetsAndJsonCannotBeActivatedByFilteredRecords() throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + TableDescriptor unsupportedMetadata = + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()), + DataTypes.FIELD( + "offset", DataTypes.BIGINT()))) + .build()) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(unsupportedMetadata.toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + TableDescriptor json = + TableDescriptor.builder(original) + .schema(Schema.newBuilder().column("business", DataTypes.STRING()).build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture(metadata(false).setTableJson(json.toJsonBytes())); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + assertThat(service.writeChecks).isZero(); + } + + private static final class RecordingTranscoder implements KafkaRecordTranscoder { + private final ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); + private final List prepared = new ArrayList<>(); + + @Override + public KafkaTopicWritePlan prepare(TableInfo table) { + KafkaTopicWritePlan plan = arrow.prepare(table); + prepared.add(plan); + return plan; + } + + @Override + public BytesView transcode(int partitionId, List records, TableInfo table) + throws Exception { + return arrow.transcode(partitionId, records, table); + } + } + + private static KafkaTopicMapper mapper() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "kafka"); + return new KafkaTopicMapper(config); + } + private static GatewayKafkaProduceBackend backend(TestingProduceService service) { - return new GatewayKafkaProduceBackend(service, service, new ArrowKafkaRecordTranscoder()); + return new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder()); } private static KafkaProduceCommand command(short acks, TopicWrite... topics) throws Exception { @@ -595,11 +899,15 @@ private static PartitionWrite good(int id) { private static GetTableInfoResponse metadata(boolean invalid) { TableDescriptor.Builder descriptor = TableDescriptor.builder() - .schema(Schema.newBuilder().column("value", DataTypes.STRING()).build()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .build()) .distributedBy(3) - .logFormat(LogFormat.ARROW); + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true); if (!invalid) { - descriptor.customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"); + descriptor.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "string"); } return new GetTableInfoResponse() .setTableId(42L) @@ -653,6 +961,9 @@ public CompletableFuture getTableInfo(GetTableInfoRequest if (metadataEntered != null) { metadataEntered.countDown(); } + if (database.equals("missing_db")) { + throw new DatabaseNotExistException("missing_db"); + } if (name.equals("missing")) { if (!asyncMissing) { throw new TableNotExistException("missing"); @@ -699,7 +1010,6 @@ public CompletableFuture produceLog(ProduceLogRequest reques @Override public void tryCompleteActions() { - assertThat(append).isNotNull(); drains++; if (appended != null) { appended.countDown(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java index 73c68ec725d..3ee3a125495 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java @@ -255,12 +255,16 @@ private static GetTableInfoResponse response( private static TableDescriptor descriptor(String version) { TableDescriptor.Builder builder = TableDescriptor.builder() - .schema(Schema.newBuilder().column("value", DataTypes.BYTES()).build()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) .distributedBy(1) .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) - .customProperty("kafka.value.format", "raw"); + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); if ("v2".equals(version)) { - builder.customProperty("application.version", "changed"); + builder.property(ConfigOptions.TABLE_KAFKA_ENABLED, false); } return builder.build(); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java index e3026f40685..ac950afeeeb 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java @@ -18,9 +18,9 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; @@ -212,14 +212,14 @@ void testCompiledAssemblerIsSafeToShare() throws Exception { () -> plan.rowAssembler() .assemble( - new Object[0], new Object[] { Integer.toString(id) .getBytes( StandardCharsets.UTF_8) }, - id, - new ArrayList<>()) + new Record( + id, null, null, new ArrayList<>()), + 0) .getField(0))); } for (int i = 0; i < futures.size(); i++) { @@ -248,7 +248,7 @@ private static KafkaTopicWritePlan compile(TableInfo tableInfo, AtomicInteger co } private static KafkaTopicWritePlan compile(TableInfo tableInfo) { - KafkaTopicSchema schema = + KafkaTableSchema schema = new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor()); return new KafkaTopicWritePlan(tableInfo, schema); } @@ -257,11 +257,18 @@ private static TableInfo tableInfo( String tableName, long tableId, int schemaId, String propertyValue) { TableDescriptor descriptor = TableDescriptor.builder() - .schema(Schema.newBuilder().column("payload", DataTypes.BYTES()).build()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .build()) .distributedBy(1) .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") - .customProperty("application.version", propertyValue) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + propertyValue.equals("v1") ? "binary" : "BINARY") .build(); return TableInfo.of( TablePath.of("kafka", tableName), tableId, schemaId, descriptor, null, 1L, 1L); From 8d5e234f83524c68facfcac9d35ac646c7ec9bda Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 16:15:53 +0800 Subject: [PATCH 27/69] [kafka] Wait for replicas before no-ack Produce tests Wait for table replica metadata and leader readiness before round-trip writes. Node discovery alone does not ensure a fresh table can accept acks=0 records, whose failures cannot be observed or retried by the producer. Include the table path in readback failures to identify format/acks cases. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 7/7 --- .../org/apache/fluss/kafka/KafkaProduceAppendITCase.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index ddbd1acf806..b331878c165 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -29,6 +29,7 @@ import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.row.InternalRow; @@ -106,6 +107,10 @@ void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception String topic = path.toString(); admin.createTable(path, descriptor(format), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); + // Node discovery does not wait for table replicas. With acks=0 the producer + // cannot observe or retry a write rejected before the leader is ready. + CLUSTER.waitUntilAllReplicaReady( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); try (KafkaProducer producer = producer(acks)) { assertThat(producer.partitionsFor(topic)).hasSize(1); ProducerRecord record = @@ -495,7 +500,7 @@ private static void assertReadback( count++; } } - assertThat(count).isEqualTo(expectedCount); + assertThat(count).as("native readback for %s", path).isEqualTo(expectedCount); } } From 0d41535cce582b4cfcb7d61cdb197398af67f7ae Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 12:04:12 +0800 Subject: [PATCH 28/69] [kafka] Decouple Produce append from delayed fetch wakeups Remove the Kafka drain-hook dependency after separating the core delayed fetch optimization into apache/fluss#3412. Continue to use native append responses for acknowledgements, including pending acks=all completions. Retain append, authorization, error mapping, ordering and shutdown coverage; move the test append signal to native submission instead of the drain hook. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 9/9 AI-Contributed/UT: 33/33 --- .../produce/GatewayKafkaProduceBackend.java | 9 +---- .../GatewayKafkaProduceBackendTest.java | 33 +++++-------------- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index fa5daca2e2d..8ab07a19de3 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -214,14 +214,7 @@ private CompletableFuture produceTopic( try { checkInterrupted(); setCurrentSession(command); - CompletableFuture appended; - try { - appended = gateway.produceLog(request); - } finally { - // produceLog appends synchronously, but acks=-1 can finish later. Wake delayed - // fetches now so replicas can fetch the append needed to complete that response. - service.tryCompleteActions(); - } + CompletableFuture appended = gateway.produceLog(request); return appended.handle( (response, failure) -> failure == null diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index 9acf78edddf..2d66b743601 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -60,7 +60,9 @@ import static org.assertj.core.api.Assertions.assertThat; -/** Tests append admission, error isolation, session propagation and delayed-fetch completion. */ +/** + * Tests append admission, error isolation, session propagation and asynchronous acknowledgements. + */ class GatewayKafkaProduceBackendTest { @Test @@ -74,7 +76,6 @@ void testAllNullValuesRequireAuthorizationWithoutAppend() throws Exception { assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); assertThat(service.writeChecks).isEqualTo(1); assertThat(service.append).isNull(); - assertThat(service.drains).isZero(); service.denyWrite = true; assertErrors( backend(service) @@ -168,7 +169,6 @@ void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exceptio assertErrors(result, Errors.NONE); assertThat(seen).containsExactly(empty, valid); assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); - assertThat(service.drains).isEqualTo(1); } private static PartitionWrite nullValue(int partition) { @@ -179,7 +179,7 @@ private static PartitionWrite nullValue(int partition) { } @Test - void testDrainsImmediatelyWhileAcksAllResponseIsPending() throws Exception { + void testAcksAllWaitsForNativeAppendResponse() throws Exception { TestingProduceService service = new TestingProduceService(); service.pendingAppend = new CompletableFuture<>(); CompletableFuture result = @@ -187,22 +187,19 @@ void testDrainsImmediatelyWhileAcksAllResponseIsPending() throws Exception { assertThat(service.append.getAcks()).isEqualTo(-1); assertThat(service.append.getTimeoutMs()).isEqualTo(4321); assertThat(service.append.getTableId()).isEqualTo(42L); - assertThat(service.drains).isEqualTo(1); assertThat(result).isNotDone(); assertThat(service.pendingAppend).isNotDone(); service.pendingAppend.complete(success(0)); assertThat(result.join().topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); - assertThat(service.drains).isEqualTo(1); } @Test - void testDrainsAfterSynchronousAppendFailure() throws Exception { + void testMapsSynchronousAppendFailure() throws Exception { TestingProduceService service = new TestingProduceService(); service.throwAppend = true; assertErrors( backend(service).write(command((short) 1, topic("kafka.topic", good(0)))).join(), Errors.UNKNOWN_SERVER_ERROR); - assertThat(service.drains).isEqualTo(1); } @Test @@ -228,7 +225,6 @@ void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception assertErrors(result, Errors.INVALID_RECORD, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); assertThat(service.append.getBucketsReqsList()).hasSize(1); assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); - assertThat(service.drains).isEqualTo(1); } @Test @@ -251,18 +247,16 @@ void testInvalidAndMissingTablesDoNotSuppressOtherTopics() throws Exception { assertThat(result.topics().get(1).partitions().get(0).error()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION); assertThat(result.topics().get(2).partitions().get(0).error()).isEqualTo(Errors.NONE); - assertThat(service.drains).isEqualTo(1); } } @Test - void testNoAppendOrDrainWhenAllPartitionsFailConversion() throws Exception { + void testNoAppendWhenAllPartitionsFailConversion() throws Exception { TestingProduceService service = new TestingProduceService(); KafkaProduceResult result = backend(service).write(command((short) 1, topic("kafka.topic", good(8)))).join(); assertErrors(result, Errors.UNKNOWN_TOPIC_OR_PARTITION); assertThat(service.append).isNull(); - assertThat(service.drains).isZero(); } @Test @@ -316,7 +310,6 @@ void testAsyncMetadataCompletionRestoresSessionAndAcksZero() throws Exception { CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))).join(); assertErrors(result.join(), Errors.NONE); assertThat(service.append.getAcks()).isZero(); - assertThat(service.drains).isEqualTo(1); } @Test @@ -566,7 +559,6 @@ void testShutdownPreservesAlreadySubmittedAppendResult() throws Exception { assertErrors(completed, Errors.NONE); assertThat(completed.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); assertThat(service.appends).hasSize(1); - assertThat(service.drains).isEqualTo(1); } finally { executor.closeAsync().get(10, TimeUnit.SECONDS); } @@ -804,7 +796,6 @@ valid, good(0).records().get(0)))))) .join(), Errors.INVALID_RECORD); assertThat(service.appends).isEmpty(); - assertThat(service.drains).isZero(); } @Test @@ -944,7 +935,6 @@ private static class TestingProduceService extends TestingTabletGatewayService { private CompletableFuture pendingMetadata; private boolean throwAppend; private boolean asyncMissing; - private int drains; private int writeChecks; private boolean denyWrite; private CountDownLatch metadataEntered; @@ -1002,18 +992,13 @@ public CompletableFuture produceLog(ProduceLogRequest reques if (throwAppend) { throw new IllegalStateException("append failed"); } + if (appended != null) { + appended.countDown(); + } return pendingAppend == null ? CompletableFuture.completedFuture( success(request.getBucketsReqsList().get(0).getBucketId())) : pendingAppend; } - - @Override - public void tryCompleteActions() { - drains++; - if (appended != null) { - appended.countDown(); - } - } } } From 0a2b80c136813e117dbd4b3ed6451d36177abd27 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 17:30:26 +0800 Subject: [PATCH 29/69] [kafka] Persist null Produce values with native offsets Append null-valued records through the native Produce path, preserving record order and offsets. Remove the separate authorization shortcut for filtered records. Cover nullable and NOT NULL values with unit and native readback tests. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 54/54 AI-Contributed/UT: 243/271 --- .../produce/GatewayKafkaProduceBackend.java | 14 +- .../backend/produce/KafkaProduceCommand.java | 23 +-- .../kafka/transcode/KafkaTopicWritePlan.java | 2 +- .../fluss/kafka/KafkaProduceAppendITCase.java | 173 +++++++++++++----- .../GatewayKafkaProduceBackendTest.java | 70 +++++-- .../produce/KafkaProduceCommandTest.java | 27 +-- .../ArrowKafkaRecordTranscoderTest.java | 1 - .../apache/fluss/rpc/RpcGatewayService.java | 8 - .../fluss/server/tablet/TabletService.java | 7 +- 9 files changed, 203 insertions(+), 122 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index 8ab07a19de3..c5dd9638dea 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -188,17 +188,9 @@ private CompletableFuture produceTopic( continue; } try { - if (partition.hasNullValues() && partition.nonNullRecords().isEmpty()) { - setCurrentSession(command); - service.authorizeTableWrite(tableInfo.getTableId()); - failures.put( - partition.partitionId(), - new PartitionResult(partition.partitionId(), Errors.NONE, -1L, null)); - continue; - } BytesView records = transcoder.transcode( - partition.partitionId(), partition.nonNullRecords(), writePlan); + partition.partitionId(), partition.records(), writePlan); request.addBucketsReq() .setBucketId(partition.partitionId()) .setRecordsBytesView(records); @@ -286,9 +278,7 @@ private static TopicResult toTopicResult( new PartitionResult( partition.partitionId(), Errors.NONE, - !partition.hasNullValues() && bucket.hasBaseOffset() - ? bucket.getBaseOffset() - : -1L, + bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, null)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index c359c6f8b21..b0d83392d39 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -109,22 +109,11 @@ public List partitions() { public static final class PartitionWrite { private final int partitionId; private final List records; - private final List nonNullRecords; /** Creates the writes for one partition. */ public PartitionWrite(int partitionId, List records) { this.partitionId = partitionId; this.records = immutableCopy(records); - List retained = new ArrayList<>(); - for (Record record : this.records) { - if (record.borrowedValue() != null) { - retained.add(record); - } - } - this.nonNullRecords = - retained.size() == records.size() - ? this.records - : Collections.unmodifiableList(retained); } /** Returns the Kafka partition ID. */ @@ -132,17 +121,7 @@ public int partitionId() { return partitionId; } - /** Returns records with non-null values, preserving their original order. */ - public List nonNullRecords() { - return nonNullRecords; - } - - /** Whether filtering changed the correspondence between Kafka and native offsets. */ - public boolean hasNullValues() { - return records.size() != nonNullRecords.size(); - } - - /** Returns copied records in append order. */ + /** Returns copied records in append order, including records with null values. */ public List records() { return records; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java index c68736e4233..d297713d5ba 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -39,7 +39,7 @@ public final class KafkaTopicWritePlan { KafkaTopicWritePlan(TableInfo tableInfo, KafkaTableSchema topicSchema) { this.tableInfo = checkNotNull(tableInfo); this.topicSchema = checkNotNull(topicSchema); - // Validate support during preparation as filtered partitions may never call transcode. + // Validate supported value formats when compiling the plan. if (topicSchema.valueType() != ValueType.BINARY && topicSchema.valueType() != ValueType.STRING) { throw new KafkaRecordEncodingException( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index b331878c165..99614f0c138 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -149,36 +149,100 @@ void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception } } - @Test - void testNullValuesRemainFilteredWithoutAdvancingNativeOffsets() throws Exception { - // Preserve the existing filtering behavior until the FIP defines its null-value contract. - TablePath path = TablePath.of(DATABASE, "produce_filtered_nulls"); + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullValuesArePersistedForAllAckModes(String format) throws Exception { + List values = Arrays.asList(null, new byte[0], VALUE, null); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - admin.createTable(path, descriptor("binary"), false).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, "produce_null_" + format + "_" + acks); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + CLUSTER.waitUntilAllReplicaReady( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); + try (KafkaProducer producer = producer(acks, 0)) { + for (int i = 0; i < values.size(); i++) { + ProducerRecord record = + new ProducerRecord<>( + i % 2 == 0 ? path.toString() : path.getTableName(), + 0, + 123L, + KEY, + values.get(i), + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + RecordMetadata metadata = producer.send(record).get(30, TimeUnit.SECONDS); + if (!acks.equals("0")) { + assertThat(metadata.offset()).isEqualTo(i); + } + } + producer.flush(); + assertReadback(connection, path, format, values); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testAllNullAndMixedBatchesPreserveOffsets(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_null_batches_" + format); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.toString()}; + assertProduceOffsets(query(socket, recordBatch(null, null), names), names, 0L); + assertProduceOffsets( + query(socket, recordBatch(VALUE, null, new byte[0], null), names), + names, + 2L); + assertProduceOffsets(query(socket, names), names, 6L); + assertReadback( + connection, + path, + format, + Arrays.asList(null, null, VALUE, null, new byte[0], null, VALUE)); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNotNullValueRejectsWholeBatchWithoutAdvancingOffsets(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_not_null_" + format); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor(format, false), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); try (KafkaProducer producer = producer("all", 0)) { - ProducerRecord filtered = - new ProducerRecord(path.toString(), 0, 123L, KEY, null); - ProducerRecord retained = - new ProducerRecord<>( - path.toString(), - 0, - 123L, - KEY, - VALUE, - Arrays.asList( - new RecordHeader("source", VALUE), - new RecordHeader("source", null))); - assertThat(producer.send(filtered).get(30, TimeUnit.SECONDS).offset()) - .isEqualTo(-1L); - assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()).isZero(); - assertThat(producer.send(filtered).get(30, TimeUnit.SECONDS).offset()) - .isEqualTo(-1L); - assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()) - .isEqualTo(1L); - assertReadback(connection, path, "binary", VALUE); + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.toString()}; + assertProduceErrors( + query(socket, recordBatch(null, null), names), + names, + Errors.INVALID_RECORD); + assertProduceErrors( + query(socket, recordBatch(VALUE, null, VALUE), names), + names, + Errors.INVALID_RECORD); + assertProduceOffsets(query(socket, names), names, 0L); + assertReadback(connection, path, format, VALUE, 1); + } } finally { admin.dropTable(path, true).get(); } @@ -375,7 +439,27 @@ private static void setEnabled(TablePath path, boolean enabled) throws Exception .get(); } + private static MemoryRecords recordBatch(byte[]... values) { + SimpleRecord[] records = new SimpleRecord[values.length]; + for (int i = 0; i < values.length; i++) { + records[i] = + new SimpleRecord( + 123L, + KEY, + values[i], + new Header[] { + new RecordHeader("source", VALUE), new RecordHeader("source", null) + }); + } + return MemoryRecords.withRecords(Compression.NONE, records); + } + private static ProduceResponse query(Socket socket, String... names) throws Exception { + return query(socket, recordBatch(VALUE), names); + } + + private static ProduceResponse query(Socket socket, MemoryRecords records, String... names) + throws Exception { List topics = new ArrayList<>(); for (String name : names) { topics.add( @@ -385,21 +469,7 @@ private static ProduceResponse query(Socket socket, String... names) throws Exce Collections.singletonList( new PartitionProduceData() .setIndex(0) - .setRecords( - MemoryRecords.withRecords( - Compression.NONE, - new SimpleRecord( - 123L, - KEY, - VALUE, - new Header[] { - new RecordHeader( - "source", - VALUE), - new RecordHeader( - "source", - null) - })))))); + .setRecords(records)))); } ProduceRequest request = new ProduceRequest( @@ -471,16 +541,26 @@ private static void assertReadback( byte[] expectedValue, int expectedCount) throws Exception { + assertReadback(connection, path, format, Collections.nCopies(expectedCount, expectedValue)); + } + + private static void assertReadback( + Connection connection, TablePath path, String format, List expectedValues) + throws Exception { try (Table table = connection.getTable(path); LogScanner scanner = table.newScan().createLogScanner()) { scanner.subscribeFromBeginning(0); int count = 0; long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - while (count < expectedCount && System.nanoTime() < deadline) { + while (count < expectedValues.size() && System.nanoTime() < deadline) { ScanRecords records = scanner.poll(Duration.ofSeconds(1)); for (ScanRecord record : records) { + assertThat(record.logOffset()).isEqualTo(count); + byte[] expectedValue = expectedValues.get(count); InternalRow row = record.getRow(); - if (format.equals("binary")) { + if (expectedValue == null) { + assertThat(row.isNullAt(0)).isTrue(); + } else if (format.equals("binary")) { assertThat(row.getBytes(0)).isEqualTo(expectedValue); } else { assertThat(row.getString(0).toString()) @@ -500,18 +580,23 @@ private static void assertReadback( count++; } } - assertThat(count).as("native readback for %s", path).isEqualTo(expectedCount); + assertThat(count).as("native readback for %s", path).isEqualTo(expectedValues.size()); } } private static TableDescriptor descriptor(String format) { + return descriptor(format, true); + } + + private static TableDescriptor descriptor(String format, boolean nullableValue) { boolean binary = format.equals("binary"); return TableDescriptor.builder() .schema( Schema.newBuilder() .column( "_kafka_value", - binary ? DataTypes.BYTES() : DataTypes.STRING()) + (binary ? DataTypes.BYTES() : DataTypes.STRING()) + .copy(nullableValue)) .column( "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index 2d66b743601..d40c294f89c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -66,23 +66,26 @@ class GatewayKafkaProduceBackendTest { @Test - void testAllNullValuesRequireAuthorizationWithoutAppend() throws Exception { + void testAllNullValuesUseNativeAppendAndAuthorization() throws Exception { TestingProduceService service = new TestingProduceService(); KafkaProduceResult result = backend(service) .write(command((short) 1, topic("kafka.topic", nullValue(0), nullValue(8)))) .join(); assertErrors(result, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); - assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); assertThat(service.writeChecks).isEqualTo(1); - assertThat(service.append).isNull(); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isZero(); service.denyWrite = true; assertErrors( backend(service) .write(command((short) 1, topic("kafka.topic", nullValue(0)))) .join(), Errors.TOPIC_AUTHORIZATION_FAILED); - assertThat(service.append).isNull(); + assertThat(service.appends).hasSize(1); + assertThat(service.writeChecks).isEqualTo(2); } @ParameterizedTest @@ -138,7 +141,7 @@ void testNullValuesCannotBypassTableAndSchemaValidation() throws Exception { } @Test - void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exception { + void testMixedNullValuesPreserveRecordOrderAndNativeOffsets() throws Exception { TestingProduceService service = new TestingProduceService(); List seen = new ArrayList<>(); ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); @@ -151,6 +154,7 @@ void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exceptio seen.addAll(records); return transcoder.transcode(partitionId, records, table); }); + Record nullRecord = nullValue(0).records().get(0); Record empty = new Record(1L, null, new byte[0], Collections.emptyList()); Record valid = good(0).records().get(0); KafkaProduceResult result = @@ -161,14 +165,11 @@ void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exceptio "kafka.topic", new PartitionWrite( 0, - Arrays.asList( - nullValue(0).records().get(0), - empty, - valid))))) + Arrays.asList(nullRecord, empty, valid))))) .join(); assertErrors(result, Errors.NONE); - assertThat(seen).containsExactly(empty, valid); - assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); + assertThat(seen).containsExactly(nullRecord, empty, valid); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); } private static PartitionWrite nullValue(int partition) { @@ -798,8 +799,48 @@ valid, good(0).records().get(0)))))) assertThat(service.appends).isEmpty(); } + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullValueViolationRejectsWholePartitionWithoutAppendingPrefix(String format) + throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + TableDescriptor notNull = + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + (format.equals("binary") + ? DataTypes.BYTES() + : DataTypes.STRING()) + .copy(false)) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(notNull.toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + + Record valid = good(0).records().get(0); + PartitionWrite invalid = + new PartitionWrite(0, Arrays.asList(valid, nullValue(0).records().get(0), valid)); + assertErrors( + backend.write(command((short) 1, topic("topic", invalid, good(1)))).join(), + Errors.INVALID_RECORD, + Errors.NONE); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + } + @Test - void testMetadataOffsetsAndJsonCannotBeActivatedByFilteredRecords() throws Exception { + void testMetadataOffsetsAndJsonCannotBeActivatedByNullValues() throws Exception { TestingProduceService service = new TestingProduceService(); TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); TableDescriptor unsupportedMetadata = @@ -969,8 +1010,7 @@ public CompletableFuture getTableInfo(GetTableInfoRequest : pendingMetadata; } - @Override - public void authorizeTableWrite(long tableId) { + private void checkWriteAuthorization(long tableId) { assertThat(tableId).isPositive(); assertThat(currentListenerName()).isEqualTo("KAFKA"); assertThat(currentSession().getInetAddress()) @@ -983,7 +1023,7 @@ public void authorizeTableWrite(long tableId) { @Override public CompletableFuture produceLog(ProduceLogRequest request) { - authorizeTableWrite(request.getTableId()); + checkWriteAuthorization(request.getTableId()); assertThat(currentListenerName()).isEqualTo("KAFKA"); assertThat(currentSession().getInetAddress()) .isEqualTo(InetAddress.getLoopbackAddress()); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java index e3624f5535b..6243bb8a900 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -163,8 +163,8 @@ void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { } @Test - void testNullFilteringPreservesRecordIdentityAndTimestampProvenance() { - Record filtered = + void testNullValuesPreserveRecordIdentityAndTimestampProvenance() { + Record nullRecord = new Record( 123L, TimestampType.CREATE_TIME, @@ -185,21 +185,22 @@ void testNullFilteringPreservesRecordIdentityAndTimestampProvenance() { null, new byte[0], Collections.emptyList()); - List source = new ArrayList<>(Arrays.asList(filtered, retained, empty)); + List source = new ArrayList<>(Arrays.asList(nullRecord, retained, empty)); PartitionWrite partition = new PartitionWrite(0, source); source.clear(); - assertThat(partition.records()).containsExactly(filtered, retained, empty); - assertThat(partition.hasNullValues()).isTrue(); - assertThat(partition.nonNullRecords()).containsExactly(retained, empty); - assertThat(partition.nonNullRecords().get(0)).isSameAs(retained); - assertThat(partition.nonNullRecords().get(0).timestampType()) + assertThat(partition.records()).containsExactly(nullRecord, retained, empty); + assertThat(partition.records().get(0)).isSameAs(nullRecord); + assertThat(partition.records().get(0).value()).isNull(); + assertThat(partition.records().get(0).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.records().get(0).timestamp()).isEqualTo(123L); + assertThat(partition.records().get(1)).isSameAs(retained); + assertThat(partition.records().get(1).timestampType()) .isEqualTo(TimestampType.LOG_APPEND_TIME); - assertThat(partition.nonNullRecords().get(0).timestamp()).isEqualTo(456L); - assertThat(partition.nonNullRecords().get(1).timestampType()) - .isEqualTo(TimestampType.CREATE_TIME); - assertThat(partition.nonNullRecords().get(1).hasTimestamp()).isFalse(); - assertThatThrownBy(() -> partition.nonNullRecords().clear()) + assertThat(partition.records().get(1).timestamp()).isEqualTo(456L); + assertThat(partition.records().get(2).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.records().get(2).hasTimestamp()).isFalse(); + assertThatThrownBy(() -> partition.records().clear()) .isInstanceOf(UnsupportedOperationException.class); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index f039715fb10..06a3841fc7b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -131,7 +131,6 @@ void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) thro @ValueSource(booleans = {false, true}) void testNullAndEmptyPayloadsRemainDistinct(boolean string) throws Exception { TableInfo table = envelope(string, 7); - // This converter retains its existing nullability checks; ingress filtering is separate. List records = Arrays.asList( new Record(Record.NO_TIMESTAMP, null, null, Collections.emptyList()), diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java index 1dc4a0da4c9..8f2365add70 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java @@ -77,14 +77,6 @@ public String currentListenerName() { } } - /** - * Authorizes a table write that may not append any records, using the current request session. - * Services must explicitly support this operation; the default never grants write access. - */ - public void authorizeTableWrite(long tableId) { - throw new UnsupportedOperationException("This service cannot authorize table writes."); - } - /** Returns the provider type of this RPC gateway service. */ public abstract ServerType providerType(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index a874c516878..da73530b13f 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -234,14 +234,9 @@ public String name() { @Override public void shutdown() {} - @Override - public void authorizeTableWrite(long tableId) { - authorizeTable(WRITE, tableId); - } - @Override public CompletableFuture produceLog(ProduceLogRequest request) { - authorizeTableWrite(request.getTableId()); + authorizeTable(WRITE, request.getTableId()); long tableId = request.getTableId(); Map routingErrors = new HashMap<>(); collectRoutingErrors( From fdaba29401b4ca7d4377915d9866d35dc2764c60 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 12:54:26 +0800 Subject: [PATCH 30/69] [kafka] Align Produce plan with current metadata API Update cached Produce write plans and tests to use the current typed schema conversion options and validate persisted table contracts before appending. Co-Authored-By: Codex AI-Model: gpt-6.1-sol AI-Contributed/Feature: 13/13 AI-Contributed/UT: 81/81 --- .../transcode/ArrowKafkaRecordTranscoder.java | 4 +- .../kafka/transcode/KafkaTopicWritePlan.java | 9 +-- .../fluss/kafka/KafkaProduceAppendITCase.java | 7 ++- .../GatewayKafkaProduceBackendTest.java | 62 ++++++++++++------- .../produce/KafkaTableInfoCacheTest.java | 5 +- .../KafkaCompiledWritePlanCacheTest.java | 7 ++- 6 files changed, 61 insertions(+), 33 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index e24a38ea652..0f84ddf7932 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -59,8 +59,8 @@ public BytesView transcode(int partitionId, List records, TableInfo tabl } @Override - public BytesView transcode( - int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { + public BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTableSchema schema = plan.topicSchema(); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java index d297713d5ba..b8855ba45ac 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -18,8 +18,8 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.metadata.KafkaTableSchema; -import org.apache.fluss.metadata.KafkaTableSchema.ValueType; import org.apache.fluss.metadata.TableInfo; import javax.annotation.concurrent.ThreadSafe; @@ -40,10 +40,11 @@ public final class KafkaTopicWritePlan { this.tableInfo = checkNotNull(tableInfo); this.topicSchema = checkNotNull(topicSchema); // Validate supported value formats when compiling the plan. - if (topicSchema.valueType() != ValueType.BINARY - && topicSchema.valueType() != ValueType.STRING) { + if (topicSchema.valueConversionType() != KafkaValueConversionType.BINARY + && topicSchema.valueConversionType() != KafkaValueConversionType.STRING) { throw new KafkaRecordEncodingException( - "Unsupported Kafka value conversion type: " + topicSchema.valueType()); + "Unsupported Kafka value conversion type: " + + topicSchema.valueConversionType()); } this.rowAssembler = new KafkaRowAssembler(topicSchema); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 99614f0c138..fbbf6b1c8ce 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -26,6 +26,7 @@ import org.apache.fluss.cluster.ServerNode; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; @@ -613,7 +614,11 @@ private static TableDescriptor descriptor(String format, boolean nullableValue) .distributedBy(1) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + format.equals("binary") + ? KafkaValueConversionType.BINARY + : KafkaValueConversionType.STRING) .build(); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index d40c294f89c..6d47af74a6e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -19,6 +19,8 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaKeyConversionType; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.exception.TableNotExistException; @@ -507,7 +509,9 @@ void testOffloadedMetadataRefreshUsesNewTableIdentityAndSchema() throws Exceptio .distributedBy(3) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) .build(); service.pendingMetadata = CompletableFuture.completedFuture( @@ -684,17 +688,25 @@ void testPropertyOnlyChangesRejectInvalidContractsAndRecompileRecoveredPlan() th new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); assertErrors( backend.write(command((short) 1, topic("topic", good(0)))).join(), Errors.NONE); - TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + TableDescriptor original = + TableDescriptor.builder( + TableDescriptor.fromJsonBytes(metadata(false).getTableJson())) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES()) + .build()) + .build(); for (TableDescriptor invalid : Arrays.asList( TableDescriptor.builder(original) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") - .build(), - TableDescriptor.builder(original) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) .build(), TableDescriptor.builder(original) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "string") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), "string") .build())) { service.pendingMetadata = CompletableFuture.completedFuture( @@ -713,9 +725,8 @@ void testPropertyOnlyChangesRejectInvalidContractsAndRecompileRecoveredPlan() th .setTableJson( TableDescriptor.builder(original) .property( - ConfigOptions - .TABLE_KAFKA_CONVERT_VALUE_TYPE, - "STRING") + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + KafkaKeyConversionType.BINARY) .build() .toJsonBytes())); assertErrors( @@ -725,7 +736,7 @@ void testPropertyOnlyChangesRejectInvalidContractsAndRecompileRecoveredPlan() th assertThat(transcoder.prepared.get(1)).isNotSameAs(transcoder.prepared.get(0)); assertThat(transcoder.prepared.get(1).tableInfo().getSchemaId()).isEqualTo(1); assertThat(service.appends).hasSize(2); - assertThat(service.metadataPaths).hasSize(8); + assertThat(service.metadataPaths).hasSize(6); } @Test @@ -762,7 +773,7 @@ void testMissingDefaultAndExplicitDatabasesDoNotFallback() throws Exception { } @Test - void testRecordNullabilityFailureDoesNotAppendValidPrefix() throws Exception { + void testNotNullKeyContractRejectsBeforeAppendingAnyRecords() throws Exception { TestingProduceService service = new TestingProduceService(); TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); service.pendingMetadata = @@ -795,13 +806,13 @@ void testRecordNullabilityFailureDoesNotAppendValidPrefix() throws Exception { Arrays.asList( valid, good(0).records().get(0)))))) .join(), - Errors.INVALID_RECORD); + Errors.INVALID_TOPIC_EXCEPTION); assertThat(service.appends).isEmpty(); } @ParameterizedTest @ValueSource(strings = {"binary", "string"}) - void testNullValueViolationRejectsWholePartitionWithoutAppendingPrefix(String format) + void testNullValueViolationRejectsNotNullTableContractBeforeAppending(String format) throws Exception { TestingProduceService service = new TestingProduceService(); TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); @@ -816,7 +827,11 @@ void testNullValueViolationRejectsWholePartitionWithoutAppendingPrefix(String fo : DataTypes.STRING()) .copy(false)) .build()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + format.equals("binary") + ? KafkaValueConversionType.BINARY + : KafkaValueConversionType.STRING) .build(); service.pendingMetadata = CompletableFuture.completedFuture( @@ -824,7 +839,7 @@ void testNullValueViolationRejectsWholePartitionWithoutAppendingPrefix(String fo GatewayKafkaProduceBackend backend = backend(service); assertErrors( backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), - Errors.INVALID_RECORD); + Errors.INVALID_TOPIC_EXCEPTION); assertThat(service.appends).isEmpty(); Record valid = good(0).records().get(0); @@ -832,11 +847,9 @@ void testNullValueViolationRejectsWholePartitionWithoutAppendingPrefix(String fo new PartitionWrite(0, Arrays.asList(valid, nullValue(0).records().get(0), valid)); assertErrors( backend.write(command((short) 1, topic("topic", invalid, good(1)))).join(), - Errors.INVALID_RECORD, - Errors.NONE); - assertThat(service.appends).hasSize(1); - assertThat(service.append.getBucketsReqsList()).hasSize(1); - assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.appends).isEmpty(); } @Test @@ -872,7 +885,9 @@ void testMetadataOffsetsAndJsonCannotBeActivatedByNullValues() throws Exception TableDescriptor json = TableDescriptor.builder(original) .schema(Schema.newBuilder().column("business", DataTypes.STRING()).build()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.PARSE_JSON) .build(); service.pendingMetadata = CompletableFuture.completedFuture(metadata(false).setTableJson(json.toJsonBytes())); @@ -939,7 +954,8 @@ private static GetTableInfoResponse metadata(boolean invalid) { .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true); if (!invalid) { - descriptor.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "string"); + descriptor.property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, KafkaValueConversionType.STRING); } return new GetTableInfoResponse() .setTableId(42L) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java index 3ee3a125495..b6c8fce097a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.backend.produce; import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; @@ -262,7 +263,9 @@ private static TableDescriptor descriptor(String version) { .distributedBy(1) .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY); if ("v2".equals(version)) { builder.property(ConfigOptions.TABLE_KAFKA_ENABLED, false); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java index ac950afeeeb..f5beaae2b61 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.metadata.KafkaTableSchema; @@ -265,9 +266,11 @@ private static TableInfo tableInfo( .distributedBy(1) .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") .property( - ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), propertyValue.equals("v1") ? "binary" : "BINARY") .build(); return TableInfo.of( From 8ca282550c853d274706bf49de124bf235dcdcd9 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 13:08:18 +0800 Subject: [PATCH 31/69] [kafka] Drain Produce RPCs before closing event loops Keep Netty event loops active until accepted requests and channels have drained, then close protocol workers. Update the integration test to assert the current nullable Kafka schema contract at table creation. Co-Authored-By: Codex AI-Model: gpt-6.1-sol AI-Contributed/Feature: 26/26 AI-Contributed/UT: 29/29 --- .../fluss/kafka/KafkaProduceAppendITCase.java | 29 +++++-------------- .../fluss/rpc/netty/server/NettyServer.java | 26 ++++++++++++----- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index fbbf6b1c8ce..2a34f8057ad 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -27,6 +27,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; @@ -82,6 +83,7 @@ import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Exercises native DDL, Kafka discovery and Produce, replication, and native Fluss readback. */ class KafkaProduceAppendITCase { @@ -222,31 +224,16 @@ void testAllNullAndMixedBatchesPreserveOffsets(String format) throws Exception { @ParameterizedTest @ValueSource(strings = {"binary", "string"}) - void testNotNullValueRejectsWholeBatchWithoutAdvancingOffsets(String format) throws Exception { + void testNotNullValueSchemaIsRejectedWhenCreatingKafkaTable(String format) throws Exception { TablePath path = TablePath.of(DATABASE, "produce_not_null_" + format); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - admin.createTable(path, descriptor(format, false), false).get(); - CLUSTER.waitUntilAllGatewayHasSameMetadata(); - try (KafkaProducer producer = producer("all", 0)) { - Node leader = producer.partitionsFor(path.toString()).get(0).leader(); - try (Socket socket = new Socket(leader.host(), leader.port())) { - String[] names = {path.toString()}; - assertProduceErrors( - query(socket, recordBatch(null, null), names), - names, - Errors.INVALID_RECORD); - assertProduceErrors( - query(socket, recordBatch(VALUE, null, VALUE), names), - names, - Errors.INVALID_RECORD); - assertProduceOffsets(query(socket, names), names, 0L); - assertReadback(connection, path, format, VALUE, 1); - } - } finally { - admin.dropTable(path, true).get(); - } + assertThatThrownBy( + () -> admin.createTable(path, descriptor(format, false), false).get()) + .hasCauseInstanceOf(InvalidTableException.class) + .rootCause() + .hasMessageContaining("Kafka requires nullable fields"); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java index 37705f2e075..7b2af609728 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java @@ -306,15 +306,25 @@ public CompletableFuture closeAsync() { } isRunning = false; + CompletableFuture channelShutdownFuture = + FutureUtils.completeAll( + bindChannels.stream() + .map(NettyUtils::shutdownChannel) + .collect(Collectors.toList())); + CompletableFuture workerShutdownFuture = + workerPool == null + ? CompletableFuture.completedFuture(null) + : workerPool.closeAsync(); + // Event loops must remain active while accepted requests drain: their responses and + // backpressure updates still use those loops. Close the loops only after both the + // listening channels and request workers have stopped. CompletableFuture transportShutdownFuture = - CompletableFuture.allOf( - shutdownGroup(acceptorGroup), - shutdownGroup(selectorGroup), - FutureUtils.completeAll( - bindChannels.stream() - .map(NettyUtils::shutdownChannel) - .collect(Collectors.toList())), - workerPool.closeAsync()); + FutureUtils.composeAfterwards( + CompletableFuture.allOf(channelShutdownFuture, workerShutdownFuture), + () -> + CompletableFuture.allOf( + shutdownGroup(acceptorGroup), + shutdownGroup(selectorGroup))); closeFuture = closeProtocolsAfter(transportShutdownFuture, protocols); return closeFuture; } From 06cec447e1387faf9e5a1dd5047c887e6b723456 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 11 Sep 2026 14:44:26 +0800 Subject: [PATCH 32/69] [kafka] Cover Produce lifecycle and replication regressions Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 364/364 --- .../kafka/KafkaProduceRegressionITCase.java | 364 ++++++++++++++++++ 1 file changed, 364 insertions(+) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java new file mode 100644 index 00000000000..0748bd81503 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.cluster.AlterConfig; +import org.apache.fluss.config.cluster.AlterConfigOpType; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.NotEnoughReplicasException; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.tuple; + +/** Regression coverage for native DDL lifecycle, metadata refresh, and Produce minimum ISR. */ +class KafkaProduceRegressionITCase { + private static final String DATABASE = "kafka"; + private static final short METADATA_VERSION = 11; + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { + String topic = "metadata_lifecycle"; + TablePath path = TablePath.of(DATABASE, topic); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, topic, 3); + MetadataResponse initial = waitForTopic(topic); + Uuid initialId = initial.data().topics().find(topic).topicId(); + assertThat(initialId).isNotEqualTo(Uuid.ZERO_UUID); + assertKafkaListenerEndpoints(initial); + assertThat(fetchMetadata(topic).data().topics().find(topic).topicId()) + .isEqualTo(initialId); + + admin.dropTable(path, false).get(); + retry( + Duration.ofMinutes(1), + () -> + assertThat(fetchMetadata(topic).errors()) + .containsEntry(topic, Errors.UNKNOWN_TOPIC_OR_PARTITION)); + + createTable(admin, topic, 3); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponseTopic recreated = + fetchMetadata(topic).data().topics().find(topic); + assertThat(recreated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(recreated.topicId()) + .isNotEqualTo(Uuid.ZERO_UUID) + .isNotEqualTo(initialId); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { + String topic = "metadata_failover"; + int stoppedLeader = -1; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, topic, 3); + MetadataResponsePartition initial = + waitForTopic(topic).data().topics().find(topic).partitions().get(0); + stoppedLeader = initial.leaderId(); + try (KafkaProducer producer = producer("1", true)) { + assertThat(send(producer, topic, "before_failover").offset()).isNotNegative(); + CLUSTER.stopTabletServer(stoppedLeader); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponseTopic metadata = + fetchMetadata(topic).data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = metadata.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(partition.leaderId()) + .isNotEqualTo(-1) + .isNotEqualTo(initial.leaderId()); + assertThat(partition.leaderEpoch()) + .isGreaterThan(initial.leaderEpoch()); + }); + // Reuse the producer's cached route so the failed request triggers a refresh. + // acks=1 permits data loss and offset reuse after failover; this verifies + // continued routing, not durability of the first acknowledged record. + assertThat(send(producer, topic, "after_failover").offset()).isNotNegative(); + } + } finally { + try { + if (stoppedLeader >= 0 && CLUSTER.getTabletServerById(stoppedLeader) == null) { + CLUSTER.startTabletServer(stoppedLeader); + CLUSTER.assertHasTabletServerNumber(3); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + } + } finally { + admin.dropTable(TablePath.of(DATABASE, topic), true).get(); + } + } + } + } + + @Test + void testClusterMinIsrPolicyAndDynamicReload() throws Exception { + String replicated = "acks_all_replicated"; + String underReplicated = "acks_all_under_replicated"; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, replicated, 3); + createTable(admin, underReplicated, 1); + try (KafkaProducer producer = producer("all", false)) { + assertThat(send(producer, replicated, "replicated").offset()).isZero(); + assertThatThrownBy(() -> send(producer, underReplicated, "rejected")) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(NotEnoughReplicasException.class); + + setMinIsr(admin, 1); + assertThat(send(producer, underReplicated, "accepted").offset()).isZero(); + // Offset zero and native readback also prove the rejected write was not stored. + assertSingleValue(connection, underReplicated, "accepted"); + } + } finally { + try { + setMinIsr(admin, 2); + } finally { + admin.dropTable(TablePath.of(DATABASE, replicated), true).get(); + admin.dropTable(TablePath.of(DATABASE, underReplicated), true).get(); + } + } + } + } + + private static void createTable(Admin admin, String topic, int replicas) throws Exception { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + TablePath path = TablePath.of(DATABASE, topic); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + } + + private static void setMinIsr(Admin admin, int minIsr) throws Exception { + admin.alterClusterConfigs( + Collections.singletonList( + new AlterConfig( + ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER.key(), + Integer.toString(minIsr), + AlterConfigOpType.SET))) + .get(30, TimeUnit.SECONDS); + retry( + Duration.ofMinutes(1), + () -> + assertThat(CLUSTER.getTabletServers()) + .allSatisfy( + server -> + assertThat( + server.getReplicaManager() + .getMinInSyncReplicas()) + .isEqualTo(minIsr))); + } + + private static void assertSingleValue(Connection connection, String topic, String expected) + throws Exception { + try (Table table = connection.getTable(TablePath.of(DATABASE, topic)); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + retry( + Duration.ofSeconds(30), + () -> { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + assertThat(records).hasSize(1); + assertThat(records.iterator().next().getRow().getBytes(0)) + .isEqualTo(expected.getBytes(StandardCharsets.UTF_8)); + }); + } + } + + private static MetadataResponse waitForTopic(String topic) { + AtomicReference result = new AtomicReference<>(); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponse response = fetchMetadata(topic); + MetadataResponseTopic metadata = response.data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(metadata.partitions()).hasSize(1); + assertThat(metadata.partitions().get(0).leaderId()).isNotEqualTo(-1); + result.set(response); + }); + return result.get(); + } + + private static void assertKafkaListenerEndpoints(MetadataResponse response) { + assertThat(response.brokers()) + .extracting( + org.apache.kafka.common.Node::id, + org.apache.kafka.common.Node::host, + org.apache.kafka.common.Node::port, + org.apache.kafka.common.Node::rack) + .containsExactlyInAnyOrderElementsOf( + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map( + node -> + tuple( + node.id(), + node.host(), + node.port(), + node.rack())) + .collect(Collectors.toList())); + } + + private static MetadataResponse fetchMetadata(String topic) throws Exception { + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList(topic), false) + .build(METADATA_VERSION); + RequestHeader header = + new RequestHeader(ApiKeys.METADATA, METADATA_VERSION, "regression-test", 1); + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + byte[] requestBytes = new byte[serialized.remaining()]; + serialized.get(requestBytes); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(node.host(), node.port()), 5000); + socket.setSoTimeout(5000); + try (DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream())) { + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + int size = input.readInt(); + assertThat(size).isBetween(1, 1024 * 1024); + byte[] response = new byte[size]; + input.readFully(response); + return (MetadataResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + } + } + + private static RecordMetadata send( + KafkaProducer producer, String topic, String value) throws Exception { + return producer.send( + new ProducerRecord<>( + topic, 0, null, value.getBytes(StandardCharsets.UTF_8))) + .get(60, TimeUnit.SECONDS); + } + + private static KafkaProducer producer(String acks, boolean retryOnFailure) { + Map config = new HashMap<>(); + config.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map(node -> node.host() + ":" + node.port()) + .collect(Collectors.joining(","))); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.RETRIES_CONFIG, retryOnFailure ? 20 : 0); + config.put(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, 200); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} From 16198b6a80bf4ab425f3c06418623b796ab01207 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 18 Sep 2026 10:34:24 +0800 Subject: [PATCH 33/69] [kafka] Cover online leader handoff and qualified topics Adapt Produce regressions to database-qualified topic names from PR07. Exercise a real leader handoff while the old node stays online, verify NOT_LEADER_OR_FOLLOWER on the existing connection, and confirm the same producer refreshes its route without disconnecting. Check acks=all offsets and native readback across the handoff. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 285/285 --- .../kafka/KafkaProduceRegressionITCase.java | 285 +++++++++++++++--- 1 file changed, 238 insertions(+), 47 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java index 0748bd81503..32b0f2a2ced 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -21,9 +21,13 @@ import org.apache.fluss.client.ConnectionFactory; import org.apache.fluss.client.admin.Admin; import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; import org.apache.fluss.client.table.scanner.log.LogScanner; import org.apache.fluss.client.table.scanner.log.ScanRecords; import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.cluster.rebalance.RebalancePlanForBucket; +import org.apache.fluss.cluster.rebalance.RebalanceProgress; +import org.apache.fluss.cluster.rebalance.RebalanceStatus; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.cluster.AlterConfig; @@ -32,9 +36,12 @@ import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.coordinator.event.RecoverRebalanceEvent; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.data.RebalanceTask; import org.apache.fluss.types.DataTypes; import org.apache.kafka.clients.producer.KafkaProducer; @@ -42,14 +49,20 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.compress.Compression; import org.apache.kafka.common.errors.NotEnoughReplicasException; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -63,8 +76,10 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -91,12 +106,12 @@ class KafkaProduceRegressionITCase { @Test void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { - String topic = "metadata_lifecycle"; - TablePath path = TablePath.of(DATABASE, topic); + TablePath path = TablePath.of(DATABASE, "metadata_lifecycle"); + String topic = path.toString(); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { - createTable(admin, topic, 3); + createTable(admin, path, 3); MetadataResponse initial = waitForTopic(topic); Uuid initialId = initial.data().topics().find(topic).topicId(); assertThat(initialId).isNotEqualTo(Uuid.ZERO_UUID); @@ -111,7 +126,7 @@ void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { assertThat(fetchMetadata(topic).errors()) .containsEntry(topic, Errors.UNKNOWN_TOPIC_OR_PARTITION)); - createTable(admin, topic, 3); + createTable(admin, path, 3); retry( Duration.ofMinutes(1), () -> { @@ -130,12 +145,13 @@ void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { @Test void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { - String topic = "metadata_failover"; + TablePath path = TablePath.of(DATABASE, "metadata_failover"); + String topic = path.toString(); int stoppedLeader = -1; try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { - createTable(admin, topic, 3); + createTable(admin, path, 3); MetadataResponsePartition initial = waitForTopic(topic).data().topics().find(topic).partitions().get(0); stoppedLeader = initial.leaderId(); @@ -156,7 +172,7 @@ void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { assertThat(partition.leaderEpoch()) .isGreaterThan(initial.leaderEpoch()); }); - // Reuse the producer's cached route so the failed request triggers a refresh. + // Reuse the same producer to verify routing recovers after leader disconnect. // acks=1 permits data loss and offset reuse after failover; this verifies // continued routing, not durability of the first acknowledged record. assertThat(send(producer, topic, "after_failover").offset()).isNotNegative(); @@ -169,7 +185,104 @@ void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { CLUSTER.waitUntilAllGatewayHasSameMetadata(); } } finally { - admin.dropTable(TablePath.of(DATABASE, topic), true).get(); + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { + TablePath path = TablePath.of(DATABASE, "metadata_online_leader_change"); + String topic = path.toString(); + String rebalanceId = "kafka-online-leader-change"; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + TableBucket bucket = + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0); + MetadataResponsePartition initial = + waitForTopic(topic).data().topics().find(topic).partitions().get(0); + int oldLeader = initial.leaderId(); + int newLeader = + initial.replicaNodes().stream() + .filter(id -> id != oldLeader) + .findFirst() + .get(); + List nodes = CLUSTER.getTabletServerNodes("KAFKA"); + ServerNode oldNode = + nodes.stream().filter(node -> node.id() == oldLeader).findFirst().get(); + + // Keep this socket open across the handoff to verify the old leader stays + // reachable. + try (Socket oldLeaderSocket = connect(oldNode); + KafkaProducer producer = producer("all", true)) { + MetadataResponse before = + (MetadataResponse) sendRequest(oldLeaderSocket, metadataRequest(topic)); + assertThat(before.data().topics().find(topic).partitions().get(0).leaderId()) + .isEqualTo(oldLeader); + assertThat(send(producer, topic, "before_handoff").offset()).isZero(); + CLUSTER.waitUntilReplicaExpandToIsr(bucket, newLeader); + double retriesBefore = producerMetric(producer, "record-retry-total"); + double disconnectsBefore = producerMetric(producer, "connection-close-total"); + + rebalanceLeader(bucket, initial, newLeader, rebalanceId); + retry( + Duration.ofMinutes(1), + () -> { + assertThat(admin.listRebalanceProgress(rebalanceId).get()) + .isPresent() + .get() + .extracting(RebalanceProgress::status) + .isEqualTo(RebalanceStatus.COMPLETED); + MetadataResponsePartition updated = + fetchMetadata(topic) + .data() + .topics() + .find(topic) + .partitions() + .get(0); + assertThat(updated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(updated.leaderId()).isEqualTo(newLeader); + assertThat(updated.leaderEpoch()) + .isGreaterThan(initial.leaderEpoch()); + }); + CLUSTER.waitAndGetFollowerReplica(bucket, oldLeader); + assertThat(CLUSTER.getTabletServerNodes("KAFKA")) + .containsExactlyInAnyOrderElementsOf(nodes); + + ProduceResponse rejected = + (ProduceResponse) + sendRequest(oldLeaderSocket, produceRequest(topic, "rejected")); + assertThat(rejected.data().responses().find(topic).partitionResponses()) + .singleElement() + .satisfies( + partition -> { + assertThat(partition.index()).isZero(); + assertThat(partition.errorCode()) + .isEqualTo(Errors.NOT_LEADER_OR_FOLLOWER.code()); + assertThat(partition.baseOffset()).isEqualTo(-1L); + }); + + // partitionsFor uses the already populated producer cache. Separate raw + // Metadata requests above must not refresh this route before the next send. + assertThat(producer.partitionsFor(topic).get(0).leader().id()) + .isEqualTo(oldLeader); + assertThat(send(producer, topic, "after_handoff").offset()).isEqualTo(1L); + assertThat(producerMetric(producer, "record-retry-total")) + .isGreaterThan(retriesBefore); + assertThat(producerMetric(producer, "connection-close-total")) + .isEqualTo(disconnectsBefore); + assertThat(producer.partitionsFor(topic).get(0).leader().id()) + .isEqualTo(newLeader); + assertValues(connection, path, "before_handoff", "after_handoff"); + } + } finally { + try { + admin.cancelRebalance(rebalanceId).get(); + } finally { + admin.dropTable(path, true).get(); } } } @@ -177,38 +290,39 @@ void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { @Test void testClusterMinIsrPolicyAndDynamicReload() throws Exception { - String replicated = "acks_all_replicated"; - String underReplicated = "acks_all_under_replicated"; + TablePath replicated = TablePath.of(DATABASE, "acks_all_replicated"); + TablePath underReplicated = TablePath.of(DATABASE, "acks_all_under_replicated"); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { createTable(admin, replicated, 3); createTable(admin, underReplicated, 1); try (KafkaProducer producer = producer("all", false)) { - assertThat(send(producer, replicated, "replicated").offset()).isZero(); - assertThatThrownBy(() -> send(producer, underReplicated, "rejected")) + assertThat(send(producer, replicated.toString(), "replicated").offset()) + .isZero(); + assertThatThrownBy(() -> send(producer, underReplicated.toString(), "rejected")) .isInstanceOf(ExecutionException.class) .hasCauseInstanceOf(NotEnoughReplicasException.class); setMinIsr(admin, 1); - assertThat(send(producer, underReplicated, "accepted").offset()).isZero(); + assertThat(send(producer, underReplicated.toString(), "accepted").offset()) + .isZero(); // Offset zero and native readback also prove the rejected write was not stored. - assertSingleValue(connection, underReplicated, "accepted"); + assertValues(connection, underReplicated, "accepted"); } } finally { try { setMinIsr(admin, 2); } finally { - admin.dropTable(TablePath.of(DATABASE, replicated), true).get(); - admin.dropTable(TablePath.of(DATABASE, underReplicated), true).get(); + admin.dropTable(replicated, true).get(); + admin.dropTable(underReplicated, true).get(); } } } } - private static void createTable(Admin admin, String topic, int replicas) throws Exception { - admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - TablePath path = TablePath.of(DATABASE, topic); + private static void createTable(Admin admin, TablePath path, int replicas) throws Exception { + admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); admin.createTable( path, TableDescriptor.builder() @@ -246,18 +360,22 @@ private static void setMinIsr(Admin admin, int minIsr) throws Exception { .isEqualTo(minIsr))); } - private static void assertSingleValue(Connection connection, String topic, String expected) + private static void assertValues(Connection connection, TablePath path, String... expected) throws Exception { - try (Table table = connection.getTable(TablePath.of(DATABASE, topic)); + try (Table table = connection.getTable(path); LogScanner scanner = table.newScan().createLogScanner()) { scanner.subscribeFromBeginning(0); + List values = new ArrayList<>(); retry( Duration.ofSeconds(30), () -> { ScanRecords records = scanner.poll(Duration.ofSeconds(1)); - assertThat(records).hasSize(1); - assertThat(records.iterator().next().getRow().getBytes(0)) - .isEqualTo(expected.getBytes(StandardCharsets.UTF_8)); + for (ScanRecord record : records) { + values.add( + new String( + record.getRow().getBytes(0), StandardCharsets.UTF_8)); + } + assertThat(values).containsExactly(expected); }); } } @@ -297,33 +415,106 @@ private static void assertKafkaListenerEndpoints(MetadataResponse response) { } private static MetadataResponse fetchMetadata(String topic) throws Exception { - MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList(topic), false) - .build(METADATA_VERSION); + try (Socket socket = connect(CLUSTER.getTabletServerNodes("KAFKA").get(0))) { + return (MetadataResponse) sendRequest(socket, metadataRequest(topic)); + } + } + + private static MetadataRequest metadataRequest(String topic) { + return new MetadataRequest.Builder(Collections.singletonList(topic), false) + .build(METADATA_VERSION); + } + + private static ProduceRequest produceRequest(String topic, String value) { + MemoryRecords records = + MemoryRecords.withRecords( + Compression.NONE, new SimpleRecord(value.getBytes(StandardCharsets.UTF_8))); + ProduceRequestData.TopicProduceData topicData = + new ProduceRequestData.TopicProduceData() + .setName(topic) + .setPartitionData( + Collections.singletonList( + new ProduceRequestData.PartitionProduceData() + .setIndex(0) + .setRecords(records))); + return new ProduceRequest( + new ProduceRequestData() + .setAcks((short) 1) + .setTimeoutMs(5000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topicData).iterator())), + (short) 11); + } + + private static Socket connect(ServerNode node) throws Exception { + Socket socket = new Socket(); + try { + socket.connect(new InetSocketAddress(node.host(), node.port()), 5000); + socket.setSoTimeout(5000); + return socket; + } catch (Exception failure) { + socket.close(); + throw failure; + } + } + + private static AbstractResponse sendRequest(Socket socket, AbstractRequest request) + throws Exception { RequestHeader header = - new RequestHeader(ApiKeys.METADATA, METADATA_VERSION, "regression-test", 1); + new RequestHeader(request.apiKey(), request.version(), "regression-test", 1); ByteBuffer serialized = RequestUtils.serialize( header.data(), header.headerVersion(), request.data(), request.version()); byte[] requestBytes = new byte[serialized.remaining()]; serialized.get(requestBytes); - ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(node.host(), node.port()), 5000); - socket.setSoTimeout(5000); - try (DataOutputStream output = new DataOutputStream(socket.getOutputStream()); - DataInputStream input = new DataInputStream(socket.getInputStream())) { - output.writeInt(requestBytes.length); - output.write(requestBytes); - output.flush(); - int size = input.readInt(); - assertThat(size).isBetween(1, 1024 * 1024); - byte[] response = new byte[size]; - input.readFully(response); - return (MetadataResponse) - AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); - } - } + // The caller owns the socket and may send another request on the same connection. + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + int size = input.readInt(); + assertThat(size).isBetween(1, 1024 * 1024); + byte[] response = new byte[size]; + input.readFully(response); + return AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static void rebalanceLeader( + TableBucket bucket, + MetadataResponsePartition initial, + int newLeader, + String rebalanceId) + throws Exception { + List replicas = new ArrayList<>(initial.replicaNodes()); + Collections.swap(replicas, 0, replicas.indexOf(newLeader)); + RebalancePlanForBucket plan = + new RebalancePlanForBucket( + bucket, initial.leaderId(), newLeader, initial.replicaNodes(), replicas); + RebalanceTask task = + new RebalanceTask( + rebalanceId, + RebalanceStatus.NOT_STARTED, + Collections.singletonMap(bucket, plan)); + // Submit a deterministic leader-only plan through the coordinator event thread. + // All replicas and their client connections remain online during the real handoff. + CLUSTER.getZooKeeperClient().registerRebalanceTask(task); + CLUSTER.getCoordinatorServer() + .getCoordinatorEventProcessor() + .getCoordinatorEventManager() + .put(new RecoverRebalanceEvent(task)); + } + + private static double producerMetric(KafkaProducer producer, String name) { + return producer.metrics().entrySet().stream() + .filter( + entry -> + entry.getKey().group().equals("producer-metrics") + && entry.getKey().name().equals(name)) + .mapToDouble(entry -> ((Number) entry.getValue().metricValue()).doubleValue()) + .findFirst() + .getAsDouble(); } private static RecordMetadata send( @@ -350,13 +541,13 @@ private static KafkaProducer producer(String acks, boolean retry config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, (int) Duration.ofHours(1).toMillis()); return new KafkaProducer<>(config); } private static Configuration clusterConfig() { Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_ENABLED, true); - config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); return config; From 7f8b7e231c17b0b9f9dee496720bf28fcba06677 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:38:54 +0800 Subject: [PATCH 34/69] [kafka] Verify null value filtering with native raw and string readback Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 67/67 --- .../fluss/kafka/KafkaProduceAppendITCase.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 2a34f8057ad..94f057ebd2c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -237,6 +237,73 @@ void testNotNullValueSchemaIsRejectedWhenCreatingKafkaTable(String format) throw } } + @Test + void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Exception { + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String format : new String[] {"raw", "string"}) { + TablePath path = TablePath.of(DATABASE, "null_values_" + format); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + String topic = path.toString(); + assertThat( + producer.send( + new ProducerRecord( + topic, + 0, + 123L, + new byte[] {(byte) 0xff}, + null)) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(-1L); + assertThat( + producer.send( + new ProducerRecord( + topic, 0, 123L, null, new byte[0])) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + assertThat( + producer.send( + new ProducerRecord( + topic, 0, 123L, null, VALUE)) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(1L); + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < 2 && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + InternalRow row = record.getRow(); + byte[] expected = count == 0 ? new byte[0] : VALUE; + if (format.equals("raw")) { + assertThat(row.getBytes(0)).isEqualTo(expected); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo( + new String(expected, StandardCharsets.UTF_8)); + } + assertThat(row.isNullAt(3)).isTrue(); + assertThat(row.getTimestampNtz(1, 3).getMillisecond()) + .isEqualTo(123L); + count++; + } + } + assertThat(count).isEqualTo(2); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + @Test void testSameNamedTablesAcrossDatabases() throws Exception { Map tables = new LinkedHashMap<>(); From 241fc6824c69457c5e638282b82359344645c3fe Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 19:08:21 +0800 Subject: [PATCH 35/69] [kafka] Wait for bucket readiness in Produce round trips Server metadata convergence only confirms node discovery, so a newly created table can still reject writes before its leader replica is ready. Wait for the real bucket leader before the all-ack-mode round trip, including acks=0 which cannot report or retry such a rejected write. Include the table and format in native readback assertion failures. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 7/7 --- .../java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 94f057ebd2c..7e527eef7be 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -635,7 +635,9 @@ private static void assertReadback( count++; } } - assertThat(count).as("native readback for %s", path).isEqualTo(expectedValues.size()); + assertThat(count) + .as("Native readback for %s (%s)", path, format) + .isEqualTo(expectedValues.size()); } } From 399369217849809ac329a163f6a8eca827fba539 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 20:45:56 +0800 Subject: [PATCH 36/69] [kafka] Validate Produce contracts with isolated client matrix Exercise persisted table contracts, alias lifecycle, invalid-record atomicity, and replicated offsets across leader loss and restart using native readback. Keep null-value filtering and online leader-transfer regressions on the PR07 base. Add checksum-verified, isolated Java producer tests for all 72 formal releases from 0.11.0.0 through 4.3.1. Verify all ACK modes and both topic names before allowing clients to close. Include the provided Jackson 2.10.2 closure required by Kafka 2.6.0 and matching diagnostic logging bindings in test processes only. Validation: cumulative reactor regressions and both full matrix phases pass; 396 distinct passing cases, with two pre-existing disabled RPC cases. Spotless, Checkstyle and RAT pass. SASL and persisted per-row metadata offsets remain outside this stage. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 1027/1027 --- fluss-kafka/src/test/client-matrix/README.md | 109 +++++++ fluss-kafka/src/test/client-matrix/prepare.py | 162 ++++++++++ .../fluss/kafka/KafkaClientMatrixITCase.java | 306 ++++++++++++++++++ .../fluss/kafka/KafkaProduceAppendITCase.java | 35 +- .../kafka/KafkaProduceRegressionITCase.java | 304 ++++++++++++++++- .../fluss/kafka/matrix/KafkaClientProbe.java | 111 +++++++ 6 files changed, 1014 insertions(+), 13 deletions(-) create mode 100644 fluss-kafka/src/test/client-matrix/README.md create mode 100644 fluss-kafka/src/test/client-matrix/prepare.py create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java diff --git a/fluss-kafka/src/test/client-matrix/README.md b/fluss-kafka/src/test/client-matrix/README.md new file mode 100644 index 00000000000..3275dc7d843 --- /dev/null +++ b/fluss-kafka/src/test/client-matrix/README.md @@ -0,0 +1,109 @@ +# Isolated Kafka producer compatibility matrix + +This test facility runs real Kafka producers against the three-node Fluss test +cluster and reads their records through the native Fluss client. Each selected +client runs in a separate JVM whose classpath contains only a standalone probe, +one `kafka-clients` JAR and that release's runtime libraries. Prepared classpaths +also include a matching SLF4J Simple binding so client network-thread failures +appear in the log. The probe +checks `AppInfoParser.getVersion()` and records its code-source JAR, Java version, +classpath and results. No production dependency is replaced. + +## Requirements and preparation + +Use Java 11 for the reactor and child JVMs, Python 3.8+, curl and Maven. Initial +artifact preparation needs access to Maven Central; subsequent reactor runs can +be offline once the project's own dependencies have been cached. + +The FIP target is every formal Java client release from **0.11.0.0 through 4.3.1**. +[Apache Kafka downloads](https://kafka.apache.org/community/downloads/) records +both boundary releases. The preparation script reads the actual formal versions +from [official Maven metadata](https://repo.maven.apache.org/maven2/org/apache/kafka/kafka-clients/maven-metadata.xml), +fetches each client POM and its runtime JARs, verifies Central SHA-1 sidecars, and +records local SHA-256 hashes. Test dependencies from the published POM are excluded. +It fails explicitly on an unfamiliar runtime dependency instead of guessing its +transitive dependency closure. + +Kafka **2.6.0** needs one explicit exception: its generated ApiVersions messages +load Jackson `JsonNode`, although its published POM declares `jackson-databind` +as `provided`. This is recorded in +[KAFKA-10378](https://issues.apache.org/jira/browse/KAFKA-10378). The script supplies +the release's exact Jackson **2.10.2** databind, core and annotations JARs, all +checksum-verified. These and the diagnostic binding are listed separately under +`test_runtime_additions` in the preparation report. Other provided dependencies +and published test dependencies are not implicitly added. These additions affect +only the isolated test processes, not Fluss production dependencies. + +Start with the boundaries and representative cached releases: + +```sh +python3 fluss-kafka/src/test/client-matrix/prepare.py \ + --output /tmp/fluss-kafka-client-matrix \ + --versions 0.11.0.0,3.6.1,3.9.1,3.9.2,4.2.0,4.3.1 + +# Set JAVA_HOME to a Java 11 installation before running Maven. +./mvnw -o -pl fluss-kafka -am verify \ + -Dtest=KafkaClientMatrixITCase \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dkafka.client.matrix.file=/tmp/fluss-kafka-client-matrix/clients.properties +``` + +After the boundary run, prepare and execute the complete formal interval: + +```sh +python3 fluss-kafka/src/test/client-matrix/prepare.py \ + --output /tmp/fluss-kafka-client-matrix --versions all + +./mvnw -o -pl fluss-kafka -am verify \ + -Dtest=KafkaClientMatrixITCase \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dkafka.client.matrix.file=/tmp/fluss-kafka-client-matrix/clients.properties +``` + +Do not remove `-am`: server, client, RPC and test artifacts must come from this +worktree's reactor, not another branch's installed SNAPSHOTs. A normal build +without `kafka.client.matrix.file` still runs an isolated smoke test using the +module's actual declared client version; that single-version run is not a matrix. + +## Coverage and result interpretation + +For each selected version the probe uses PLAINTEXT and +`enable.idempotence=false`, negotiates APIs through the real producer, and tests +`acks=0`, `acks=1` and `acks=all`. The same producer alternates canonical and bare +names for the same table, checking acknowledged topic names and offsets. Native +readback verifies all 12 records, their real storage offsets, keys, last-header +wins semantics and null binary rescue fields. Bucket leaders are ready before +unacknowledged writes begin. Producer connections stay open until the parent +confirms native readback; a stdin handshake then permits the probe to close. This +avoids racing unacknowledged appends against connection-close cancellation. + +Preparation is not execution. `preparation.json` lists every requested version as +prepared, unavailable or not executed, including exact artifacts and any errors. +The script deletes the previous executable manifest at startup, including when +metadata retrieval or parsing fails. It only publishes `clients.properties` when +all requested versions are prepared. If downloads are blocked, keep that failure +report and make an explicit, separately recorded subset request for the available +versions; never call the subset a full interval pass. + +Runtime evidence lives under: + +- `fluss-kafka/target/client-matrix-results//command.txt` +- `fluss-kafka/target/client-matrix-results//client.log` +- `fluss-kafka/target/client-matrix-results//result.txt` +- `fluss-kafka/target/surefire-reports/TEST-org.apache.fluss.kafka.KafkaClientMatrixITCase.xml` + +Each invocation deletes the old result and log, writes `FAIL` with the error on +failure, and only writes `PASS` after successful native readback. A client failure +fails its parameterized test; other prepared versions still run. A missing JAR, +version mismatch, process timeout or unsupported JDK is a failure, not a skipped +compatibility result. Archive the preparation report, manifest, Maven log, runtime +logs and current Surefire XML together before `clean` or another matrix run. +The POM runs the selected tests in both Surefire phases; count distinct versions +once and require both executions to succeed. + +This is a bounded Produce compatibility acceptance flow, not proof of every +operation in the FIP. Leadership, lifecycle, min-ISR, invalid-record atomicity and +null-value filtering run separately in the cumulative integration tests using the +module client. They are not yet repeated for every matrix version. SASL is reserved +for PR19. Idempotence, other client SDKs, pre-releases, versions outside the target +interval, and persisted per-row `_kafka_metadata.offset` are not covered here. diff --git a/fluss-kafka/src/test/client-matrix/prepare.py b/fluss-kafka/src/test/client-matrix/prepare.py new file mode 100644 index 00000000000..448949bf003 --- /dev/null +++ b/fluss-kafka/src/test/client-matrix/prepare.py @@ -0,0 +1,162 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Prepare isolated, checksum-verified client classpaths from Maven Central.""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from pathlib import Path + +CENTRAL = "https://repo.maven.apache.org/maven2/" +NS = {"m": "http://maven.apache.org/POM/4.0.0"} +RUNTIME = {"slf4j-api", "lz4", "lz4-java", "snappy-java", "zstd-jni"} + + +def fetch(relative, destination, refresh=False): + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists() and not refresh: + return + temporary = destination.with_name(destination.name + ".download") + try: + subprocess.run(["curl", "--fail", "--location", "--silent", "--show-error", + "--connect-timeout", "10", "--max-time", "120", "--retry", "2", + CENTRAL + relative, "--output", str(temporary)], check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + + +def jar(cache, group, artifact, version): + relative = "/".join([group.replace(".", "/"), artifact, version, + artifact + "-" + version + ".jar"]) + target = cache / relative + fetch(relative, target) + checksum = target.with_suffix(".jar.sha1") + fetch(relative + ".sha1", checksum) + expected = checksum.read_text().strip().split()[0].lower() + actual = hashlib.sha1(target.read_bytes()).hexdigest() + if actual != expected: + raise ValueError("Maven Central checksum mismatch: " + str(target)) + return target.resolve() + + +def prepare(output, version): + cache = output / "artifacts" + relative = "org/apache/kafka/kafka-clients/" + version + "/kafka-clients-" + version + ".pom" + pom = cache / relative + fetch(relative, pom) + root = ET.parse(pom).getroot() + paths = [jar(cache, "org.apache.kafka", "kafka-clients", version)] + dependencies = [] + test_runtime = [] + for dep in root.findall("m:dependencies/m:dependency", NS): + scope = dep.findtext("m:scope", "compile", NS) + group = dep.findtext("m:groupId", namespaces=NS) + artifact = dep.findtext("m:artifactId", namespaces=NS) + dep_version = dep.findtext("m:version", namespaces=NS) + # Kafka 2.6.0 generated ApiVersions messages require JsonNode even for + # byte-array producers, but its POM marks databind provided (KAFKA-10378). + # Supply the release's exact Jackson version and its two runtime dependencies. + if version == "2.6.0" and group == "com.fasterxml.jackson.core" and artifact == "jackson-databind": + if scope != "provided" or dep_version != "2.10.2": + raise ValueError("Unexpected Kafka 2.6.0 Jackson dependency: " + ET.tostring(dep).decode()) + for jackson in ("jackson-databind", "jackson-core", "jackson-annotations"): + paths.append(jar(cache, group, jackson, dep_version)) + test_runtime.append({"coordinate": ":".join([group, jackson, dep_version]), + "reason": "Kafka 2.6.0 provided Jackson closure; KAFKA-10378"}) + continue + if scope not in ("compile", "runtime") or dep.findtext("m:optional", "false", NS) == "true": + continue + # These published Kafka runtime libraries are standalone JARs. Fail on a new + # dependency rather than silently constructing an incomplete classpath. + if artifact not in RUNTIME or not dep_version or "${" in dep_version: + raise ValueError("Unsupported dependency; resolve its runtime closure explicitly: " + ET.tostring(dep).decode()) + paths.append(jar(cache, group, artifact, dep_version)) + dependencies.append(":".join([group, artifact, dep_version])) + if group == "org.slf4j" and artifact == "slf4j-api": + # Without a binding, Kafka's network-thread failures disappear into NOP logging. + paths.append(jar(cache, group, "slf4j-simple", dep_version)) + test_runtime.append({"coordinate": ":".join([group, "slf4j-simple", dep_version]), + "reason": "Isolated client diagnostics; matches published slf4j-api"}) + return paths, dependencies, test_runtime + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--versions", default="0.11.0.0,3.6.1,3.9.1,3.9.2,4.2.0,4.3.1", + help="Comma-separated formal releases, or 'all' for the FIP interval") + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + manifest = output / "clients.properties" + manifest.unlink(missing_ok=True) + metadata_path = output / "maven-metadata.xml" + try: + fetch("org/apache/kafka/kafka-clients/maven-metadata.xml", metadata_path, refresh=True) + except Exception as exc: + (output / "preparation.json").write_text(json.dumps( + {"status": "METADATA_UNAVAILABLE_NOT_EXECUTED", "error": str(exc)}, indent=2) + "\n") + return 1 + root = ET.parse(metadata_path).getroot() + versions = [v.text for v in root.findall("versioning/versions/version") + if re.fullmatch(r"[0-9]+(?:\.[0-9]+){2,3}", v.text)] + def number(version): + return tuple(int(x) for x in version.split(".")) + eligible = sorted([v for v in versions if number("0.11.0.0") <= number(v) <= number("4.3.1")], key=number) + requested = eligible if args.versions == "all" else args.versions.split(",") + report = {"source": CENTRAL + "org/apache/kafka/kafka-clients/maven-metadata.xml", + "prepared_at": datetime.now(timezone.utc).isoformat(), + "metadata_updated": root.findtext("versioning/lastUpdated"), + "formal_releases_in_scope": eligible, "requested": requested, + "not_requested": [v for v in eligible if v not in requested], + "clients": {v: {"status": "NOT_EXECUTED"} for v in requested}} + entries = [] + for version in requested: + try: + if version not in eligible: + raise ValueError("Version is absent from the official formal-release interval") + paths, dependencies, test_runtime = prepare(output, version) + entries.append(version + "=" + os.pathsep.join(str(p) for p in paths)) + report["clients"][version] = {"status": "PREPARED_NOT_EXECUTED", "dependencies": dependencies, + "test_runtime_additions": test_runtime, + "jars": [{"path": str(p), "sha256": hashlib.sha256(p.read_bytes()).hexdigest()} for p in paths]} + except Exception as exc: + detail = getattr(exc, "stderr", b"") + report["clients"][version] = {"status": "UNAVAILABLE_NOT_EXECUTED", + "error": str(exc) + " " + (detail.decode(errors="replace") if isinstance(detail, bytes) else str(detail))} + print(version, report["clients"][version]["status"], flush=True) + (output / "preparation.json").write_text(json.dumps(report, indent=2) + "\n") + # Do not publish a partial executable matrix when a requested version is missing. + manifest.unlink(missing_ok=True) + if any(v["status"] == "UNAVAILABLE_NOT_EXECUTED" for v in report["clients"].values()): + return 1 + manifest.write_text("\n".join(entries) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java new file mode 100644 index 00000000000..dd378dece44 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.matrix.KafkaClientProbe; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.utils.AppInfoParser; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.File; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Runs actual, version-checked Kafka clients in isolated JVMs and verifies native Fluss storage. + */ +class KafkaClientMatrixITCase { + private static final String DATABASE = "client_matrix"; + private static final Path OUTPUT = + Paths.get(System.getProperty("project.basedir"), "target", "client-matrix-results"); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + static Stream clients() throws Exception { + return installations().stringPropertyNames().stream().sorted(); + } + + private static Properties installations() throws Exception { + Properties properties = new Properties(); + String manifest = System.getProperty("kafka.client.matrix.file"); + if (manifest != null) { + try (InputStream input = Files.newInputStream(Paths.get(manifest))) { + properties.load(input); + } + assertThat(properties).as("The requested matrix must not be empty").isNotEmpty(); + } else { + // Normal builds still exercise isolation. Other versions require an explicit manifest. + String classpath = + Arrays.stream( + System.getProperty("java.class.path") + .split(Pattern.quote(File.pathSeparator))) + .filter( + entry -> + Paths.get(entry) + .getFileName() + .toString() + .matches( + "(kafka-clients|slf4j-api|zstd-jni|snappy-java|lz4|lz4-java)-.+\\.jar")) + .collect(Collectors.joining(File.pathSeparator)); + properties.setProperty(AppInfoParser.getVersion(), classpath); + } + return properties; + } + + @ParameterizedTest(name = "Kafka client {0}") + @MethodSource("clients") + void testIsolatedClientWithAllAckModesAndBothNames(String version) throws Exception { + Path output = OUTPUT.resolve(version); + Files.createDirectories(output); + Files.deleteIfExists(output.resolve("result.txt")); + Files.deleteIfExists(output.resolve("client.log")); + Files.deleteIfExists(output.resolve("command.txt")); + try { + runClient(version, output); + } catch (Exception | AssertionError failure) { + Files.write( + output.resolve("result.txt"), + Arrays.asList("FAIL " + version, failure.toString()), + StandardCharsets.UTF_8); + throw failure; + } + } + + private static void runClient(String version, Path output) throws Exception { + String classpath = installations().getProperty(version); + List jars = Arrays.asList(classpath.split(Pattern.quote(File.pathSeparator))); + assertThat(jars).allSatisfy(jar -> assertThat(Paths.get(jar)).isRegularFile()); + assertThat( + jars.stream() + .filter( + jar -> + Paths.get(jar) + .getFileName() + .toString() + .startsWith("kafka-clients-")) + .collect(Collectors.toList())) + .singleElement() + .satisfies( + jar -> + assertThat(Paths.get(jar).getFileName().toString()) + .isEqualTo("kafka-clients-" + version + ".jar")); + // Export just the probe bytecode, never the server/test classpath, to the child JVM. + String resource = KafkaClientProbe.class.getName().replace('.', '/') + ".class"; + Path probe = output.resolve("probe").resolve(resource); + Files.createDirectories(probe.getParent()); + try (InputStream input = + KafkaClientProbe.class.getClassLoader().getResourceAsStream(resource)) { + assertThat(input).isNotNull(); + Files.copy(input, probe, StandardCopyOption.REPLACE_EXISTING); + } + String prefix = "client_" + version.replace('.', '_'); + List paths = new ArrayList<>(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, prefix + "_" + acks); + paths.add(path); + admin.createTable(path, descriptor(), false).get(); + CLUSTER.waitAndGetLeaderReplica( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); + } + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + List command = + Arrays.asList( + Paths.get(System.getProperty("java.home"), "bin", "java") + .toString(), + "-cp", + output.resolve("probe") + File.pathSeparator + classpath, + KafkaClientProbe.class.getName(), + version, + node.host() + ":" + node.port(), + DATABASE, + prefix); + Files.write(output.resolve("command.txt"), command, StandardCharsets.UTF_8); + Path log = output.resolve("client.log"); + Process process = + new ProcessBuilder(command) + .redirectErrorStream(true) + .redirectOutput(log.toFile()) + .start(); + try { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(120); + String text = ""; + while (process.isAlive() && System.nanoTime() < deadline) { + text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + if (text.contains("WRITES_COMPLETE")) { + break; + } + process.waitFor(100, TimeUnit.MILLISECONDS); + } + text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + assertThat(text) + .as("Client %s log: %s", version, log) + .contains( + "VERSION=" + version, + "JAR=", + "JAVA=", + "ACKS=0:SENT", + "ACKS=1:SENT", + "ACKS=all:SENT", + "WRITES_COMPLETE"); + assertThat(process.isAlive()) + .as("Client must stay connected during native readback") + .isTrue(); + for (String acks : Arrays.asList("0", "1", "all")) { + assertNativeReadback( + connection, + TablePath.of(DATABASE, prefix + "_" + acks), + version, + acks); + } + process.getOutputStream().write('\n'); + process.getOutputStream().flush(); + assertThat(process.waitFor(30, TimeUnit.SECONDS)) + .as("Client log: %s", log) + .isTrue(); + text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Client %s: %s", version, text).isZero(); + assertThat(text).contains("NATIVE_READBACK_CONFIRMED"); + Files.write( + output.resolve("result.txt"), + Collections.singletonList( + "PASS " + + version + + " PLAINTEXT non-idempotent; acks=0,1,all; same producer full/alias; 12 native rows and offsets verified"), + StandardCharsets.UTF_8); + } finally { + if (process.isAlive()) { + process.destroyForcibly(); + process.waitFor(10, TimeUnit.SECONDS); + } + } + } finally { + for (TablePath path : paths) { + admin.dropTable(path, true).get(); + } + } + } + } + + private static void assertNativeReadback( + Connection connection, TablePath path, String version, String acks) throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < 4 && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + InternalRow row = record.getRow(); + assertThat(row.getBytes(0)) + .isEqualTo( + (version + ":" + acks + ":" + count) + .getBytes(StandardCharsets.UTF_8)); + assertThat(row.getBytes(1)).isEqualTo("key".getBytes(StandardCharsets.UTF_8)); + assertThat(row.getMap(2).size()).isEqualTo(1); + assertThat(row.getMap(2).keyArray().getString(0).toString()) + .isEqualTo("source"); + assertThat(row.getMap(2).valueArray().getBytes(0)).containsExactly((byte) 2); + assertThat(row.isNullAt(3)).isTrue(); + assertThat(record.logOffset()).isEqualTo(count); + count++; + } + } + assertThat(count) + .as("Native readback for client %s, acks %s", version, acks) + .isEqualTo(4); + } + } + + private static TableDescriptor descriptor() { + return TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 7e527eef7be..e65a18cd1f2 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -242,7 +242,7 @@ void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Excepti try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - for (String format : new String[] {"raw", "string"}) { + for (String format : new String[] {"binary", "string"}) { TablePath path = TablePath.of(DATABASE, "null_values_" + format); admin.createTable(path, descriptor(format), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); @@ -282,7 +282,7 @@ void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Excepti for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { InternalRow row = record.getRow(); byte[] expected = count == 0 ? new byte[0] : VALUE; - if (format.equals("raw")) { + if (format.equals("binary")) { assertThat(row.getBytes(0)).isEqualTo(expected); } else { assertThat(row.getString(0).toString()) @@ -290,8 +290,9 @@ void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Excepti new String(expected, StandardCharsets.UTF_8)); } assertThat(row.isNullAt(3)).isTrue(); - assertThat(row.getTimestampNtz(1, 3).getMillisecond()) - .isEqualTo(123L); + assertThat(row.getMap(1).size()).isZero(); + assertThat(row.isNullAt(2)).isTrue(); + assertThat(record.logOffset()).isEqualTo(count); count++; } } @@ -471,7 +472,25 @@ void testDeletedDefaultDatabaseDoesNotFallbackOrRecreateDatabase() throws Except query(socket, existing.toString()), new String[] {existing.toString()}, 0L); assertProduceOffsets( query(socket, existing.toString()), new String[] {existing.toString()}, 1L); - assertReadback(connection, existing, "binary", VALUE); + try (KafkaProducer producer = producer(node, "all", 0)) { + assertThat( + producer.send( + new ProducerRecord<>( + existing.toString(), + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader( + "source", VALUE), + new RecordHeader( + "source", null)))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(2L); + } + assertReadback(connection, existing, "binary", VALUE, 3); assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); assertThat(admin.databaseExists("another_missing_db").get()).isFalse(); } @@ -683,7 +702,11 @@ private static KafkaProducer producer(String acks) { } private static KafkaProducer producer(String acks, int batchSize) { - ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + return producer(CLUSTER.getTabletServerNodes("KAFKA").get(0), acks, batchSize); + } + + private static KafkaProducer producer( + ServerNode node, String acks, int batchSize) { Map config = new HashMap<>(); config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java index 32b0f2a2ced..8066f396991 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -32,11 +32,11 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.cluster.AlterConfig; import org.apache.fluss.config.cluster.AlterConfigOpType; -import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.server.coordinator.event.RecoverRebalanceEvent; @@ -93,7 +93,7 @@ /** Regression coverage for native DDL lifecycle, metadata refresh, and Produce minimum ISR. */ class KafkaProduceRegressionITCase { - private static final String DATABASE = "kafka"; + private static final String DATABASE = "fluss"; private static final short METADATA_VERSION = 11; @RegisterExtension @@ -321,6 +321,292 @@ void testClusterMinIsrPolicyAndDynamicReload() throws Exception { } } + @Test + void testSameProducerAliasesFollowEnablementPropertiesAndRecreation() throws Exception { + TablePath path = TablePath.of(DATABASE, "cached_lifecycle"); + String full = path.toString(); + String alias = path.getTableName(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + Uuid fullId = topicId(full); + Uuid aliasId = topicId(alias); + assertThat(aliasId).isNotEqualTo(Uuid.ZERO_UUID).isNotEqualTo(fullId); + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, full, "full").offset()).isZero(); + assertThat(send(producer, alias, "alias").offset()).isEqualTo(1L); + setTableEnabled(admin, path, false); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setTableEnabled(admin, path, true); + assertThat(topicId(full)).isEqualTo(fullId); + assertThat(topicId(alias)).isEqualTo(aliasId); + assertThat(send(producer, alias, "enabled").offset()).isEqualTo(2L); + Map properties = + admin.getTableInfo(path).get().getProperties().toMap(); + assertThatThrownBy( + () -> + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE + .key(), + "string")), + false) + .get()) + .isInstanceOf(ExecutionException.class); + assertThat(admin.getTableInfo(path).get().getProperties().toMap()) + .isEqualTo(properties); + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE + .key(), + "BINARY")), + false) + .get(); + assertThat(admin.getTableInfo(path).get().getProperties().toMap()) + .containsEntry( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "BINARY"); + assertThat(send(producer, full, "property").offset()).isEqualTo(3L); + assertValues(connection, path, "full", "alias", "enabled", "property"); + admin.dropTable(path, false).get(); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + createTable(admin, path, 3); + assertThat(topicId(full)).isNotEqualTo(fullId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(alias)).isNotEqualTo(aliasId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(full)).isNotEqualTo(topicId(alias)); + assertThat(send(producer, full, "recreated_full").offset()).isZero(); + assertThat(send(producer, alias, "recreated_alias").offset()).isEqualTo(1L); + assertValues(connection, path, "recreated_full", "recreated_alias"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testNeverEnabledTableRejectsBothNamesWithoutAppending() throws Exception { + TablePath path = TablePath.of(DATABASE, "never_enabled"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + "binary") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setTableEnabled(admin, path, true); + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, path.getTableName(), "accepted").offset()).isZero(); + assertValues(connection, path, "accepted"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testInvalidRecordRejectsWholePartitionBeforeNativeAppend() throws Exception { + TablePath path = TablePath.of(DATABASE, "atomic_conversion"); + byte[] key = new byte[] {1}; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_key", + DataTypes.BYTES().copy(false)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + "binary") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + try (KafkaProducer producer = producer("all", true)) { + int leader = producer.partitionsFor(path.toString()).get(0).leader().id(); + ServerNode node = + CLUSTER.getTabletServerNodes("KAFKA").stream() + .filter(n -> n.id() == leader) + .findFirst() + .get(); + try (Socket socket = connect(node)) { + ProduceResponse rejected = + (ProduceResponse) + sendRequest( + socket, + produceRequest( + path.toString(), + new SimpleRecord( + key, + "valid_prefix" + .getBytes( + StandardCharsets + .UTF_8)), + new SimpleRecord( + null, + "invalid_key" + .getBytes( + StandardCharsets + .UTF_8)))); + assertThat(rejected.errorCounts()).containsEntry(Errors.INVALID_RECORD, 1); + assertThat( + rejected.data() + .responses() + .find(path.toString()) + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(-1L); + } + assertThat( + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + key, + "accepted" + .getBytes( + StandardCharsets + .UTF_8))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + assertValues(connection, path, "accepted"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testReplicatedOffsetsSurviveLeaderLossAndRestart() throws Exception { + TablePath path = TablePath.of(DATABASE, "durable_restart"); + int stopped = -1; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + TableBucket bucket = + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0); + MetadataResponsePartition initial = + waitForTopic(path.toString()) + .data() + .topics() + .find(path.toString()) + .partitions() + .get(0); + for (int replica : initial.replicaNodes()) { + CLUSTER.waitUntilReplicaExpandToIsr(bucket, replica); + } + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, path.toString(), "before").offset()).isZero(); + stopped = initial.leaderId(); + CLUSTER.stopTabletServer(stopped); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + waitForTopic(path.toString()) + .data() + .topics() + .find(path.toString()) + .partitions() + .get(0) + .leaderId()) + .isNotEqualTo(initial.leaderId())); + assertThat(send(producer, path.toString(), "after_loss").offset()) + .isEqualTo(1L); + CLUSTER.startTabletServer(stopped); + CLUSTER.assertHasTabletServerNumber(3); + CLUSTER.waitUntilReplicaExpandToIsr(bucket, stopped); + assertThat(send(producer, path.getTableName(), "after_restart").offset()) + .isEqualTo(2L); + assertValues(connection, path, "before", "after_loss", "after_restart"); + } + } finally { + if (stopped >= 0 && CLUSTER.getTabletServerById(stopped) == null) { + CLUSTER.startTabletServer(stopped); + CLUSTER.assertHasTabletServerNumber(3); + } + admin.dropTable(path, true).get(); + } + } + } + + private static void setTableEnabled(Admin admin, TablePath path, boolean enabled) + throws Exception { + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled))), + false) + .get(); + } + + private static Uuid topicId(String topic) { + return waitForTopic(topic).data().topics().find(topic).topicId(); + } + + private static void assertRejectedNames(TablePath path, Errors error) throws Exception { + retry( + Duration.ofMinutes(1), + () -> { + for (String topic : new String[] {path.toString(), path.getTableName()}) { + for (ServerNode node : CLUSTER.getTabletServerNodes("KAFKA")) { + try (Socket socket = connect(node)) { + MetadataResponse response = + (MetadataResponse) + sendRequest(socket, metadataRequest(topic)); + assertThat(response.errors()).containsEntry(topic, error); + } + } + } + }); + for (String topic : new String[] {path.toString(), path.getTableName()}) { + for (ServerNode node : CLUSTER.getTabletServerNodes("KAFKA")) { + try (Socket socket = connect(node)) { + ProduceResponse response = + (ProduceResponse) + sendRequest(socket, produceRequest(topic, "rejected")); + assertThat(response.errorCounts()).containsEntry(error, 1); + } + } + } + } + private static void createTable(Admin admin, TablePath path, int replicas) throws Exception { admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); admin.createTable( @@ -328,12 +614,13 @@ private static void createTable(Admin admin, TablePath path, int replicas) throw TableDescriptor.builder() .schema( Schema.newBuilder() - .column("value", DataTypes.BYTES()) + .column("_kafka_value", DataTypes.BYTES()) .build()) .distributedBy(1) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") .build(), false) .get(); @@ -371,6 +658,7 @@ private static void assertValues(Connection connection, TablePath path, String.. () -> { ScanRecords records = scanner.poll(Duration.ofSeconds(1)); for (ScanRecord record : records) { + assertThat(record.logOffset()).isEqualTo(values.size()); values.add( new String( record.getRow().getBytes(0), StandardCharsets.UTF_8)); @@ -426,9 +714,11 @@ private static MetadataRequest metadataRequest(String topic) { } private static ProduceRequest produceRequest(String topic, String value) { - MemoryRecords records = - MemoryRecords.withRecords( - Compression.NONE, new SimpleRecord(value.getBytes(StandardCharsets.UTF_8))); + return produceRequest(topic, new SimpleRecord(value.getBytes(StandardCharsets.UTF_8))); + } + + private static ProduceRequest produceRequest(String topic, SimpleRecord... values) { + MemoryRecords records = MemoryRecords.withRecords(Compression.NONE, values); ProduceRequestData.TopicProduceData topicData = new ProduceRequestData.TopicProduceData() .setName(topic) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java new file mode 100644 index 00000000000..b0dd5a0e146 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.matrix; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.utils.AppInfoParser; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Standalone client process: deliberately depends only on the JDK and the selected Kafka client. + */ +public final class KafkaClientProbe { + private KafkaClientProbe() {} + + /** Writes through a real endpoint; a failed check terminates the isolated process. */ + public static void main(String[] args) throws Exception { + String expectedVersion = args[0]; + String actualVersion = AppInfoParser.getVersion(); + if (!expectedVersion.equals(actualVersion)) { + throw new IllegalStateException( + "Expected " + expectedVersion + " but loaded " + actualVersion); + } + System.out.println("VERSION=" + actualVersion); + System.out.println( + "JAR=" + KafkaProducer.class.getProtectionDomain().getCodeSource().getLocation()); + System.out.println("JAVA=" + System.getProperty("java.version")); + System.out.println("CLASSPATH=" + System.getProperty("java.class.path")); + List> producers = new ArrayList<>(); + try { + for (String acks : Arrays.asList("0", "1", "all")) { + Map config = new HashMap<>(); + config.put("bootstrap.servers", args[1]); + config.put("key.serializer", ByteArraySerializer.class); + config.put("value.serializer", ByteArraySerializer.class); + config.put("enable.idempotence", false); + config.put("security.protocol", "PLAINTEXT"); + config.put("acks", acks); + config.put("max.in.flight.requests.per.connection", 1); + config.put("request.timeout.ms", 10000); + config.put("max.block.ms", 30000); + config.put("retries", 3); + config.put("compression.type", "none"); + config.put("linger.ms", 0); + String alias = args[3] + "_" + acks; + KafkaProducer producer = new KafkaProducer<>(config); + producers.add(producer); + for (int i = 0; i < 4; i++) { + String topic = i % 2 == 0 ? args[2] + "." + alias : alias; + RecordMetadata result = + producer.send( + new ProducerRecord<>( + topic, + 0, + 123L, + "key".getBytes(StandardCharsets.UTF_8), + (actualVersion + ":" + acks + ":" + i) + .getBytes(StandardCharsets.UTF_8), + Arrays.asList( + new RecordHeader( + "source", new byte[] {1}), + new RecordHeader( + "source", new byte[] {2})))) + .get(30, TimeUnit.SECONDS); + if (!result.topic().equals(topic) + || (!acks.equals("0") && result.offset() != i)) { + throw new IllegalStateException("Unexpected acknowledgement " + result); + } + } + producer.flush(); + System.out.println("ACKS=" + acks + ":SENT"); + } + // acks=0 only confirms a socket write. Keep connections open until the + // parent verifies native storage, rather than racing cancellation on close. + System.out.println("WRITES_COMPLETE"); + if (System.in.read() != '\n') { + throw new IllegalStateException("Native readback was not confirmed by the parent"); + } + System.out.println("NATIVE_READBACK_CONFIRMED"); + } finally { + for (KafkaProducer producer : producers) { + producer.close(); + } + } + } +} From 2410d0441aab76440e2e5fbf766ca107141a377f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 15 Oct 2026 22:40:15 +0800 Subject: [PATCH 37/69] [kafka] Verify metadata across schema changes and leader recovery Verify that a cached producer using both topic names picks up an added metadata column. Read back the schema ID and all three metadata fields through the native client after online leader transfer, leader loss, and restart. Transfer leadership back to the recovered replica to prove it retains records written while it was offline. The source matches the verified PR07-based snapshot byte for byte: 325 distinct tests pass, with two pre-existing disabled RPC cases. All nine Produce regression cases pass in both execution phases; Spotless, Checkstyle, and RAT pass. The isolated client check used 3.9.2; the full 72-version matrix was not rerun for this snapshot. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 325/325 --- .../kafka/KafkaProduceRegressionITCase.java | 325 +++++++++++++++--- 1 file changed, 277 insertions(+), 48 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java index 8066f396991..5d016ac5c9b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -38,11 +38,14 @@ import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; import org.apache.fluss.server.coordinator.event.RecoverRebalanceEvent; import org.apache.fluss.server.testutils.FlussClusterExtension; import org.apache.fluss.server.zk.data.RebalanceTask; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; @@ -66,6 +69,7 @@ import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.assertj.core.groups.Tuple; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -199,9 +203,9 @@ void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { - createTable(admin, path, 3); - TableBucket bucket = - new TableBucket(admin.getTableInfo(path).get().getTableId(), 0); + createTable(admin, path, 3, metadataSchema(), 1); + TableInfo tableInfo = admin.getTableInfo(path).get(); + TableBucket bucket = new TableBucket(tableInfo.getTableId(), 0); MetadataResponsePartition initial = waitForTopic(topic).data().topics().find(topic).partitions().get(0); int oldLeader = initial.leaderId(); @@ -222,32 +226,21 @@ void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { (MetadataResponse) sendRequest(oldLeaderSocket, metadataRequest(topic)); assertThat(before.data().topics().find(topic).partitions().get(0).leaderId()) .isEqualTo(oldLeader); - assertThat(send(producer, topic, "before_handoff").offset()).isZero(); + assertThat(send(producer, topic, 0, 123L, "before_handoff").offset()).isZero(); + assertRecordsWithMetadata( + connection, + path, + 0, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before_handoff", 123L, 0, 0)); CLUSTER.waitUntilReplicaExpandToIsr(bucket, newLeader); double retriesBefore = producerMetric(producer, "record-retry-total"); double disconnectsBefore = producerMetric(producer, "connection-close-total"); rebalanceLeader(bucket, initial, newLeader, rebalanceId); - retry( - Duration.ofMinutes(1), - () -> { - assertThat(admin.listRebalanceProgress(rebalanceId).get()) - .isPresent() - .get() - .extracting(RebalanceProgress::status) - .isEqualTo(RebalanceStatus.COMPLETED); - MetadataResponsePartition updated = - fetchMetadata(topic) - .data() - .topics() - .find(topic) - .partitions() - .get(0); - assertThat(updated.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(updated.leaderId()).isEqualTo(newLeader); - assertThat(updated.leaderEpoch()) - .isGreaterThan(initial.leaderEpoch()); - }); + waitForLeaderChange( + admin, topic, 0, newLeader, initial.leaderEpoch(), rebalanceId); CLUSTER.waitAndGetFollowerReplica(bucket, oldLeader); assertThat(CLUSTER.getTabletServerNodes("KAFKA")) .containsExactlyInAnyOrderElementsOf(nodes); @@ -269,14 +262,22 @@ void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { // Metadata requests above must not refresh this route before the next send. assertThat(producer.partitionsFor(topic).get(0).leader().id()) .isEqualTo(oldLeader); - assertThat(send(producer, topic, "after_handoff").offset()).isEqualTo(1L); + assertThat(send(producer, topic, 0, 456L, "after_handoff").offset()) + .isEqualTo(1L); assertThat(producerMetric(producer, "record-retry-total")) .isGreaterThan(retriesBefore); assertThat(producerMetric(producer, "connection-close-total")) .isEqualTo(disconnectsBefore); assertThat(producer.partitionsFor(topic).get(0).leader().id()) .isEqualTo(newLeader); - assertValues(connection, path, "before_handoff", "after_handoff"); + assertRecordsWithMetadata( + connection, + path, + 0, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before_handoff", 123L, 0, 0), + tuple(1L, "after_handoff", 456L, 0, 0)); } } finally { try { @@ -389,6 +390,68 @@ void testSameProducerAliasesFollowEnablementPropertiesAndRecreation() throws Exc } } + @Test + void testSameProducerAliasesUseAddedMetadataColumn() throws Exception { + TablePath path = TablePath.of(DATABASE, "cached_schema_alter"); + String full = path.toString(); + String alias = path.getTableName(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + TableInfo before = admin.getTableInfo(path).get(); + Uuid fullId = topicId(full); + Uuid aliasId = topicId(alias); + try (KafkaProducer producer = producer("all", true)) { + // Populate both name mappings and the write plan before changing the schema. + assertThat(send(producer, full, "old_full").offset()).isZero(); + assertThat(send(producer, alias, "old_alias").offset()).isEqualTo(1L); + assertValues(connection, path, "old_full", "old_alias"); + + admin.alterTable( + path, + Collections.singletonList( + TableChange.addColumn( + "_kafka_metadata", + metadataType(), + null, + TableChange.ColumnPosition.last())), + false) + .get(); + TableInfo after = admin.getTableInfo(path).get(); + assertThat(after.getTableId()).isEqualTo(before.getTableId()); + assertThat(after.getSchemaId()).isGreaterThan(before.getSchemaId()); + assertThat(after.getProperties().toMap()) + .isEqualTo(before.getProperties().toMap()); + assertThat(after.getSchema().getColumns()).hasSize(2); + assertThat(after.getSchema().getColumn("_kafka_metadata").getDataType()) + .isEqualTo(metadataType()); + + // Neither producer recreation nor a Kafka Metadata refresh should be needed. + assertThat(send(producer, full, 0, 789L, "new_full").offset()).isEqualTo(2L); + assertThat(send(producer, alias, 0, 987L, "new_alias").offset()).isEqualTo(3L); + assertThat(topicId(full)).isEqualTo(fullId); + assertThat(topicId(alias)).isEqualTo(aliasId); + // Read the new batches using the new native schema, independently of the + // native client's earlier cached table handle. + try (Connection reader = + ConnectionFactory.createConnection(CLUSTER.getClientConfig())) { + assertRecordsWithMetadata( + reader, + path, + 0, + 2L, + after.getSchemaId(), + tuple(2L, "new_full", 789L, 0, 0), + tuple(3L, "new_alias", 987L, 0, 0)); + } + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + @Test void testNeverEnabledTableRejectsBothNamesWithoutAppending() throws Exception { TablePath path = TablePath.of(DATABASE, "never_enabled"); @@ -510,56 +573,108 @@ void testInvalidRecordRejectsWholePartitionBeforeNativeAppend() throws Exception } @Test - void testReplicatedOffsetsSurviveLeaderLossAndRestart() throws Exception { + void testReplicatedOffsetsAndMetadataSurviveLeaderLossAndRestart() throws Exception { TablePath path = TablePath.of(DATABASE, "durable_restart"); + String topic = path.toString(); + String rebalanceId = "kafka-recovered-leader"; + int partition = 1; int stopped = -1; try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { - createTable(admin, path, 3); - TableBucket bucket = - new TableBucket(admin.getTableInfo(path).get().getTableId(), 0); + createTable(admin, path, 3, metadataSchema(), 2); + TableInfo tableInfo = admin.getTableInfo(path).get(); + TableBucket bucket = new TableBucket(tableInfo.getTableId(), partition); MetadataResponsePartition initial = - waitForTopic(path.toString()) + waitForTopic(topic, 2) .data() .topics() - .find(path.toString()) + .find(topic) .partitions() - .get(0); + .get(partition); for (int replica : initial.replicaNodes()) { CLUSTER.waitUntilReplicaExpandToIsr(bucket, replica); } try (KafkaProducer producer = producer("all", true)) { - assertThat(send(producer, path.toString(), "before").offset()).isZero(); + assertThat(send(producer, topic, partition, 123L, "before").offset()).isZero(); + assertRecordsWithMetadata( + connection, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition)); stopped = initial.leaderId(); CLUSTER.stopTabletServer(stopped); retry( Duration.ofMinutes(1), () -> assertThat( - waitForTopic(path.toString()) + waitForTopic(topic, 2) .data() .topics() - .find(path.toString()) + .find(topic) .partitions() - .get(0) + .get(partition) .leaderId()) .isNotEqualTo(initial.leaderId())); - assertThat(send(producer, path.toString(), "after_loss").offset()) + assertThat(send(producer, topic, partition, 456L, "after_loss").offset()) .isEqualTo(1L); + assertRecordsWithMetadata( + connection, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition), + tuple(1L, "after_loss", 456L, 0, partition)); CLUSTER.startTabletServer(stopped); CLUSTER.assertHasTabletServerNumber(3); CLUSTER.waitUntilReplicaExpandToIsr(bucket, stopped); - assertThat(send(producer, path.getTableName(), "after_restart").offset()) + // Make the recovered replica serve reads and writes, including the record + // produced while it was offline. ISR membership alone does not prove readback. + MetadataResponsePartition current = + waitForTopic(topic, 2) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + rebalanceLeader(bucket, current, stopped, rebalanceId); + waitForLeaderChange( + admin, topic, partition, stopped, current.leaderEpoch(), rebalanceId); + assertThat( + send( + producer, + path.getTableName(), + partition, + 789L, + "after_restart") + .offset()) .isEqualTo(2L); - assertValues(connection, path, "before", "after_loss", "after_restart"); + try (Connection reader = + ConnectionFactory.createConnection(CLUSTER.getClientConfig())) { + assertRecordsWithMetadata( + reader, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition), + tuple(1L, "after_loss", 456L, 0, partition), + tuple(2L, "after_restart", 789L, 0, partition)); + } } } finally { if (stopped >= 0 && CLUSTER.getTabletServerById(stopped) == null) { CLUSTER.startTabletServer(stopped); CLUSTER.assertHasTabletServerNumber(3); } - admin.dropTable(path, true).get(); + try { + admin.cancelRebalance(rebalanceId).get(); + } finally { + admin.dropTable(path, true).get(); + } } } } @@ -608,15 +723,23 @@ private static void assertRejectedNames(TablePath path, Errors error) throws Exc } private static void createTable(Admin admin, TablePath path, int replicas) throws Exception { + createTable( + admin, + path, + replicas, + Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build(), + 1); + } + + private static void createTable( + Admin admin, TablePath path, int replicas, Schema schema, int buckets) + throws Exception { admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); admin.createTable( path, TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .distributedBy(1) + .schema(schema) + .distributedBy(buckets) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) @@ -627,6 +750,20 @@ private static void createTable(Admin admin, TablePath path, int replicas) throw CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); } + private static Schema metadataSchema() { + return Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_metadata", metadataType()) + .build(); + } + + private static RowType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + private static void setMinIsr(Admin admin, int minIsr) throws Exception { admin.alterClusterConfigs( Collections.singletonList( @@ -668,7 +805,52 @@ private static void assertValues(Connection connection, TablePath path, String.. } } + private static void assertRecordsWithMetadata( + Connection connection, + TablePath path, + int partition, + long startOffset, + int schemaId, + Tuple... expected) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribe(partition, startOffset); + List actual = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (actual.size() < expected.length && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.getSchemaId()) + .as("Schema at offset %s", record.logOffset()) + .isEqualTo(schemaId); + InternalRow row = record.getRow(); + assertThat(row.getFieldCount()).isEqualTo(2); + assertThat(row.isNullAt(1)).isFalse(); + InternalRow metadata = row.getRow(1, 3); + assertThat(metadata.getFieldCount()).isEqualTo(3); + assertThat(metadata.isNullAt(0)).isFalse(); + assertThat(metadata.isNullAt(1)).isFalse(); + assertThat(metadata.isNullAt(2)).isFalse(); + actual.add( + tuple( + record.logOffset(), + new String(row.getBytes(0), StandardCharsets.UTF_8), + metadata.getTimestampLtz(0, 3).getEpochMillisecond(), + metadata.getInt(1), + metadata.getInt(2))); + } + } + assertThat(actual) + .as("Native metadata readback for %s bucket %s", path, partition) + .containsExactly(expected); + } + } + private static MetadataResponse waitForTopic(String topic) { + return waitForTopic(topic, 1); + } + + private static MetadataResponse waitForTopic(String topic, int partitions) { AtomicReference result = new AtomicReference<>(); retry( Duration.ofMinutes(1), @@ -676,8 +858,10 @@ private static MetadataResponse waitForTopic(String topic) { MetadataResponse response = fetchMetadata(topic); MetadataResponseTopic metadata = response.data().topics().find(topic); assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(metadata.partitions()).hasSize(1); - assertThat(metadata.partitions().get(0).leaderId()).isNotEqualTo(-1); + assertThat(metadata.partitions()) + .hasSize(partitions) + .allSatisfy( + partition -> assertThat(partition.leaderId()).isNotEqualTo(-1)); result.set(response); }); return result.get(); @@ -796,6 +980,34 @@ private static void rebalanceLeader( .put(new RecoverRebalanceEvent(task)); } + private static void waitForLeaderChange( + Admin admin, + String topic, + int partition, + int newLeader, + int oldEpoch, + String rebalanceId) { + retry( + Duration.ofMinutes(1), + () -> { + assertThat(admin.listRebalanceProgress(rebalanceId).get()) + .isPresent() + .get() + .extracting(RebalanceProgress::status) + .isEqualTo(RebalanceStatus.COMPLETED); + MetadataResponsePartition updated = + fetchMetadata(topic) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + assertThat(updated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(updated.leaderId()).isEqualTo(newLeader); + assertThat(updated.leaderEpoch()).isGreaterThan(oldEpoch); + }); + } + private static double producerMetric(KafkaProducer producer, String name) { return producer.metrics().entrySet().stream() .filter( @@ -815,6 +1027,23 @@ private static RecordMetadata send( .get(60, TimeUnit.SECONDS); } + private static RecordMetadata send( + KafkaProducer producer, + String topic, + int partition, + long timestamp, + String value) + throws Exception { + return producer.send( + new ProducerRecord<>( + topic, + partition, + timestamp, + null, + value.getBytes(StandardCharsets.UTF_8))) + .get(60, TimeUnit.SECONDS); + } + private static KafkaProducer producer(String acks, boolean retryOnFailure) { Map config = new HashMap<>(); config.put( From 6609679139c81705b581336f7d850ef8bb2e537b Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 19:18:34 +0800 Subject: [PATCH 38/69] [kafka] Verify null-value persistence and native offsets Update the Produce regression to expect null values to be persisted in binary and string tables. Verify contiguous offsets and native readback for the null record with a binary key, followed by empty and nonempty values with null keys. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 32/32 --- .../fluss/kafka/KafkaProduceAppendITCase.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index e65a18cd1f2..79679e2f5dd 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -238,7 +238,7 @@ void testNotNullValueSchemaIsRejectedWhenCreatingKafkaTable(String format) throw } @Test - void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Exception { + void testNullAndEmptyValuesPreserveKeysAndOffsets() throws Exception { try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); @@ -258,45 +258,51 @@ void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Excepti null)) .get(30, TimeUnit.SECONDS) .offset()) - .isEqualTo(-1L); + .isZero(); assertThat( producer.send( new ProducerRecord( topic, 0, 123L, null, new byte[0])) .get(30, TimeUnit.SECONDS) .offset()) - .isZero(); + .isEqualTo(1L); assertThat( producer.send( new ProducerRecord( topic, 0, 123L, null, VALUE)) .get(30, TimeUnit.SECONDS) .offset()) - .isEqualTo(1L); + .isEqualTo(2L); try (Table table = connection.getTable(path); LogScanner scanner = table.newScan().createLogScanner()) { scanner.subscribeFromBeginning(0); int count = 0; long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - while (count < 2 && System.nanoTime() < deadline) { + while (count < 3 && System.nanoTime() < deadline) { for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { InternalRow row = record.getRow(); - byte[] expected = count == 0 ? new byte[0] : VALUE; - if (format.equals("binary")) { - assertThat(row.getBytes(0)).isEqualTo(expected); + if (count == 0) { + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.getBytes(3)).isEqualTo(new byte[] {(byte) 0xff}); } else { - assertThat(row.getString(0).toString()) - .isEqualTo( - new String(expected, StandardCharsets.UTF_8)); + byte[] expected = count == 1 ? new byte[0] : VALUE; + if (format.equals("binary")) { + assertThat(row.getBytes(0)).isEqualTo(expected); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo( + new String( + expected, StandardCharsets.UTF_8)); + } + assertThat(row.isNullAt(3)).isTrue(); } - assertThat(row.isNullAt(3)).isTrue(); assertThat(row.getMap(1).size()).isZero(); assertThat(row.isNullAt(2)).isTrue(); assertThat(record.logOffset()).isEqualTo(count); count++; } } - assertThat(count).isEqualTo(2); + assertThat(count).isEqualTo(3); } } finally { admin.dropTable(path, true).get(); From 3590ef04120eeb2663e0bcad4cc8f248a219bcca Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 17 Oct 2026 00:57:40 +0800 Subject: [PATCH 39/69] [kafka] Keep focused Produce integration regressions Remove the client matrix runner and probe, and keep dependency preparation scripts and documentation outside the repository. Consolidate lifecycle checks and parameterize null-value readback without reducing coverage. Validation: clean verify passed for 22 remaining integration test cases in both test phases, together with Spotless, Checkstyle and RAT. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 847/847 --- fluss-kafka/src/test/client-matrix/README.md | 109 ------- fluss-kafka/src/test/client-matrix/prepare.py | 162 ---------- .../fluss/kafka/KafkaClientMatrixITCase.java | 306 ------------------ .../fluss/kafka/KafkaProduceAppendITCase.java | 112 +++---- .../kafka/KafkaProduceRegressionITCase.java | 47 +-- .../fluss/kafka/matrix/KafkaClientProbe.java | 111 ------- 6 files changed, 55 insertions(+), 792 deletions(-) delete mode 100644 fluss-kafka/src/test/client-matrix/README.md delete mode 100644 fluss-kafka/src/test/client-matrix/prepare.py delete mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java delete mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java diff --git a/fluss-kafka/src/test/client-matrix/README.md b/fluss-kafka/src/test/client-matrix/README.md deleted file mode 100644 index 3275dc7d843..00000000000 --- a/fluss-kafka/src/test/client-matrix/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# Isolated Kafka producer compatibility matrix - -This test facility runs real Kafka producers against the three-node Fluss test -cluster and reads their records through the native Fluss client. Each selected -client runs in a separate JVM whose classpath contains only a standalone probe, -one `kafka-clients` JAR and that release's runtime libraries. Prepared classpaths -also include a matching SLF4J Simple binding so client network-thread failures -appear in the log. The probe -checks `AppInfoParser.getVersion()` and records its code-source JAR, Java version, -classpath and results. No production dependency is replaced. - -## Requirements and preparation - -Use Java 11 for the reactor and child JVMs, Python 3.8+, curl and Maven. Initial -artifact preparation needs access to Maven Central; subsequent reactor runs can -be offline once the project's own dependencies have been cached. - -The FIP target is every formal Java client release from **0.11.0.0 through 4.3.1**. -[Apache Kafka downloads](https://kafka.apache.org/community/downloads/) records -both boundary releases. The preparation script reads the actual formal versions -from [official Maven metadata](https://repo.maven.apache.org/maven2/org/apache/kafka/kafka-clients/maven-metadata.xml), -fetches each client POM and its runtime JARs, verifies Central SHA-1 sidecars, and -records local SHA-256 hashes. Test dependencies from the published POM are excluded. -It fails explicitly on an unfamiliar runtime dependency instead of guessing its -transitive dependency closure. - -Kafka **2.6.0** needs one explicit exception: its generated ApiVersions messages -load Jackson `JsonNode`, although its published POM declares `jackson-databind` -as `provided`. This is recorded in -[KAFKA-10378](https://issues.apache.org/jira/browse/KAFKA-10378). The script supplies -the release's exact Jackson **2.10.2** databind, core and annotations JARs, all -checksum-verified. These and the diagnostic binding are listed separately under -`test_runtime_additions` in the preparation report. Other provided dependencies -and published test dependencies are not implicitly added. These additions affect -only the isolated test processes, not Fluss production dependencies. - -Start with the boundaries and representative cached releases: - -```sh -python3 fluss-kafka/src/test/client-matrix/prepare.py \ - --output /tmp/fluss-kafka-client-matrix \ - --versions 0.11.0.0,3.6.1,3.9.1,3.9.2,4.2.0,4.3.1 - -# Set JAVA_HOME to a Java 11 installation before running Maven. -./mvnw -o -pl fluss-kafka -am verify \ - -Dtest=KafkaClientMatrixITCase \ - -Dsurefire.failIfNoSpecifiedTests=false \ - -Dkafka.client.matrix.file=/tmp/fluss-kafka-client-matrix/clients.properties -``` - -After the boundary run, prepare and execute the complete formal interval: - -```sh -python3 fluss-kafka/src/test/client-matrix/prepare.py \ - --output /tmp/fluss-kafka-client-matrix --versions all - -./mvnw -o -pl fluss-kafka -am verify \ - -Dtest=KafkaClientMatrixITCase \ - -Dsurefire.failIfNoSpecifiedTests=false \ - -Dkafka.client.matrix.file=/tmp/fluss-kafka-client-matrix/clients.properties -``` - -Do not remove `-am`: server, client, RPC and test artifacts must come from this -worktree's reactor, not another branch's installed SNAPSHOTs. A normal build -without `kafka.client.matrix.file` still runs an isolated smoke test using the -module's actual declared client version; that single-version run is not a matrix. - -## Coverage and result interpretation - -For each selected version the probe uses PLAINTEXT and -`enable.idempotence=false`, negotiates APIs through the real producer, and tests -`acks=0`, `acks=1` and `acks=all`. The same producer alternates canonical and bare -names for the same table, checking acknowledged topic names and offsets. Native -readback verifies all 12 records, their real storage offsets, keys, last-header -wins semantics and null binary rescue fields. Bucket leaders are ready before -unacknowledged writes begin. Producer connections stay open until the parent -confirms native readback; a stdin handshake then permits the probe to close. This -avoids racing unacknowledged appends against connection-close cancellation. - -Preparation is not execution. `preparation.json` lists every requested version as -prepared, unavailable or not executed, including exact artifacts and any errors. -The script deletes the previous executable manifest at startup, including when -metadata retrieval or parsing fails. It only publishes `clients.properties` when -all requested versions are prepared. If downloads are blocked, keep that failure -report and make an explicit, separately recorded subset request for the available -versions; never call the subset a full interval pass. - -Runtime evidence lives under: - -- `fluss-kafka/target/client-matrix-results//command.txt` -- `fluss-kafka/target/client-matrix-results//client.log` -- `fluss-kafka/target/client-matrix-results//result.txt` -- `fluss-kafka/target/surefire-reports/TEST-org.apache.fluss.kafka.KafkaClientMatrixITCase.xml` - -Each invocation deletes the old result and log, writes `FAIL` with the error on -failure, and only writes `PASS` after successful native readback. A client failure -fails its parameterized test; other prepared versions still run. A missing JAR, -version mismatch, process timeout or unsupported JDK is a failure, not a skipped -compatibility result. Archive the preparation report, manifest, Maven log, runtime -logs and current Surefire XML together before `clean` or another matrix run. -The POM runs the selected tests in both Surefire phases; count distinct versions -once and require both executions to succeed. - -This is a bounded Produce compatibility acceptance flow, not proof of every -operation in the FIP. Leadership, lifecycle, min-ISR, invalid-record atomicity and -null-value filtering run separately in the cumulative integration tests using the -module client. They are not yet repeated for every matrix version. SASL is reserved -for PR19. Idempotence, other client SDKs, pre-releases, versions outside the target -interval, and persisted per-row `_kafka_metadata.offset` are not covered here. diff --git a/fluss-kafka/src/test/client-matrix/prepare.py b/fluss-kafka/src/test/client-matrix/prepare.py deleted file mode 100644 index 448949bf003..00000000000 --- a/fluss-kafka/src/test/client-matrix/prepare.py +++ /dev/null @@ -1,162 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Prepare isolated, checksum-verified client classpaths from Maven Central.""" - -import argparse -import hashlib -import json -import os -import re -import subprocess -import sys -import xml.etree.ElementTree as ET -from datetime import datetime, timezone -from pathlib import Path - -CENTRAL = "https://repo.maven.apache.org/maven2/" -NS = {"m": "http://maven.apache.org/POM/4.0.0"} -RUNTIME = {"slf4j-api", "lz4", "lz4-java", "snappy-java", "zstd-jni"} - - -def fetch(relative, destination, refresh=False): - destination.parent.mkdir(parents=True, exist_ok=True) - if destination.exists() and not refresh: - return - temporary = destination.with_name(destination.name + ".download") - try: - subprocess.run(["curl", "--fail", "--location", "--silent", "--show-error", - "--connect-timeout", "10", "--max-time", "120", "--retry", "2", - CENTRAL + relative, "--output", str(temporary)], check=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - temporary.replace(destination) - finally: - temporary.unlink(missing_ok=True) - - -def jar(cache, group, artifact, version): - relative = "/".join([group.replace(".", "/"), artifact, version, - artifact + "-" + version + ".jar"]) - target = cache / relative - fetch(relative, target) - checksum = target.with_suffix(".jar.sha1") - fetch(relative + ".sha1", checksum) - expected = checksum.read_text().strip().split()[0].lower() - actual = hashlib.sha1(target.read_bytes()).hexdigest() - if actual != expected: - raise ValueError("Maven Central checksum mismatch: " + str(target)) - return target.resolve() - - -def prepare(output, version): - cache = output / "artifacts" - relative = "org/apache/kafka/kafka-clients/" + version + "/kafka-clients-" + version + ".pom" - pom = cache / relative - fetch(relative, pom) - root = ET.parse(pom).getroot() - paths = [jar(cache, "org.apache.kafka", "kafka-clients", version)] - dependencies = [] - test_runtime = [] - for dep in root.findall("m:dependencies/m:dependency", NS): - scope = dep.findtext("m:scope", "compile", NS) - group = dep.findtext("m:groupId", namespaces=NS) - artifact = dep.findtext("m:artifactId", namespaces=NS) - dep_version = dep.findtext("m:version", namespaces=NS) - # Kafka 2.6.0 generated ApiVersions messages require JsonNode even for - # byte-array producers, but its POM marks databind provided (KAFKA-10378). - # Supply the release's exact Jackson version and its two runtime dependencies. - if version == "2.6.0" and group == "com.fasterxml.jackson.core" and artifact == "jackson-databind": - if scope != "provided" or dep_version != "2.10.2": - raise ValueError("Unexpected Kafka 2.6.0 Jackson dependency: " + ET.tostring(dep).decode()) - for jackson in ("jackson-databind", "jackson-core", "jackson-annotations"): - paths.append(jar(cache, group, jackson, dep_version)) - test_runtime.append({"coordinate": ":".join([group, jackson, dep_version]), - "reason": "Kafka 2.6.0 provided Jackson closure; KAFKA-10378"}) - continue - if scope not in ("compile", "runtime") or dep.findtext("m:optional", "false", NS) == "true": - continue - # These published Kafka runtime libraries are standalone JARs. Fail on a new - # dependency rather than silently constructing an incomplete classpath. - if artifact not in RUNTIME or not dep_version or "${" in dep_version: - raise ValueError("Unsupported dependency; resolve its runtime closure explicitly: " + ET.tostring(dep).decode()) - paths.append(jar(cache, group, artifact, dep_version)) - dependencies.append(":".join([group, artifact, dep_version])) - if group == "org.slf4j" and artifact == "slf4j-api": - # Without a binding, Kafka's network-thread failures disappear into NOP logging. - paths.append(jar(cache, group, "slf4j-simple", dep_version)) - test_runtime.append({"coordinate": ":".join([group, "slf4j-simple", dep_version]), - "reason": "Isolated client diagnostics; matches published slf4j-api"}) - return paths, dependencies, test_runtime - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--versions", default="0.11.0.0,3.6.1,3.9.1,3.9.2,4.2.0,4.3.1", - help="Comma-separated formal releases, or 'all' for the FIP interval") - args = parser.parse_args() - output = args.output.resolve() - output.mkdir(parents=True, exist_ok=True) - manifest = output / "clients.properties" - manifest.unlink(missing_ok=True) - metadata_path = output / "maven-metadata.xml" - try: - fetch("org/apache/kafka/kafka-clients/maven-metadata.xml", metadata_path, refresh=True) - except Exception as exc: - (output / "preparation.json").write_text(json.dumps( - {"status": "METADATA_UNAVAILABLE_NOT_EXECUTED", "error": str(exc)}, indent=2) + "\n") - return 1 - root = ET.parse(metadata_path).getroot() - versions = [v.text for v in root.findall("versioning/versions/version") - if re.fullmatch(r"[0-9]+(?:\.[0-9]+){2,3}", v.text)] - def number(version): - return tuple(int(x) for x in version.split(".")) - eligible = sorted([v for v in versions if number("0.11.0.0") <= number(v) <= number("4.3.1")], key=number) - requested = eligible if args.versions == "all" else args.versions.split(",") - report = {"source": CENTRAL + "org/apache/kafka/kafka-clients/maven-metadata.xml", - "prepared_at": datetime.now(timezone.utc).isoformat(), - "metadata_updated": root.findtext("versioning/lastUpdated"), - "formal_releases_in_scope": eligible, "requested": requested, - "not_requested": [v for v in eligible if v not in requested], - "clients": {v: {"status": "NOT_EXECUTED"} for v in requested}} - entries = [] - for version in requested: - try: - if version not in eligible: - raise ValueError("Version is absent from the official formal-release interval") - paths, dependencies, test_runtime = prepare(output, version) - entries.append(version + "=" + os.pathsep.join(str(p) for p in paths)) - report["clients"][version] = {"status": "PREPARED_NOT_EXECUTED", "dependencies": dependencies, - "test_runtime_additions": test_runtime, - "jars": [{"path": str(p), "sha256": hashlib.sha256(p.read_bytes()).hexdigest()} for p in paths]} - except Exception as exc: - detail = getattr(exc, "stderr", b"") - report["clients"][version] = {"status": "UNAVAILABLE_NOT_EXECUTED", - "error": str(exc) + " " + (detail.decode(errors="replace") if isinstance(detail, bytes) else str(detail))} - print(version, report["clients"][version]["status"], flush=True) - (output / "preparation.json").write_text(json.dumps(report, indent=2) + "\n") - # Do not publish a partial executable matrix when a requested version is missing. - manifest.unlink(missing_ok=True) - if any(v["status"] == "UNAVAILABLE_NOT_EXECUTED" for v in report["clients"].values()): - return 1 - manifest.write_text("\n".join(entries) + "\n") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java deleted file mode 100644 index dd378dece44..00000000000 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka; - -import org.apache.fluss.client.Connection; -import org.apache.fluss.client.ConnectionFactory; -import org.apache.fluss.client.admin.Admin; -import org.apache.fluss.client.table.Table; -import org.apache.fluss.client.table.scanner.ScanRecord; -import org.apache.fluss.client.table.scanner.log.LogScanner; -import org.apache.fluss.cluster.ServerNode; -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.kafka.matrix.KafkaClientProbe; -import org.apache.fluss.metadata.DatabaseDescriptor; -import org.apache.fluss.metadata.LogFormat; -import org.apache.fluss.metadata.Schema; -import org.apache.fluss.metadata.TableBucket; -import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.row.InternalRow; -import org.apache.fluss.server.testutils.FlussClusterExtension; -import org.apache.fluss.types.DataTypes; - -import org.apache.kafka.common.utils.AppInfoParser; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; - -import java.io.File; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Runs actual, version-checked Kafka clients in isolated JVMs and verifies native Fluss storage. - */ -class KafkaClientMatrixITCase { - private static final String DATABASE = "client_matrix"; - private static final Path OUTPUT = - Paths.get(System.getProperty("project.basedir"), "target", "client-matrix-results"); - - @RegisterExtension - static final FlussClusterExtension CLUSTER = - FlussClusterExtension.builder() - .setNumOfTabletServers(3) - .setClusterConf(clusterConfig()) - .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") - .build(); - - static Stream clients() throws Exception { - return installations().stringPropertyNames().stream().sorted(); - } - - private static Properties installations() throws Exception { - Properties properties = new Properties(); - String manifest = System.getProperty("kafka.client.matrix.file"); - if (manifest != null) { - try (InputStream input = Files.newInputStream(Paths.get(manifest))) { - properties.load(input); - } - assertThat(properties).as("The requested matrix must not be empty").isNotEmpty(); - } else { - // Normal builds still exercise isolation. Other versions require an explicit manifest. - String classpath = - Arrays.stream( - System.getProperty("java.class.path") - .split(Pattern.quote(File.pathSeparator))) - .filter( - entry -> - Paths.get(entry) - .getFileName() - .toString() - .matches( - "(kafka-clients|slf4j-api|zstd-jni|snappy-java|lz4|lz4-java)-.+\\.jar")) - .collect(Collectors.joining(File.pathSeparator)); - properties.setProperty(AppInfoParser.getVersion(), classpath); - } - return properties; - } - - @ParameterizedTest(name = "Kafka client {0}") - @MethodSource("clients") - void testIsolatedClientWithAllAckModesAndBothNames(String version) throws Exception { - Path output = OUTPUT.resolve(version); - Files.createDirectories(output); - Files.deleteIfExists(output.resolve("result.txt")); - Files.deleteIfExists(output.resolve("client.log")); - Files.deleteIfExists(output.resolve("command.txt")); - try { - runClient(version, output); - } catch (Exception | AssertionError failure) { - Files.write( - output.resolve("result.txt"), - Arrays.asList("FAIL " + version, failure.toString()), - StandardCharsets.UTF_8); - throw failure; - } - } - - private static void runClient(String version, Path output) throws Exception { - String classpath = installations().getProperty(version); - List jars = Arrays.asList(classpath.split(Pattern.quote(File.pathSeparator))); - assertThat(jars).allSatisfy(jar -> assertThat(Paths.get(jar)).isRegularFile()); - assertThat( - jars.stream() - .filter( - jar -> - Paths.get(jar) - .getFileName() - .toString() - .startsWith("kafka-clients-")) - .collect(Collectors.toList())) - .singleElement() - .satisfies( - jar -> - assertThat(Paths.get(jar).getFileName().toString()) - .isEqualTo("kafka-clients-" + version + ".jar")); - // Export just the probe bytecode, never the server/test classpath, to the child JVM. - String resource = KafkaClientProbe.class.getName().replace('.', '/') + ".class"; - Path probe = output.resolve("probe").resolve(resource); - Files.createDirectories(probe.getParent()); - try (InputStream input = - KafkaClientProbe.class.getClassLoader().getResourceAsStream(resource)) { - assertThat(input).isNotNull(); - Files.copy(input, probe, StandardCopyOption.REPLACE_EXISTING); - } - String prefix = "client_" + version.replace('.', '_'); - List paths = new ArrayList<>(); - try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); - Admin admin = connection.getAdmin()) { - try { - admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - for (String acks : Arrays.asList("0", "1", "all")) { - TablePath path = TablePath.of(DATABASE, prefix + "_" + acks); - paths.add(path); - admin.createTable(path, descriptor(), false).get(); - CLUSTER.waitAndGetLeaderReplica( - new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); - } - ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); - List command = - Arrays.asList( - Paths.get(System.getProperty("java.home"), "bin", "java") - .toString(), - "-cp", - output.resolve("probe") + File.pathSeparator + classpath, - KafkaClientProbe.class.getName(), - version, - node.host() + ":" + node.port(), - DATABASE, - prefix); - Files.write(output.resolve("command.txt"), command, StandardCharsets.UTF_8); - Path log = output.resolve("client.log"); - Process process = - new ProcessBuilder(command) - .redirectErrorStream(true) - .redirectOutput(log.toFile()) - .start(); - try { - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(120); - String text = ""; - while (process.isAlive() && System.nanoTime() < deadline) { - text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); - if (text.contains("WRITES_COMPLETE")) { - break; - } - process.waitFor(100, TimeUnit.MILLISECONDS); - } - text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); - assertThat(text) - .as("Client %s log: %s", version, log) - .contains( - "VERSION=" + version, - "JAR=", - "JAVA=", - "ACKS=0:SENT", - "ACKS=1:SENT", - "ACKS=all:SENT", - "WRITES_COMPLETE"); - assertThat(process.isAlive()) - .as("Client must stay connected during native readback") - .isTrue(); - for (String acks : Arrays.asList("0", "1", "all")) { - assertNativeReadback( - connection, - TablePath.of(DATABASE, prefix + "_" + acks), - version, - acks); - } - process.getOutputStream().write('\n'); - process.getOutputStream().flush(); - assertThat(process.waitFor(30, TimeUnit.SECONDS)) - .as("Client log: %s", log) - .isTrue(); - text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); - assertThat(process.exitValue()).as("Client %s: %s", version, text).isZero(); - assertThat(text).contains("NATIVE_READBACK_CONFIRMED"); - Files.write( - output.resolve("result.txt"), - Collections.singletonList( - "PASS " - + version - + " PLAINTEXT non-idempotent; acks=0,1,all; same producer full/alias; 12 native rows and offsets verified"), - StandardCharsets.UTF_8); - } finally { - if (process.isAlive()) { - process.destroyForcibly(); - process.waitFor(10, TimeUnit.SECONDS); - } - } - } finally { - for (TablePath path : paths) { - admin.dropTable(path, true).get(); - } - } - } - } - - private static void assertNativeReadback( - Connection connection, TablePath path, String version, String acks) throws Exception { - try (Table table = connection.getTable(path); - LogScanner scanner = table.newScan().createLogScanner()) { - scanner.subscribeFromBeginning(0); - int count = 0; - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - while (count < 4 && System.nanoTime() < deadline) { - for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { - InternalRow row = record.getRow(); - assertThat(row.getBytes(0)) - .isEqualTo( - (version + ":" + acks + ":" + count) - .getBytes(StandardCharsets.UTF_8)); - assertThat(row.getBytes(1)).isEqualTo("key".getBytes(StandardCharsets.UTF_8)); - assertThat(row.getMap(2).size()).isEqualTo(1); - assertThat(row.getMap(2).keyArray().getString(0).toString()) - .isEqualTo("source"); - assertThat(row.getMap(2).valueArray().getBytes(0)).containsExactly((byte) 2); - assertThat(row.isNullAt(3)).isTrue(); - assertThat(record.logOffset()).isEqualTo(count); - count++; - } - } - assertThat(count) - .as("Native readback for client %s, acks %s", version, acks) - .isEqualTo(4); - } - } - - private static TableDescriptor descriptor() { - return TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column("_kafka_key", DataTypes.BYTES()) - .column( - "_kafka_headers", - DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) - .column("_kafka_rescue", DataTypes.STRING()) - .build()) - .distributedBy(1) - .logFormat(LogFormat.ARROW) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") - .build(); - } - - private static Configuration clusterConfig() { - Configuration config = new Configuration(); - config.set(ConfigOptions.KAFKA_ENABLED, true); - config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, DATABASE); - config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); - config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); - return config; - } -} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 79679e2f5dd..88f62125eb0 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -237,76 +237,62 @@ void testNotNullValueSchemaIsRejectedWhenCreatingKafkaTable(String format) throw } } - @Test - void testNullAndEmptyValuesPreserveKeysAndOffsets() throws Exception { + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullAndEmptyValuesPreserveKeysAndOffsets(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "null_values_" + format); + List values = Arrays.asList(null, new byte[0], VALUE); + List keys = Arrays.asList(new byte[] {(byte) 0xff}, null, null); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - for (String format : new String[] {"binary", "string"}) { - TablePath path = TablePath.of(DATABASE, "null_values_" + format); - admin.createTable(path, descriptor(format), false).get(); - CLUSTER.waitUntilAllGatewayHasSameMetadata(); - try (KafkaProducer producer = producer("all", 0)) { - String topic = path.toString(); - assertThat( - producer.send( - new ProducerRecord( - topic, - 0, - 123L, - new byte[] {(byte) 0xff}, - null)) - .get(30, TimeUnit.SECONDS) - .offset()) - .isZero(); - assertThat( - producer.send( - new ProducerRecord( - topic, 0, 123L, null, new byte[0])) - .get(30, TimeUnit.SECONDS) - .offset()) - .isEqualTo(1L); - assertThat( - producer.send( - new ProducerRecord( - topic, 0, 123L, null, VALUE)) - .get(30, TimeUnit.SECONDS) - .offset()) - .isEqualTo(2L); - try (Table table = connection.getTable(path); - LogScanner scanner = table.newScan().createLogScanner()) { - scanner.subscribeFromBeginning(0); - int count = 0; - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - while (count < 3 && System.nanoTime() < deadline) { - for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { - InternalRow row = record.getRow(); - if (count == 0) { - assertThat(row.isNullAt(0)).isTrue(); - assertThat(row.getBytes(3)).isEqualTo(new byte[] {(byte) 0xff}); - } else { - byte[] expected = count == 1 ? new byte[0] : VALUE; - if (format.equals("binary")) { - assertThat(row.getBytes(0)).isEqualTo(expected); - } else { - assertThat(row.getString(0).toString()) - .isEqualTo( - new String( - expected, StandardCharsets.UTF_8)); - } - assertThat(row.isNullAt(3)).isTrue(); - } - assertThat(row.getMap(1).size()).isZero(); - assertThat(row.isNullAt(2)).isTrue(); - assertThat(record.logOffset()).isEqualTo(count); - count++; + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + for (int i = 0; i < values.size(); i++) { + RecordMetadata metadata = + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + 123L, + keys.get(i), + values.get(i))) + .get(30, TimeUnit.SECONDS); + assertThat(metadata.offset()).isEqualTo(i); + } + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < values.size() && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + InternalRow row = record.getRow(); + byte[] expected = values.get(count); + if (expected == null) { + assertThat(row.isNullAt(0)).isTrue(); + } else if (format.equals("binary")) { + assertThat(row.getBytes(0)).isEqualTo(expected); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo(new String(expected, StandardCharsets.UTF_8)); } + if (keys.get(count) == null) { + assertThat(row.isNullAt(3)).isTrue(); + } else { + assertThat(row.getBytes(3)).isEqualTo(keys.get(count)); + } + assertThat(row.getMap(1).size()).isZero(); + assertThat(row.isNullAt(2)).isTrue(); + assertThat(record.logOffset()).isEqualTo(count); + count++; } - assertThat(count).isEqualTo(3); } - } finally { - admin.dropTable(path, true).get(); + assertThat(count).isEqualTo(values.size()); } + } finally { + admin.dropTable(path, true).get(); } } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java index 5d016ac5c9b..e35d658f211 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -108,45 +108,6 @@ class KafkaProduceRegressionITCase { .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") .build(); - @Test - void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { - TablePath path = TablePath.of(DATABASE, "metadata_lifecycle"); - String topic = path.toString(); - try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); - Admin admin = connection.getAdmin()) { - try { - createTable(admin, path, 3); - MetadataResponse initial = waitForTopic(topic); - Uuid initialId = initial.data().topics().find(topic).topicId(); - assertThat(initialId).isNotEqualTo(Uuid.ZERO_UUID); - assertKafkaListenerEndpoints(initial); - assertThat(fetchMetadata(topic).data().topics().find(topic).topicId()) - .isEqualTo(initialId); - - admin.dropTable(path, false).get(); - retry( - Duration.ofMinutes(1), - () -> - assertThat(fetchMetadata(topic).errors()) - .containsEntry(topic, Errors.UNKNOWN_TOPIC_OR_PARTITION)); - - createTable(admin, path, 3); - retry( - Duration.ofMinutes(1), - () -> { - MetadataResponseTopic recreated = - fetchMetadata(topic).data().topics().find(topic); - assertThat(recreated.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(recreated.topicId()) - .isNotEqualTo(Uuid.ZERO_UUID) - .isNotEqualTo(initialId); - }); - } finally { - admin.dropTable(path, true).get(); - } - } - } - @Test void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { TablePath path = TablePath.of(DATABASE, "metadata_failover"); @@ -323,7 +284,7 @@ void testClusterMinIsrPolicyAndDynamicReload() throws Exception { } @Test - void testSameProducerAliasesFollowEnablementPropertiesAndRecreation() throws Exception { + void testSameProducerAliasesFollowTableLifecycle() throws Exception { TablePath path = TablePath.of(DATABASE, "cached_lifecycle"); String full = path.toString(); String alias = path.getTableName(); @@ -331,7 +292,11 @@ void testSameProducerAliasesFollowEnablementPropertiesAndRecreation() throws Exc Admin admin = connection.getAdmin()) { try { createTable(admin, path, 3); - Uuid fullId = topicId(full); + MetadataResponse initial = waitForTopic(full); + assertKafkaListenerEndpoints(initial); + Uuid fullId = initial.data().topics().find(full).topicId(); + assertThat(fullId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(full)).isEqualTo(fullId); Uuid aliasId = topicId(alias); assertThat(aliasId).isNotEqualTo(Uuid.ZERO_UUID).isNotEqualTo(fullId); try (KafkaProducer producer = producer("all", true)) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java deleted file mode 100644 index b0dd5a0e146..00000000000 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka.matrix; - -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.header.internals.RecordHeader; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.utils.AppInfoParser; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * Standalone client process: deliberately depends only on the JDK and the selected Kafka client. - */ -public final class KafkaClientProbe { - private KafkaClientProbe() {} - - /** Writes through a real endpoint; a failed check terminates the isolated process. */ - public static void main(String[] args) throws Exception { - String expectedVersion = args[0]; - String actualVersion = AppInfoParser.getVersion(); - if (!expectedVersion.equals(actualVersion)) { - throw new IllegalStateException( - "Expected " + expectedVersion + " but loaded " + actualVersion); - } - System.out.println("VERSION=" + actualVersion); - System.out.println( - "JAR=" + KafkaProducer.class.getProtectionDomain().getCodeSource().getLocation()); - System.out.println("JAVA=" + System.getProperty("java.version")); - System.out.println("CLASSPATH=" + System.getProperty("java.class.path")); - List> producers = new ArrayList<>(); - try { - for (String acks : Arrays.asList("0", "1", "all")) { - Map config = new HashMap<>(); - config.put("bootstrap.servers", args[1]); - config.put("key.serializer", ByteArraySerializer.class); - config.put("value.serializer", ByteArraySerializer.class); - config.put("enable.idempotence", false); - config.put("security.protocol", "PLAINTEXT"); - config.put("acks", acks); - config.put("max.in.flight.requests.per.connection", 1); - config.put("request.timeout.ms", 10000); - config.put("max.block.ms", 30000); - config.put("retries", 3); - config.put("compression.type", "none"); - config.put("linger.ms", 0); - String alias = args[3] + "_" + acks; - KafkaProducer producer = new KafkaProducer<>(config); - producers.add(producer); - for (int i = 0; i < 4; i++) { - String topic = i % 2 == 0 ? args[2] + "." + alias : alias; - RecordMetadata result = - producer.send( - new ProducerRecord<>( - topic, - 0, - 123L, - "key".getBytes(StandardCharsets.UTF_8), - (actualVersion + ":" + acks + ":" + i) - .getBytes(StandardCharsets.UTF_8), - Arrays.asList( - new RecordHeader( - "source", new byte[] {1}), - new RecordHeader( - "source", new byte[] {2})))) - .get(30, TimeUnit.SECONDS); - if (!result.topic().equals(topic) - || (!acks.equals("0") && result.offset() != i)) { - throw new IllegalStateException("Unexpected acknowledgement " + result); - } - } - producer.flush(); - System.out.println("ACKS=" + acks + ":SENT"); - } - // acks=0 only confirms a socket write. Keep connections open until the - // parent verifies native storage, rather than racing cancellation on close. - System.out.println("WRITES_COMPLETE"); - if (System.in.read() != '\n') { - throw new IllegalStateException("Native readback was not confirmed by the parent"); - } - System.out.println("NATIVE_READBACK_CONFIRMED"); - } finally { - for (KafkaProducer producer : producers) { - producer.close(); - } - } - } -} From a39b97485c8d9c3a0bc2ea95a7882a1fda5ffc70 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 13:09:03 +0800 Subject: [PATCH 40/69] [kafka] Use typed conversion options in regression tests Update integration table descriptors to use the current typed Kafka value conversion API. Co-Authored-By: Codex AI-Model: gpt-6.1-sol AI-Contributed/Feature: 0/0 AI-Contributed/UT: 9/9 --- .../apache/fluss/kafka/KafkaProduceRegressionITCase.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java index e35d658f211..3f686d6a74e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -30,6 +30,7 @@ import org.apache.fluss.cluster.rebalance.RebalanceStatus; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.config.cluster.AlterConfig; import org.apache.fluss.config.cluster.AlterConfigOpType; import org.apache.fluss.metadata.DatabaseDescriptor; @@ -435,7 +436,7 @@ void testNeverEnabledTableRejectsBothNamesWithoutAppending() throws Exception { .logFormat(LogFormat.ARROW) .property( ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, - "binary") + KafkaValueConversionType.BINARY) .build(), false) .get(); @@ -475,7 +476,7 @@ void testInvalidRecordRejectsWholePartitionBeforeNativeAppend() throws Exception .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) .property( ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, - "binary") + KafkaValueConversionType.BINARY) .build(), false) .get(); @@ -708,7 +709,9 @@ private static void createTable( .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) .build(), false) .get(); From 72ca847aabc9e0d8e87f4fda500ae7c545d6c1fd Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 13:12:49 +0800 Subject: [PATCH 41/69] [kafka] Test nullable key schema validation Replace the stale invalid-record case that depended on a NOT NULL Kafka key with an assertion that table creation rejects the unsupported schema contract. Co-Authored-By: Codex AI-Model: gpt-6.1-sol AI-Contributed/Feature: 0/0 AI-Contributed/UT: 93/93 --- .../kafka/KafkaProduceRegressionITCase.java | 93 ++++--------------- 1 file changed, 20 insertions(+), 73 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java index 3f686d6a74e..8601ae69661 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -33,6 +33,7 @@ import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.config.cluster.AlterConfig; import org.apache.fluss.config.cluster.AlterConfigOpType; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; @@ -454,84 +455,30 @@ void testNeverEnabledTableRejectsBothNamesWithoutAppending() throws Exception { } @Test - void testInvalidRecordRejectsWholePartitionBeforeNativeAppend() throws Exception { + void testNotNullKeySchemaIsRejectedWhenCreatingKafkaTable() throws Exception { TablePath path = TablePath.of(DATABASE, "atomic_conversion"); - byte[] key = new byte[] {1}; try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - admin.createTable( - path, - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column( - "_kafka_key", - DataTypes.BYTES().copy(false)) - .build()) - .distributedBy(1) - .logFormat(LogFormat.ARROW) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property( - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, - KafkaValueConversionType.BINARY) - .build(), - false) - .get(); - CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); - try (KafkaProducer producer = producer("all", true)) { - int leader = producer.partitionsFor(path.toString()).get(0).leader().id(); - ServerNode node = - CLUSTER.getTabletServerNodes("KAFKA").stream() - .filter(n -> n.id() == leader) - .findFirst() - .get(); - try (Socket socket = connect(node)) { - ProduceResponse rejected = - (ProduceResponse) - sendRequest( - socket, - produceRequest( - path.toString(), - new SimpleRecord( - key, - "valid_prefix" - .getBytes( - StandardCharsets - .UTF_8)), - new SimpleRecord( - null, - "invalid_key" - .getBytes( - StandardCharsets - .UTF_8)))); - assertThat(rejected.errorCounts()).containsEntry(Errors.INVALID_RECORD, 1); - assertThat( - rejected.data() - .responses() - .find(path.toString()) - .partitionResponses() - .get(0) - .baseOffset()) - .isEqualTo(-1L); - } - assertThat( - producer.send( - new ProducerRecord<>( - path.toString(), - 0, - key, - "accepted" - .getBytes( - StandardCharsets - .UTF_8))) - .get(30, TimeUnit.SECONDS) - .offset()) - .isZero(); - assertValues(connection, path, "accepted"); - } + TableDescriptor descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES().copy(false)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(); + assertThatThrownBy(() -> admin.createTable(path, descriptor, false).get()) + .hasCauseInstanceOf(InvalidTableException.class) + .rootCause() + .hasMessageContaining("Kafka requires nullable fields"); } finally { admin.dropTable(path, true).get(); } From 50abb2dc6fd7ad52e3cdab4f323c8c6368c62360 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 11 Sep 2026 15:42:27 +0800 Subject: [PATCH 42/69] [kafka] Separate value decoding and preserve record metadata Use the shared table contract for binary and strict UTF-8 string value decoders. Preserve the partition-aware transcoder API and pass each full record to row assembly so optional key, headers and three-field metadata retain their input semantics. Binary values keep borrowed array storage. Cover invalid UTF-8 and nullability failures with declared metadata: failed partitions append no prefix, and reused plans preserve missing timestamps, batch types and the next request's partition after failure. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 274/274 AI-Contributed/UT: 268/268 --- .../format/BinaryKafkaFormatFactory.java | 46 +++++ .../fluss/kafka/format/KafkaFieldDecoder.java | 35 ++++ .../kafka/format/KafkaFormatFactory.java | 33 ++++ .../format/KafkaFormatFactoryRegistry.java | 55 ++++++ .../format/StringKafkaFormatFactory.java | 72 +++++++ .../transcode/ArrowKafkaRecordTranscoder.java | 42 +--- .../GatewayKafkaProduceBackendTest.java | 63 ++++++ .../KafkaFormatFactoryRegistryTest.java | 185 ++++++++++++++++++ .../ArrowKafkaRecordTranscoderTest.java | 51 +++-- 9 files changed, 535 insertions(+), 47 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java new file mode 100644 index 00000000000..bfa8e1288c7 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.types.BytesType; + +/** Factory for the Kafka binary byte format. */ +@Internal +public final class BinaryKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public KafkaValueConversionType format() { + return KafkaValueConversionType.BINARY; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof BytesType)) { + throw new KafkaTopicSchemaException( + "Kafka binary format requires exactly one _kafka_value BYTES field."); + } + return bytes -> new Object[] {bytes}; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java new file mode 100644 index 00000000000..b70c85f17d4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +/** Decodes one nullable Kafka record value into its ordered Fluss field projection. */ +@Internal +public interface KafkaFieldDecoder { + + /** + * Decodes Kafka value bytes into internal Fluss field values. + * + *

The input is borrowed and must not be modified. Returned values may borrow its storage; + * callers must encode them synchronously before the input can be reused. + */ + Object[] decode(@Nullable byte[] bytes); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java new file mode 100644 index 00000000000..bb444610726 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.metadata.KafkaFieldProjection; + +/** Creates a decoder for one Kafka value format and Fluss field projection. */ +@Internal +public interface KafkaFormatFactory { + + /** Returns the format implemented by this factory. */ + KafkaValueConversionType format(); + + /** Creates and validates a decoder for the projection. */ + KafkaFieldDecoder createDecoder(KafkaFieldProjection projection); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java new file mode 100644 index 00000000000..36ff64a9b69 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaFieldProjection; + +import java.util.EnumMap; +import java.util.Map; + +/** Registry of built-in Kafka record format factories. */ +@Internal +public final class KafkaFormatFactoryRegistry { + + private final Map factories; + + /** Creates a registry containing all built-in formats. */ + public KafkaFormatFactoryRegistry() { + factories = new EnumMap<>(KafkaValueConversionType.class); + register(new BinaryKafkaFormatFactory()); + register(new StringKafkaFormatFactory()); + } + + /** Creates a decoder for the requested format and projection. */ + public KafkaFieldDecoder createDecoder( + KafkaValueConversionType format, KafkaFieldProjection projection) { + KafkaFormatFactory factory = factories.get(format); + if (factory == null) { + throw new KafkaTopicSchemaException( + "No Kafka format factory is registered for '" + format + "'."); + } + return factory.createDecoder(projection); + } + + private void register(KafkaFormatFactory factory) { + factories.put(factory.format(), factory); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java new file mode 100644 index 00000000000..ff6fd544397 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.types.StringType; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** Factory for strict UTF-8 Kafka string decoding. */ +@Internal +public final class StringKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public KafkaValueConversionType format() { + return KafkaValueConversionType.STRING; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof StringType)) { + throw new KafkaTopicSchemaException( + "Kafka string format requires exactly one _kafka_value STRING field."); + } + final String fieldName = projection.nameAt(0); + return bytes -> new Object[] {decode(bytes, fieldName)}; + } + + private static BinaryString decode(byte[] bytes, String fieldName) { + if (bytes == null) { + return null; + } + try { + return BinaryString.fromString( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString()); + } catch (CharacterCodingException e) { + throw new KafkaRecordEncodingException( + "Kafka field '" + fieldName + "' is not valid UTF-8.", e); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 0f84ddf7932..4eb770c2272 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -18,20 +18,14 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.annotation.Internal; -import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.format.KafkaFieldDecoder; +import org.apache.fluss.kafka.format.KafkaFormatFactoryRegistry; import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; -import org.apache.fluss.row.BinaryString; -import javax.annotation.Nullable; - -import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.StandardCharsets; import java.util.List; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -39,6 +33,7 @@ /** Converts binary/string Kafka values and fixed system columns into owned Arrow log bytes. */ @Internal public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { + private final KafkaFormatFactoryRegistry formats = new KafkaFormatFactoryRegistry(); private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); private final KafkaCompiledWritePlanCache writePlanCache = new KafkaCompiledWritePlanCache(); @@ -64,44 +59,19 @@ public BytesView transcode(int partitionId, List records, KafkaTopicWrit checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTableSchema schema = plan.topicSchema(); - if (schema.valueConversionType() != KafkaValueConversionType.BINARY - && schema.valueConversionType() != KafkaValueConversionType.STRING) { - throw new KafkaRecordEncodingException( - "Unsupported Kafka value conversion type: " + schema.valueConversionType()); - } + KafkaFieldDecoder valueDecoder = + formats.createDecoder(schema.valueConversionType(), schema.valueProjection()); KafkaRowAssembler assembler = plan.rowAssembler(); return arrowRecordEncoder.encodeStreaming( consumer -> { for (Record record : records) { consumer.append( assembler.assemble( - new Object[] { - decode( - schema.valueConversionType(), - record.borrowedValue()) - }, + valueDecoder.decode(record.borrowedValue()), record, partitionId)); } }, plan.tableInfo()); } - - private static @Nullable Object decode( - KafkaValueConversionType format, @Nullable byte[] bytes) { - if (bytes == null || format == KafkaValueConversionType.BINARY) { - return bytes; - } - try { - return BinaryString.fromString( - StandardCharsets.UTF_8 - .newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(bytes)) - .toString()); - } catch (CharacterCodingException e) { - throw new KafkaRecordEncodingException("Kafka string field is not valid UTF-8.", e); - } - } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index 6d47af74a6e..34ab0ad5edc 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -810,6 +810,69 @@ valid, good(0).records().get(0)))))) assertThat(service.appends).isEmpty(); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testInvalidRecordDoesNotAppendValidPrefixWithMetadata(boolean malformedUtf8) + throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .schema(schema) + .build() + .toJsonBytes())); + Record valid = new Record(1L, new byte[] {1}, new byte[] {65}, Collections.emptyList()); + Record invalid = + malformedUtf8 + ? new Record( + 2L, + new byte[] {1}, + new byte[] {(byte) 0xc3, 0x28}, + Collections.emptyList()) + : new Record( + 2L, + new byte[] {1}, + new byte[] {(byte) 0xff}, + Collections.emptyList()); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 2, Arrays.asList(valid, invalid))))) + .join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + assertErrors( + backend.write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 1, Collections.singletonList(valid))))) + .join(), + Errors.NONE); + assertThat(service.appends).hasSize(1); + } + @ParameterizedTest @ValueSource(strings = {"binary", "string"}) void testNullValueViolationRejectsNotNullTableContractBeforeAppending(String format) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java new file mode 100644 index 00000000000..c7051c0f816 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests value decoders against the shared fixed-column table contract. */ +class KafkaFormatFactoryRegistryTest { + private final KafkaFormatFactoryRegistry formats = new KafkaFormatFactoryRegistry(); + + @Test + void testBinaryBorrowsOpaqueBytesAndPreservesNullAndEmpty() { + KafkaFieldDecoder decoder = + formats.createDecoder( + KafkaValueConversionType.BINARY, projection(DataTypes.BYTES())); + byte[] opaque = new byte[] {(byte) 0xff, 0}; + assertThat(decoder.decode(opaque)[0]).isSameAs(opaque); + assertThat(opaque).containsExactly((byte) 0xff, (byte) 0); + assertThat(decoder.decode(null)).containsExactly((Object) null); + byte[] empty = new byte[0]; + assertThat(decoder.decode(empty)[0]).isSameAs(empty); + } + + @Test + void testStringRejectsMalformedUtf8AndCanBeReused() { + KafkaFieldDecoder decoder = + formats.createDecoder( + KafkaValueConversionType.STRING, projection(DataTypes.STRING())); + for (byte[] invalid : + new byte[][] { + {(byte) 0xc3, 0x28}, + {(byte) 0xc0, (byte) 0x80}, + {(byte) 0xed, (byte) 0xa0, (byte) 0x80}, + {(byte) 0xe2, (byte) 0x82} + }) { + byte[] original = invalid.clone(); + assertThatThrownBy(() -> decoder.decode(invalid)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("_kafka_value") + .hasMessageContaining("UTF-8"); + assertThat(invalid).isEqualTo(original); + byte[] valid = "键".getBytes(StandardCharsets.UTF_8); + assertThat(decoder.decode(valid)).containsExactly(BinaryString.fromString("键")); + assertThat(valid).isEqualTo("键".getBytes(StandardCharsets.UTF_8)); + } + assertThat(decoder.decode(null)).containsExactly((Object) null); + assertThat(decoder.decode(new byte[0])).containsExactly(BinaryString.fromString("")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesUseThePublicValueProjection(boolean string) { + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + TableDescriptor table = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES()) + .column(KafkaTableSchema.VALUE_COLUMN, type) + .column( + KafkaTableSchema.HEADERS_COLUMN, + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + string + ? KafkaValueConversionType.STRING + : KafkaValueConversionType.BINARY) + .build(); + KafkaTableSchema schema = KafkaTableSchema.resolve(table); + assertThat(schema.valueProjection().positions()).containsExactly(1); + KafkaFieldDecoder decoder = + formats.createDecoder(schema.valueConversionType(), schema.valueProjection()); + byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + assertThat(decoder.decode(payload)) + .containsExactly(string ? BinaryString.fromString("payload") : payload); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesRejectIncompatibleProjections(boolean string) { + KafkaValueConversionType format = + string ? KafkaValueConversionType.STRING : KafkaValueConversionType.BINARY; + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + assertThatThrownBy( + () -> + formats.createDecoder( + format, + projection( + string ? DataTypes.BYTES() : DataTypes.STRING()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining(string ? "STRING" : "BYTES"); + KafkaFieldProjection multiple = + new KafkaFieldProjection( + DataTypes.ROW( + DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type), + DataTypes.FIELD(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES())), + Arrays.asList(0, 1)); + KafkaFieldProjection empty = + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type)), + Collections.emptyList()); + for (KafkaFieldProjection invalid : Arrays.asList(multiple, empty)) { + assertThatThrownBy(() -> formats.createDecoder(format, invalid)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("exactly one"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesRejectKeyAndArbitraryValueColumnNames(boolean string) { + KafkaValueConversionType format = + string ? KafkaValueConversionType.STRING : KafkaValueConversionType.BINARY; + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + for (String name : Arrays.asList(KafkaTableSchema.KEY_COLUMN, "body")) { + KafkaFieldProjection invalid = + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(name, type)), + Collections.singletonList(0)); + assertThatThrownBy(() -> formats.createDecoder(format, invalid)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value"); + } + } + + @Test + void testJsonHasNoDecoderUntilItsImplementationIsRegistered() { + assertThatThrownBy( + () -> + formats.createDecoder( + KafkaValueConversionType.PARSE_JSON, + projection(DataTypes.STRING()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("PARSE_JSON"); + } + + private static KafkaFieldProjection projection(DataType type) { + return new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type)), + Collections.singletonList(0)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 06a3841fc7b..59e33ad2ea0 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -677,10 +677,23 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce }); } - @Test - void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { - TableInfo table = valueTable(false); - Record valid = new Record(1L, null, bytes("valid"), Collections.emptyList()); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable(boolean metadata) + throws Exception { + Schema.Builder schema = Schema.newBuilder().column("_kafka_value", DataTypes.STRING()); + if (metadata) { + schema.column("_kafka_metadata", metadataType(true)); + } + TableInfo table = table(descriptor(schema.build(), true).build()); + KafkaTopicWritePlan plan = transcoder.prepare(table); + Record valid = + new Record( + Record.NO_TIMESTAMP, + TimestampType.LOG_APPEND_TIME, + null, + bytes("valid"), + Collections.emptyList()); for (byte[] invalid : new byte[][] {{(byte) 0xc3, 0x28}, {(byte) 0xff}}) { AtomicReference output = new AtomicReference<>(); Throwable failure = @@ -688,33 +701,49 @@ void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw () -> output.set( transcoder.transcode( - 0, + 7, Arrays.asList( valid, new Record( 2L, + TimestampType.LOG_APPEND_TIME, null, invalid, Collections.emptyList())), - table))); + plan))); assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); assertThat(failure.getSuppressed()).isEmpty(); assertThat(output).hasNullValue(); } read( - transcoder.transcode(0, Collections.singletonList(valid), table), + transcoder.transcode(3, Collections.singletonList(valid), plan), table, 1, - row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); + row -> { + assertThat(row.getString(0).toString()).isEqualTo("valid"); + if (metadata) { + assertMetadata(row, 1, Record.NO_TIMESTAMP, 1, 3); + } + }); read( transcoder.transcode( 0, Collections.singletonList( - new Record(2L, null, null, Collections.emptyList())), - table), + new Record( + 2L, + TimestampType.LOG_APPEND_TIME, + null, + null, + Collections.emptyList())), + plan), table, 1, - row -> assertThat(row.isNullAt(0)).isTrue()); + row -> { + assertThat(row.isNullAt(0)).isTrue(); + if (metadata) { + assertMetadata(row, 1, 2L, 1, 0); + } + }); } @Test From 4a7434253f829cfdc67f6d68ab2f8c46515fa89d Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 11 Sep 2026 15:45:31 +0800 Subject: [PATCH 43/69] [kafka] Add strict JSON scalar conversion Local JSON schema-aware Produce split; no remote publication. Co-Authored-By: Codex AI-Model: gpt-6-astra AI-Contributed/Feature: 363/363 AI-Contributed/UT: 140/140 --- fluss-kafka/pom.xml | 5 + .../format/json/JsonToFlussConverter.java | 32 ++ .../format/json/JsonToFlussConverters.java | 326 ++++++++++++++++++ .../json/JsonToFlussConvertersTest.java | 140 ++++++++ 4 files changed, 503 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index e38487db564..e2ed458a1a3 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -63,6 +63,11 @@ ${project.version} + + org.apache.fluss + fluss-shaded-jackson + + ${project.groupId} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java new file mode 100644 index 00000000000..e1bbf8ae52b --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; + +import javax.annotation.Nullable; + +/** Converts one JSON node into the internal representation of one Fluss field. */ +@Internal +interface JsonToFlussConverter { + + /** Converts the node, which is null when the JSON property is missing. */ + @Nullable + Object convert(@Nullable JsonNode node, String path); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java new file mode 100644 index 00000000000..b3dc7e478b6 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.types.BinaryType; +import org.apache.fluss.types.CharType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DecimalType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.TimeType; +import org.apache.fluss.types.TimestampType; + +import javax.annotation.Nullable; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.DateTimeException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeParseException; +import java.util.Base64; + +/** Strict JSON-to-Fluss scalar converter construction. */ +final class JsonToFlussConverters { + + private JsonToFlussConverters() {} + + static JsonToFlussConverter create(DataType dataType) { + final JsonToFlussConverter notNullConverter = createNotNull(dataType); + return (node, path) -> { + if (node == null || node.isNull()) { + if (!dataType.isNullable()) { + throw invalid(path, dataType, "field is missing or null"); + } + return null; + } + try { + return notNullConverter.convert(node, path); + } catch (KafkaRecordEncodingException e) { + throw e; + } catch (RuntimeException e) { + throw invalid(path, dataType, "value cannot be converted", e); + } + }; + } + + private static JsonToFlussConverter createNotNull(DataType dataType) { + switch (dataType.getTypeRoot()) { + case BOOLEAN: + return (node, path) -> { + require(node.isBoolean(), path, dataType, "expected a JSON boolean"); + return node.booleanValue(); + }; + case TINYINT: + return (node, path) -> { + long value = integralValue(node, path, dataType); + require( + value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE, + path, + dataType, + "integer overflow"); + return (byte) value; + }; + case SMALLINT: + return (node, path) -> { + long value = integralValue(node, path, dataType); + require( + value >= Short.MIN_VALUE && value <= Short.MAX_VALUE, + path, + dataType, + "integer overflow"); + return (short) value; + }; + case INTEGER: + return (node, path) -> { + long value = integralValue(node, path, dataType); + require( + value >= Integer.MIN_VALUE && value <= Integer.MAX_VALUE, + path, + dataType, + "integer overflow"); + return (int) value; + }; + case BIGINT: + return (node, path) -> integralValue(node, path, dataType); + case FLOAT: + return (node, path) -> { + require(node.isNumber(), path, dataType, "expected a JSON number"); + double doubleValue = node.doubleValue(); + float value = (float) doubleValue; + require( + Double.isFinite(doubleValue) && Float.isFinite(value), + path, + dataType, + "non-finite or overflowing number"); + return value; + }; + case DOUBLE: + return (node, path) -> { + require(node.isNumber(), path, dataType, "expected a JSON number"); + double value = node.doubleValue(); + require( + Double.isFinite(value), + path, + dataType, + "non-finite or overflowing number"); + return value; + }; + case DECIMAL: + return decimalConverter((DecimalType) dataType); + case CHAR: + return charConverter((CharType) dataType); + case STRING: + return (node, path) -> { + require(node.isTextual(), path, dataType, "expected a JSON string"); + return BinaryString.fromString(node.textValue()); + }; + case BINARY: + return binaryConverter((BinaryType) dataType); + case BYTES: + return (node, path) -> decodeBase64(node, path, dataType); + case DATE: + return (node, path) -> { + String value = textualValue(node, path, dataType); + try { + return Math.toIntExact(LocalDate.parse(value).toEpochDay()); + } catch (DateTimeException | ArithmeticException e) { + throw invalid(path, dataType, "invalid ISO-8601 date", e); + } + }; + case TIME_WITHOUT_TIME_ZONE: + return timeConverter((TimeType) dataType); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return timestampConverter((TimestampType) dataType); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return timestampLtzConverter((LocalZonedTimestampType) dataType); + default: + throw new KafkaTopicSchemaException( + "Kafka JSON format does not support Fluss type " + + dataType.asSummaryString() + + "."); + } + } + + private static JsonToFlussConverter decimalConverter(DecimalType dataType) { + return (node, path) -> { + require(node.isNumber(), path, dataType, "expected a JSON number"); + BigDecimal value = node.decimalValue(); + final BigDecimal scaled; + try { + scaled = value.setScale(dataType.getScale(), RoundingMode.UNNECESSARY); + } catch (ArithmeticException e) { + throw invalid(path, dataType, "decimal scale exceeds target", e); + } + Decimal decimal = + Decimal.fromBigDecimal(scaled, dataType.getPrecision(), dataType.getScale()); + if (decimal == null) { + throw invalid(path, dataType, "decimal precision exceeds target"); + } + return decimal; + }; + } + + private static JsonToFlussConverter charConverter(CharType dataType) { + return (node, path) -> { + String value = textualValue(node, path, dataType); + int codePoints = value.codePointCount(0, value.length()); + require( + codePoints <= dataType.getLength(), + path, + dataType, + "character length exceeds target"); + return BinaryString.fromString(value); + }; + } + + private static JsonToFlussConverter binaryConverter(BinaryType dataType) { + return (node, path) -> { + byte[] value = decodeBase64(node, path, dataType); + require( + value.length == dataType.getLength(), + path, + dataType, + "decoded binary length must equal " + dataType.getLength()); + return value; + }; + } + + private static JsonToFlussConverter timeConverter(TimeType dataType) { + return (node, path) -> { + String value = textualValue(node, path, dataType); + final LocalTime time; + try { + time = LocalTime.parse(value); + } catch (DateTimeParseException e) { + throw invalid(path, dataType, "invalid ISO-8601 local time", e); + } + validatePrecision(time.getNano(), dataType.getPrecision(), path, dataType); + require( + time.getNano() % 1_000_000 == 0, + path, + dataType, + "Fluss TIME storage currently requires millisecond precision"); + return Math.toIntExact(time.toNanoOfDay() / 1_000_000); + }; + } + + private static JsonToFlussConverter timestampConverter(TimestampType dataType) { + return (node, path) -> { + String value = textualValue(node, path, dataType); + final LocalDateTime timestamp; + try { + timestamp = LocalDateTime.parse(value); + } catch (DateTimeParseException e) { + throw invalid(path, dataType, "invalid ISO-8601 local timestamp", e); + } + validatePrecision(timestamp.getNano(), dataType.getPrecision(), path, dataType); + return TimestampNtz.fromLocalDateTime(timestamp); + }; + } + + private static JsonToFlussConverter timestampLtzConverter(LocalZonedTimestampType dataType) { + return (node, path) -> { + String value = textualValue(node, path, dataType); + final OffsetDateTime timestamp; + try { + timestamp = OffsetDateTime.parse(value); + } catch (DateTimeParseException e) { + throw invalid( + path, + dataType, + "invalid ISO-8601 timestamp; an offset or Z is required", + e); + } + validatePrecision(timestamp.getNano(), dataType.getPrecision(), path, dataType); + return TimestampLtz.fromInstant(timestamp.toInstant()); + }; + } + + private static long integralValue(JsonNode node, String path, DataType dataType) { + require(node.isIntegralNumber(), path, dataType, "expected a JSON integer"); + require(node.canConvertToLong(), path, dataType, "integer overflow"); + return node.longValue(); + } + + private static String textualValue(JsonNode node, String path, DataType dataType) { + require(node.isTextual(), path, dataType, "expected a JSON string"); + return node.textValue(); + } + + private static byte[] decodeBase64(JsonNode node, String path, DataType dataType) { + String value = textualValue(node, path, dataType); + try { + return Base64.getDecoder().decode(value); + } catch (IllegalArgumentException e) { + throw invalid(path, dataType, "invalid Base64 string", e); + } + } + + private static void validatePrecision( + int nanos, int precision, String path, DataType dataType) { + if (precision >= 9) { + return; + } + int unit = 1; + for (int i = precision; i < 9; i++) { + unit *= 10; + } + require(nanos % unit == 0, path, dataType, "fractional seconds exceed target precision"); + } + + private static void require(boolean condition, String path, DataType dataType, String reason) { + if (!condition) { + throw invalid(path, dataType, reason); + } + } + + private static KafkaRecordEncodingException invalid( + String path, DataType dataType, String reason) { + return new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + " for " + + dataType.asSummaryString() + + ": " + + reason + + "."); + } + + private static KafkaRecordEncodingException invalid( + String path, DataType dataType, String reason, @Nullable Throwable cause) { + return new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + " for " + + dataType.asSummaryString() + + ": " + + reason + + ".", + cause); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java new file mode 100644 index 00000000000..785edd9e5e4 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies strict scalar conversion without transport or Arrow dependencies. */ +class JsonToFlussConvertersTest { + private static final ObjectMapper MAPPER = + new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + + @Test + void testNumericRepresentationsAndExactDecimals() throws Exception { + assertThat(convert(DataTypes.BOOLEAN(), "true")).isEqualTo(true); + assertThat(convert(DataTypes.TINYINT(), "-128")).isEqualTo((byte) -128); + assertThat(convert(DataTypes.SMALLINT(), "32767")).isEqualTo((short) 32767); + assertThat(convert(DataTypes.INT(), "2147483647")).isEqualTo(Integer.MAX_VALUE); + assertThat(convert(DataTypes.BIGINT(), "9223372036854775807")).isEqualTo(Long.MAX_VALUE); + assertThat(convert(DataTypes.FLOAT(), "1.25")).isEqualTo(1.25F); + assertThat(convert(DataTypes.DOUBLE(), "2.5")).isEqualTo(2.5D); + Decimal decimal = + (Decimal) convert(DataTypes.DECIMAL(30, 10), "12345678901234567890.1234567890"); + assertThat(decimal.toBigDecimal()) + .isEqualByComparingTo(new BigDecimal("12345678901234567890.1234567890")); + } + + @Test + void testStringsBinaryAndTemporalRepresentations() throws Exception { + assertThat(convert(DataTypes.CHAR(2), "\"😀a\"")).isEqualTo(BinaryString.fromString("😀a")); + assertThat(convert(DataTypes.STRING(), "\"文本\"")).isEqualTo(BinaryString.fromString("文本")); + assertThat((byte[]) convert(DataTypes.BINARY(2), "\"AQI=\"")).containsExactly(1, 2); + assertThat((byte[]) convert(DataTypes.BYTES(), "\"\"")).isEmpty(); + assertThat(convert(DataTypes.DATE(), "\"1970-01-02\"")) + .isEqualTo((int) LocalDate.of(1970, 1, 2).toEpochDay()); + assertThat(convert(DataTypes.TIME(3), "\"00:00:01.123\"")).isEqualTo(1123); + assertThat(convert(DataTypes.TIMESTAMP(3), "\"2026-09-11T12:00:00.123\"")) + .isEqualTo( + TimestampNtz.fromLocalDateTime( + LocalDateTime.of(2026, 9, 11, 12, 0, 0, 123000000))); + assertThat(convert(DataTypes.TIMESTAMP_LTZ(3), "\"2026-09-11T12:00:00.123+08:00\"")) + .isEqualTo(TimestampLtz.fromInstant(Instant.parse("2026-09-11T04:00:00.123Z"))); + } + + @Test + void testMissingAndNullRespectNullability() throws Exception { + assertThat(JsonToFlussConverters.create(DataTypes.INT()).convert(null, "$.field")).isNull(); + assertThat(convert(DataTypes.INT(), "null")).isNull(); + assertThatThrownBy( + () -> + JsonToFlussConverters.create(DataTypes.INT().copy(false)) + .convert(null, "$.field")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("missing or null"); + assertThatThrownBy(() -> convert(DataTypes.INT().copy(false), "null")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("missing or null"); + } + + @ParameterizedTest + @MethodSource("invalidScalars") + void testRejectsCoercionOverflowAndPrecisionLoss(DataType type, String json) { + assertThatThrownBy(() -> convert(type, json)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field"); + } + + private static Stream invalidScalars() { + return Stream.of( + Arguments.of(DataTypes.BOOLEAN(), "1"), + Arguments.of(DataTypes.INT(), "\"1\""), + Arguments.of(DataTypes.INT(), "1.0"), + Arguments.of(DataTypes.INT(), "2147483648"), + Arguments.of(DataTypes.TINYINT(), "128"), + Arguments.of(DataTypes.SMALLINT(), "-32769"), + Arguments.of(DataTypes.BIGINT(), "9223372036854775808"), + Arguments.of(DataTypes.FLOAT(), "1e39"), + Arguments.of(DataTypes.DOUBLE(), "1e309"), + Arguments.of(DataTypes.DECIMAL(5, 2), "1.234"), + Arguments.of(DataTypes.DECIMAL(5, 2), "1000"), + Arguments.of(DataTypes.CHAR(1), "\"ab\""), + Arguments.of(DataTypes.STRING(), "false"), + Arguments.of(DataTypes.BINARY(2), "\"AQ==\""), + Arguments.of(DataTypes.BYTES(), "\"!invalid!\""), + Arguments.of(DataTypes.DATE(), "\"2026-02-30\""), + Arguments.of(DataTypes.TIME(0), "\"00:00:00.001\""), + Arguments.of(DataTypes.TIME(3), "\"00:00:00.0001\""), + Arguments.of(DataTypes.TIMESTAMP(3), "\"2026-09-11T00:00:00.0001\""), + Arguments.of(DataTypes.TIMESTAMP_LTZ(3), "\"2026-09-11T00:00:00\"")); + } + + @Test + void testComplexTypesRemainUnsupportedAtThisStage() { + assertThatThrownBy(() -> JsonToFlussConverters.create(DataTypes.ARRAY(DataTypes.INT()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("does not support"); + } + + private static Object convert(DataType type, String json) throws Exception { + return JsonToFlussConverters.create(type).convert(MAPPER.readTree(json), "$.field"); + } +} From 238ac26715bbf74571e65fcc207396c66e1cd3b7 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 20 Sep 2026 11:54:43 +0800 Subject: [PATCH 44/69] [kafka] Fix JSON scalar and timestamp boundary conversions Bound decimal rescaling, reject unpaired Unicode surrogates, and convert FLOAT directly to avoid double rounding. Check timestamp storage overflow and preserve negative and large-date timestamps when reading them back. Add regression coverage for the conversion and Arrow round-trip boundaries. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 147/147 AI-Contributed/UT: 341/341 --- .../org/apache/fluss/row/TimestampNtz.java | 8 +- .../ArrowTimestampLtzColumnVector.java | 6 +- .../ArrowTimestampNtzColumnVector.java | 6 +- .../writers/ArrowTimestampLtzWriter.java | 9 +- .../writers/ArrowTimestampNtzWriter.java | 6 +- .../org/apache/fluss/utils/DateTimeUtils.java | 29 ++++ .../apache/fluss/row/TimestampNtzTest.java | 25 +++ .../row/arrow/ArrowTimestampWriterTest.java | 153 ++++++++++++++++ .../format/json/JsonToFlussConverters.java | 83 +++++++-- .../json/JsonToFlussConvertersTest.java | 163 ++++++++++++++++++ 10 files changed, 463 insertions(+), 25 deletions(-) create mode 100644 fluss-common/src/test/java/org/apache/fluss/row/arrow/ArrowTimestampWriterTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/row/TimestampNtz.java b/fluss-common/src/main/java/org/apache/fluss/row/TimestampNtz.java index c58141da950..c7dd14b9387 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/TimestampNtz.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/TimestampNtz.java @@ -137,12 +137,8 @@ public long toEpochMicros() { /** Converts this {@link TimestampNtz} object to a {@link LocalDateTime}. */ public LocalDateTime toLocalDateTime() { - int date = (int) (millisecond / MILLIS_PER_DAY); - long time = millisecond % MILLIS_PER_DAY; - if (time < 0) { - --date; - time += MILLIS_PER_DAY; - } + long date = Math.floorDiv(millisecond, MILLIS_PER_DAY); + long time = Math.floorMod(millisecond, MILLIS_PER_DAY); long nanoOfDay = time * 1_000_000L + nanoOfMillisecond; LocalDate localDate = LocalDate.ofEpochDay(date); LocalTime localTime = LocalTime.ofNanoOfDay(nanoOfDay); diff --git a/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampLtzColumnVector.java b/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampLtzColumnVector.java index 9b870cdf1ff..9af6601bc53 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampLtzColumnVector.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampLtzColumnVector.java @@ -57,10 +57,12 @@ public TimestampLtz getTimestampLtz(int i, int precision) { return TimestampLtz.fromEpochMillis(((TimeStampMilliVector) valueVector).get(i)); } else if (valueVector instanceof TimeStampMicroVector) { long micros = ((TimeStampMicroVector) valueVector).get(i); - return TimestampLtz.fromEpochMillis(micros / 1000, (int) (micros % 1000) * 1000); + return TimestampLtz.fromEpochMillis( + Math.floorDiv(micros, 1_000L), (int) Math.floorMod(micros, 1_000L) * 1_000); } else { long nanos = ((TimeStampNanoVector) valueVector).get(i); - return TimestampLtz.fromEpochMillis(nanos / 1_000_000, (int) (nanos % 1_000_000)); + return TimestampLtz.fromEpochMillis( + Math.floorDiv(nanos, 1_000_000L), (int) Math.floorMod(nanos, 1_000_000L)); } } diff --git a/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampNtzColumnVector.java b/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampNtzColumnVector.java index 37bc9f7b9df..19ac10479c1 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampNtzColumnVector.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampNtzColumnVector.java @@ -56,10 +56,12 @@ public TimestampNtz getTimestampNtz(int i, int precision) { return TimestampNtz.fromMillis(((TimeStampMilliVector) valueVector).get(i)); } else if (valueVector instanceof TimeStampMicroVector) { long micros = ((TimeStampMicroVector) valueVector).get(i); - return TimestampNtz.fromMillis(micros / 1000, (int) (micros % 1000) * 1000); + return TimestampNtz.fromMillis( + Math.floorDiv(micros, 1_000L), (int) Math.floorMod(micros, 1_000L) * 1_000); } else { long nanos = ((TimeStampNanoVector) valueVector).get(i); - return TimestampNtz.fromMillis(nanos / 1_000_000, (int) (nanos % 1_000_000)); + return TimestampNtz.fromMillis( + Math.floorDiv(nanos, 1_000_000L), (int) Math.floorMod(nanos, 1_000_000L)); } } diff --git a/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampLtzWriter.java b/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampLtzWriter.java index fe10b374e34..42a4bbc582f 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampLtzWriter.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampLtzWriter.java @@ -27,6 +27,8 @@ import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.TimeStampVector; import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.types.pojo.ArrowType; +import static org.apache.fluss.utils.DateTimeUtils.timestampToMicros; +import static org.apache.fluss.utils.DateTimeUtils.timestampToNanos; import static org.apache.fluss.utils.Preconditions.checkState; /** {@link ArrowFieldWriter} for TimestampLtz. */ @@ -64,8 +66,8 @@ public void doWrite(int rowIndex, DataGetters row, int ordinal, boolean handleSa } } else if (vector instanceof TimeStampMicroVector) { long microSec = - timestamp.getEpochMillisecond() * 1000 - + timestamp.getNanoOfMillisecond() / 1000; + timestampToMicros( + timestamp.getEpochMillisecond(), timestamp.getNanoOfMillisecond()); if (handleSafe) { vector.setSafe(rowIndex, microSec); } else { @@ -73,7 +75,8 @@ public void doWrite(int rowIndex, DataGetters row, int ordinal, boolean handleSa } } else { long nanoSec = - timestamp.getEpochMillisecond() * 1_000_000 + timestamp.getNanoOfMillisecond(); + timestampToNanos( + timestamp.getEpochMillisecond(), timestamp.getNanoOfMillisecond()); if (handleSafe) { vector.setSafe(rowIndex, nanoSec); } else { diff --git a/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampNtzWriter.java b/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampNtzWriter.java index 89f965ad362..5d79bff0d1b 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampNtzWriter.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampNtzWriter.java @@ -27,6 +27,8 @@ import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.TimeStampVector; import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.types.pojo.ArrowType; +import static org.apache.fluss.utils.DateTimeUtils.timestampToMicros; +import static org.apache.fluss.utils.DateTimeUtils.timestampToNanos; import static org.apache.fluss.utils.Preconditions.checkState; /** {@link ArrowFieldWriter} for TimestampNtz. */ @@ -64,7 +66,7 @@ public void doWrite(int rowIndex, DataGetters row, int ordinal, boolean handleSa } } else if (vector instanceof TimeStampMicroVector) { long microSec = - timestamp.getMillisecond() * 1000 + timestamp.getNanoOfMillisecond() / 1000; + timestampToMicros(timestamp.getMillisecond(), timestamp.getNanoOfMillisecond()); if (handleSafe) { vector.setSafe(rowIndex, microSec); } else { @@ -72,7 +74,7 @@ public void doWrite(int rowIndex, DataGetters row, int ordinal, boolean handleSa } } else { long nanoSec = - timestamp.getMillisecond() * 1_000_000 + timestamp.getNanoOfMillisecond(); + timestampToNanos(timestamp.getMillisecond(), timestamp.getNanoOfMillisecond()); if (handleSafe) { vector.setSafe(rowIndex, nanoSec); } else { diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/DateTimeUtils.java b/fluss-common/src/main/java/org/apache/fluss/utils/DateTimeUtils.java index 3019df4e33c..1140c8c1165 100644 --- a/fluss-common/src/main/java/org/apache/fluss/utils/DateTimeUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/utils/DateTimeUtils.java @@ -71,6 +71,35 @@ public class DateTimeUtils { .optionalEnd() .toFormatter(); + /** + * Converts a timestamp's milliseconds and nanoseconds within the millisecond to microseconds. + * Sub-microsecond digits are discarded. + * + * @throws ArithmeticException if the result does not fit in a long + */ + public static long timestampToMicros(long millisecond, int nanoOfMillisecond) { + return scaleTimestamp(millisecond, MICROS_PER_MILLIS, nanoOfMillisecond / NANOS_PER_MICROS); + } + + /** + * Converts a timestamp's milliseconds and nanoseconds within the millisecond to nanoseconds. + * + * @throws ArithmeticException if the result does not fit in a long + */ + public static long timestampToNanos(long millisecond, int nanoOfMillisecond) { + return scaleTimestamp(millisecond, 1_000_000L, nanoOfMillisecond); + } + + private static long scaleTimestamp(long millisecond, long unitsPerMillis, long fraction) { + if (millisecond < 0) { + // The non-negative fraction can bring the final value back into range even when + // multiplying the floor-rounded millisecond would underflow at Long.MIN_VALUE. + return Math.addExact( + Math.multiplyExact(millisecond + 1, unitsPerMillis), fraction - unitsPerMillis); + } + return Math.addExact(Math.multiplyExact(millisecond, unitsPerMillis), fraction); + } + public static LocalDate toLocalDate(int date) { return julianToLocalDate(date + EPOCH_JULIAN); } diff --git a/fluss-common/src/test/java/org/apache/fluss/row/TimestampNtzTest.java b/fluss-common/src/test/java/org/apache/fluss/row/TimestampNtzTest.java index 1a939842450..dcd4df406b9 100644 --- a/fluss-common/src/test/java/org/apache/fluss/row/TimestampNtzTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/row/TimestampNtzTest.java @@ -19,7 +19,9 @@ import org.junit.jupiter.api.Test; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.TimeZone; import static org.assertj.core.api.Assertions.assertThat; @@ -46,6 +48,29 @@ public void testNormal() { assertThat(TimestampNtz.fromLocalDateTime(ldt3).toLocalDateTime()).isEqualTo(ldt3); } + @Test + void testToLocalDateTimeAcrossMillisecondRange() { + long[] milliseconds = { + Long.MIN_VALUE, + (long) Integer.MIN_VALUE * 86_400_000 - 1, + -86_400_001, + -1, + 0, + 86_400_000, + ((long) Integer.MAX_VALUE + 1) * 86_400_000, + Long.MAX_VALUE + }; + for (long millis : milliseconds) { + for (int nanos : new int[] {0, 999_999}) { + LocalDateTime expected = + LocalDateTime.ofInstant( + Instant.ofEpochMilli(millis).plusNanos(nanos), ZoneOffset.UTC); + assertThat(TimestampNtz.fromMillis(millis, nanos).toLocalDateTime()) + .isEqualTo(expected); + } + } + } + @Test public void testDaylightSavingTime() { TimeZone tz = TimeZone.getDefault(); diff --git a/fluss-common/src/test/java/org/apache/fluss/row/arrow/ArrowTimestampWriterTest.java b/fluss-common/src/test/java/org/apache/fluss/row/arrow/ArrowTimestampWriterTest.java new file mode 100644 index 00000000000..5c5becd1a8e --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/row/arrow/ArrowTimestampWriterTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.row.arrow; + +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.row.arrow.writers.ArrowFieldWriter; +import org.apache.fluss.row.columnar.ColumnVector; +import org.apache.fluss.row.columnar.ColumnarRow; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator; +import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.TimeStampVector; +import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.ArrowUtils; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies timestamp range checks and round trips through Arrow vectors. */ +class ArrowTimestampWriterTest { + + @ParameterizedTest + @MethodSource("timestampWriters") + void testRoundTripsTimestampBoundaries(int precision, boolean withZone, boolean handleSafe) { + DataType type = + withZone ? DataTypes.TIMESTAMP_LTZ(precision) : DataTypes.TIMESTAMP(precision); + RowType rowType = DataTypes.ROW(type); + long unitsPerMilli = precision == 6 ? 1_000 : 1_000_000; + int nanosPerUnit = precision == 6 ? 1_000 : 1; + try (RootAllocator allocator = new RootAllocator(1024 * 1024); + VectorSchemaRoot root = + VectorSchemaRoot.create(ArrowUtils.toArrowSchema(rowType), allocator)) { + root.allocateNew(); + TimeStampVector vector = (TimeStampVector) root.getVector(0); + ArrowFieldWriter writer = ArrowUtils.createArrowFieldWriter(vector, type); + long[] values = { + Long.MIN_VALUE, + -1_000_001, + -unitsPerMilli, + -unitsPerMilli + 1, + -1, + 0, + 1, + 1_000_001, + Long.MAX_VALUE + }; + for (int i = 0; i < values.length; i++) { + long millis = Math.floorDiv(values[i], unitsPerMilli); + int nanos = (int) Math.floorMod(values[i], unitsPerMilli) * nanosPerUnit; + writer.write(i, timestampRow(millis, nanos, withZone), 0, handleSafe); + assertThat(vector.get(i)).isEqualTo(values[i]); + } + root.setRowCount(values.length); + ArrowReader reader = + new ArrowReader( + new ColumnVector[] {ArrowUtils.createArrowColumnVector(vector, type)}, + root.getRowCount()); + for (int i = 0; i < values.length; i++) { + long millis = Math.floorDiv(values[i], unitsPerMilli); + int nanos = (int) Math.floorMod(values[i], unitsPerMilli) * nanosPerUnit; + ColumnarRow row = reader.read(i); + if (withZone) { + assertThat(row.getTimestampLtz(0, precision)) + .isEqualTo(TimestampLtz.fromEpochMillis(millis, nanos)); + } else { + assertThat(row.getTimestampNtz(0, precision)) + .isEqualTo(TimestampNtz.fromMillis(millis, nanos)); + } + } + } + } + + @ParameterizedTest + @MethodSource("timestampWriters") + void testRejectsOverflowBeforeChangingVector( + int precision, boolean withZone, boolean handleSafe) { + DataType type = + withZone ? DataTypes.TIMESTAMP_LTZ(precision) : DataTypes.TIMESTAMP(precision); + RowType rowType = DataTypes.ROW(type); + long unitsPerMilli = precision == 6 ? 1_000 : 1_000_000; + int nanosPerUnit = precision == 6 ? 1_000 : 1; + try (RootAllocator allocator = new RootAllocator(1024 * 1024); + VectorSchemaRoot root = + VectorSchemaRoot.create(ArrowUtils.toArrowSchema(rowType), allocator)) { + root.allocateNew(); + TimeStampVector vector = (TimeStampVector) root.getVector(0); + ArrowFieldWriter writer = ArrowUtils.createArrowFieldWriter(vector, type); + vector.set(0, 42); + for (long limit : new long[] {Long.MIN_VALUE, Long.MAX_VALUE}) { + long millis = Math.floorDiv(limit, unitsPerMilli); + int nanos = (int) Math.floorMod(limit, unitsPerMilli) * nanosPerUnit; + int outsideNanos = nanos + (limit < 0 ? -nanosPerUnit : nanosPerUnit); + assertThatThrownBy( + () -> + writer.write( + 0, + timestampRow(millis, outsideNanos, withZone), + 0, + handleSafe)) + .isInstanceOf(ArithmeticException.class); + long outsideMillis = millis + (limit < 0 ? -1 : 1); + assertThatThrownBy( + () -> + writer.write( + 0, + timestampRow(outsideMillis, nanos, withZone), + 0, + handleSafe)) + .isInstanceOf(ArithmeticException.class); + assertThat(vector.get(0)).isEqualTo(42); + } + } + } + + private static GenericRow timestampRow(long millis, int nanos, boolean withZone) { + return GenericRow.of( + withZone + ? TimestampLtz.fromEpochMillis(millis, nanos) + : TimestampNtz.fromMillis(millis, nanos)); + } + + private static Stream timestampWriters() { + return Stream.of( + Arguments.of(6, false, false), Arguments.of(6, false, true), + Arguments.of(6, true, false), Arguments.of(6, true, true), + Arguments.of(9, false, false), Arguments.of(9, false, true), + Arguments.of(9, true, false), Arguments.of(9, true, true)); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java index b3dc7e478b6..79fb79c98d1 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java @@ -37,13 +37,18 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.time.DateTimeException; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeParseException; import java.util.Base64; +import static org.apache.fluss.utils.DateTimeUtils.timestampToMicros; +import static org.apache.fluss.utils.DateTimeUtils.timestampToNanos; + /** Strict JSON-to-Fluss scalar converter construction. */ final class JsonToFlussConverters { @@ -110,10 +115,9 @@ private static JsonToFlussConverter createNotNull(DataType dataType) { case FLOAT: return (node, path) -> { require(node.isNumber(), path, dataType, "expected a JSON number"); - double doubleValue = node.doubleValue(); - float value = (float) doubleValue; + float value = node.floatValue(); require( - Double.isFinite(doubleValue) && Float.isFinite(value), + Float.isFinite(value), path, dataType, "non-finite or overflowing number"); @@ -135,10 +139,7 @@ private static JsonToFlussConverter createNotNull(DataType dataType) { case CHAR: return charConverter((CharType) dataType); case STRING: - return (node, path) -> { - require(node.isTextual(), path, dataType, "expected a JSON string"); - return BinaryString.fromString(node.textValue()); - }; + return (node, path) -> BinaryString.fromString(textualValue(node, path, dataType)); case BINARY: return binaryConverter((BinaryType) dataType); case BYTES: @@ -170,6 +171,22 @@ private static JsonToFlussConverter decimalConverter(DecimalType dataType) { return (node, path) -> { require(node.isNumber(), path, dataType, "expected a JSON number"); BigDecimal value = node.decimalValue(); + if (value.signum() == 0) { + return Decimal.zero(dataType.getPrecision(), dataType.getScale()); + } + // Check the resulting digits before setScale can expand a large JSON exponent. + // Use long arithmetic because BigDecimal scales may span the entire int range. + long scaledPrecision = (long) value.precision() - value.scale() + dataType.getScale(); + require( + scaledPrecision <= dataType.getPrecision(), + path, + dataType, + "decimal precision exceeds target"); + require( + value.stripTrailingZeros().scale() <= dataType.getScale(), + path, + dataType, + "decimal scale exceeds target"); final BigDecimal scaled; try { scaled = value.setScale(dataType.getScale(), RoundingMode.UNNECESSARY); @@ -239,7 +256,13 @@ private static JsonToFlussConverter timestampConverter(TimestampType dataType) { throw invalid(path, dataType, "invalid ISO-8601 local timestamp", e); } validatePrecision(timestamp.getNano(), dataType.getPrecision(), path, dataType); - return TimestampNtz.fromLocalDateTime(timestamp); + long millis = + timestampMillis( + timestamp.toInstant(ZoneOffset.UTC), + dataType.getPrecision(), + path, + dataType); + return TimestampNtz.fromMillis(millis, timestamp.getNano() % 1_000_000); }; } @@ -257,10 +280,30 @@ private static JsonToFlussConverter timestampLtzConverter(LocalZonedTimestampTyp e); } validatePrecision(timestamp.getNano(), dataType.getPrecision(), path, dataType); - return TimestampLtz.fromInstant(timestamp.toInstant()); + long millis = + timestampMillis(timestamp.toInstant(), dataType.getPrecision(), path, dataType); + return TimestampLtz.fromEpochMillis(millis, timestamp.getNano() % 1_000_000); }; } + private static long timestampMillis( + Instant timestamp, int precision, String path, DataType dataType) { + try { + long millis = timestamp.toEpochMilli(); + int nanos = timestamp.getNano() % 1_000_000; + // Arrow stores precision 4-6 in microseconds and precision 7-9 in nanoseconds. + // Check here as well so Kafka reports the offending JSON field before encoding. + if (precision > 6) { + timestampToNanos(millis, nanos); + } else if (precision > 3) { + timestampToMicros(millis, nanos); + } + return millis; + } catch (ArithmeticException e) { + throw invalid(path, dataType, "timestamp exceeds storage range", e); + } + } + private static long integralValue(JsonNode node, String path, DataType dataType) { require(node.isIntegralNumber(), path, dataType, "expected a JSON integer"); require(node.canConvertToLong(), path, dataType, "integer overflow"); @@ -269,7 +312,27 @@ private static long integralValue(JsonNode node, String path, DataType dataType) private static String textualValue(JsonNode node, String path, DataType dataType) { require(node.isTextual(), path, dataType, "expected a JSON string"); - return node.textValue(); + String value = node.textValue(); + // Jackson can retain unpaired JSON escapes in Java strings. Reject them before + // BinaryString's UTF-8 encoding silently replaces them with a question mark. + for (int i = 0; i < value.length(); i++) { + char character = value.charAt(i); + if (Character.isHighSurrogate(character)) { + require( + i + 1 < value.length() && Character.isLowSurrogate(value.charAt(i + 1)), + path, + dataType, + "unpaired UTF-16 surrogate"); + i++; + } else { + require( + !Character.isLowSurrogate(character), + path, + dataType, + "unpaired UTF-16 surrogate"); + } + } + return value; } private static byte[] decodeBase64(JsonNode node, String path, DataType dataType) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java index 785edd9e5e4..05a18072c89 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java @@ -25,6 +25,7 @@ import org.apache.fluss.row.TimestampNtz; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DecimalNode; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; @@ -32,11 +33,14 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import java.math.BigDecimal; +import java.math.BigInteger; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -62,6 +66,29 @@ void testNumericRepresentationsAndExactDecimals() throws Exception { .isEqualByComparingTo(new BigDecimal("12345678901234567890.1234567890")); } + @ParameterizedTest + @ValueSource(strings = {"", "-"}) + void testFloatRoundsDirectlyWithoutDoubleRounding(String sign) throws Exception { + float direction = sign.isEmpty() ? 1.0F : -1.0F; + assertThat(convert(DataTypes.FLOAT(), sign + "1.0000000596046448")) + .isEqualTo(direction * 1.0000001F); + assertThat(convert(DataTypes.FLOAT(), sign + "1.0000000596046447")).isEqualTo(direction); + // The integer and decimal nodes must both round to the largest finite float. + String belowOverflow = "340282356779733661637539395458142568447"; + assertThat(convert(DataTypes.FLOAT(), sign + belowOverflow)) + .isEqualTo(direction * Float.MAX_VALUE); + assertThat(convert(DataTypes.FLOAT(), sign + belowOverflow + ".0")) + .isEqualTo(direction * Float.MAX_VALUE); + assertThatThrownBy( + () -> + convert( + DataTypes.FLOAT(), + sign + "340282356779733661637539395458142568448")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("overflowing number"); + } + @Test void testStringsBinaryAndTemporalRepresentations() throws Exception { assertThat(convert(DataTypes.CHAR(2), "\"😀a\"")).isEqualTo(BinaryString.fromString("😀a")); @@ -95,6 +122,142 @@ void testMissingAndNullRespectNullability() throws Exception { .hasMessageContaining("missing or null"); } + @Test + void testDecimalBoundsBeforeRescaling() throws Exception { + for (int scale : new int[] {100_000_000, Integer.MAX_VALUE, Integer.MIN_VALUE}) { + assertThatThrownBy( + () -> + JsonToFlussConverters.create(DataTypes.DECIMAL(5, 2)) + .convert( + DecimalNode.valueOf( + new BigDecimal(BigInteger.ONE, scale)), + "$.amount")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.amount") + .hasMessageContaining(scale > 0 ? "scale" : "precision"); + Decimal zero = + (Decimal) + JsonToFlussConverters.create(DataTypes.DECIMAL(5, 2)) + .convert( + DecimalNode.valueOf( + new BigDecimal(BigInteger.ZERO, scale)), + "$.amount"); + assertThat(zero.toBigDecimal()).isEqualTo(new BigDecimal("0.00")); + } + for (String json : new String[] {"12.3400", "1.234e1", "-12.3400", "1e2", "1e-2"}) { + Decimal decimal = (Decimal) convert(DataTypes.DECIMAL(5, 2), json); + assertThat(decimal.toBigDecimal()).isEqualByComparingTo(new BigDecimal(json)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"\\uD800", "\\uDC00", "a\\uD800b", "\\uD800\\uD800", "\\uDC00\\uD800"}) + void testRejectsUnpairedSurrogates(String escaped) { + for (DataType type : new DataType[] {DataTypes.STRING(), DataTypes.CHAR(8)}) { + assertThatThrownBy(() -> convert(type, "\"" + escaped + "\"")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("surrogate"); + } + } + + @Test + void testAcceptsPairedSurrogatesAndLiteralEscape() throws Exception { + String escapedEmoji = "\"\\uD83D\\uDE00\""; + assertThat(convert(DataTypes.STRING(), escapedEmoji)) + .isEqualTo(BinaryString.fromString("😀")); + assertThat(convert(DataTypes.CHAR(1), escapedEmoji)) + .isEqualTo(BinaryString.fromString("😀")); + assertThat(convert(DataTypes.STRING(), "\"\\\\uD800\"")) + .isEqualTo(BinaryString.fromString("\\uD800")); + } + + @ParameterizedTest + @ValueSource(ints = {3, 6, 9}) + void testTimestampStorageBoundaries(int precision) throws Exception { + long unitsPerSecond = precision == 3 ? 1_000 : precision == 6 ? 1_000_000 : 1_000_000_000; + long nanosPerUnit = 1_000_000_000 / unitsPerSecond; + for (long units : new long[] {Long.MIN_VALUE, -1, 0, 1, Long.MAX_VALUE}) { + Instant instant = + Instant.ofEpochSecond( + Math.floorDiv(units, unitsPerSecond), + Math.floorMod(units, unitsPerSecond) * nanosPerUnit); + for (boolean withZone : new boolean[] {false, true}) { + Object actual = + convert( + timestampType(precision, withZone), + timestampJson(instant, withZone)); + Object expected = + withZone + ? TimestampLtz.fromEpochMillis( + instant.toEpochMilli(), instant.getNano() % 1_000_000) + : TimestampNtz.fromMillis( + instant.toEpochMilli(), instant.getNano() % 1_000_000); + assertThat(actual).isEqualTo(expected); + if (withZone) { + assertThat(((TimestampLtz) actual).toInstant()).isEqualTo(instant); + } else { + assertThat(((TimestampNtz) actual).toLocalDateTime()) + .isEqualTo(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); + } + } + } + for (long limit : new long[] {Long.MIN_VALUE, Long.MAX_VALUE}) { + Instant outside = + Instant.ofEpochSecond( + Math.floorDiv(limit, unitsPerSecond), + Math.floorMod(limit, unitsPerSecond) * nanosPerUnit) + .plusNanos(limit < 0 ? -nanosPerUnit : nanosPerUnit); + for (boolean withZone : new boolean[] {false, true}) { + assertThatThrownBy( + () -> + convert( + timestampType(precision, withZone), + timestampJson(outside, withZone))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("timestamp exceeds storage range"); + } + } + } + + @Test + void testRejectsTimestampOverflowAndKeepsRepresentableDates() throws Exception { + for (boolean withZone : new boolean[] {false, true}) { + String suffix = withZone ? "Z" : ""; + for (String date : new String[] {"3000-01-01T00:00:00", "1600-01-01T00:00:00"}) { + assertThatThrownBy( + () -> + convert( + timestampType(9, withZone), + "\"" + date + suffix + "\"")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("timestamp exceeds storage range"); + assertThat(convert(timestampType(6, withZone), "\"" + date + suffix + "\"")) + .isNotNull(); + } + assertThatThrownBy( + () -> + convert( + timestampType(3, withZone), + "\"+999999999-01-01T00:00:00" + suffix + "\"")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("timestamp exceeds storage range"); + } + } + + private static DataType timestampType(int precision, boolean withZone) { + return withZone ? DataTypes.TIMESTAMP_LTZ(precision) : DataTypes.TIMESTAMP(precision); + } + + private static String timestampJson(Instant instant, boolean withZone) { + return "\"" + + (withZone + ? instant.toString() + : LocalDateTime.ofInstant(instant, ZoneOffset.UTC).toString()) + + "\""; + } + @ParameterizedTest @MethodSource("invalidScalars") void testRejectsCoercionOverflowAndPrecisionLoss(DataType type, String json) { From aa6158f4cb4a0f51f93b8c7709565f8ada026a90 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 21:14:56 +0800 Subject: [PATCH 45/69] [kafka] Reuse public JSON schema validation for scalar converters Validate schemas with KafkaTableSchema before converter construction and translate failures to KafkaTopicSchemaException. Cover shared map-key and depth rules while retaining explicit rejection of complex converters. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 12/12 AI-Contributed/UT: 67/67 --- .../format/json/JsonToFlussConverters.java | 12 ++++ .../json/JsonToFlussConvertersTest.java | 67 ++++++++++++++++++- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java index 79fb79c98d1..eb1433f4e78 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java @@ -17,8 +17,10 @@ package org.apache.fluss.kafka.format.json; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.Decimal; import org.apache.fluss.row.TimestampLtz; @@ -54,7 +56,17 @@ final class JsonToFlussConverters { private JsonToFlussConverters() {} + /** + * Creates a converter for one top-level JSON business field, excluding the JSON root container. + * The shared schema contract is validated before checking decoder support; complex types still + * require their own converter implementations. + */ static JsonToFlussConverter create(DataType dataType) { + try { + KafkaTableSchema.validateJsonType(dataType); + } catch (InvalidTableException e) { + throw new KafkaTopicSchemaException(e.getMessage()); + } final JsonToFlussConverter notNullConverter = createNotNull(dataType); return (node, path) -> { if (node == null || node.isNull()) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java index 05a18072c89..f8d35148564 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java @@ -17,8 +17,10 @@ package org.apache.fluss.kafka.format.json; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.Decimal; import org.apache.fluss.row.TimestampLtz; @@ -292,9 +294,72 @@ private static Stream invalidScalars() { @Test void testComplexTypesRemainUnsupportedAtThisStage() { - assertThatThrownBy(() -> JsonToFlussConverters.create(DataTypes.ARRAY(DataTypes.INT()))) + for (DataType type : + new DataType[] { + DataTypes.ARRAY(DataTypes.INT()), + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + DataTypes.ROW(DataTypes.INT()) + }) { + KafkaTableSchema.validateJsonType(type); + assertThatThrownBy(() -> JsonToFlussConverters.create(type)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("does not support"); + } + } + + @ParameterizedTest + @MethodSource("invalidMapKeyTypes") + void testInvalidMapKeysUseThePublicSchemaError(DataType type) { + assertThatThrownBy(() -> KafkaTableSchema.validateJsonType(type)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("only supports STRING map keys"); + assertThatThrownBy(() -> JsonToFlussConverters.create(type)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("only supports STRING map keys"); + } + + private static Stream invalidMapKeyTypes() { + return Stream.of( + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()), + DataTypes.MAP(DataTypes.CHAR(8), DataTypes.STRING()), + DataTypes.ROW(DataTypes.ARRAY(DataTypes.MAP(DataTypes.BYTES(), DataTypes.INT()))), + DataTypes.MAP( + DataTypes.STRING(), DataTypes.MAP(DataTypes.BOOLEAN(), DataTypes.INT()))); + } + + @Test + void testSchemaDepthUsesTheSharedBusinessFieldLimit() { + DataType type = DataTypes.INT(); + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + switch (depth % 3) { + case 0: + type = DataTypes.ROW(type); + break; + case 1: + type = DataTypes.ARRAY(type); + break; + default: + type = DataTypes.MAP(DataTypes.STRING(), type); + } + } + // The top-level business field starts at zero; each child consumes one level. + DataType atLimit = type; + KafkaTableSchema.validateJsonType(atLimit); + assertThatThrownBy(() -> JsonToFlussConverters.create(atLimit)) .isInstanceOf(KafkaTopicSchemaException.class) .hasMessageContaining("does not support"); + + DataType tooDeep = DataTypes.ROW(atLimit); + String message = + "Kafka JSON schema exceeds the maximum nesting depth of " + + KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH + + "."; + assertThatThrownBy(() -> KafkaTableSchema.validateJsonType(tooDeep)) + .isInstanceOf(InvalidTableException.class) + .hasMessage(message); + assertThatThrownBy(() -> JsonToFlussConverters.create(tooDeep)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessage(message); } private static Object convert(DataType type, String json) throws Exception { From 3c4662f7bbf493d16f0a5c32b9118fe3e127ced4 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 15 Oct 2026 23:29:09 +0800 Subject: [PATCH 46/69] [kafka] Cover JSON scalar edge cases on the updated FIP base Verify signed floating-point zero and underflow, rejection of non-finite numeric nodes and malformed Base64, and timestamp offsets before epoch. Preserve the existing scalar conversion and Arrow timestamp behavior. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 74/74 --- .../json/JsonToFlussConvertersTest.java | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java index f8d35148564..c7273ca7401 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java @@ -26,8 +26,11 @@ import org.apache.fluss.row.TimestampLtz; import org.apache.fluss.row.TimestampNtz; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DecimalNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DoubleNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.FloatNode; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; @@ -91,6 +94,77 @@ void testFloatRoundsDirectlyWithoutDoubleRounding(String sign) throws Exception .hasMessageContaining("overflowing number"); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFloatingPointPreservesZeroSignIncludingUnderflow(boolean negative) { + double zero = negative ? -0.0D : 0.0D; + for (JsonNode node : + new JsonNode[] { + DoubleNode.valueOf(zero), + FloatNode.valueOf((float) zero), + DecimalNode.valueOf(new BigDecimal(negative ? "-1e-1000" : "1e-1000")) + }) { + float floatValue = + (float) JsonToFlussConverters.create(DataTypes.FLOAT()).convert(node, "$.f"); + double doubleValue = + (double) JsonToFlussConverters.create(DataTypes.DOUBLE()).convert(node, "$.d"); + assertThat(Float.floatToRawIntBits(floatValue)) + .isEqualTo(Float.floatToRawIntBits((float) zero)); + assertThat(Double.doubleToRawLongBits(doubleValue)) + .isEqualTo(Double.doubleToRawLongBits(zero)); + } + } + + @ParameterizedTest + @MethodSource("nonFiniteNumbers") + void testRejectsNonFiniteNumericNodes(JsonNode node) { + for (DataType type : + new DataType[] {DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.DECIMAL(38, 18)}) { + assertThatThrownBy(() -> JsonToFlussConverters.create(type).convert(node, "$.number")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.number") + .hasMessageContaining(type.asSummaryString()); + } + } + + private static Stream nonFiniteNumbers() { + return Stream.of( + DoubleNode.valueOf(Double.NaN), + DoubleNode.valueOf(Double.NEGATIVE_INFINITY), + DoubleNode.valueOf(Double.POSITIVE_INFINITY), + FloatNode.valueOf(Float.NaN), + FloatNode.valueOf(Float.NEGATIVE_INFINITY), + FloatNode.valueOf(Float.POSITIVE_INFINITY)); + } + + @ParameterizedTest + @ValueSource(strings = {"A", "AQ===", "AQ==x", "A Q==", "AQ==\n", "_w=="}) + void testBase64RejectsInvalidPaddingAlphabetAndWhitespace(String value) { + for (DataType type : new DataType[] {DataTypes.BYTES(), DataTypes.BINARY(1)}) { + assertThatThrownBy( + () -> + JsonToFlussConverters.create(type) + .convert( + MAPPER.getNodeFactory().textNode(value), "$.b")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.b") + .hasMessageContaining("invalid Base64"); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "1969-12-31T23:59:59.999999999Z", + "1970-01-01T05:44:59.999999999+05:45", + "1969-12-31T20:29:59.999999999-03:30" + }) + void testTimestampLtzNormalizesOffsetsBeforeEpoch(String value) throws Exception { + TimestampLtz timestamp = + (TimestampLtz) convert(DataTypes.TIMESTAMP_LTZ(9), "\"" + value + "\""); + assertThat(timestamp.toInstant()).isEqualTo(Instant.EPOCH.minusNanos(1)); + } + @Test void testStringsBinaryAndTemporalRepresentations() throws Exception { assertThat(convert(DataTypes.CHAR(2), "\"😀a\"")).isEqualTo(BinaryString.fromString("😀a")); From e5dfff2b65fc3df64fbc3534a79b9a63c02780ea Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 11 Sep 2026 15:54:09 +0800 Subject: [PATCH 47/69] [kafka] Enable schema-aware JSON Produce for scalar tables Local JSON schema-aware Produce split; no remote publication. Co-Authored-By: Codex AI-Model: gpt-6-astra AI-Contributed/Feature: 295/295 AI-Contributed/UT: 517/517 --- .../format/KafkaFormatFactoryRegistry.java | 2 + .../format/json/JsonKafkaFieldDecoder.java | 141 +++++++++++++ .../format/json/JsonKafkaFormatFactory.java | 39 ++++ .../fluss/kafka/format/json/JsonPath.java | 97 +++++++++ .../fluss/kafka/KafkaJsonProduceITCase.java | 194 ++++++++++++++++++ .../json/JsonKafkaFieldDecoderTest.java | 149 ++++++++++++++ .../kafka/schema/KafkaJsonSchemaTest.java | 95 +++++++++ 7 files changed, 717 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonPath.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java index 36ff64a9b69..52fab5ab58e 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java @@ -19,6 +19,7 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.format.json.JsonKafkaFormatFactory; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.metadata.KafkaFieldProjection; @@ -36,6 +37,7 @@ public KafkaFormatFactoryRegistry() { factories = new EnumMap<>(KafkaValueConversionType.class); register(new BinaryKafkaFormatFactory()); register(new StringKafkaFormatFactory()); + register(new JsonKafkaFormatFactory()); } /** Creates a decoder for the requested format and projection. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java new file mode 100644 index 00000000000..bdaefa2d8a8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.format.KafkaFieldDecoder; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonParser; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.StreamReadConstraints; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.types.DataType; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +/** Decodes a strict UTF-8 JSON object according to the projected Fluss fields. */ +@Internal +public final class JsonKafkaFieldDecoder implements KafkaFieldDecoder { + static final int MAX_JSON_NESTING_DEPTH = 64; + static final int MAX_CONTAINER_ELEMENTS = 10_000; + private static final ObjectMapper OBJECT_MAPPER = createObjectMapper(); + + private final KafkaFieldProjection projection; + private final Set projectedFieldNames = new HashSet<>(); + private final JsonToFlussConverter[] converters; + + /** Creates a JSON decoder and validates every projected field type. */ + public JsonKafkaFieldDecoder(KafkaFieldProjection projection) { + this.projection = projection; + converters = new JsonToFlussConverter[projection.size()]; + for (int i = 0; i < projection.size(); i++) { + if (!projectedFieldNames.add(projection.nameAt(i))) { + throw new KafkaTopicSchemaException( + "Duplicate Kafka JSON field '" + projection.nameAt(i) + "'."); + } + converters[i] = JsonToFlussConverters.create(projection.dataTypeAt(i)); + } + } + + @Override + public Object[] decode(@Nullable byte[] bytes) { + if (bytes == null) { + return nullValues(); + } + final JsonNode root; + try { + String json = + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString(); + root = OBJECT_MAPPER.readTree(json); + } catch (IOException | RuntimeException e) { + throw new KafkaRecordEncodingException( + "Kafka record value is not valid strict UTF-8 JSON.", e); + } + if (root == null || !root.isObject()) { + throw new KafkaRecordEncodingException( + "Kafka JSON record value must have an object root."); + } + if (root.size() > MAX_CONTAINER_ELEMENTS) { + throw new KafkaRecordEncodingException( + "Kafka JSON object exceeds the maximum field count of " + + MAX_CONTAINER_ELEMENTS + + "."); + } + Iterator fieldNames = root.fieldNames(); + while (fieldNames.hasNext()) { + String fieldName = fieldNames.next(); + if (!projectedFieldNames.contains(fieldName)) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + JsonPath.field(JsonPath.ROOT, fieldName) + + ": unknown field."); + } + } + Object[] values = new Object[projection.size()]; + for (int i = 0; i < projection.size(); i++) { + String name = projection.nameAt(i); + values[i] = converters[i].convert(root.get(name), JsonPath.field(JsonPath.ROOT, name)); + } + return values; + } + + private Object[] nullValues() { + Object[] values = new Object[projection.size()]; + for (int i = 0; i < projection.size(); i++) { + DataType dataType = projection.dataTypeAt(i); + if (!dataType.isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka null value cannot populate NOT NULL field '" + + projection.nameAt(i) + + "'."); + } + } + return values; + } + + private static ObjectMapper createObjectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.getFactory() + .setStreamReadConstraints( + StreamReadConstraints.builder() + .maxNestingDepth(MAX_JSON_NESTING_DEPTH) + .maxNumberLength(StreamReadConstraints.DEFAULT_MAX_NUM_LEN) + .maxStringLength(StreamReadConstraints.DEFAULT_MAX_STRING_LEN) + .build()); + mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + mapper.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); + mapper.enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + return mapper; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java new file mode 100644 index 00000000000..01540dc0a70 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.kafka.format.KafkaFieldDecoder; +import org.apache.fluss.kafka.format.KafkaFormatFactory; +import org.apache.fluss.metadata.KafkaFieldProjection; + +/** Factory for schema-aware Kafka JSON value decoding. */ +@Internal +public final class JsonKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public ValueType format() { + return ValueType.PARSE_JSON; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + return new JsonKafkaFieldDecoder(projection); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonPath.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonPath.java new file mode 100644 index 00000000000..c6a8eb5b958 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonPath.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +/** Builds bounded, escaped JSON paths for Kafka record errors. */ +final class JsonPath { + + static final String ROOT = "$"; + + private static final int MAX_SEGMENT_LENGTH = 128; + private static final int MAX_PATH_LENGTH = 512; + private static final String TRUNCATED = "..."; + + private JsonPath() {} + + static String field(String path, String fieldName) { + return append(path, "[\"" + escapeAndTruncate(fieldName) + "\"]"); + } + + static String index(String path, int index) { + return append(path, "[" + index + "]"); + } + + private static String append(String path, String segment) { + if (path.length() + segment.length() <= MAX_PATH_LENGTH) { + return path + segment; + } + int prefixLength = MAX_PATH_LENGTH - TRUNCATED.length() - segment.length(); + if (prefixLength <= ROOT.length()) { + return (ROOT + TRUNCATED + segment).substring(0, MAX_PATH_LENGTH); + } + return path.substring(0, Math.min(path.length(), prefixLength)) + TRUNCATED + segment; + } + + private static String escapeAndTruncate(String value) { + StringBuilder escaped = new StringBuilder(MAX_SEGMENT_LENGTH); + int contentLimit = MAX_SEGMENT_LENGTH - TRUNCATED.length(); + int index = 0; + while (index < value.length()) { + String encodedCharacter; + char character = value.charAt(index); + switch (character) { + case '\\': + encodedCharacter = "\\\\"; + break; + case '"': + encodedCharacter = "\\\""; + break; + case '\b': + encodedCharacter = "\\b"; + break; + case '\f': + encodedCharacter = "\\f"; + break; + case '\n': + encodedCharacter = "\\n"; + break; + case '\r': + encodedCharacter = "\\r"; + break; + case '\t': + encodedCharacter = "\\t"; + break; + default: + if (character < 0x20) { + encodedCharacter = String.format("\\u%04x", (int) character); + } else { + encodedCharacter = String.valueOf(character); + } + } + if (escaped.length() + encodedCharacter.length() > contentLimit) { + break; + } + escaped.append(encodedCharacter); + index++; + } + if (index < value.length()) { + escaped.append(TRUNCATED); + } + return escaped.toString(); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java new file mode 100644 index 00000000000..447cf3f6ee2 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.InvalidRecordException; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies DDL-discovered JSON writes and native Arrow readback. */ +class KafkaJsonProduceITCase { + private static final String DATABASE = "kafka"; + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + void testScalarJsonRoundTripAndRejectedWritesDoNotAppend() throws Exception { + TableDescriptor descriptor = + descriptor( + Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("message_key", DataTypes.STRING()) + .column("amount", DataTypes.DECIMAL(30, 10)) + .column( + "received_at", + DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .build()) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") + .build(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + TablePath path = TablePath.of(DATABASE, "json_scalars"); + create(admin, path, descriptor); + try (KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(1); + assertThatThrownBy( + () -> + producer.send(record(path, "{\"id\":\"bad\"}")) + .get(30, TimeUnit.SECONDS)) + .hasCauseInstanceOf(InvalidRecordException.class); + assertThat( + producer.send( + record( + path, + "{\"id\":7,\"amount\":12345678901234567890.1234567890}")) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + read( + connection, + path, + row -> { + assertThat(row.getInt(0)).isEqualTo(7); + assertThat(row.getString(1).toString()).isEqualTo("key"); + assertThat(row.getDecimal(2, 30, 10).toBigDecimal()) + .isEqualByComparingTo( + new BigDecimal("12345678901234567890.1234567890")); + assertThat(row.getTimestampLtz(3, 3).getEpochMillisecond()) + .isEqualTo(123L); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + private static void create(Admin admin, TablePath path, TableDescriptor descriptor) + throws Exception { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor, false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + } + + private static TableDescriptor.Builder descriptor(Schema schema) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json"); + } + + private static ProducerRecord record(TablePath path, String json) { + return new ProducerRecord<>( + path.toString(), + 0, + 123L, + "key".getBytes(StandardCharsets.UTF_8), + json.getBytes(StandardCharsets.UTF_8)); + } + + private static void read(Connection connection, TablePath path, Consumer check) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (System.nanoTime() < deadline) { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + int count = 0; + for (ScanRecord record : records) { + assertThat(record.logOffset()).isZero(); + check.accept(record.getRow()); + count++; + } + if (count > 0) { + assertThat(count).isEqualTo(1); + return; + } + } + throw new AssertionError("JSON record was not visible to the native scanner."); + } + } + + private static KafkaProducer producer() { + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, "all"); + config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 1); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java new file mode 100644 index 00000000000..5e2438c09c8 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests strict schema-aware JSON scalar conversion. */ +class JsonKafkaFieldDecoderTest { + + @Test + void testRejectsMalformedAndAmbiguousJson() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.BIGINT().copy(false)), + DataTypes.FIELD("optional", DataTypes.STRING())); + JsonKafkaFieldDecoder decoder = decoder(rowType); + + assertThatThrownBy(() -> decoder.decode(bytes("[]"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("object root"); + assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":1,\"id\":2}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":1} {}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + assertThatThrownBy(() -> decoder.decode(bytes("{"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + } + + @Test + void testRejectsMissingTypeOverflowAndPrecisionLoss() { + assertFailure( + DataTypes.ROW(DataTypes.FIELD("id", DataTypes.BIGINT().copy(false))), + "{}", + "$[\"id\"]", + "missing or null"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT())), + "{\"id\":2147483648}", + "$[\"id\"]", + "integer overflow"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("amount", DataTypes.DECIMAL(5, 2))), + "{\"amount\":1.234}", + "$[\"amount\"]", + "scale exceeds"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3))), + "{\"timestamp\":\"2026-07-27T19:30:12.123\"}", + "$[\"timestamp\"]", + "offset or Z is required"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("fixed", DataTypes.BINARY(2))), + "{\"fixed\":\"AQ==\"}", + "$[\"fixed\"]", + "length must equal 2"); + } + + @Test + void testNullKafkaValueRequiresNullableProjection() { + JsonKafkaFieldDecoder nullableDecoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING()))); + assertThat(nullableDecoder.decode(null)).containsExactly((Object) null); + + JsonKafkaFieldDecoder notNullDecoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING().copy(false)))); + assertThatThrownBy(() -> notNullDecoder.decode(null)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL field 'value'"); + } + + @Test + void testExactDecimalParsingAndUnknownFields() { + JsonKafkaFieldDecoder decoder = + decoder(DataTypes.ROW(DataTypes.FIELD("amount", DataTypes.DECIMAL(30, 10)))); + Decimal value = + (Decimal) decoder.decode(bytes("{\"amount\":12345678901234567890.1234567890}"))[0]; + assertThat(value.toBigDecimal()) + .isEqualByComparingTo(new BigDecimal("12345678901234567890.1234567890")); + assertThatThrownBy(() -> decoder.decode(bytes("{\"extra\":1}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("unknown field"); + assertThat(decoder.decode(bytes("{}"))).containsExactly((Object) null); + } + + @Test + void testRejectsInvalidUtf8AndAlternateEncodings() { + JsonKafkaFieldDecoder decoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING()))); + for (byte[] input : + new byte[][] { + new byte[] {(byte) 0xff}, "{\"value\":\"ok\"}".getBytes(StandardCharsets.UTF_16) + }) { + assertThatThrownBy(() -> decoder.decode(input)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + } + } + + private static void assertFailure(RowType rowType, String json, String path, String reason) { + assertThatThrownBy(() -> decoder(rowType).decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining(path) + .hasMessageContaining(reason); + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType) { + List positions = new ArrayList<>(); + for (int i = 0; i < rowType.getFieldCount(); i++) { + positions.add(i); + } + return new JsonKafkaFieldDecoder(new KafkaFieldProjection(rowType, positions)); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java new file mode 100644 index 00000000000..ee3269c9b89 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests JSON table admission shared by Metadata and Produce. */ +class KafkaJsonSchemaTest { + private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); + + @Test + void testJsonProjectsValuesAroundKeyAndMetadataColumns() { + KafkaTopicSchema schema = + resolver.resolve( + descriptor() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "key") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "time") + .build()); + assertThat(schema.keyProjection().positions()).containsExactly(1); + assertThat(schema.valueProjection().positions()).containsExactly(0, 3); + assertThat(schema.timestampPosition()).isEqualTo(2); + } + + @Test + void testRejectsJsonKeysAndUnsupportedFieldTypes() { + assertThatThrownBy( + () -> + resolver.resolve( + descriptor() + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, "json") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("only supported for record values"); + assertThatThrownBy( + () -> + resolver.resolve( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column( + "items", + DataTypes.ARRAY( + DataTypes.INT())) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty( + KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("does not support"); + } + + private static TableDescriptor.Builder descriptor() { + return TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("key", DataTypes.STRING()) + .column("time", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("amount", DataTypes.DECIMAL(8, 2)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json"); + } +} From 54eed7c94ad13fb78c23de928141741590a2f840 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 20 Sep 2026 13:01:10 +0800 Subject: [PATCH 48/69] [kafka] Bound JSON parsing and preserve floating-point zero Decode scalar JSON values as a stream, rejecting containers and excess fields before materialization. Parse FLOAT and DOUBLE from original tokens to preserve signed zero and avoid double rounding while keeping exact DECIMAL decoding. Cover early rejection, field limits, strict syntax, numeric boundaries, and signed-zero round trips through Kafka Produce and native Arrow reads. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 121/121 AI-Contributed/UT: 204/204 --- .../format/json/JsonKafkaFieldDecoder.java | 121 +++++++---- .../fluss/kafka/KafkaJsonProduceITCase.java | 9 +- .../json/JsonKafkaFieldDecoderTest.java | 195 ++++++++++++++++++ 3 files changed, 283 insertions(+), 42 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java index bdaefa2d8a8..3a09123dc65 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java @@ -23,21 +23,25 @@ import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonParser; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonToken; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.StreamReadConstraints; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DoubleNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.FloatNode; import org.apache.fluss.types.DataType; import javax.annotation.Nullable; +import java.io.ByteArrayInputStream; import java.io.IOException; -import java.nio.ByteBuffer; +import java.io.InputStreamReader; +import java.io.Reader; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; +import java.util.HashMap; +import java.util.Map; /** Decodes a strict UTF-8 JSON object according to the projected Fluss fields. */ @Internal @@ -47,7 +51,7 @@ public final class JsonKafkaFieldDecoder implements KafkaFieldDecoder { private static final ObjectMapper OBJECT_MAPPER = createObjectMapper(); private final KafkaFieldProjection projection; - private final Set projectedFieldNames = new HashSet<>(); + private final Map projectedFields = new HashMap<>(); private final JsonToFlussConverter[] converters; /** Creates a JSON decoder and validates every projected field type. */ @@ -55,7 +59,7 @@ public JsonKafkaFieldDecoder(KafkaFieldProjection projection) { this.projection = projection; converters = new JsonToFlussConverter[projection.size()]; for (int i = 0; i < projection.size(); i++) { - if (!projectedFieldNames.add(projection.nameAt(i))) { + if (projectedFields.put(projection.nameAt(i), i) != null) { throw new KafkaTopicSchemaException( "Duplicate Kafka JSON field '" + projection.nameAt(i) + "'."); } @@ -68,46 +72,81 @@ public Object[] decode(@Nullable byte[] bytes) { if (bytes == null) { return nullValues(); } - final JsonNode root; - try { - String json = - StandardCharsets.UTF_8 - .newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(bytes)) - .toString(); - root = OBJECT_MAPPER.readTree(json); + try (Reader reader = + new InputStreamReader( + new ByteArrayInputStream(bytes), + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT)); + JsonParser parser = OBJECT_MAPPER.getFactory().createParser(reader)) { + if (parser.nextToken() != JsonToken.START_OBJECT) { + throw new KafkaRecordEncodingException( + "Kafka JSON record value must have an object root."); + } + Object[] values = new Object[projection.size()]; + boolean[] present = new boolean[projection.size()]; + int fieldCount = 0; + while (parser.nextToken() != JsonToken.END_OBJECT) { + if (++fieldCount > MAX_CONTAINER_ELEMENTS) { + throw new KafkaRecordEncodingException( + "Kafka JSON object exceeds the maximum field count of " + + MAX_CONTAINER_ELEMENTS + + "."); + } + String fieldName = parser.currentName(); + String path = JsonPath.field(JsonPath.ROOT, fieldName); + Integer position = projectedFields.get(fieldName); + if (position == null) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + path + ": unknown field."); + } + JsonToken token = parser.nextToken(); + // This stage supports scalar fields only. Reject containers before readTree + // can allocate their contents, even when the container is wide and shallow. + if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + path + ": expected a JSON scalar."); + } + JsonNode value = readScalar(parser, projection.dataTypeAt(position)); + values[position] = converters[position].convert(value, path); + present[position] = true; + } + if (parser.nextToken() != null) { + throw new KafkaRecordEncodingException( + "Kafka record value is not valid strict UTF-8 JSON: trailing content."); + } + for (int i = 0; i < projection.size(); i++) { + if (!present[i]) { + values[i] = + converters[i].convert( + null, JsonPath.field(JsonPath.ROOT, projection.nameAt(i))); + } + } + return values; + } catch (KafkaRecordEncodingException e) { + throw e; } catch (IOException | RuntimeException e) { throw new KafkaRecordEncodingException( "Kafka record value is not valid strict UTF-8 JSON.", e); } - if (root == null || !root.isObject()) { - throw new KafkaRecordEncodingException( - "Kafka JSON record value must have an object root."); - } - if (root.size() > MAX_CONTAINER_ELEMENTS) { - throw new KafkaRecordEncodingException( - "Kafka JSON object exceeds the maximum field count of " - + MAX_CONTAINER_ELEMENTS - + "."); - } - Iterator fieldNames = root.fieldNames(); - while (fieldNames.hasNext()) { - String fieldName = fieldNames.next(); - if (!projectedFieldNames.contains(fieldName)) { - throw new KafkaRecordEncodingException( - "Invalid Kafka record value at " - + JsonPath.field(JsonPath.ROOT, fieldName) - + ": unknown field."); + } + + private static JsonNode readScalar(JsonParser parser, DataType dataType) throws IOException { + if (parser.currentToken().isNumeric()) { + // Parse floating-point targets from the original token: BigDecimal loses negative + // zero, and parsing FLOAT through a double can round twice. Other numeric targets + // retain exact Jackson integer/BigDecimal nodes and their strict conversion rules. + switch (dataType.getTypeRoot()) { + case FLOAT: + return FloatNode.valueOf(Float.parseFloat(parser.getText())); + case DOUBLE: + return DoubleNode.valueOf(Double.parseDouble(parser.getText())); + default: + break; } } - Object[] values = new Object[projection.size()]; - for (int i = 0; i < projection.size(); i++) { - String name = projection.nameAt(i); - values[i] = converters[i].convert(root.get(name), JsonPath.field(JsonPath.ROOT, name)); - } - return values; + return OBJECT_MAPPER.readTree(parser); } private Object[] nullValues() { @@ -135,7 +174,7 @@ private static ObjectMapper createObjectMapper() { .build()); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); mapper.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); - mapper.enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + // Trailing content is checked after the root object, not after individual scalar values. return mapper; } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 447cf3f6ee2..fd4376c4ad7 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -79,6 +79,8 @@ void testScalarJsonRoundTripAndRejectedWritesDoNotAppend() throws Exception { .column( "received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("float_value", DataTypes.FLOAT()) + .column("double_value", DataTypes.DOUBLE()) .build()) .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") @@ -100,7 +102,8 @@ void testScalarJsonRoundTripAndRejectedWritesDoNotAppend() throws Exception { producer.send( record( path, - "{\"id\":7,\"amount\":12345678901234567890.1234567890}")) + "{\"id\":7,\"amount\":12345678901234567890.1234567890," + + "\"float_value\":-0.0,\"double_value\":-0.0}")) .get(30, TimeUnit.SECONDS) .offset()) .isZero(); @@ -115,6 +118,10 @@ void testScalarJsonRoundTripAndRejectedWritesDoNotAppend() throws Exception { new BigDecimal("12345678901234567890.1234567890")); assertThat(row.getTimestampLtz(3, 3).getEpochMillisecond()) .isEqualTo(123L); + assertThat(Float.floatToRawIntBits(row.getFloat(4))) + .isEqualTo(Float.floatToRawIntBits(-0.0F)); + assertThat(Double.doubleToRawLongBits(row.getDouble(5))) + .isEqualTo(Double.doubleToRawLongBits(-0.0D)); }); } finally { admin.dropTable(path, true).get(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java index 5e2438c09c8..3797a996af9 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java @@ -20,14 +20,18 @@ import org.apache.fluss.metadata.KafkaFieldProjection; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; import org.apache.fluss.row.Decimal; +import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -114,6 +118,197 @@ void testExactDecimalParsingAndUnknownFields() { assertThat(decoder.decode(bytes("{}"))).containsExactly((Object) null); } + @ParameterizedTest + @ValueSource(strings = {"[", "{"}) + void testRejectsContainersBeforeReadingTheirContents(String container) { + // The unfinished value must fail the scalar check before parsing its contents. + assertFailure( + DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING())), + "{\"value\":" + container, + "$[\"value\"]", + "scalar"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING())), + "{\"extra\":" + container, + "$[\"extra\"]", + "unknown field"); + } + + @Test + void testFieldCountLimitBeforeReadingExcessFieldValue() { + int maximumFields = JsonKafkaFieldDecoder.MAX_CONTAINER_ELEMENTS; + List fields = new ArrayList<>(); + StringBuilder json = new StringBuilder("{"); + Object[] expected = new Object[maximumFields + 1]; + for (int i = 0; i <= maximumFields; i++) { + fields.add(DataTypes.FIELD("field" + i, DataTypes.INT())); + if (i < maximumFields) { + if (i > 0) { + json.append(','); + } + json.append("\"field").append(i).append("\":").append(i); + expected[i] = i; + } + } + JsonKafkaFieldDecoder decoder = decoder(new RowType(fields)); + assertThat(decoder.decode(bytes(json + "}"))).containsExactly(expected); + + String excessField = json + ",\"field" + maximumFields + "\":"; + for (String value : new String[] {"0}", "["}) { + assertThatThrownBy(() -> decoder.decode(bytes(excessField + value))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("maximum field count of " + maximumFields); + } + } + + @ParameterizedTest + @ValueSource(strings = {"-0", "-0.0", "-0e3", "-0e-3", "0", "0.0", "0e3", "0e-3"}) + void testPreservesFloatingPointZeroSign(String number) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE()))); + Object[] values = + decoder.decode( + bytes("{\"floatValue\":" + number + ",\"doubleValue\":" + number + "}")); + float expectedFloat = number.startsWith("-") ? -0.0F : 0.0F; + double expectedDouble = number.startsWith("-") ? -0.0D : 0.0D; + assertThat(Float.floatToRawIntBits((Float) values[0])) + .isEqualTo(Float.floatToRawIntBits(expectedFloat)); + assertThat(Double.doubleToRawLongBits((Double) values[1])) + .isEqualTo(Double.doubleToRawLongBits(expectedDouble)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "-"}) + void testFloatingPointRoundingAndExactDecimalParsing(String sign) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE()), + DataTypes.FIELD("amount", DataTypes.DECIMAL(38, 18)))); + String amount = sign + "12345678901234567890.123456789012345678"; + Object[] values = + decoder.decode( + bytes( + "{\"floatValue\":" + + sign + + "1.0000000596046448,\"doubleValue\":" + + sign + + "1.00000000000000011102230246251565404236316680908203126," + + "\"amount\":" + + amount + + "}")); + float direction = sign.isEmpty() ? 1.0F : -1.0F; + assertThat(values[0]).isEqualTo(direction * 1.0000001F); + assertThat(values[1]).isEqualTo(direction * 1.0000000000000002D); + assertThat(((Decimal) values[2]).toBigDecimal()).isEqualTo(new BigDecimal(amount)); + + assertThat( + decoder.decode( + bytes( + "{\"floatValue\":" + + sign + + "1.0000000596046447,\"doubleValue\":" + + sign + + "1.00000000000000011102230246251565404236316680908203124}"))) + .containsExactly(direction, (double) direction, null); + } + + @ParameterizedTest + @ValueSource(strings = {"", "-"}) + void testFloatingPointOverflowBoundaries(String sign) { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE())); + JsonKafkaFieldDecoder decoder = decoder(rowType); + float direction = sign.isEmpty() ? 1.0F : -1.0F; + String largestFloat = sign + "340282356779733661637539395458142568447"; + for (String suffix : new String[] {"", ".0"}) { + assertThat(decoder.decode(bytes("{\"floatValue\":" + largestFloat + suffix + "}"))) + .containsExactly(direction * Float.MAX_VALUE, null); + assertFailure( + rowType, + "{\"floatValue\":" + + sign + + "340282356779733661637539395458142568448" + + suffix + + "}", + "$[\"floatValue\"]", + "overflowing number"); + } + assertThat(decoder.decode(bytes("{\"doubleValue\":" + sign + "1.7976931348623157e308}"))) + .containsExactly(null, direction * Double.MAX_VALUE); + assertFailure( + rowType, + "{\"doubleValue\":" + sign + "1e309}", + "$[\"doubleValue\"]", + "overflowing number"); + } + + @Test + void testMixedScalarTokensPreserveProjectionAndMissingFields() { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("text", DataTypes.STRING()), + DataTypes.FIELD("amount", DataTypes.DECIMAL(5, 2)), + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE()), + DataTypes.FIELD("optional", DataTypes.INT()))); + Object[] values = + decoder.decode( + bytes( + "{\"floatValue\":-0.0,\"text\":\"hello\",\"amount\":12.34," + + "\"doubleValue\":2.5,\"optional\":null}")); + assertThat(values[0].toString()).isEqualTo("hello"); + assertThat(((Decimal) values[1]).toBigDecimal()).isEqualTo(new BigDecimal("12.34")); + assertThat(Float.floatToRawIntBits((Float) values[2])) + .isEqualTo(Float.floatToRawIntBits(-0.0F)); + assertThat(values[3]).isEqualTo(2.5D); + assertThat(values[4]).isNull(); + assertThat(decoder.decode(bytes("{\"doubleValue\":2.5}"))) + .containsExactly(null, null, null, 2.5D, null); + } + + @Test + void testFloatingPointParsingStillRequiresStrictJsonSyntax() { + JsonKafkaFieldDecoder decoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.DOUBLE()))); + for (String json : + new String[] { + "{\"value\":}", + "{\"value\":-0.0 \"value\":1}", + "{\"value\":-0.0,}", + "{\"value\":01}", + "{\"value\":NaN}", + "{\"value\":-Infinity}", + "{\"value\":0x1.0p0}", + "{\"value\":1f}", + "{\"value\":1}null" + }) { + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + } + } + + @Test + void testRejectsInvalidUtf8AfterTrailingWhitespaceAcrossReaderBuffers() { + JsonKafkaFieldDecoder decoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING()))); + byte[] json = bytes("{\"value\":\"ok\"}"); + byte[] input = Arrays.copyOf(json, json.length + 32_768); + Arrays.fill(input, json.length, input.length - 1, (byte) ' '); + input[input.length - 1] = (byte) 0xff; + assertThatThrownBy(() -> decoder.decode(input)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + } + @Test void testRejectsInvalidUtf8AndAlternateEncodings() { JsonKafkaFieldDecoder decoder = From 18afb5e640a80418bd2cc51ef89fd81e520885c0 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:41:17 +0800 Subject: [PATCH 49/69] [kafka] Verify JSON null filtering and timestamp readback Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 48/48 --- .../fluss/kafka/KafkaJsonProduceITCase.java | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index fd4376c4ad7..85c3fc38df9 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -76,9 +76,7 @@ void testScalarJsonRoundTripAndRejectedWritesDoNotAppend() throws Exception { .column("id", DataTypes.INT().copy(false)) .column("message_key", DataTypes.STRING()) .column("amount", DataTypes.DECIMAL(30, 10)) - .column( - "received_at", - DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) .column("float_value", DataTypes.FLOAT()) .column("double_value", DataTypes.DOUBLE()) .build()) @@ -116,8 +114,7 @@ void testScalarJsonRoundTripAndRejectedWritesDoNotAppend() throws Exception { assertThat(row.getDecimal(2, 30, 10).toBigDecimal()) .isEqualByComparingTo( new BigDecimal("12345678901234567890.1234567890")); - assertThat(row.getTimestampLtz(3, 3).getEpochMillisecond()) - .isEqualTo(123L); + assertThat(row.getTimestampNtz(3, 3).getMillisecond()).isEqualTo(123L); assertThat(Float.floatToRawIntBits(row.getFloat(4))) .isEqualTo(Float.floatToRawIntBits(-0.0F)); assertThat(Double.doubleToRawLongBits(row.getDouble(5))) @@ -129,6 +126,47 @@ void testScalarJsonRoundTripAndRejectedWritesDoNotAppend() throws Exception { } } + @Test + void testNullJsonValuesSkipDecodingWhileNullFieldsRemainRows() throws Exception { + TableDescriptor descriptor = + descriptor( + Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("optional", DataTypes.STRING()) + .build()) + .build(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + TablePath path = TablePath.of(DATABASE, "json_null_values"); + create(admin, path, descriptor); + try (KafkaProducer producer = producer()) { + assertThat( + producer.send( + new ProducerRecord( + path.toString(), 0, null, null)) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(-1L); + assertThatThrownBy(() -> producer.send(record(path, "")).get(30, TimeUnit.SECONDS)) + .hasCauseInstanceOf(InvalidRecordException.class); + assertThat( + producer.send(record(path, "{\"id\":7,\"optional\":null}")) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + read( + connection, + path, + row -> { + assertThat(row.getInt(0)).isEqualTo(7); + assertThat(row.isNullAt(1)).isTrue(); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + private static void create(Admin admin, TablePath path, TableDescriptor descriptor) throws Exception { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); From 025c590afc782b253e299c42eef9eaab9e797163 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:47:43 +0800 Subject: [PATCH 50/69] [kafka] Align JSON schema test metadata with TIMESTAMP contract Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 2/2 --- .../java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java index ee3269c9b89..0b94ab0c934 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java @@ -85,7 +85,7 @@ private static TableDescriptor.Builder descriptor() { Schema.newBuilder() .column("id", DataTypes.INT().copy(false)) .column("key", DataTypes.STRING()) - .column("time", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("time", DataTypes.TIMESTAMP(3).copy(false)) .column("amount", DataTypes.DECIMAL(8, 2)) .build()) .distributedBy(1) From 0c5c2ffaa1fb8830ae34edb5d9394d788cda220d Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 22:13:01 +0800 Subject: [PATCH 51/69] [kafka] Support flat and nested JSON table contracts Use the shared Kafka table schema and physical value projections. Decode nested JSON roots into the fixed value ROW while preserving streaming limits, strict scalar conversion, floating-point zero and null filtering. Reject reserved root members and unsupported schemas before append. Verify native readback for both layouts, optional system columns and topic aliases, and cover partition atomicity plus schema and cache transitions. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 121/121 AI-Contributed/UT: 1082/1082 --- .../kafka/format/KafkaFormatFactory.java | 9 + .../format/KafkaFormatFactoryRegistry.java | 13 +- .../format/json/JsonKafkaFieldDecoder.java | 62 ++- .../format/json/JsonKafkaFormatFactory.java | 15 +- .../schema/KafkaTopicSchemaResolver.java | 10 +- .../transcode/ArrowKafkaRecordTranscoder.java | 7 +- .../kafka/transcode/KafkaTopicWritePlan.java | 19 +- .../fluss/kafka/KafkaJsonProduceITCase.java | 439 ++++++++++++++---- .../fluss/kafka/KafkaMetadataHandlerTest.java | 110 +++++ .../GatewayKafkaProduceBackendTest.java | 177 ++++++- .../KafkaFormatFactoryRegistryTest.java | 36 +- .../json/JsonKafkaFieldDecoderTest.java | 107 ++++- .../kafka/schema/KafkaJsonSchemaTest.java | 202 ++++++-- .../kafka/schema/KafkaTableMappingITCase.java | 1 + .../ArrowKafkaRecordTranscoderTest.java | 11 +- 15 files changed, 1048 insertions(+), 170 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java index bb444610726..02b3dc2846d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java @@ -18,9 +18,12 @@ package org.apache.fluss.kafka.format; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.metadata.KafkaFieldProjection; +import javax.annotation.Nullable; + /** Creates a decoder for one Kafka value format and Fluss field projection. */ @Internal public interface KafkaFormatFactory { @@ -30,4 +33,10 @@ public interface KafkaFormatFactory { /** Creates and validates a decoder for the projection. */ KafkaFieldDecoder createDecoder(KafkaFieldProjection projection); + + /** Creates a decoder with the JSON layout, or no layout for binary/string values. */ + default KafkaFieldDecoder createDecoder( + KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { + return createDecoder(projection); + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java index 52fab5ab58e..29339483bd0 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java @@ -18,11 +18,14 @@ package org.apache.fluss.kafka.format; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.kafka.format.json.JsonKafkaFormatFactory; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.metadata.KafkaFieldProjection; +import javax.annotation.Nullable; + import java.util.EnumMap; import java.util.Map; @@ -43,12 +46,20 @@ public KafkaFormatFactoryRegistry() { /** Creates a decoder for the requested format and projection. */ public KafkaFieldDecoder createDecoder( KafkaValueConversionType format, KafkaFieldProjection projection) { + return createDecoder(format, projection, null); + } + + /** Creates and validates a decoder using the table's physical projection and value layout. */ + public KafkaFieldDecoder createDecoder( + KafkaValueConversionType format, + KafkaFieldProjection projection, + @Nullable KafkaValueConversionLayout layout) { KafkaFormatFactory factory = factories.get(format); if (factory == null) { throw new KafkaTopicSchemaException( "No Kafka format factory is registered for '" + format + "'."); } - return factory.createDecoder(projection); + return factory.createDecoder(projection, layout); } private void register(KafkaFormatFactory factory) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java index 3a09123dc65..cad4e2ddec9 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java @@ -18,10 +18,13 @@ package org.apache.fluss.kafka.format.json; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.kafka.format.KafkaFieldDecoder; -import org.apache.fluss.metadata.KafkaFieldProjection; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.GenericRow; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonParser; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonToken; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.StreamReadConstraints; @@ -31,6 +34,7 @@ import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DoubleNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.FloatNode; import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; import javax.annotation.Nullable; @@ -40,30 +44,61 @@ import java.io.Reader; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; /** Decodes a strict UTF-8 JSON object according to the projected Fluss fields. */ @Internal public final class JsonKafkaFieldDecoder implements KafkaFieldDecoder { + // Jackson counts the JSON root; the shared business-schema depth starts below that root. + // The physical nested ROW adds no JSON level. This stage rejects all business containers. static final int MAX_JSON_NESTING_DEPTH = 64; static final int MAX_CONTAINER_ELEMENTS = 10_000; private static final ObjectMapper OBJECT_MAPPER = createObjectMapper(); private final KafkaFieldProjection projection; + private final @Nullable RowType nestedType; private final Map projectedFields = new HashMap<>(); private final JsonToFlussConverter[] converters; /** Creates a JSON decoder and validates every projected field type. */ public JsonKafkaFieldDecoder(KafkaFieldProjection projection) { - this.projection = projection; - converters = new JsonToFlussConverter[projection.size()]; - for (int i = 0; i < projection.size(); i++) { - if (projectedFields.put(projection.nameAt(i), i) != null) { + this(projection, KafkaValueConversionLayout.FLAT); + } + + /** Creates a decoder whose nested physical ROW represents the JSON object root. */ + public JsonKafkaFieldDecoder( + KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { + if (layout == KafkaValueConversionLayout.NESTED) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof RowType)) { throw new KafkaTopicSchemaException( - "Duplicate Kafka JSON field '" + projection.nameAt(i) + "'."); + "Nested Kafka JSON requires exactly one _kafka_value ROW."); + } + nestedType = (RowType) projection.dataTypeAt(0); + List positions = new ArrayList<>(); + for (int i = 0; i < nestedType.getFieldCount(); i++) { + positions.add(i); } - converters[i] = JsonToFlussConverters.create(projection.dataTypeAt(i)); + this.projection = new KafkaFieldProjection(nestedType, positions); + } else { + nestedType = null; + this.projection = projection; + } + converters = new JsonToFlussConverter[this.projection.size()]; + for (int i = 0; i < this.projection.size(); i++) { + String name = this.projection.nameAt(i); + if (KafkaTableSchema.isReservedName(name)) { + throw new KafkaTopicSchemaException( + "Reserved name '" + name + "' cannot be a Kafka JSON business field."); + } + if (projectedFields.put(name, i) != null) { + throw new KafkaTopicSchemaException("Duplicate Kafka JSON field '" + name + "'."); + } + converters[i] = JsonToFlussConverters.create(this.projection.dataTypeAt(i)); } } @@ -96,6 +131,10 @@ public Object[] decode(@Nullable byte[] bytes) { } String fieldName = parser.currentName(); String path = JsonPath.field(JsonPath.ROOT, fieldName); + if (KafkaTableSchema.isReservedName(fieldName)) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + path + ": reserved field."); + } Integer position = projectedFields.get(fieldName); if (position == null) { throw new KafkaRecordEncodingException( @@ -123,7 +162,7 @@ public Object[] decode(@Nullable byte[] bytes) { null, JsonPath.field(JsonPath.ROOT, projection.nameAt(i))); } } - return values; + return nestedType == null ? values : new Object[] {GenericRow.of(values)}; } catch (KafkaRecordEncodingException e) { throw e; } catch (IOException | RuntimeException e) { @@ -150,6 +189,13 @@ private static JsonNode readScalar(JsonParser parser, DataType dataType) throws } private Object[] nullValues() { + if (nestedType != null) { + if (!nestedType.isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka null value cannot populate NOT NULL field '_kafka_value'."); + } + return new Object[] {null}; + } Object[] values = new Object[projection.size()]; for (int i = 0; i < projection.size(); i++) { DataType dataType = projection.dataTypeAt(i); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java index 01540dc0a70..a09be5218d4 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java @@ -18,22 +18,31 @@ package org.apache.fluss.kafka.format.json; import org.apache.fluss.annotation.Internal; -import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.kafka.format.KafkaFieldDecoder; import org.apache.fluss.kafka.format.KafkaFormatFactory; import org.apache.fluss.metadata.KafkaFieldProjection; +import javax.annotation.Nullable; + /** Factory for schema-aware Kafka JSON value decoding. */ @Internal public final class JsonKafkaFormatFactory implements KafkaFormatFactory { @Override - public ValueType format() { - return ValueType.PARSE_JSON; + public KafkaValueConversionType format() { + return KafkaValueConversionType.PARSE_JSON; } @Override public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { return new JsonKafkaFieldDecoder(projection); } + + @Override + public KafkaFieldDecoder createDecoder( + KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { + return new JsonKafkaFieldDecoder(projection, layout); + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index c02879e922a..5de300ccf81 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -18,7 +18,9 @@ package org.apache.fluss.kafka.schema; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.kafka.format.json.JsonKafkaFieldDecoder; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TableDescriptor; @@ -38,7 +40,13 @@ public boolean isEnabled(TableDescriptor table) { /** Resolves one enabled table's contract; invalid enabled tables remain visible as errors. */ public KafkaTableSchema resolve(TableDescriptor table) { try { - return KafkaTableSchema.resolve(table); + KafkaTableSchema schema = KafkaTableSchema.resolve(table); + if (schema.valueConversionType() == KafkaValueConversionType.PARSE_JSON) { + // Metadata and Produce must reject schemas beyond the installed decoder support. + new JsonKafkaFieldDecoder( + schema.valueProjection(), schema.valueConversionLayout()); + } + return schema; } catch (InvalidTableException | IllegalArgumentException e) { throw new KafkaTopicSchemaException(e.getMessage()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 4eb770c2272..3f9bb7b53ef 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -30,7 +30,7 @@ import static org.apache.fluss.utils.Preconditions.checkArgument; -/** Converts binary/string Kafka values and fixed system columns into owned Arrow log bytes. */ +/** Converts schema-mapped Kafka values and fixed system columns into owned Arrow log bytes. */ @Internal public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { private final KafkaFormatFactoryRegistry formats = new KafkaFormatFactoryRegistry(); @@ -60,7 +60,10 @@ public BytesView transcode(int partitionId, List records, KafkaTopicWrit checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTableSchema schema = plan.topicSchema(); KafkaFieldDecoder valueDecoder = - formats.createDecoder(schema.valueConversionType(), schema.valueProjection()); + formats.createDecoder( + schema.valueConversionType(), + schema.valueProjection(), + schema.valueConversionLayout()); KafkaRowAssembler assembler = plan.rowAssembler(); return arrowRecordEncoder.encodeStreaming( consumer -> { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java index b8855ba45ac..4f8c5d90312 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -18,7 +18,7 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.annotation.Internal; -import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.format.KafkaFormatFactoryRegistry; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TableInfo; @@ -26,9 +26,7 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** - * Immutable binary/string conversion plan; mutable Arrow resources are owned by each conversion. - */ +/** Immutable value conversion plan; mutable Arrow resources are owned by each conversion. */ @Internal @ThreadSafe public final class KafkaTopicWritePlan { @@ -39,13 +37,12 @@ public final class KafkaTopicWritePlan { KafkaTopicWritePlan(TableInfo tableInfo, KafkaTableSchema topicSchema) { this.tableInfo = checkNotNull(tableInfo); this.topicSchema = checkNotNull(topicSchema); - // Validate supported value formats when compiling the plan. - if (topicSchema.valueConversionType() != KafkaValueConversionType.BINARY - && topicSchema.valueConversionType() != KafkaValueConversionType.STRING) { - throw new KafkaRecordEncodingException( - "Unsupported Kafka value conversion type: " - + topicSchema.valueConversionType()); - } + // Validate supported value formats and projections when compiling the plan. + new KafkaFormatFactoryRegistry() + .createDecoder( + topicSchema.valueConversionType(), + topicSchema.valueProjection(), + topicSchema.valueConversionLayout()); this.rowAssembler = new KafkaRowAssembler(topicSchema); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 85c3fc38df9..3de480e557b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -23,42 +23,82 @@ import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.scanner.ScanRecord; import org.apache.fluss.client.table.scanner.log.LogScanner; -import org.apache.fluss.client.table.scanner.log.ScanRecords; import org.apache.fluss.cluster.ServerNode; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalMap; import org.apache.fluss.row.InternalRow; import org.apache.fluss.server.testutils.FlussClusterExtension; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.InvalidRecordException; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.math.BigDecimal; +import java.net.Socket; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; +import java.util.function.BiConsumer; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Verifies DDL-discovered JSON writes and native Arrow readback. */ class KafkaJsonProduceITCase { - private static final String DATABASE = "kafka"; + private static final String DATABASE = "fluss"; + private static final byte[] KEY = new byte[] {(byte) 0xff, 1}; + private static final RowType SCALAR_TYPE = + RowType.builder() + .field("id", DataTypes.INT().copy(false)) + .field("amount", DataTypes.DECIMAL(30, 10)) + .field("received_at", DataTypes.TIMESTAMP(6).copy(false)) + .field("emitted_at", DataTypes.TIMESTAMP_LTZ(6).copy(false)) + .field("float_value", DataTypes.FLOAT()) + .field("double_value", DataTypes.DOUBLE()) + .field("optional", DataTypes.STRING()) + .field("rounded_float", DataTypes.FLOAT()) + .build(); @RegisterExtension static final FlussClusterExtension CLUSTER = @@ -68,57 +108,88 @@ class KafkaJsonProduceITCase { .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") .build(); - @Test - void testScalarJsonRoundTripAndRejectedWritesDoNotAppend() throws Exception { + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( + String layout, boolean withSystemColumns) throws Exception { + TablePath path = TablePath.of(DATABASE, "json_scalars_" + layout + "_" + withSystemColumns); TableDescriptor descriptor = - descriptor( - Schema.newBuilder() - .column("id", DataTypes.INT().copy(false)) - .column("message_key", DataTypes.STRING()) - .column("amount", DataTypes.DECIMAL(30, 10)) - .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) - .column("float_value", DataTypes.FLOAT()) - .column("double_value", DataTypes.DOUBLE()) - .build()) - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") - .build(); + descriptor(SCALAR_TYPE, layout, withSystemColumns, withSystemColumns, 1); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { - TablePath path = TablePath.of(DATABASE, "json_scalars"); create(admin, path, descriptor); try (KafkaProducer producer = producer()) { assertThat(producer.partitionsFor(path.toString())).hasSize(1); - assertThatThrownBy( - () -> - producer.send(record(path, "{\"id\":\"bad\"}")) - .get(30, TimeUnit.SECONDS)) - .hasCauseInstanceOf(InvalidRecordException.class); + assertThat(producer.partitionsFor(path.getTableName())).hasSize(1); + // Null values are filtered before validating required JSON business fields. assertThat( producer.send( - record( - path, - "{\"id\":7,\"amount\":12345678901234567890.1234567890," - + "\"float_value\":-0.0,\"double_value\":-0.0}")) + new ProducerRecord( + path.toString(), 0, null, null)) .get(30, TimeUnit.SECONDS) .offset()) - .isZero(); + .isEqualTo(-1L); + RecordMetadata first = + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + 123L, + KEY, + bytes(scalarJson(7, false)), + Arrays.asList( + new RecordHeader("source", bytes("old")), + new RecordHeader( + "source", new byte[] {1, 2}), + new RecordHeader("nullable", null)))) + .get(30, TimeUnit.SECONDS); + assertThat(first.topic()).isEqualTo(path.toString()); + assertThat(first.offset()).isZero(); + assertThat( + producer.send( + new ProducerRecord( + path.getTableName(), 0, KEY, null)) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(-1L); + // A bare alias and the full name append to the same native table and bucket. + RecordMetadata second = + producer.send( + new ProducerRecord( + path.getTableName(), + 0, + null, + bytes(scalarJson(8, true)))) + .get(30, TimeUnit.SECONDS); + assertThat(second.topic()).isEqualTo(path.getTableName()); + assertThat(second.offset()).isEqualTo(1L); read( connection, path, - row -> { - assertThat(row.getInt(0)).isEqualTo(7); - assertThat(row.getString(1).toString()).isEqualTo("key"); - assertThat(row.getDecimal(2, 30, 10).toBigDecimal()) - .isEqualByComparingTo( - new BigDecimal("12345678901234567890.1234567890")); - assertThat(row.getTimestampNtz(3, 3).getMillisecond()).isEqualTo(123L); - assertThat(Float.floatToRawIntBits(row.getFloat(4))) - .isEqualTo(Float.floatToRawIntBits(-0.0F)); - assertThat(Double.doubleToRawLongBits(row.getDouble(5))) - .isEqualTo(Double.doubleToRawLongBits(-0.0D)); + 0, + 2, + (row, index) -> { + int valuePosition = withSystemColumns ? 1 : 0; + InternalRow value = + layout.equals("nested") + ? row.getRow(valuePosition, SCALAR_TYPE.getFieldCount()) + : row; + int fieldOffset = layout.equals("nested") ? 0 : valuePosition; + assertScalars(value, fieldOffset, 7 + index); + if (withSystemColumns) { + if (index == 0) { + assertThat(row.getBytes(0)).isEqualTo(KEY); + } else { + assertThat(row.isNullAt(0)).isTrue(); + } + int headersPosition = + valuePosition + + (layout.equals("nested") + ? 1 + : SCALAR_TYPE.getFieldCount()); + assertHeaders(row.getMap(headersPosition), index == 0); + assertThat(row.isNullAt(headersPosition + 1)).isTrue(); + } }); } finally { admin.dropTable(path, true).get(); @@ -126,91 +197,271 @@ void testScalarJsonRoundTripAndRejectedWritesDoNotAppend() throws Exception { } } - @Test - void testNullJsonValuesSkipDecodingWhileNullFieldsRemainRows() throws Exception { - TableDescriptor descriptor = - descriptor( - Schema.newBuilder() - .column("id", DataTypes.INT().copy(false)) - .column("optional", DataTypes.STRING()) - .build()) - .build(); + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( + String layout, boolean withRescue) throws Exception { + TablePath path = TablePath.of(DATABASE, "json_rejections_" + layout + "_" + withRescue); + RowType valueType = RowType.builder().field("id", DataTypes.INT().copy(false)).build(); + List invalidValues = + new ArrayList<>( + Arrays.asList( + "", + "{", + "null", + "{\"id\":\"bad\"}", + "{\"id\":null}", + "{\"id\":1,\"unknown\":2}", + "{\"id\":1} {}", + "{\"id\":1,\"id\":2}")); + for (String reserved : + new String[] { + KafkaTableSchema.KEY_COLUMN, + KafkaTableSchema.HEADERS_COLUMN, + KafkaTableSchema.METADATA_COLUMN, + KafkaTableSchema.RESCUE_COLUMN, + KafkaTableSchema.VALUE_COLUMN + }) { + // This also rejects the erroneous nested payload wrapper {"_kafka_value":{...}}. + invalidValues.add("{\"id\":1,\"" + reserved + "\":{\"id\":2}}"); + } try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { - TablePath path = TablePath.of(DATABASE, "json_null_values"); - create(admin, path, descriptor); - try (KafkaProducer producer = producer()) { + create(admin, path, descriptor(valueType, layout, true, withRescue, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port()); + KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(2); + for (int index = 0; index < invalidValues.size(); index++) { + String name = index % 2 == 0 ? path.toString() : path.getTableName(); + ProduceResponse response = + produce( + socket, + name, + MemoryRecords.withRecords( + Compression.NONE, + record("{\"id\":1000}"), + record(invalidValues.get(index))), + MemoryRecords.withRecords( + Compression.NONE, record("{\"id\":" + index + "}"))); + assertThat(response.data().responses()).hasSize(1); + assertThat(response.data().responses().find(name).partitionResponses()) + .hasSize(2); + PartitionProduceResponse rejected = partition(response, name, 0); + assertThat(rejected.errorCode()) + .as( + "rejection for %s with rescue=%s", + invalidValues.get(index), withRescue) + .isEqualTo(Errors.INVALID_RECORD.code()); + assertThat(rejected.baseOffset()).isEqualTo(-1L); + PartitionProduceResponse accepted = partition(response, name, 1); + assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(accepted.baseOffset()).isEqualTo(index); + } + // Even when the schema declares rescue, unsupported capture must not lose data. + // A later successful write must be the first actual append to the rejected bucket. assertThat( producer.send( - new ProducerRecord( - path.toString(), 0, null, null)) - .get(30, TimeUnit.SECONDS) - .offset()) - .isEqualTo(-1L); - assertThatThrownBy(() -> producer.send(record(path, "")).get(30, TimeUnit.SECONDS)) - .hasCauseInstanceOf(InvalidRecordException.class); - assertThat( - producer.send(record(path, "{\"id\":7,\"optional\":null}")) + new ProducerRecord<>( + path.toString(), + 0, + KEY, + bytes("{\"id\":-1}"))) .get(30, TimeUnit.SECONDS) .offset()) .isZero(); read( connection, path, - row -> { - assertThat(row.getInt(0)).isEqualTo(7); - assertThat(row.isNullAt(1)).isTrue(); - }); + 0, + 1, + (row, index) -> assertThat(valueId(row, layout)).isEqualTo(-1)); + read( + connection, + path, + 1, + invalidValues.size(), + (row, index) -> assertThat(valueId(row, layout)).isEqualTo(index)); } finally { admin.dropTable(path, true).get(); } } } + private static int valueId(InternalRow row, String layout) { + return layout.equals("nested") ? row.getRow(1, 1).getInt(0) : row.getInt(1); + } + + private static void assertScalars(InternalRow row, int start, int id) { + assertThat(row.getInt(start)).isEqualTo(id); + assertThat(row.getDecimal(start + 1, 30, 10).toBigDecimal()) + .isEqualByComparingTo(new BigDecimal("12345678901234567890.1234567890")); + assertThat(row.getTimestampNtz(start + 2, 6).toLocalDateTime()) + .isEqualTo(LocalDateTime.parse("1969-12-31T23:59:59.999999")); + assertThat(row.getTimestampLtz(start + 3, 6).toInstant()) + .isEqualTo(Instant.parse("1969-12-31T23:59:59.999999Z")); + assertThat(Float.floatToRawIntBits(row.getFloat(start + 4))) + .isEqualTo(Float.floatToRawIntBits(-0.0F)); + assertThat(Double.doubleToRawLongBits(row.getDouble(start + 5))) + .isEqualTo(Double.doubleToRawLongBits(-0.0D)); + assertThat(row.isNullAt(start + 6)).isTrue(); + assertThat(row.getFloat(start + 7)).isEqualTo(1.0000001F); + } + + private static void assertHeaders(InternalMap headers, boolean populated) { + if (!populated) { + assertThat(headers.size()).isZero(); + return; + } + Map values = new HashMap<>(); + for (int index = 0; index < headers.size(); index++) { + values.put( + headers.keyArray().getString(index).toString(), + headers.valueArray().isNullAt(index) + ? null + : headers.valueArray().getBytes(index)); + } + assertThat(values).containsOnlyKeys("source", "nullable"); + assertThat(values.get("source")).containsExactly(1, 2); + assertThat(values.get("nullable")).isNull(); + } + + private static String scalarJson(int id, boolean explicitNull) { + // Both layouts consume this same root object; nested does not require a value wrapper. + return "{\"id\":" + + id + + ",\"amount\":12345678901234567890.1234567890," + + "\"received_at\":\"1969-12-31T23:59:59.999999\"," + + "\"emitted_at\":\"1970-01-01T07:59:59.999999+08:00\"," + + "\"float_value\":-0.0,\"double_value\":-0.0," + + "\"rounded_float\":1.0000000596046448" + + (explicitNull ? ",\"optional\":null" : "") + + "}"; + } + private static void create(Admin admin, TablePath path, TableDescriptor descriptor) throws Exception { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); admin.createTable(path, descriptor, false).get(); + TableInfo table = admin.getTableInfo(path).get(); + CLUSTER.waitUntilTableReady(table.getTableId()); + for (int bucket = 0; bucket < table.getNumBuckets(); bucket++) { + CLUSTER.waitAndGetLeaderReplica(new TableBucket(table.getTableId(), bucket)); + } CLUSTER.waitUntilAllGatewayHasSameMetadata(); } - private static TableDescriptor.Builder descriptor(Schema schema) { + private static TableDescriptor descriptor( + RowType valueType, + String layout, + boolean withSystemColumns, + boolean withRescue, + int buckets) { + Schema.Builder schema = Schema.newBuilder(); + if (withSystemColumns) { + schema.column(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES()); + } + if (layout.equals("nested")) { + schema.column(KafkaTableSchema.VALUE_COLUMN, valueType); + } else { + for (int index = 0; index < valueType.getFieldCount(); index++) { + schema.column(valueType.getFieldNames().get(index), valueType.getTypeAt(index)); + } + } + if (withSystemColumns) { + schema.column( + KafkaTableSchema.HEADERS_COLUMN, + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + } + if (withRescue) { + schema.column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()); + } return TableDescriptor.builder() - .schema(schema) - .distributedBy(1) + .schema(schema.build()) + .distributedBy(buckets) .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json"); + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout) + .build(); + } + + private static SimpleRecord record(String json) { + return new SimpleRecord(123L, KEY, bytes(json), new Header[0]); } - private static ProducerRecord record(TablePath path, String json) { - return new ProducerRecord<>( - path.toString(), - 0, - 123L, - "key".getBytes(StandardCharsets.UTF_8), - json.getBytes(StandardCharsets.UTF_8)); + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); } - private static void read(Connection connection, TablePath path, Consumer check) + private static ProduceResponse produce( + Socket socket, String topic, MemoryRecords rejected, MemoryRecords accepted) + throws Exception { + TopicProduceData topicData = + new TopicProduceData() + .setName(topic) + .setPartitionData( + Arrays.asList( + new PartitionProduceData().setIndex(0).setRecords(rejected), + new PartitionProduceData() + .setIndex(1) + .setRecords(accepted))); + ProduceRequest request = + new ProduceRequest( + new ProduceRequestData() + .setAcks((short) -1) + .setTimeoutMs(10000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topicData).iterator())), + (short) 9); + RequestHeader header = + new RequestHeader(ApiKeys.PRODUCE, request.version(), "json-produce-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] requestBytes = new byte[encoded.remaining()]; + encoded.get(requestBytes); + socket.setSoTimeout(15000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + int responseSize = input.readInt(); + assertThat(responseSize).isBetween(1, 1024 * 1024); + byte[] response = new byte[responseSize]; + input.readFully(response); + return (ProduceResponse) AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static PartitionProduceResponse partition( + ProduceResponse response, String topic, int index) { + return response.data().responses().find(topic).partitionResponses().stream() + .filter(partition -> partition.index() == index) + .findFirst() + .get(); + } + + private static void read( + Connection connection, + TablePath path, + int bucket, + int expectedCount, + BiConsumer check) throws Exception { try (Table table = connection.getTable(path); LogScanner scanner = table.newScan().createLogScanner()) { - scanner.subscribeFromBeginning(0); + scanner.subscribeFromBeginning(bucket); long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - while (System.nanoTime() < deadline) { - ScanRecords records = scanner.poll(Duration.ofSeconds(1)); - int count = 0; - for (ScanRecord record : records) { - assertThat(record.logOffset()).isZero(); - check.accept(record.getRow()); + int count = 0; + while (count < expectedCount && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.logOffset()).isEqualTo(count); + check.accept(record.getRow(), count); count++; } - if (count > 0) { - assertThat(count).isEqualTo(1); - return; - } } - throw new AssertionError("JSON record was not visible to the native scanner."); + assertThat(count).isEqualTo(expectedCount); + assertThat(scanner.poll(Duration.ofMillis(100))).isEmpty(); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index f41134c5e47..9c83311d215 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -289,6 +289,116 @@ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { assertThat(response.brokers()).isEmpty(); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testJsonTableIsDiscoverableForEveryMetadataVersion(boolean nested) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Schema schema = + nested + ? Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT()), + DataTypes.FIELD("name", DataTypes.STRING()))) + .column("_kafka_key", DataTypes.BYTES()) + .build() + : Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("_kafka_key", DataTypes.BYTES()) + .column("name", DataTypes.STRING()) + .build(); + service.putTable( + "fluss.json_topic", + 140L, + TableDescriptor.builder(defaultDescriptor()) + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + nested ? "nested" : "flat") + .build()); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + handle( + service, + topicsRequest(version, "json_topic", "fluss.json_topic"), + version); + for (String name : Arrays.asList("json_topic", "fluss.json_topic")) { + assertThat(response.data().topics().find(name).errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(response.data().topics().find(name).partitions()).hasSize(2); + } + if (version >= 10) { + assertThat(response.data().topics().find("json_topic").topicId()) + .isEqualTo(new Uuid(0x466c757373410000L, 140L)); + assertThat(response.data().topics().find("fluss.json_topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 140L)); + } + assertThat(handle(service, allTopicsRequest(version), version).data().topics()) + .extracting(MetadataResponseTopic::name) + .contains("fluss.json_topic") + .doesNotContain("json_topic"); + } + } + + @Test + public void testJsonAdmissionReflectsAuthoritativeSchemaAndProperties() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + TableDescriptor flat = + TableDescriptor.builder(defaultDescriptor()) + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build(); + service.putTable("fluss.json_topic", 140L, flat); + assertThat( + handle(service, namedTopicRequest("json_topic"), (short) 11) + .data() + .topics() + .find("json_topic") + .errorCode()) + .isEqualTo(Errors.NONE.code()); + // Changing the layout without moving business columns invalidates the same table ID. + service.putTable( + "fluss.json_topic", + 140L, + TableDescriptor.builder(flat) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build()); + assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) + .containsEntry("json_topic", Errors.INVALID_TOPIC_EXCEPTION); + service.putTable( + "fluss.json_topic", + 140L, + TableDescriptor.builder(flat) + .schema( + Schema.newBuilder() + .column("items", DataTypes.ARRAY(DataTypes.INT())) + .build()) + .build()); + assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) + .containsEntry("json_topic", Errors.INVALID_TOPIC_EXCEPTION); + service.putTable( + "fluss.json_topic", + 140L, + TableDescriptor.builder(flat) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) + .containsEntry("json_topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + service.putTable("fluss.json_topic", 141L, flat); + MetadataResponse recreated = + handle( + service, + topicsRequest((short) 11, "json_topic", "fluss.json_topic"), + (short) 11); + assertThat(recreated.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(recreated.data().topics().find("json_topic").topicId()) + .isEqualTo(new Uuid(0x466c757373410000L, 141L)); + assertThat(recreated.data().topics().find("fluss.json_topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 141L)); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) public void testMetadataColumnContractForEveryVersion(boolean legacyOffset) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index 34ab0ad5edc..ad7e3946b93 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -916,7 +916,7 @@ void testNullValueViolationRejectsNotNullTableContractBeforeAppending(String for } @Test - void testMetadataOffsetsAndJsonCannotBeActivatedByNullValues() throws Exception { + void testLegacyMetadataOffsetsRejectedAndJsonNullValuesPersisted() throws Exception { TestingProduceService service = new TestingProduceService(); TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); TableDescriptor unsupportedMetadata = @@ -956,11 +956,184 @@ void testMetadataOffsetsAndJsonCannotBeActivatedByNullValues() throws Exception CompletableFuture.completedFuture(metadata(false).setTableJson(json.toJsonBytes())); assertErrors( backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), - Errors.INVALID_RECORD); + Errors.NONE); + assertThat(service.appends).isEmpty(); + assertThat(service.writeChecks).isEqualTo(1); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testJsonConversionRejectsWholeBadPartitionAndPreservesOtherPartitions(boolean nested) + throws Exception { + List invalidValues = + new ArrayList<>( + Arrays.asList( + "{", "[]", "{\"id\":\"bad\"}", "{}", "{\"id\":1,\"unknown\":2}")); + for (String reserved : + Arrays.asList( + "_kafka_key", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue", + "_kafka_value")) { + invalidValues.add("{\"id\":1,\"" + reserved + "\":null}"); + } + for (boolean rescue : new boolean[] {false, true}) { + for (String invalid : invalidValues) { + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + jsonDescriptor(nested, rescue).toJsonBytes())); + PartitionWrite bad = + new PartitionWrite( + 0, + Arrays.asList( + json(0, "{\"id\":0}").records().get(0), + json(0, invalid).records().get(0))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic( + "topic", + bad, + json(1, "{\"id\":1}"), + json(8, "{\"id\":8}")))) + .join(); + assertErrors( + result, + Errors.INVALID_RECORD, + Errors.NONE, + Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + assertThat(result.topics().get(0).partitions().get(1).baseOffset()).isEqualTo(17L); + assertThat(service.drains).isEqualTo(1); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testUnsupportedJsonSchemaIsInvalidTopicEvenForFilteredRecords(boolean nested) + throws Exception { + TestingProduceService service = new TestingProduceService(); + Schema schema = + nested + ? Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD( + "id", DataTypes.ARRAY(DataTypes.INT())))) + .build() + : Schema.newBuilder() + .column("id", DataTypes.ARRAY(DataTypes.INT())) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(jsonDescriptor(nested, false)) + .schema(schema) + .build() + .toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + for (PartitionWrite partition : Arrays.asList(nullValue(0), json(0, "{\"id\":[1]}"))) { + assertErrors( + backend.write(command((short) 1, topic("topic", partition))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + } assertThat(service.appends).isEmpty(); assertThat(service.writeChecks).isZero(); } + @Test + void testJsonCacheUsesAuthoritativeLayoutSchemaAndTableIdentity() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + TableDescriptor flat = jsonDescriptor(false, false); + service.pendingMetadata = + CompletableFuture.completedFuture(metadata(false).setTableJson(flat.toJsonBytes())); + KafkaProduceCommand bare = command((short) 1, topic("topic", json(0, "{\"id\":1}"))); + KafkaProduceCommand full = command((short) 1, topic("kafka.topic", json(0, "{\"id\":1}"))); + assertErrors(backend.write(bare).join(), Errors.NONE); + assertErrors(backend.write(full).join(), Errors.NONE); + assertThat(transcoder.prepared.get(1)).isSameAs(transcoder.prepared.get(0)); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(flat) + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + "nested") + .build() + .toJsonBytes())); + assertErrors(backend.write(bare).join(), Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.appends).hasSize(2); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setSchemaId(2) + .setTableJson(jsonDescriptor(true, false).toJsonBytes())); + assertErrors(backend.write(full).join(), Errors.NONE); + assertThat(transcoder.prepared.get(2)).isNotSameAs(transcoder.prepared.get(0)); + assertThat(transcoder.prepared.get(2).tableInfo().getSchemaId()).isEqualTo(2); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableId(84L).setTableJson(flat.toJsonBytes())); + assertErrors(backend.write(bare).join(), Errors.NONE); + assertThat(transcoder.prepared.get(3)).isNotSameAs(transcoder.prepared.get(0)); + assertThat(transcoder.prepared.get(3).tableInfo().getTableId()).isEqualTo(84L); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L, 42L, 84L); + assertThat(service.metadataPaths).hasSize(5); + } + + private static TableDescriptor jsonDescriptor(boolean nested, boolean rescue) { + Schema.Builder schema = Schema.newBuilder(); + if (nested) { + schema.column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT().copy(false)))); + } else { + schema.column("id", DataTypes.INT().copy(false)); + } + schema.column("_kafka_key", DataTypes.BYTES()); + if (rescue) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + return TableDescriptor.builder() + .schema(schema.build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, nested ? "nested" : "flat") + .build(); + } + + private static PartitionWrite json(int id, String value) { + return new PartitionWrite( + id, + Collections.singletonList( + new Record( + 123L, + null, + value.getBytes(java.nio.charset.StandardCharsets.UTF_8), + Collections.emptyList()))); + } + private static final class RecordingTranscoder implements KafkaRecordTranscoder { private final ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); private final List prepared = new ArrayList<>(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java index c7051c0f816..77f5b445b24 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.format; import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; @@ -27,8 +28,10 @@ import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.InternalRow; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -166,15 +169,42 @@ void testFactoriesRejectKeyAndArbitraryValueColumnNames(boolean string) { } } - @Test - void testJsonHasNoDecoderUntilItsImplementationIsRegistered() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testJsonLayoutDeterminesPhysicalValueContainer(boolean nested) { + RowType business = DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT())); + KafkaFieldProjection projection = + nested + ? projection(business) + : new KafkaFieldProjection(business, Collections.singletonList(0)); + KafkaFieldDecoder decoder = + formats.createDecoder( + KafkaValueConversionType.PARSE_JSON, + projection, + nested + ? KafkaValueConversionLayout.NESTED + : KafkaValueConversionLayout.FLAT); + Object[] values = decoder.decode("{\"id\":7}".getBytes(StandardCharsets.UTF_8)); + assertThat(values).hasSize(1); + if (nested) { + assertThat(((InternalRow) values[0]).getInt(0)).isEqualTo(7); + } else { + assertThat(values).containsExactly(7); + } assertThatThrownBy( () -> formats.createDecoder( KafkaValueConversionType.PARSE_JSON, projection(DataTypes.STRING()))) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("PARSE_JSON"); + .hasMessageContaining("Reserved name"); + assertThatThrownBy( + () -> + decoder.decode( + "{\"_kafka_value\":{\"id\":7}}" + .getBytes(StandardCharsets.UTF_8))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("reserved field"); } private static KafkaFieldProjection projection(DataType type) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java index 3797a996af9..2078f034317 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java @@ -17,9 +17,13 @@ package org.apache.fluss.kafka.format.json; -import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericRow; import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; @@ -32,6 +36,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -134,8 +139,9 @@ void testRejectsContainersBeforeReadingTheirContents(String container) { "unknown field"); } - @Test - void testFieldCountLimitBeforeReadingExcessFieldValue() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFieldCountLimitBeforeReadingExcessFieldValue(boolean nested) { int maximumFields = JsonKafkaFieldDecoder.MAX_CONTAINER_ELEMENTS; List fields = new ArrayList<>(); StringBuilder json = new StringBuilder("{"); @@ -150,8 +156,10 @@ void testFieldCountLimitBeforeReadingExcessFieldValue() { expected[i] = i; } } - JsonKafkaFieldDecoder decoder = decoder(new RowType(fields)); - assertThat(decoder.decode(bytes(json + "}"))).containsExactly(expected); + JsonKafkaFieldDecoder decoder = decoder(new RowType(fields), nested); + Object[] result = decoder.decode(bytes(json + "}")); + assertThat(result) + .containsExactly(nested ? new Object[] {GenericRow.of(expected)} : expected); String excessField = json + ",\"field" + maximumFields + "\":"; for (String value : new String[] {"0}", "["}) { @@ -331,6 +339,17 @@ private static void assertFailure(RowType rowType, String json, String path, Str } private static JsonKafkaFieldDecoder decoder(RowType rowType) { + return decoder(rowType, false); + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType, boolean nested) { + if (nested) { + return new JsonKafkaFieldDecoder( + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, rowType)), + Collections.singletonList(0)), + KafkaValueConversionLayout.NESTED); + } List positions = new ArrayList<>(); for (int i = 0; i < rowType.getFieldCount(); i++) { positions.add(i); @@ -338,6 +357,84 @@ private static JsonKafkaFieldDecoder decoder(RowType rowType) { return new JsonKafkaFieldDecoder(new KafkaFieldProjection(rowType, positions)); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRootContractAndEarlyRejectionAreIndependentOfLayout(boolean nested) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT().copy(false)), + DataTypes.FIELD("optional", DataTypes.STRING())), + nested); + for (String name : + new String[] { + "_kafka_key", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue", + "_kafka_value" + }) { + assertThatThrownBy(() -> decoder.decode(bytes("{\"" + name + "\":["))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("reserved field"); + assertThatThrownBy( + () -> + decoder( + DataTypes.ROW(DataTypes.FIELD(name, DataTypes.INT())), + nested)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("Reserved name"); + } + assertThatThrownBy(() -> decoder.decode(bytes("{\"unknown\":["))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("unknown field"); + assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":["))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("expected a JSON scalar"); + for (byte[] invalid : + new byte[][] { + bytes("null"), + bytes("[]"), + bytes("{}"), + bytes("{\"id\":1,\"id\":2}"), + bytes("{\"id\":1} {}"), + {(byte) 0xff} + }) { + assertThatThrownBy(() -> decoder.decode(invalid)) + .isInstanceOf(KafkaRecordEncodingException.class); + } + Object[] result = decoder.decode(bytes("{\"optional\":null,\"id\":7}")); + assertThat(result) + .containsExactly( + nested ? new Object[] {GenericRow.of(7, null)} : new Object[] {7, null}); + } + + @Test + void testNestedRootPreservesScalarValuesAndContainerNullability() { + RowType business = + DataTypes.ROW( + DataTypes.FIELD("f", DataTypes.FLOAT()), + DataTypes.FIELD("d", DataTypes.DOUBLE()), + DataTypes.FIELD("amount", DataTypes.DECIMAL(38, 10).copy(false))); + JsonKafkaFieldDecoder decoder = decoder(business, true); + GenericRow row = + (GenericRow) + decoder.decode( + bytes( + "{\"amount\":1234567890123456789012345678.1234567890,\"d\":-0.0,\"f\":-0.0}"))[ + 0]; + assertThat(Float.floatToRawIntBits(row.getFloat(0))) + .isEqualTo(Float.floatToRawIntBits(-0.0F)); + assertThat(Double.doubleToRawLongBits(row.getDouble(1))) + .isEqualTo(Double.doubleToRawLongBits(-0.0D)); + assertThat(row.getDecimal(2, 38, 10).toBigDecimal()) + .isEqualTo(new BigDecimal("1234567890123456789012345678.1234567890")); + assertThat(decoder.decode(null)).containsExactly((Object) null); + assertThatThrownBy(() -> decoder((RowType) business.copy(false), true).decode(null)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL field '_kafka_value'"); + } + private static byte[] bytes(String value) { return value.getBytes(StandardCharsets.UTF_8); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java index 0b94ab0c934..bab37c29118 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java @@ -17,13 +17,22 @@ package org.apache.fluss.kafka.schema; -import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -33,63 +42,184 @@ class KafkaJsonSchemaTest { private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); @Test - void testJsonProjectsValuesAroundKeyAndMetadataColumns() { - KafkaTopicSchema schema = + void testFlatJsonProjectsOnlyBusinessColumnsAroundOptionalSystemColumns() { + TableDescriptor descriptor = + descriptor( + Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column("amount", DataTypes.DECIMAL(8, 2)) + .column("_kafka_rescue", DataTypes.STRING()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "binary") + .build(); + KafkaTableSchema schema = resolver.resolve(descriptor); + assertThat(schema).isEqualTo(KafkaTableSchema.resolve(descriptor)); + assertThat(schema.valueConversionType()).isEqualTo(KafkaValueConversionType.PARSE_JSON); + assertThat(schema.valueConversionLayout()).isEqualTo(KafkaValueConversionLayout.FLAT); + assertThat(schema.valueProjection().positions()).containsExactly(0, 3); + assertThat(schema.keyPosition()).isEqualTo(1); + assertThat(schema.headersPosition()).isEqualTo(2); + assertThat(schema.rescuePosition()).isEqualTo(4); + assertThat(schema.rowType()).isEqualTo(descriptor.getSchema().getRowType()); + } + + @Test + void testNestedJsonProjectsItsPhysicalValueContainer() { + RowType value = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT().copy(false)), + DataTypes.FIELD("name", DataTypes.STRING())); + KafkaTableSchema schema = resolver.resolve( - descriptor() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "key") - .customProperty( - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "time") + descriptor( + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES()) + .column("_kafka_value", value) + .column("_kafka_rescue", DataTypes.STRING()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") .build()); - assertThat(schema.keyProjection().positions()).containsExactly(1); - assertThat(schema.valueProjection().positions()).containsExactly(0, 3); - assertThat(schema.timestampPosition()).isEqualTo(2); + assertThat(schema.valueConversionLayout()).isEqualTo(KafkaValueConversionLayout.NESTED); + assertThat(schema.valueProjection().positions()).containsExactly(1); + assertThat(schema.valueProjection().dataTypeAt(0)).isEqualTo(value); + assertThat(schema.keyPosition()).isZero(); + assertThat(schema.rescuePosition()).isEqualTo(2); + } + + @Test + void testJsonContractRequiresExplicitEnablementAndRejectsPrototypeOptions() { + TableDescriptor.Builder table = + descriptor(Schema.newBuilder().column("id", DataTypes.INT()).build()); + TableDescriptor disabled = table.property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + assertThat(resolver.isEnabled(disabled)).isFalse(); + assertThatThrownBy(() -> resolver.resolve(disabled)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); + for (String option : + Arrays.asList( + "kafka.value.format", + "kafka.key.fields", + "table.kafka.convert.value.type")) { + assertThatThrownBy( + () -> + resolver.resolve( + descriptor(disabled.getSchema()) + .customProperty(option, "json") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("custom property"); + } + assertThatThrownBy( + () -> + resolver.resolve( + descriptor(disabled.getSchema()) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + "string") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("only supports binary"); } @Test - void testRejectsJsonKeysAndUnsupportedFieldTypes() { + void testNestedJsonRejectsUnmappedTopLevelBusinessColumns() { assertThatThrownBy( () -> resolver.resolve( - descriptor() - .customProperty( - KafkaDataFormat.KEY_FORMAT_CONFIG, "json") + descriptor( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD( + "id", + DataTypes + .INT()))) + .column("other", DataTypes.STRING()) + .build()) + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + "nested") .build())) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("only supported for record values"); + .hasMessageContaining("no value mapping"); + } + + @ParameterizedTest + @ValueSource( + strings = { + "_kafka_key", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue", + "_kafka_value" + }) + void testNestedJsonRejectsReservedRootBusinessNames(String name) { assertThatThrownBy( () -> resolver.resolve( - TableDescriptor.builder() - .schema( + descriptor( Schema.newBuilder() .column( - "items", - DataTypes.ARRAY( - DataTypes.INT())) + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD( + name, + DataTypes + .STRING()))) .build()) - .distributedBy(1) - .logFormat(LogFormat.ARROW) - .customProperty( - KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + "nested") .build())) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("does not support"); + .hasMessageContaining("Reserved name"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testComplexBusinessFieldsRemainUnavailableUntilRecursiveConversion(boolean nested) { + for (DataType type : + Arrays.asList( + DataTypes.ARRAY(DataTypes.INT()), + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + DataTypes.ROW(DataTypes.FIELD("child", DataTypes.INT())))) { + Schema schema = + nested + ? Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("items", type))) + .build() + : Schema.newBuilder().column("items", type).build(); + TableDescriptor table = + descriptor(schema) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + nested ? "nested" : "flat") + .build(); + // The persisted contract is valid, but this gateway stage must report INVALID_TOPIC + // before accepting records for a decoder that it does not yet implement. + assertThat(KafkaTableSchema.resolve(table).valueConversionType()) + .isEqualTo(KafkaValueConversionType.PARSE_JSON); + assertThatThrownBy(() -> resolver.resolve(table)) + .isInstanceOf(KafkaTopicSchemaException.class); + } } - private static TableDescriptor.Builder descriptor() { + private static TableDescriptor.Builder descriptor(Schema schema) { return TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("id", DataTypes.INT().copy(false)) - .column("key", DataTypes.STRING()) - .column("time", DataTypes.TIMESTAMP(3).copy(false)) - .column("amount", DataTypes.DECIMAL(8, 2)) - .build()) + .schema(schema) .distributedBy(1) .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json"); + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 43eb7c9a40e..2686df6ecad 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -337,6 +337,7 @@ void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { assertThat(stored.getCustomProperties()).isEqualTo(descriptor.getCustomProperties()); assertThat(kafkaProperties(stored)) .isEqualTo(Collections.singletonMap("table.kafka.enabled", "true")); + // This checks persisted DDL mapping; recursive gateway decoding is a later stage. assertThat(new KafkaTopicSchemaResolver().resolve(stored).valueProjection().positions()) .containsExactly(1); set(path, Collections.singletonMap("table.kafka.convert.key.type", "binary")); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 59e33ad2ea0..38b8c1128e3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -542,13 +542,13 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() throws Exception } @Test - void testJsonIsNotActivatedEvenForNullValues() { + void testUnsupportedJsonBusinessTypesFailDuringPreparation() { TableInfo table = table( TableDescriptor.builder() .schema( Schema.newBuilder() - .column("body", DataTypes.STRING()) + .column("body", DataTypes.ARRAY(DataTypes.INT())) .build()) .distributedBy(1) .logFormat(LogFormat.ARROW) @@ -557,6 +557,9 @@ void testJsonIsNotActivatedEvenForNullValues() { ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, KafkaValueConversionType.PARSE_JSON) .build()); + assertThatThrownBy(() -> transcoder.prepare(table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("ARRAY"); for (byte[] value : new byte[][] {null, bytes("{}")}) { assertThatThrownBy( () -> @@ -569,8 +572,8 @@ void testJsonIsNotActivatedEvenForNullValues() { value, Collections.emptyList())), table)) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("PARSE_JSON"); + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("ARRAY"); } } From 933ac05f40f0d2dc0ab348f489225278e51b0e6e Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 15 Oct 2026 23:52:11 +0800 Subject: [PATCH 52/69] [kafka] Verify JSON metadata and floating-point boundary contracts Cover native metadata readback for flat and nested JSON tables, including batch timestamp provenance, absent timestamps and nonzero partitions. Verify partition atomicity for required metadata and reserved JSON fields, and preserve signed underflow with single-rounding FLOAT/DOUBLE parsing. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 248/248 --- .../fluss/kafka/KafkaJsonProduceITCase.java | 205 +++++++++++++++++- .../json/JsonKafkaFieldDecoderTest.java | 43 ++++ 2 files changed, 244 insertions(+), 4 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 3de480e557b..345ce6337b6 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -54,7 +54,10 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.MemoryRecordsBuilder; +import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ProduceRequest; import org.apache.kafka.common.requests.ProduceResponse; @@ -63,6 +66,7 @@ import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -158,6 +162,7 @@ void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( new ProducerRecord( path.getTableName(), 0, + 456L, null, bytes(scalarJson(8, true)))) .get(30, TimeUnit.SECONDS); @@ -188,7 +193,9 @@ void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( ? 1 : SCALAR_TYPE.getFieldCount()); assertHeaders(row.getMap(headersPosition), index == 0); - assertThat(row.isNullAt(headersPosition + 1)).isTrue(); + assertMetadata( + row, headersPosition + 1, index == 0 ? 123L : 456L, 0, 0); + assertThat(row.isNullAt(headersPosition + 2)).isTrue(); } }); } finally { @@ -223,7 +230,11 @@ void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( KafkaTableSchema.VALUE_COLUMN }) { // This also rejects the erroneous nested payload wrapper {"_kafka_value":{...}}. - invalidValues.add("{\"id\":1,\"" + reserved + "\":{\"id\":2}}"); + String value = + reserved.equals(KafkaTableSchema.METADATA_COLUMN) + ? "{\"timestamp\":0,\"timestamp_type\":1,\"partition\":99}" + : "{\"id\":2}"; + invalidValues.add("{\"id\":1,\"" + reserved + "\":" + value + "}"); } try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { @@ -265,6 +276,7 @@ void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( new ProducerRecord<>( path.toString(), 0, + 123L, KEY, bytes("{\"id\":-1}"))) .get(30, TimeUnit.SECONDS) @@ -275,19 +287,168 @@ void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( path, 0, 1, - (row, index) -> assertThat(valueId(row, layout)).isEqualTo(-1)); + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(-1); + assertMetadata(row, 3, 123L, 0, 0); + }); read( connection, path, 1, invalidValues.size(), - (row, index) -> assertThat(valueId(row, layout)).isEqualTo(index)); + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(index); + assertMetadata(row, 3, 123L, 0, 1); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout) throws Exception { + TablePath path = TablePath.of(DATABASE, "json_metadata_" + layout); + RowType valueType = RowType.builder().field("id", DataTypes.INT().copy(false)).build(); + // Magic v2 batches carry CREATE_TIME or LOG_APPEND_TIME even without a timestamp. + TimestampType[] types = { + TimestampType.CREATE_TIME, + TimestampType.LOG_APPEND_TIME, + TimestampType.CREATE_TIME, + TimestampType.LOG_APPEND_TIME + }; + long[] expectedTimestamps = {0L, 456L, 123L, 123L, -1L, -1L, -1L, -1L}; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(valueType, layout, true, false, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port())) { + for (int batch = 0; batch < types.length; batch++) { + long firstTimestamp = batch < 2 ? 0L : RecordBatch.NO_TIMESTAMP; + long secondTimestamp = batch < 2 ? 456L : RecordBatch.NO_TIMESTAMP; + long appendTimestamp = batch == 1 ? 123L : RecordBatch.NO_TIMESTAMP; + MemoryRecords records = + timestampedRecords( + types[batch], + appendTimestamp, + new SimpleRecord( + firstTimestamp, + KEY, + bytes("{\"id\":" + (batch * 2) + "}")), + new SimpleRecord( + secondTimestamp, + KEY, + bytes("{\"id\":" + (batch * 2 + 1) + "}"))); + String topic = batch % 2 == 0 ? path.toString() : path.getTableName(); + ProduceResponse response = produce(socket, topic, records, records); + for (int partition = 0; partition < 2; partition++) { + PartitionProduceResponse result = partition(response, topic, partition); + assertThat(result.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(result.baseOffset()).isEqualTo(batch * 2L); + } + } + for (int partition = 0; partition < 2; partition++) { + int expectedPartition = partition; + read( + connection, + path, + partition, + expectedTimestamps.length, + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(index); + assertMetadata( + row, + 3, + expectedTimestamps[index], + (index / 2) % 2, + expectedPartition); + }); + } } finally { admin.dropTable(path, true).get(); } } } + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testMissingRequiredMetadataTimestampIsPartitionAtomic(String layout) throws Exception { + TablePath path = TablePath.of(DATABASE, "json_required_timestamp_" + layout); + RowType valueType = RowType.builder().field("id", DataTypes.INT().copy(false)).build(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(valueType, layout, true, false, 2, false)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port()); + KafkaProducer producer = producer()) { + ProduceResponse response = + produce( + socket, + path.toString(), + timestampedRecords( + TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + record("{\"id\":1000}"), + new SimpleRecord( + RecordBatch.NO_TIMESTAMP, + KEY, + bytes("{\"id\":1001}"))), + MemoryRecords.withRecords(Compression.NONE, record("{\"id\":1}"))); + PartitionProduceResponse rejected = partition(response, path.toString(), 0); + assertThat(rejected.errorCode()).isEqualTo(Errors.INVALID_RECORD.code()); + assertThat(rejected.baseOffset()).isEqualTo(-1L); + PartitionProduceResponse accepted = partition(response, path.toString(), 1); + assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(accepted.baseOffset()).isZero(); + assertThat( + producer.send( + new ProducerRecord<>( + path.getTableName(), + 0, + 789L, + KEY, + bytes("{\"id\":-1}"))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + read( + connection, + path, + 0, + 1, + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(-1); + assertMetadata(row, 3, 789L, 0, 0); + }); + read( + connection, + path, + 1, + 1, + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(1); + assertMetadata(row, 3, 123L, 0, 1); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + private static void assertMetadata( + InternalRow row, int position, long timestamp, int timestampType, int partition) { + assertThat(row.isNullAt(position)).isFalse(); + InternalRow metadata = row.getRow(position, 3); + if (timestamp == RecordBatch.NO_TIMESTAMP) { + assertThat(metadata.isNullAt(0)).isTrue(); + } else { + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(timestamp); + } + assertThat(metadata.getInt(1)).isEqualTo(timestampType); + assertThat(metadata.getInt(2)).isEqualTo(partition); + } + private static int valueId(InternalRow row, String layout) { return layout.equals("nested") ? row.getRow(1, 1).getInt(0) : row.getInt(1); } @@ -357,6 +518,16 @@ private static TableDescriptor descriptor( boolean withSystemColumns, boolean withRescue, int buckets) { + return descriptor(valueType, layout, withSystemColumns, withRescue, buckets, true); + } + + private static TableDescriptor descriptor( + RowType valueType, + String layout, + boolean withSystemColumns, + boolean withRescue, + int buckets, + boolean nullableTimestamp) { Schema.Builder schema = Schema.newBuilder(); if (withSystemColumns) { schema.column(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES()); @@ -372,6 +543,15 @@ private static TableDescriptor descriptor( schema.column( KafkaTableSchema.HEADERS_COLUMN, DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + schema.column( + KafkaTableSchema.METADATA_COLUMN, + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3).copy(nullableTimestamp)), + DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))) + .copy(false)); } if (withRescue) { schema.column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()); @@ -386,6 +566,23 @@ private static TableDescriptor descriptor( .build(); } + private static MemoryRecords timestampedRecords( + TimestampType timestampType, long logAppendTime, SimpleRecord... records) { + try (MemoryRecordsBuilder builder = + MemoryRecords.builder( + ByteBuffer.allocate(1024), + RecordBatch.MAGIC_VALUE_V2, + Compression.NONE, + timestampType, + 0L, + logAppendTime)) { + for (SimpleRecord record : records) { + builder.append(record); + } + return builder.build(); + } + } + private static SimpleRecord record(String json) { return new SimpleRecord(123L, KEY, bytes(json), new Header[0]); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java index 2078f034317..0fa69e90eca 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java @@ -188,6 +188,49 @@ void testPreservesFloatingPointZeroSign(String number) { .isEqualTo(Double.doubleToRawLongBits(expectedDouble)); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFloatingPointUnderflowPreservesSignAndRoundsOnce(boolean nested) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE())), + nested); + // The final two pairs straddle half of the smallest subnormal for each target. + // Converting the FLOAT token through DOUBLE first rounds both to the same midpoint. + String[][] tokens = { + {"0", "0"}, + {"0.0", "0.0"}, + {"1e-1000", "1e-1000"}, + {"7.006492321624085e-46", "2.4703282292062327e-324"}, + {"7.006492321624086e-46", "2.4703282292062328e-324"} + }; + for (boolean negative : new boolean[] {false, true}) { + String sign = negative ? "-" : ""; + for (int i = 0; i < tokens.length; i++) { + Object[] decoded = + decoder.decode( + bytes( + "{\"floatValue\":" + + sign + + tokens[i][0] + + ",\"doubleValue\":" + + sign + + tokens[i][1] + + "}")); + GenericRow row = nested ? (GenericRow) decoded[0] : GenericRow.of(decoded); + int magnitude = i == tokens.length - 1 ? 1 : 0; + assertThat(Float.floatToRawIntBits(row.getFloat(0))) + .as("FLOAT %s%s, nested=%s", sign, tokens[i][0], nested) + .isEqualTo((negative ? Integer.MIN_VALUE : 0) | magnitude); + assertThat(Double.doubleToRawLongBits(row.getDouble(1))) + .as("DOUBLE %s%s, nested=%s", sign, tokens[i][1], nested) + .isEqualTo((negative ? Long.MIN_VALUE : 0L) | magnitude); + } + } + } + @ParameterizedTest @ValueSource(strings = {"", "-"}) void testFloatingPointRoundingAndExactDecimalParsing(String sign) { From 4f41194364c8c4fb6a177573cfa10e9df20e3158 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 20:04:19 +0800 Subject: [PATCH 53/69] [kafka] Verify JSON null persistence on the updated base Align JSON tests with native persistence of nullable values and real offsets. Cover all-null and mixed batches, both layouts, system fields and atomic NOT NULL failures. Check that direct floating-point parsing retains the parser number-length limit; remove obsolete null-filtering/drain assertions. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 236/236 --- .../fluss/kafka/KafkaJsonProduceITCase.java | 172 ++++++++++++++++-- .../GatewayKafkaProduceBackendTest.java | 40 +++- .../json/JsonKafkaFieldDecoderTest.java | 25 +++ 3 files changed, 211 insertions(+), 26 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 345ce6337b6..633aed55c6d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -125,14 +125,6 @@ void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( try (KafkaProducer producer = producer()) { assertThat(producer.partitionsFor(path.toString())).hasSize(1); assertThat(producer.partitionsFor(path.getTableName())).hasSize(1); - // Null values are filtered before validating required JSON business fields. - assertThat( - producer.send( - new ProducerRecord( - path.toString(), 0, null, null)) - .get(30, TimeUnit.SECONDS) - .offset()) - .isEqualTo(-1L); RecordMetadata first = producer.send( new ProducerRecord<>( @@ -149,13 +141,6 @@ void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( .get(30, TimeUnit.SECONDS); assertThat(first.topic()).isEqualTo(path.toString()); assertThat(first.offset()).isZero(); - assertThat( - producer.send( - new ProducerRecord( - path.getTableName(), 0, KEY, null)) - .get(30, TimeUnit.SECONDS) - .offset()) - .isEqualTo(-1L); // A bare alias and the full name append to the same native table and bucket. RecordMetadata second = producer.send( @@ -204,6 +189,161 @@ void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( } } + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout) + throws Exception { + TablePath path = TablePath.of(DATABASE, "json_null_values_" + layout); + // A nullable nested parent accepts null even when its business child is NOT NULL. + RowType valueType = + RowType.builder() + .field("id", DataTypes.INT().copy(layout.equals("flat"))) + .field("label", DataTypes.STRING()) + .build(); + Header[] headers = { + new RecordHeader("source", bytes("old")), + new RecordHeader("source", new byte[] {1, 2}), + new RecordHeader("nullable", null) + }; + SimpleRecord[] records = new SimpleRecord[5]; + for (int index = 0; index < records.length; index++) { + String json = + index == 2 || index == 4 + ? "{\"id\":" + index + ",\"label\":\"v" + index + "\"}" + : null; + records[index] = + new SimpleRecord( + (index + 1) * 10L, + index % 2 == 0 ? KEY : null, + json == null ? null : bytes(json), + headers); + } + MemoryRecords[] batches = { + MemoryRecords.withRecords(Compression.NONE, records[0], records[1]), + MemoryRecords.withRecords(Compression.NONE, records[2], records[3], records[4]) + }; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(valueType, layout, true, true, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port())) { + for (int batch = 0; batch < batches.length; batch++) { + String topic = batch == 0 ? path.toString() : path.getTableName(); + ProduceResponse response = + produce(socket, topic, batches[batch], batches[batch]); + for (int partition = 0; partition < 2; partition++) { + PartitionProduceResponse result = partition(response, topic, partition); + assertThat(result.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(result.baseOffset()).isEqualTo(batch == 0 ? 0L : 2L); + } + } + for (int partition = 0; partition < 2; partition++) { + int expectedPartition = partition; + read( + connection, + path, + partition, + records.length, + (row, index) -> { + if (index % 2 == 0) { + assertThat(row.getBytes(0)).isEqualTo(KEY); + } else { + assertThat(row.isNullAt(0)).isTrue(); + } + if (index == 2 || index == 4) { + InternalRow value = + layout.equals("nested") ? row.getRow(1, 2) : row; + int start = layout.equals("nested") ? 0 : 1; + assertThat(value.getInt(start)).isEqualTo(index); + assertThat(value.getString(start + 1).toString()) + .isEqualTo("v" + index); + } else { + assertThat(row.isNullAt(1)).isTrue(); + if (layout.equals("flat")) { + assertThat(row.isNullAt(2)).isTrue(); + } + } + int headersPosition = layout.equals("nested") ? 2 : 3; + assertHeaders(row.getMap(headersPosition), true); + assertMetadata( + row, + headersPosition + 1, + (index + 1) * 10L, + 0, + expectedPartition); + assertThat(row.isNullAt(headersPosition + 2)).isTrue(); + }); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testNullValueViolatingRequiredBusinessTypeIsPartitionAtomic(String layout) + throws Exception { + TablePath path = TablePath.of(DATABASE, "json_required_value_" + layout); + // Flat checks the business field; nested checks the parent even with nullable children. + RowType valueType = + RowType.builder(false) + .field("id", DataTypes.INT().copy(layout.equals("nested"))) + .build(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(valueType, layout, true, false, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port()); + KafkaProducer producer = producer()) { + ProduceResponse response = + produce( + socket, + path.toString(), + MemoryRecords.withRecords( + Compression.NONE, record("{\"id\":1000}"), record(null)), + MemoryRecords.withRecords(Compression.NONE, record("{\"id\":1}"))); + PartitionProduceResponse rejected = partition(response, path.toString(), 0); + assertThat(rejected.errorCode()).isEqualTo(Errors.INVALID_RECORD.code()); + assertThat(rejected.baseOffset()).isEqualTo(-1L); + PartitionProduceResponse accepted = partition(response, path.toString(), 1); + assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(accepted.baseOffset()).isZero(); + assertThat( + producer.send( + new ProducerRecord<>( + path.getTableName(), + 0, + 789L, + KEY, + bytes("{\"id\":-1}"))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + read( + connection, + path, + 0, + 1, + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(-1); + assertMetadata(row, 3, 789L, 0, 0); + }); + read( + connection, + path, + 1, + 1, + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(1); + assertMetadata(row, 3, 123L, 0, 1); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + @ParameterizedTest @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( @@ -584,7 +724,7 @@ private static MemoryRecords timestampedRecords( } private static SimpleRecord record(String json) { - return new SimpleRecord(123L, KEY, bytes(json), new Header[0]); + return new SimpleRecord(123L, KEY, json == null ? null : bytes(json), new Header[0]); } private static byte[] bytes(String value) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index ad7e3946b93..ed06efa34c1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -20,6 +20,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.KafkaKeyConversionType; +import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.DatabaseNotExistException; @@ -915,8 +916,10 @@ void testNullValueViolationRejectsNotNullTableContractBeforeAppending(String for assertThat(service.appends).isEmpty(); } - @Test - void testLegacyMetadataOffsetsRejectedAndJsonNullValuesPersisted() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testLegacyMetadataOffsetsRejectedAndJsonNullValuesPersisted(boolean nested) + throws Exception { TestingProduceService service = new TestingProduceService(); TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); TableDescriptor unsupportedMetadata = @@ -945,19 +948,38 @@ void testLegacyMetadataOffsetsRejectedAndJsonNullValuesPersisted() throws Except assertErrors( backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), Errors.INVALID_TOPIC_EXCEPTION); + Schema jsonSchema = + nested + ? Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD( + "business", + DataTypes.STRING().copy(false)))) + .build() + : Schema.newBuilder().column("business", DataTypes.STRING()).build(); TableDescriptor json = TableDescriptor.builder(original) - .schema(Schema.newBuilder().column("business", DataTypes.STRING()).build()) + .schema(jsonSchema) .property( ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, KafkaValueConversionType.PARSE_JSON) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + nested + ? KafkaValueConversionLayout.NESTED + : KafkaValueConversionLayout.FLAT) .build(); service.pendingMetadata = CompletableFuture.completedFuture(metadata(false).setTableJson(json.toJsonBytes())); - assertErrors( - backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), - Errors.NONE); - assertThat(service.appends).isEmpty(); + KafkaProduceResult result = + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(); + assertErrors(result, Errors.NONE); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isZero(); assertThat(service.writeChecks).isEqualTo(1); } @@ -1012,15 +1034,13 @@ void testJsonConversionRejectsWholeBadPartitionAndPreservesOtherPartitions(boole assertThat(service.append.getBucketsReqsList()).hasSize(1); assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); assertThat(result.topics().get(0).partitions().get(1).baseOffset()).isEqualTo(17L); - assertThat(service.drains).isEqualTo(1); } } } @ParameterizedTest @ValueSource(booleans = {false, true}) - void testUnsupportedJsonSchemaIsInvalidTopicEvenForFilteredRecords(boolean nested) - throws Exception { + void testUnsupportedJsonSchemaIsInvalidTopicEvenForNullValues(boolean nested) throws Exception { TestingProduceService service = new TestingProduceService(); Schema schema = nested diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java index 0fa69e90eca..78083b6932d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java @@ -24,6 +24,8 @@ import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.Decimal; import org.apache.fluss.row.GenericRow; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.StreamReadConstraints; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.exc.StreamConstraintsException; import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; @@ -300,6 +302,29 @@ void testFloatingPointOverflowBoundaries(String sign) { "overflowing number"); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRawFloatingPointTokensStillEnforceNumberLengthLimit(boolean nested) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE())), + nested); + // This token underflows to zero rather than overflowing the target. It must still be + // rejected by the parser before the direct FLOAT/DOUBLE token conversion. + StringBuilder number = new StringBuilder("-0."); + for (int i = 0; i < StreamReadConstraints.DEFAULT_MAX_NUM_LEN; i++) { + number.append('0'); + } + number.append('1'); + for (String field : new String[] {"floatValue", "doubleValue"}) { + assertThatThrownBy(() -> decoder.decode(bytes("{\"" + field + "\":" + number + "}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasRootCauseInstanceOf(StreamConstraintsException.class); + } + } + @Test void testMixedScalarTokensPreserveProjectionAndMissingFields() { JsonKafkaFieldDecoder decoder = From dc3d305034537303ca0c57d1d6587288fea1c4e8 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 20:28:15 +0800 Subject: [PATCH 54/69] [kafka] Align JSON Produce tests with current table contracts Adapt schema and integration tests to the nullable field contract and typed Kafka configuration introduced by preceding PRs. Cover not-null rejection at DDL and persistence for null values and absent timestamps. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 3/3 AI-Contributed/UT: 207/207 --- .../schema/KafkaTopicSchemaResolver.java | 3 +- .../fluss/kafka/KafkaJsonProduceITCase.java | 134 ++++++------------ .../fluss/kafka/KafkaMetadataHandlerTest.java | 8 +- .../GatewayKafkaProduceBackendTest.java | 21 ++- .../kafka/schema/KafkaJsonSchemaTest.java | 40 +++--- .../kafka/schema/KafkaTableMappingITCase.java | 4 +- 6 files changed, 79 insertions(+), 131 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index 5de300ccf81..6b84fe7c675 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -43,8 +43,7 @@ public KafkaTableSchema resolve(TableDescriptor table) { KafkaTableSchema schema = KafkaTableSchema.resolve(table); if (schema.valueConversionType() == KafkaValueConversionType.PARSE_JSON) { // Metadata and Produce must reject schemas beyond the installed decoder support. - new JsonKafkaFieldDecoder( - schema.valueProjection(), schema.valueConversionLayout()); + new JsonKafkaFieldDecoder(schema.valueProjection(), schema.valueConversionLayout()); } return schema; } catch (InvalidTableException | IllegalArgumentException e) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 633aed55c6d..8afadffd44e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -26,6 +26,7 @@ import org.apache.fluss.cluster.ServerNode; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; @@ -87,6 +88,7 @@ import java.util.function.BiConsumer; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Verifies DDL-discovered JSON writes and native Arrow readback. */ class KafkaJsonProduceITCase { @@ -94,10 +96,10 @@ class KafkaJsonProduceITCase { private static final byte[] KEY = new byte[] {(byte) 0xff, 1}; private static final RowType SCALAR_TYPE = RowType.builder() - .field("id", DataTypes.INT().copy(false)) + .field("id", DataTypes.INT()) .field("amount", DataTypes.DECIMAL(30, 10)) - .field("received_at", DataTypes.TIMESTAMP(6).copy(false)) - .field("emitted_at", DataTypes.TIMESTAMP_LTZ(6).copy(false)) + .field("received_at", DataTypes.TIMESTAMP(6)) + .field("emitted_at", DataTypes.TIMESTAMP_LTZ(6)) .field("float_value", DataTypes.FLOAT()) .field("double_value", DataTypes.DOUBLE()) .field("optional", DataTypes.STRING()) @@ -194,10 +196,10 @@ void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout) throws Exception { TablePath path = TablePath.of(DATABASE, "json_null_values_" + layout); - // A nullable nested parent accepts null even when its business child is NOT NULL. + // Null Kafka values populate nullable flat fields or the nullable nested value container. RowType valueType = RowType.builder() - .field("id", DataTypes.INT().copy(layout.equals("flat"))) + .field("id", DataTypes.INT()) .field("label", DataTypes.STRING()) .build(); Header[] headers = { @@ -282,65 +284,19 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout) @ParameterizedTest @ValueSource(strings = {"flat", "nested"}) - void testNullValueViolatingRequiredBusinessTypeIsPartitionAtomic(String layout) - throws Exception { + void testNotNullBusinessFieldsAreRejectedAtTableCreation(String layout) throws Exception { TablePath path = TablePath.of(DATABASE, "json_required_value_" + layout); - // Flat checks the business field; nested checks the parent even with nullable children. - RowType valueType = - RowType.builder(false) - .field("id", DataTypes.INT().copy(layout.equals("nested"))) - .build(); + RowType valueType = RowType.builder(false).field("id", DataTypes.INT().copy(false)).build(); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { - create(admin, path, descriptor(valueType, layout, true, false, 2)); - ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); - try (Socket socket = new Socket(node.host(), node.port()); - KafkaProducer producer = producer()) { - ProduceResponse response = - produce( - socket, - path.toString(), - MemoryRecords.withRecords( - Compression.NONE, record("{\"id\":1000}"), record(null)), - MemoryRecords.withRecords(Compression.NONE, record("{\"id\":1}"))); - PartitionProduceResponse rejected = partition(response, path.toString(), 0); - assertThat(rejected.errorCode()).isEqualTo(Errors.INVALID_RECORD.code()); - assertThat(rejected.baseOffset()).isEqualTo(-1L); - PartitionProduceResponse accepted = partition(response, path.toString(), 1); - assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(accepted.baseOffset()).isZero(); - assertThat( - producer.send( - new ProducerRecord<>( - path.getTableName(), - 0, - 789L, - KEY, - bytes("{\"id\":-1}"))) - .get(30, TimeUnit.SECONDS) - .offset()) - .isZero(); - read( - connection, - path, - 0, - 1, - (row, index) -> { - assertThat(valueId(row, layout)).isEqualTo(-1); - assertMetadata(row, 3, 789L, 0, 0); - }); - read( - connection, - path, - 1, - 1, - (row, index) -> { - assertThat(valueId(row, layout)).isEqualTo(1); - assertMetadata(row, 3, 123L, 0, 1); - }); - } finally { - admin.dropTable(path, true).get(); - } + assertThatThrownBy( + () -> + create( + admin, + path, + descriptor(valueType, layout, true, false, 2))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("nullable fields"); } } @@ -349,7 +305,7 @@ Compression.NONE, record("{\"id\":1000}"), record(null)), void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( String layout, boolean withRescue) throws Exception { TablePath path = TablePath.of(DATABASE, "json_rejections_" + layout + "_" + withRescue); - RowType valueType = RowType.builder().field("id", DataTypes.INT().copy(false)).build(); + RowType valueType = RowType.builder().field("id", DataTypes.INT()).build(); List invalidValues = new ArrayList<>( Arrays.asList( @@ -357,7 +313,6 @@ void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( "{", "null", "{\"id\":\"bad\"}", - "{\"id\":null}", "{\"id\":1,\"unknown\":2}", "{\"id\":1} {}", "{\"id\":1,\"id\":2}")); @@ -450,7 +405,7 @@ void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( @ValueSource(strings = {"flat", "nested"}) void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout) throws Exception { TablePath path = TablePath.of(DATABASE, "json_metadata_" + layout); - RowType valueType = RowType.builder().field("id", DataTypes.INT().copy(false)).build(); + RowType valueType = RowType.builder().field("id", DataTypes.INT()).build(); // Magic v2 batches carry CREATE_TIME or LOG_APPEND_TIME even without a timestamp. TimestampType[] types = { TimestampType.CREATE_TIME, @@ -513,12 +468,12 @@ void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout) throws Ex @ParameterizedTest @ValueSource(strings = {"flat", "nested"}) - void testMissingRequiredMetadataTimestampIsPartitionAtomic(String layout) throws Exception { + void testMissingMetadataTimestampPersistsAsNull(String layout) throws Exception { TablePath path = TablePath.of(DATABASE, "json_required_timestamp_" + layout); - RowType valueType = RowType.builder().field("id", DataTypes.INT().copy(false)).build(); + RowType valueType = RowType.builder().field("id", DataTypes.INT()).build(); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { - create(admin, path, descriptor(valueType, layout, true, false, 2, false)); + create(admin, path, descriptor(valueType, layout, true, false, 2)); ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); try (Socket socket = new Socket(node.host(), node.port()); KafkaProducer producer = producer()) { @@ -535,9 +490,9 @@ void testMissingRequiredMetadataTimestampIsPartitionAtomic(String layout) throws KEY, bytes("{\"id\":1001}"))), MemoryRecords.withRecords(Compression.NONE, record("{\"id\":1}"))); - PartitionProduceResponse rejected = partition(response, path.toString(), 0); - assertThat(rejected.errorCode()).isEqualTo(Errors.INVALID_RECORD.code()); - assertThat(rejected.baseOffset()).isEqualTo(-1L); + PartitionProduceResponse missingTimestamp = partition(response, path.toString(), 0); + assertThat(missingTimestamp.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(missingTimestamp.baseOffset()).isZero(); PartitionProduceResponse accepted = partition(response, path.toString(), 1); assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); assertThat(accepted.baseOffset()).isZero(); @@ -551,15 +506,23 @@ void testMissingRequiredMetadataTimestampIsPartitionAtomic(String layout) throws bytes("{\"id\":-1}"))) .get(30, TimeUnit.SECONDS) .offset()) - .isZero(); + .isEqualTo(2L); read( connection, path, 0, - 1, + 3, (row, index) -> { - assertThat(valueId(row, layout)).isEqualTo(-1); - assertMetadata(row, 3, 789L, 0, 0); + if (index == 0) { + assertThat(valueId(row, layout)).isEqualTo(1000); + assertMetadata(row, 3, 123L, 0, 0); + } else if (index == 1) { + assertThat(valueId(row, layout)).isEqualTo(1001); + assertMetadata(row, 3, RecordBatch.NO_TIMESTAMP, 0, 0); + } else { + assertThat(valueId(row, layout)).isEqualTo(-1); + assertMetadata(row, 3, 789L, 0, 0); + } }); read( connection, @@ -658,16 +621,6 @@ private static TableDescriptor descriptor( boolean withSystemColumns, boolean withRescue, int buckets) { - return descriptor(valueType, layout, withSystemColumns, withRescue, buckets, true); - } - - private static TableDescriptor descriptor( - RowType valueType, - String layout, - boolean withSystemColumns, - boolean withRescue, - int buckets, - boolean nullableTimestamp) { Schema.Builder schema = Schema.newBuilder(); if (withSystemColumns) { schema.column(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES()); @@ -686,12 +639,9 @@ private static TableDescriptor descriptor( schema.column( KafkaTableSchema.METADATA_COLUMN, DataTypes.ROW( - DataTypes.FIELD( - "timestamp", - DataTypes.TIMESTAMP_LTZ(3).copy(nullableTimestamp)), - DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), - DataTypes.FIELD("partition", DataTypes.INT().copy(false))) - .copy(false)); + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))); } if (withRescue) { schema.column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()); @@ -701,8 +651,8 @@ private static TableDescriptor descriptor( .distributedBy(buckets) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), layout) .build(); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 9c83311d215..a8288620d7e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -313,9 +313,9 @@ public void testJsonTableIsDiscoverableForEveryMetadataVersion(boolean nested) { 140L, TableDescriptor.builder(defaultDescriptor()) .schema(schema) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") .property( - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), nested ? "nested" : "flat") .build()); for (short version = 0; version <= 11; version++) { @@ -348,7 +348,7 @@ public void testJsonAdmissionReflectsAuthoritativeSchemaAndProperties() { TableDescriptor flat = TableDescriptor.builder(defaultDescriptor()) .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") .build(); service.putTable("fluss.json_topic", 140L, flat); assertThat( @@ -363,7 +363,7 @@ public void testJsonAdmissionReflectsAuthoritativeSchemaAndProperties() { "fluss.json_topic", 140L, TableDescriptor.builder(flat) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), "nested") .build()); assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) .containsEntry("json_topic", Errors.INVALID_TOPIC_EXCEPTION); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index ed06efa34c1..b3c356eb00a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -954,9 +954,7 @@ void testLegacyMetadataOffsetsRejectedAndJsonNullValuesPersisted(boolean nested) .column( "_kafka_value", DataTypes.ROW( - DataTypes.FIELD( - "business", - DataTypes.STRING().copy(false)))) + DataTypes.FIELD("business", DataTypes.STRING()))) .build() : Schema.newBuilder().column("business", DataTypes.STRING()).build(); TableDescriptor json = @@ -989,8 +987,7 @@ void testJsonConversionRejectsWholeBadPartitionAndPreservesOtherPartitions(boole throws Exception { List invalidValues = new ArrayList<>( - Arrays.asList( - "{", "[]", "{\"id\":\"bad\"}", "{}", "{\"id\":1,\"unknown\":2}")); + Arrays.asList("{", "[]", "{\"id\":\"bad\"}", "{\"id\":1,\"unknown\":2}")); for (String reserved : Arrays.asList( "_kafka_key", @@ -1093,7 +1090,8 @@ void testJsonCacheUsesAuthoritativeLayoutSchemaAndTableIdentity() throws Excepti TableDescriptor.builder(flat) .property( ConfigOptions - .TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + .TABLE_KAFKA_CONVERT_VALUE_LAYOUT + .key(), "nested") .build() .toJsonBytes())); @@ -1122,11 +1120,9 @@ void testJsonCacheUsesAuthoritativeLayoutSchemaAndTableIdentity() throws Excepti private static TableDescriptor jsonDescriptor(boolean nested, boolean rescue) { Schema.Builder schema = Schema.newBuilder(); if (nested) { - schema.column( - "_kafka_value", - DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT().copy(false)))); + schema.column("_kafka_value", DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT()))); } else { - schema.column("id", DataTypes.INT().copy(false)); + schema.column("id", DataTypes.INT()); } schema.column("_kafka_key", DataTypes.BYTES()); if (rescue) { @@ -1137,9 +1133,10 @@ private static TableDescriptor jsonDescriptor(boolean nested, boolean rescue) { .distributedBy(2) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") .property( - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, nested ? "nested" : "flat") + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + nested ? "nested" : "flat") .build(); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java index bab37c29118..bf72ebcca07 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java @@ -46,7 +46,7 @@ void testFlatJsonProjectsOnlyBusinessColumnsAroundOptionalSystemColumns() { TableDescriptor descriptor = descriptor( Schema.newBuilder() - .column("id", DataTypes.INT().copy(false)) + .column("id", DataTypes.INT()) .column("_kafka_key", DataTypes.BYTES()) .column( "_kafka_headers", @@ -55,7 +55,7 @@ void testFlatJsonProjectsOnlyBusinessColumnsAroundOptionalSystemColumns() { .column("amount", DataTypes.DECIMAL(8, 2)) .column("_kafka_rescue", DataTypes.STRING()) .build()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), "binary") .build(); KafkaTableSchema schema = resolver.resolve(descriptor); assertThat(schema).isEqualTo(KafkaTableSchema.resolve(descriptor)); @@ -72,7 +72,7 @@ void testFlatJsonProjectsOnlyBusinessColumnsAroundOptionalSystemColumns() { void testNestedJsonProjectsItsPhysicalValueContainer() { RowType value = DataTypes.ROW( - DataTypes.FIELD("id", DataTypes.INT().copy(false)), + DataTypes.FIELD("id", DataTypes.INT()), DataTypes.FIELD("name", DataTypes.STRING())); KafkaTableSchema schema = resolver.resolve( @@ -82,7 +82,9 @@ void testNestedJsonProjectsItsPhysicalValueContainer() { .column("_kafka_value", value) .column("_kafka_rescue", DataTypes.STRING()) .build()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + "nested") .build()); assertThat(schema.valueConversionLayout()).isEqualTo(KafkaValueConversionLayout.NESTED); assertThat(schema.valueProjection().positions()).containsExactly(1); @@ -92,7 +94,7 @@ void testNestedJsonProjectsItsPhysicalValueContainer() { } @Test - void testJsonContractRequiresExplicitEnablementAndRejectsPrototypeOptions() { + void testJsonContractRequiresExplicitEnablementAndIgnoresCustomProperties() { TableDescriptor.Builder table = descriptor(Schema.newBuilder().column("id", DataTypes.INT()).build()); TableDescriptor disabled = table.property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); @@ -105,25 +107,23 @@ void testJsonContractRequiresExplicitEnablementAndRejectsPrototypeOptions() { "kafka.value.format", "kafka.key.fields", "table.kafka.convert.value.type")) { - assertThatThrownBy( - () -> - resolver.resolve( - descriptor(disabled.getSchema()) - .customProperty(option, "json") - .build())) - .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("custom property"); + TableDescriptor withCustomProperty = + descriptor(disabled.getSchema()).customProperty(option, "json").build(); + assertThat(withCustomProperty.getCustomProperties()).containsEntry(option, "json"); + assertThat(resolver.resolve(withCustomProperty).valueConversionType()) + .isEqualTo(KafkaValueConversionType.PARSE_JSON); } assertThatThrownBy( () -> resolver.resolve( descriptor(disabled.getSchema()) .property( - ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE + .key(), "string") .build())) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("only supports binary"); + .hasMessageContaining("Could not parse value 'string'"); } @Test @@ -144,7 +144,8 @@ void testNestedJsonRejectsUnmappedTopLevelBusinessColumns() { .build()) .property( ConfigOptions - .TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + .TABLE_KAFKA_CONVERT_VALUE_LAYOUT + .key(), "nested") .build())) .isInstanceOf(KafkaTopicSchemaException.class) @@ -176,7 +177,8 @@ void testNestedJsonRejectsReservedRootBusinessNames(String name) { .build()) .property( ConfigOptions - .TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + .TABLE_KAFKA_CONVERT_VALUE_LAYOUT + .key(), "nested") .build())) .isInstanceOf(KafkaTopicSchemaException.class) @@ -202,7 +204,7 @@ void testComplexBusinessFieldsRemainUnavailableUntilRecursiveConversion(boolean TableDescriptor table = descriptor(schema) .property( - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), nested ? "nested" : "flat") .build(); // The persisted contract is valid, but this gateway stage must report INVALID_TOPIC @@ -220,6 +222,6 @@ private static TableDescriptor.Builder descriptor(Schema schema) { .distributedBy(1) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json"); + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 2686df6ecad..f2f5f847347 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -327,7 +327,7 @@ void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { .schema(schema) .distributedBy(1) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .customProperty("table.kafka.convert.value.type", "custom-value") + .customProperty("custom.kafka.convert.value.type", "custom-value") .customProperty("kafka.value.format", "custom-format") .customProperty("fluss.owner", "team") .build(); @@ -338,7 +338,7 @@ void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { assertThat(kafkaProperties(stored)) .isEqualTo(Collections.singletonMap("table.kafka.enabled", "true")); // This checks persisted DDL mapping; recursive gateway decoding is a later stage. - assertThat(new KafkaTopicSchemaResolver().resolve(stored).valueProjection().positions()) + assertThat(KafkaTableSchema.resolve(stored).valueProjection().positions()) .containsExactly(1); set(path, Collections.singletonMap("table.kafka.convert.key.type", "binary")); assertThat(KafkaTableSchema.resolve(stored(path))) From acdc53e2c790245a28c2ec22db67713cc7a7bc42 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 11 Sep 2026 15:57:53 +0800 Subject: [PATCH 55/69] [kafka] Support nested JSON values with resource bounds Rebase the local complex JSON split on the latest scalar JSON Produce implementation, preserving streaming scalar parsing and precise numeric conversion. Adapt qualified-topic and JSON-depth integration coverage. Co-Authored-By: Codex AI-Model: gpt-6-astra AI-Contributed/Feature: 148/148 AI-Contributed/UT: 344/344 --- .../format/json/JsonKafkaFieldDecoder.java | 29 +- .../format/json/JsonToFlussConverters.java | 113 +++++++- .../fluss/kafka/KafkaJsonProduceITCase.java | 63 +++++ .../json/JsonKafkaComplexTypesTest.java | 265 ++++++++++++++++++ 4 files changed, 456 insertions(+), 14 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java index cad4e2ddec9..f581220b984 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java @@ -55,7 +55,7 @@ public final class JsonKafkaFieldDecoder implements KafkaFieldDecoder { // Jackson counts the JSON root; the shared business-schema depth starts below that root. // The physical nested ROW adds no JSON level. This stage rejects all business containers. static final int MAX_JSON_NESTING_DEPTH = 64; - static final int MAX_CONTAINER_ELEMENTS = 10_000; + static final int MAX_CONTAINER_ELEMENTS = JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; private static final ObjectMapper OBJECT_MAPPER = createObjectMapper(); private final KafkaFieldProjection projection; @@ -140,14 +140,8 @@ public Object[] decode(@Nullable byte[] bytes) { throw new KafkaRecordEncodingException( "Invalid Kafka record value at " + path + ": unknown field."); } - JsonToken token = parser.nextToken(); - // This stage supports scalar fields only. Reject containers before readTree - // can allocate their contents, even when the container is wide and shallow. - if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { - throw new KafkaRecordEncodingException( - "Invalid Kafka record value at " + path + ": expected a JSON scalar."); - } - JsonNode value = readScalar(parser, projection.dataTypeAt(position)); + parser.nextToken(); + JsonNode value = readValue(parser, projection.dataTypeAt(position), path); values[position] = converters[position].convert(value, path); present[position] = true; } @@ -171,7 +165,20 @@ public Object[] decode(@Nullable byte[] bytes) { } } - private static JsonNode readScalar(JsonParser parser, DataType dataType) throws IOException { + private static JsonNode readValue(JsonParser parser, DataType dataType, String path) + throws IOException { + if (parser.currentToken() == JsonToken.START_OBJECT + || parser.currentToken() == JsonToken.START_ARRAY) { + switch (dataType.getTypeRoot()) { + case ROW: + case ARRAY: + case MAP: + break; + default: + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + path + ": expected a JSON scalar."); + } + } if (parser.currentToken().isNumeric()) { // Parse floating-point targets from the original token: BigDecimal loses negative // zero, and parsing FLOAT through a double can round twice. Other numeric targets @@ -220,7 +227,7 @@ private static ObjectMapper createObjectMapper() { .build()); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); mapper.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); - // Trailing content is checked after the root object, not after individual scalar values. + // Trailing content is checked after the root object, not after individual field values. return mapper; } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java index eb1433f4e78..d9280b75fcc 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java @@ -23,14 +23,22 @@ import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.TimestampLtz; import org.apache.fluss.row.TimestampNtz; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BinaryType; import org.apache.fluss.types.CharType; +import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; import org.apache.fluss.types.DecimalType; import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; import org.apache.fluss.types.TimeType; import org.apache.fluss.types.TimestampType; @@ -47,13 +55,22 @@ import java.time.ZoneOffset; import java.time.format.DateTimeParseException; import java.util.Base64; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import static org.apache.fluss.utils.DateTimeUtils.timestampToMicros; import static org.apache.fluss.utils.DateTimeUtils.timestampToNanos; -/** Strict JSON-to-Fluss scalar converter construction. */ +/** Strict recursive JSON-to-Fluss converter construction. */ final class JsonToFlussConverters { + static final int MAX_SCHEMA_NESTING_DEPTH = KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; + static final int MAX_CONTAINER_ELEMENTS = 10_000; + private JsonToFlussConverters() {} /** @@ -67,7 +84,11 @@ static JsonToFlussConverter create(DataType dataType) { } catch (InvalidTableException e) { throw new KafkaTopicSchemaException(e.getMessage()); } - final JsonToFlussConverter notNullConverter = createNotNull(dataType); + return create(dataType, 0); + } + + private static JsonToFlussConverter create(DataType dataType, int nestingDepth) { + final JsonToFlussConverter notNullConverter = createNotNull(dataType, nestingDepth); return (node, path) -> { if (node == null || node.isNull()) { if (!dataType.isNullable()) { @@ -85,7 +106,7 @@ static JsonToFlussConverter create(DataType dataType) { }; } - private static JsonToFlussConverter createNotNull(DataType dataType) { + private static JsonToFlussConverter createNotNull(DataType dataType, int nestingDepth) { switch (dataType.getTypeRoot()) { case BOOLEAN: return (node, path) -> { @@ -171,6 +192,12 @@ private static JsonToFlussConverter createNotNull(DataType dataType) { return timestampConverter((TimestampType) dataType); case TIMESTAMP_WITH_LOCAL_TIME_ZONE: return timestampLtzConverter((LocalZonedTimestampType) dataType); + case ROW: + return rowConverter((RowType) dataType, nestingDepth); + case ARRAY: + return arrayConverter((ArrayType) dataType, nestingDepth); + case MAP: + return mapConverter((MapType) dataType, nestingDepth); default: throw new KafkaTopicSchemaException( "Kafka JSON format does not support Fluss type " @@ -316,6 +343,86 @@ private static long timestampMillis( } } + private static JsonToFlussConverter rowConverter(RowType dataType, int nestingDepth) { + List fields = dataType.getFields(); + String[] fieldNames = new String[fields.size()]; + Set declaredFieldNames = new HashSet<>(); + JsonToFlussConverter[] fieldConverters = new JsonToFlussConverter[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + DataField field = fields.get(i); + fieldNames[i] = field.getName(); + if (!declaredFieldNames.add(field.getName())) { + throw new KafkaTopicSchemaException( + "Duplicate Kafka JSON row field '" + field.getName() + "'."); + } + fieldConverters[i] = create(field.getType(), nestingDepth + 1); + } + return (node, path) -> { + require(node.isObject(), path, dataType, "expected a JSON object"); + requireContainerSize(node.size(), path, dataType); + Iterator inputFieldNames = node.fieldNames(); + while (inputFieldNames.hasNext()) { + String inputFieldName = inputFieldNames.next(); + require( + declaredFieldNames.contains(inputFieldName), + JsonPath.field(path, inputFieldName), + dataType, + "unknown field"); + } + GenericRow row = new GenericRow(fieldNames.length); + for (int i = 0; i < fieldNames.length; i++) { + String fieldPath = JsonPath.field(path, fieldNames[i]); + row.setField(i, fieldConverters[i].convert(node.get(fieldNames[i]), fieldPath)); + } + return row; + }; + } + + private static JsonToFlussConverter arrayConverter(ArrayType dataType, int nestingDepth) { + JsonToFlussConverter elementConverter = create(dataType.getElementType(), nestingDepth + 1); + return (node, path) -> { + require(node.isArray(), path, dataType, "expected a JSON array"); + requireContainerSize(node.size(), path, dataType); + Object[] elements = new Object[node.size()]; + for (int i = 0; i < node.size(); i++) { + elements[i] = elementConverter.convert(node.get(i), JsonPath.index(path, i)); + } + return new GenericArray(elements); + }; + } + + private static JsonToFlussConverter mapConverter(MapType dataType, int nestingDepth) { + if (dataType.getKeyType().getTypeRoot() != DataTypeRoot.STRING) { + throw new KafkaTopicSchemaException( + "Kafka JSON format only supports STRING map keys, but found " + + dataType.asSummaryString() + + "."); + } + JsonToFlussConverter valueConverter = create(dataType.getValueType(), nestingDepth + 1); + return (node, path) -> { + require(node.isObject(), path, dataType, "expected a JSON object"); + requireContainerSize(node.size(), path, dataType); + Map values = new LinkedHashMap<>(); + Iterator> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + values.put( + BinaryString.fromString(field.getKey()), + valueConverter.convert( + field.getValue(), JsonPath.field(path, field.getKey()))); + } + return new GenericMap(values); + }; + } + + private static void requireContainerSize(int size, String path, DataType dataType) { + require( + size <= MAX_CONTAINER_ELEMENTS, + path, + dataType, + "container size exceeds " + MAX_CONTAINER_ELEMENTS); + } + private static long integralValue(JsonNode node, String path, DataType dataType) { require(node.isIntegralNumber(), path, dataType, "expected a JSON integer"); require(node.canConvertToLong(), path, dataType, "integer overflow"); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 8afadffd44e..2cc839c99d4 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -114,6 +114,69 @@ class KafkaJsonProduceITCase { .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") .build(); + @Test + void testNestedJsonRoundTrip() throws Exception { + TableDescriptor descriptor = + descriptor( + Schema.newBuilder() + .column( + "customer", + DataTypes.ROW( + DataTypes.FIELD( + "name", + DataTypes.STRING().copy(false)))) + .column( + "items", + DataTypes.ARRAY(DataTypes.INT().copy(false))) + .column( + "attributes", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW( + DataTypes.FIELD( + "count", DataTypes.INT())))) + .build()) + .build(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + TablePath path = TablePath.of(DATABASE, "json_complex"); + create(admin, path, descriptor); + try (KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(1); + assertThatThrownBy( + () -> + producer.send( + record( + path, + "{\"customer\":{\"name\":\"Alice\",\"extra\":1}}")) + .get(30, TimeUnit.SECONDS)) + .hasCauseInstanceOf(InvalidRecordException.class); + assertThat( + producer.send( + record( + path, + "{\"customer\":{\"name\":\"Alice\"},\"items\":[1,2],\"attributes\":{\"dynamic\":{\"count\":3}}}")) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + read( + connection, + path, + row -> { + assertThat(row.getRow(0, 1).getString(0).toString()).isEqualTo("Alice"); + assertThat(row.getArray(1).size()).isEqualTo(2); + assertThat(row.getArray(1).getInt(1)).isEqualTo(2); + assertThat(row.getMap(2).keyArray().getString(0).toString()) + .isEqualTo("dynamic"); + assertThat(row.getMap(2).valueArray().getRow(0, 1).getInt(0)) + .isEqualTo(3); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + @ParameterizedTest @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java new file mode 100644 index 00000000000..1cb83a29f0a --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.kafka.schema.KafkaFieldProjection; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests recursive JSON conversion, nested validation and resource bounds. */ +class JsonKafkaComplexTypesTest { + + @Test + void testDecodesRecursiveRowArrayAndMapTypes() { + RowType rowType = complexRowType(); + + Object[] values = + decoder(rowType) + .decode( + bytes( + "{" + + "\"customer\":{\"name\":\"Alice\"," + + "\"address\":{\"city\":\"Hangzhou\"}}," + + "\"items\":[{\"sku\":\"A-100\"," + + "\"quantity\":2}]," + + "\"attributes\":{\"scores\":[7,null,9]}" + + "}")); + + GenericRow customer = (GenericRow) values[0]; + assertThat(customer.getString(0).toString()).isEqualTo("Alice"); + assertThat(customer.getRow(1, 1).getString(0).toString()).isEqualTo("Hangzhou"); + + GenericArray items = (GenericArray) values[1]; + assertThat(items.size()).isEqualTo(1); + assertThat(items.getRow(0, 2).getString(0).toString()).isEqualTo("A-100"); + assertThat(items.getRow(0, 2).getInt(1)).isEqualTo(2); + + GenericMap attributes = (GenericMap) values[2]; + GenericArray scores = (GenericArray) attributes.get(BinaryString.fromString("scores")); + assertThat(scores.toObjectArray()).containsExactly(7, null, 9); + } + + @Test + void testRejectsInvalidNestedValuesWithPrecisePaths() { + assertFailure( + complexRowType(), + "{\"customer\":{\"address\":{\"city\":\"Hangzhou\"}}," + + "\"items\":[],\"attributes\":{}}", + "$[\"customer\"][\"name\"]", + "missing or null"); + assertFailure( + complexRowType(), + "{\"customer\":{\"name\":\"Alice\"," + + "\"address\":{\"city\":\"Hangzhou\"}}," + + "\"items\":[{\"sku\":\"A-100\",\"quantity\":2}," + + "{\"sku\":\"A-200\",\"quantity\":\"bad\"}]," + + "\"attributes\":{}}", + "$[\"items\"][1][\"quantity\"]", + "expected a JSON integer"); + assertFailure( + complexRowType(), + "{\"customer\":{\"name\":\"Alice\"," + + "\"address\":{\"city\":\"Hangzhou\"}}," + + "\"items\":[],\"attributes\":{\"bad\\\"key\":[\"bad\"]}}", + "$[\"attributes\"][\"bad\\\"key\"][0]", + "expected a JSON integer"); + } + + @Test + void testRejectsUnknownRootAndNestedRowFieldsButAllowsMapKeys() { + assertFailure( + DataTypes.ROW(DataTypes.FIELD("id", DataTypes.BIGINT())), + "{\"id\":1,\"extra\":\"unexpected\"}", + "$[\"extra\"]", + "unknown field"); + assertFailure( + complexRowType(), + "{\"customer\":{\"name\":\"Alice\"," + + "\"address\":{\"city\":\"Hangzhou\",\"extra\":1}}," + + "\"items\":[],\"attributes\":{}}", + "$[\"customer\"][\"address\"][\"extra\"]", + "unknown field"); + + Object[] values = + decoder( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.INT())))) + .decode(bytes("{\"attributes\":{\"dynamic-key\":1}}")); + GenericMap attributes = (GenericMap) values[0]; + assertThat(attributes.size()).isEqualTo(1); + assertThat(attributes.get(BinaryString.fromString("dynamic-key"))).isEqualTo(1); + } + + @Test + void testRejectsNonStringMapKeyBeforeDecodingRecords() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD( + "attributes", DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()))); + + assertThatThrownBy(() -> decoder(rowType)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("only supports STRING map keys") + .hasMessageContaining("MAP decoder(DataTypes.ROW(DataTypes.FIELD("nested", tooDeepType)))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("maximum nesting depth"); + + StringBuilder nestedJson = new StringBuilder("{\"value\":"); + for (int i = 0; i < JsonKafkaFieldDecoder.MAX_JSON_NESTING_DEPTH; i++) { + nestedJson.append("{\"nested\":"); + } + nestedJson.append('0'); + for (int i = 0; i < JsonKafkaFieldDecoder.MAX_JSON_NESTING_DEPTH; i++) { + nestedJson.append('}'); + } + nestedJson.append('}'); + JsonKafkaFieldDecoder complexDecoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD( + "value", + DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING())))); + assertThatThrownBy(() -> complexDecoder.decode(bytes(nestedJson.toString()))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("not valid strict UTF-8 JSON"); + + StringBuilder oversizedArray = new StringBuilder("{\"values\":["); + for (int i = 0; i <= JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; i++) { + if (i > 0) { + oversizedArray.append(','); + } + oversizedArray.append('0'); + } + oversizedArray.append("]}"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("values", DataTypes.ARRAY(DataTypes.INT()))), + oversizedArray.toString(), + "$[\"values\"]", + "container size exceeds"); + } + + @Test + void testBoundsAndEscapesMapKeysInErrorPaths() { + StringBuilder mapKey = new StringBuilder("quote\""); + for (int i = 0; i < 300; i++) { + mapKey.append('x'); + } + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD( + "attributes", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))); + + assertThatThrownBy( + () -> + decoder(rowType) + .decode( + bytes( + "{\"attributes\":{" + + quoteJson(mapKey.toString()) + + ":\"bad\"}}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$[\"attributes\"][\"quote\\\"") + .hasMessageContaining("...") + .satisfies( + error -> assertThat(error.getMessage()).doesNotContain(mapKey.toString())); + + StringBuilder controlCharacters = new StringBuilder(); + for (int i = 0; i < 300; i++) { + controlCharacters.append('\u0001'); + } + String boundedPath = JsonPath.field(JsonPath.ROOT, controlCharacters.toString()); + assertThat(boundedPath) + .hasSizeLessThanOrEqualTo(512) + .endsWith("\"]") + .doesNotContain("\\u0..."); + } + + private static void assertFailure(RowType rowType, String json, String path, String reason) { + assertThatThrownBy(() -> decoder(rowType).decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining(path) + .hasMessageContaining(reason); + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType) { + List positions = new ArrayList<>(); + for (int i = 0; i < rowType.getFieldCount(); i++) { + positions.add(i); + } + return new JsonKafkaFieldDecoder(new KafkaFieldProjection(rowType, positions)); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static RowType complexRowType() { + return DataTypes.ROW( + DataTypes.FIELD( + "customer", + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING().copy(false)), + DataTypes.FIELD( + "address", + DataTypes.ROW( + DataTypes.FIELD( + "city", DataTypes.STRING().copy(false)))))), + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("sku", DataTypes.STRING().copy(false)), + DataTypes.FIELD("quantity", DataTypes.INT().copy(false))))), + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(DataTypes.INT())))); + } + + private static String quoteJson(String value) { + return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; + } +} From 0d56349a72ce02a0cbc3d902911d841972ab590b Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 20 Sep 2026 14:56:28 +0800 Subject: [PATCH 56/69] [kafka] Bound nested JSON containers during parsing Check each ARRAY, ROW and MAP container while consuming JSON tokens, before materializing any child beyond the 10,000-element limit. Cover early rejection, exact boundaries, independent sibling limits and strict parsing of nested JSON. Co-Authored-By: Codex AI-Model: gpt-6-astra AI-Contributed/Feature: 40/40 AI-Contributed/UT: 138/138 --- .../format/json/JsonKafkaFieldDecoder.java | 40 ++++- .../json/JsonKafkaComplexTypesTest.java | 138 ++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java index f581220b984..d021bc463f2 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java @@ -31,8 +31,10 @@ import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DoubleNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.FloatNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.fluss.types.DataType; import org.apache.fluss.types.RowType; @@ -192,7 +194,43 @@ private static JsonNode readValue(JsonParser parser, DataType dataType, String p break; } } - return OBJECT_MAPPER.readTree(parser); + return readBoundedTree(parser, path); + } + + private static JsonNode readBoundedTree(JsonParser parser, String path) throws IOException { + // Bound every container while consuming tokens, before allocating excess children. + // The parser independently enforces nesting depth and rejects duplicate object fields. + switch (parser.currentToken()) { + case START_ARRAY: + ArrayNode array = OBJECT_MAPPER.createArrayNode(); + while (parser.nextToken() != JsonToken.END_ARRAY) { + checkContainerSize(array.size() + 1, path); + array.add(readBoundedTree(parser, JsonPath.index(path, array.size()))); + } + return array; + case START_OBJECT: + ObjectNode object = OBJECT_MAPPER.createObjectNode(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + checkContainerSize(object.size() + 1, path); + String fieldName = parser.currentName(); + parser.nextToken(); + object.set(fieldName, readBoundedTree(parser, JsonPath.field(path, fieldName))); + } + return object; + default: + return OBJECT_MAPPER.readTree(parser); + } + } + + private static void checkContainerSize(int size, String path) { + if (size > MAX_CONTAINER_ELEMENTS) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + ": container size exceeds " + + MAX_CONTAINER_ELEMENTS + + "."); + } } private Object[] nullValues() { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java index 1cb83a29f0a..ff2a18c5344 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java @@ -24,15 +24,21 @@ import org.apache.fluss.row.GenericArray; import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; +import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -182,6 +188,138 @@ void testEnforcesSchemaJsonAndContainerResourceBounds() { "container size exceeds"); } + @ParameterizedTest(name = "{index}: {0}") + @MethodSource("oversizedContainers") + void testRejectsOversizedContainersBeforeReadingExcessChild( + String description, DataType dataType, String json, String path) { + // The excess child is deliberately unfinished. Reading it before checking the + // containing ARRAY/MAP/ROW limit would fail with a syntax error instead. + assertFailure( + DataTypes.ROW(DataTypes.FIELD("value", dataType)), + "{\"value\":" + json, + path, + "container size exceeds " + JsonToFlussConverters.MAX_CONTAINER_ELEMENTS); + } + + private static Stream oversizedContainers() { + String array = containerAtLimit(false); + String object = containerAtLimit(true); + String oversizedArray = array.substring(0, array.length() - 1) + ",["; + String oversizedObject = object.substring(0, object.length() - 1) + ",\"excess\":{"; + DataType arrayType = DataTypes.ARRAY(DataTypes.INT()); + DataType mapType = DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()); + RowType rowType = rowTypeAtLimit(); + return Stream.of( + Arguments.of("ARRAY", arrayType, oversizedArray, "$[\"value\"]"), + Arguments.of("MAP", mapType, oversizedObject, "$[\"value\"]"), + Arguments.of("ROW", rowType, oversizedObject, "$[\"value\"]"), + Arguments.of( + "ARRAY inside ROW", + DataTypes.ROW(DataTypes.FIELD("nested", arrayType)), + "{\"nested\":" + oversizedArray, + "$[\"value\"][\"nested\"]"), + Arguments.of( + "MAP inside ARRAY", + DataTypes.ARRAY(mapType), + "[" + oversizedObject, + "$[\"value\"][0]"), + Arguments.of( + "ROW inside MAP", + DataTypes.MAP(DataTypes.STRING(), rowType), + "{\"nested\":" + oversizedObject, + "$[\"value\"][\"nested\"]")); + } + + @Test + void testAcceptsLimitForEachSiblingContainer() { + String array = containerAtLimit(false); + String object = containerAtLimit(true); + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD( + "arrays", DataTypes.ARRAY(DataTypes.ARRAY(DataTypes.INT()))), + DataTypes.FIELD( + "maps", + DataTypes.ARRAY( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))), + DataTypes.FIELD("rows", DataTypes.ARRAY(rowTypeAtLimit()))); + Object[] values = + decoder(rowType) + .decode( + bytes( + "{\"arrays\":[" + + array + + "," + + array + + "],\"maps\":[" + + object + + "," + + object + + "],\"rows\":[" + + object + + "," + + object + + "]}")); + + int limit = JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; + for (int i = 0; i < 2; i++) { + GenericArray arrayValue = (GenericArray) ((GenericArray) values[0]).getArray(i); + assertThat(arrayValue.size()).isEqualTo(limit); + assertThat(arrayValue.getInt(limit - 1)).isZero(); + GenericMap mapValue = (GenericMap) ((GenericArray) values[1]).getMap(i); + assertThat(mapValue.size()).isEqualTo(limit); + assertThat(mapValue.get(BinaryString.fromString("k" + (limit - 1)))).isEqualTo(0); + GenericRow rowValue = (GenericRow) ((GenericArray) values[2]).getRow(i, limit); + assertThat(rowValue.getFieldCount()).isEqualTo(limit); + assertThat(rowValue.getInt(limit - 1)).isZero(); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "{\"value\":[{\"a\":[],\"a\":[]}]}", + "{\"value\":[{\"a\":[1,]}]}", + "{\"value\":[{\"a\":[1]", + "{\"value\":[{\"a\":[1]}]} {}", + }) + void testRetainsStrictParsingForNestedContainers(String json) { + JsonKafkaFieldDecoder nestedDecoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD( + "value", + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ARRAY(DataTypes.INT())))))); + assertThatThrownBy(() -> nestedDecoder.decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("not valid strict UTF-8 JSON"); + } + + private static RowType rowTypeAtLimit() { + DataField[] fields = new DataField[JsonToFlussConverters.MAX_CONTAINER_ELEMENTS]; + for (int i = 0; i < fields.length; i++) { + fields[i] = DataTypes.FIELD("k" + i, DataTypes.INT()); + } + return DataTypes.ROW(fields); + } + + private static String containerAtLimit(boolean object) { + StringBuilder json = new StringBuilder(object ? "{" : "["); + for (int i = 0; i < JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; i++) { + if (i > 0) { + json.append(','); + } + if (object) { + json.append('"').append('k').append(i).append("\":"); + } + json.append('0'); + } + return json.append(object ? '}' : ']').toString(); + } + @Test void testBoundsAndEscapesMapKeysInErrorPaths() { StringBuilder mapKey = new StringBuilder("quote\""); From e3918d27543c8c4f8b399bda6a6cc64b2f9d4d72 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 22:18:05 +0800 Subject: [PATCH 57/69] [kafka] Format JSON produce integration test Normalize timestamp schema and assertion wrapping to satisfy Spotless on the updated PR11 baseline. No behavior changes. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 7/7 # Conflicts: # fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java From 831264840a93e26f13ec3032812f7dc1a7a03fe0 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 22:35:47 +0800 Subject: [PATCH 58/69] [kafka] Align complex JSON conversion with public table contracts Support recursive ROW, ARRAY and MAP values in flat and nested layouts using shared schema validation and consistent root depth accounting. Preserve nested numeric tokens and reject malformed Unicode map keys, while retaining early container limits and partition-atomic conversion. Verify both layouts through strict decoder, Arrow and native readback regressions: 512 unique Java 11 reactor cases pass. Changed sources also compile against the Java 8 API. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 92/92 AI-Contributed/UT: 829/829 --- .../format/json/JsonKafkaFieldDecoder.java | 45 ++- .../format/json/JsonToFlussConverters.java | 47 ++- .../fluss/kafka/KafkaJsonProduceITCase.java | 277 +++++++++++--- .../fluss/kafka/KafkaMetadataHandlerTest.java | 64 +++- .../GatewayKafkaProduceBackendTest.java | 9 +- .../json/JsonKafkaComplexTypesTest.java | 342 ++++++++++++++++-- .../json/JsonToFlussConvertersTest.java | 40 +- .../kafka/schema/KafkaJsonSchemaTest.java | 9 +- .../ArrowKafkaRecordTranscoderTest.java | 88 ++++- 9 files changed, 767 insertions(+), 154 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java index d021bc463f2..9ea06f5ee08 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java @@ -35,7 +35,10 @@ import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DoubleNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.FloatNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataType; +import org.apache.fluss.types.MapType; import org.apache.fluss.types.RowType; import javax.annotation.Nullable; @@ -54,9 +57,9 @@ /** Decodes a strict UTF-8 JSON object according to the projected Fluss fields. */ @Internal public final class JsonKafkaFieldDecoder implements KafkaFieldDecoder { - // Jackson counts the JSON root; the shared business-schema depth starts below that root. - // The physical nested ROW adds no JSON level. This stage rejects all business containers. - static final int MAX_JSON_NESTING_DEPTH = 64; + // Jackson counts both the JSON root and a container at business depth zero. An empty ROW + // at the maximum business depth is still a container. The nested physical ROW is the root. + static final int MAX_JSON_NESTING_DEPTH = KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH + 2; static final int MAX_CONTAINER_ELEMENTS = JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; private static final ObjectMapper OBJECT_MAPPER = createObjectMapper(); @@ -194,27 +197,53 @@ private static JsonNode readValue(JsonParser parser, DataType dataType, String p break; } } - return readBoundedTree(parser, path); + return readBoundedTree(parser, dataType, path); } - private static JsonNode readBoundedTree(JsonParser parser, String path) throws IOException { + private static JsonNode readBoundedTree(JsonParser parser, DataType dataType, String path) + throws IOException { // Bound every container while consuming tokens, before allocating excess children. - // The parser independently enforces nesting depth and rejects duplicate object fields. + // Read each child with its schema so numeric tokens retain their original precision + // and floating-point sign. The parser also bounds depth and rejects duplicate fields. switch (parser.currentToken()) { case START_ARRAY: + if (!(dataType instanceof ArrayType)) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + path + ": expected a JSON object."); + } + DataType elementType = ((ArrayType) dataType).getElementType(); ArrayNode array = OBJECT_MAPPER.createArrayNode(); while (parser.nextToken() != JsonToken.END_ARRAY) { checkContainerSize(array.size() + 1, path); - array.add(readBoundedTree(parser, JsonPath.index(path, array.size()))); + array.add(readValue(parser, elementType, JsonPath.index(path, array.size()))); } return array; case START_OBJECT: + if (dataType instanceof ArrayType) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + path + ": expected a JSON array."); + } + Map rowFields = new HashMap<>(); + if (dataType instanceof RowType) { + for (DataField field : ((RowType) dataType).getFields()) { + rowFields.put(field.getName(), field.getType()); + } + } ObjectNode object = OBJECT_MAPPER.createObjectNode(); while (parser.nextToken() != JsonToken.END_OBJECT) { checkContainerSize(object.size() + 1, path); String fieldName = parser.currentName(); + String fieldPath = JsonPath.field(path, fieldName); + DataType fieldType = + dataType instanceof MapType + ? ((MapType) dataType).getValueType() + : rowFields.get(fieldName); + if (fieldType == null) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + fieldPath + ": unknown field."); + } parser.nextToken(); - object.set(fieldName, readBoundedTree(parser, JsonPath.field(path, fieldName))); + object.set(fieldName, readValue(parser, fieldType, fieldPath)); } return object; default: diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java index d9280b75fcc..e030889a076 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java @@ -34,7 +34,6 @@ import org.apache.fluss.types.CharType; import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataType; -import org.apache.fluss.types.DataTypeRoot; import org.apache.fluss.types.DecimalType; import org.apache.fluss.types.LocalZonedTimestampType; import org.apache.fluss.types.MapType; @@ -75,8 +74,7 @@ private JsonToFlussConverters() {} /** * Creates a converter for one top-level JSON business field, excluding the JSON root container. - * The shared schema contract is validated before checking decoder support; complex types still - * require their own converter implementations. + * The shared schema contract validates the entire type before recursive converter construction. */ static JsonToFlussConverter create(DataType dataType) { try { @@ -84,11 +82,11 @@ static JsonToFlussConverter create(DataType dataType) { } catch (InvalidTableException e) { throw new KafkaTopicSchemaException(e.getMessage()); } - return create(dataType, 0); + return createValidated(dataType); } - private static JsonToFlussConverter create(DataType dataType, int nestingDepth) { - final JsonToFlussConverter notNullConverter = createNotNull(dataType, nestingDepth); + private static JsonToFlussConverter createValidated(DataType dataType) { + final JsonToFlussConverter notNullConverter = createNotNull(dataType); return (node, path) -> { if (node == null || node.isNull()) { if (!dataType.isNullable()) { @@ -106,7 +104,7 @@ private static JsonToFlussConverter create(DataType dataType, int nestingDepth) }; } - private static JsonToFlussConverter createNotNull(DataType dataType, int nestingDepth) { + private static JsonToFlussConverter createNotNull(DataType dataType) { switch (dataType.getTypeRoot()) { case BOOLEAN: return (node, path) -> { @@ -193,11 +191,11 @@ private static JsonToFlussConverter createNotNull(DataType dataType, int nesting case TIMESTAMP_WITH_LOCAL_TIME_ZONE: return timestampLtzConverter((LocalZonedTimestampType) dataType); case ROW: - return rowConverter((RowType) dataType, nestingDepth); + return rowConverter((RowType) dataType); case ARRAY: - return arrayConverter((ArrayType) dataType, nestingDepth); + return arrayConverter((ArrayType) dataType); case MAP: - return mapConverter((MapType) dataType, nestingDepth); + return mapConverter((MapType) dataType); default: throw new KafkaTopicSchemaException( "Kafka JSON format does not support Fluss type " @@ -343,7 +341,7 @@ private static long timestampMillis( } } - private static JsonToFlussConverter rowConverter(RowType dataType, int nestingDepth) { + private static JsonToFlussConverter rowConverter(RowType dataType) { List fields = dataType.getFields(); String[] fieldNames = new String[fields.size()]; Set declaredFieldNames = new HashSet<>(); @@ -355,7 +353,7 @@ private static JsonToFlussConverter rowConverter(RowType dataType, int nestingDe throw new KafkaTopicSchemaException( "Duplicate Kafka JSON row field '" + field.getName() + "'."); } - fieldConverters[i] = create(field.getType(), nestingDepth + 1); + fieldConverters[i] = createValidated(field.getType()); } return (node, path) -> { require(node.isObject(), path, dataType, "expected a JSON object"); @@ -378,8 +376,8 @@ private static JsonToFlussConverter rowConverter(RowType dataType, int nestingDe }; } - private static JsonToFlussConverter arrayConverter(ArrayType dataType, int nestingDepth) { - JsonToFlussConverter elementConverter = create(dataType.getElementType(), nestingDepth + 1); + private static JsonToFlussConverter arrayConverter(ArrayType dataType) { + JsonToFlussConverter elementConverter = createValidated(dataType.getElementType()); return (node, path) -> { require(node.isArray(), path, dataType, "expected a JSON array"); requireContainerSize(node.size(), path, dataType); @@ -391,14 +389,8 @@ private static JsonToFlussConverter arrayConverter(ArrayType dataType, int nesti }; } - private static JsonToFlussConverter mapConverter(MapType dataType, int nestingDepth) { - if (dataType.getKeyType().getTypeRoot() != DataTypeRoot.STRING) { - throw new KafkaTopicSchemaException( - "Kafka JSON format only supports STRING map keys, but found " - + dataType.asSummaryString() - + "."); - } - JsonToFlussConverter valueConverter = create(dataType.getValueType(), nestingDepth + 1); + private static JsonToFlussConverter mapConverter(MapType dataType) { + JsonToFlussConverter valueConverter = createValidated(dataType.getValueType()); return (node, path) -> { require(node.isObject(), path, dataType, "expected a JSON object"); requireContainerSize(node.size(), path, dataType); @@ -406,10 +398,11 @@ private static JsonToFlussConverter mapConverter(MapType dataType, int nestingDe Iterator> fields = node.fields(); while (fields.hasNext()) { Map.Entry field = fields.next(); + String fieldPath = JsonPath.field(path, field.getKey()); + validateUnicode(field.getKey(), fieldPath, dataType.getKeyType()); values.put( BinaryString.fromString(field.getKey()), - valueConverter.convert( - field.getValue(), JsonPath.field(path, field.getKey()))); + valueConverter.convert(field.getValue(), fieldPath)); } return new GenericMap(values); }; @@ -432,6 +425,11 @@ private static long integralValue(JsonNode node, String path, DataType dataType) private static String textualValue(JsonNode node, String path, DataType dataType) { require(node.isTextual(), path, dataType, "expected a JSON string"); String value = node.textValue(); + validateUnicode(value, path, dataType); + return value; + } + + private static void validateUnicode(String value, String path, DataType dataType) { // Jackson can retain unpaired JSON escapes in Java strings. Reject them before // BinaryString's UTF-8 encoding silently replaces them with a question mark. for (int i = 0; i < value.length(); i++) { @@ -451,7 +449,6 @@ private static String textualValue(JsonNode node, String path, DataType dataType "unpaired UTF-16 surrogate"); } } - return value; } private static byte[] decodeBase64(JsonNode node, String path, DataType dataType) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 2cc839c99d4..cdb2c1222b8 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -106,6 +106,39 @@ class KafkaJsonProduceITCase { .field("rounded_float", DataTypes.FLOAT()) .build(); + private static final RowType NUMBER_TYPE = + RowType.builder() + .field("float_value", DataTypes.FLOAT().copy(false)) + .field("double_value", DataTypes.DOUBLE().copy(false)) + .field("rounded_float", DataTypes.FLOAT().copy(false)) + .field("amount", DataTypes.DECIMAL(30, 10).copy(false)) + .build(); + private static final RowType CUSTOMER_TYPE = + RowType.builder() + .field("name", DataTypes.STRING().copy(false)) + .field(KafkaTableSchema.KEY_COLUMN, DataTypes.INT()) + .field(KafkaTableSchema.VALUE_COLUMN, DataTypes.INT()) + .field(KafkaTableSchema.HEADERS_COLUMN, DataTypes.INT()) + .field(KafkaTableSchema.METADATA_COLUMN, DataTypes.INT()) + .field(KafkaTableSchema.RESCUE_COLUMN, DataTypes.INT()) + .build(); + private static final RowType COMPLEX_TYPE = + RowType.builder() + .field("id", DataTypes.INT().copy(false)) + .field("customer", CUSTOMER_TYPE) + .field("measurements", NUMBER_TYPE) + .field("float_values", DataTypes.ARRAY(DataTypes.FLOAT().copy(false))) + .field("double_values", DataTypes.ARRAY(DataTypes.DOUBLE().copy(false))) + .field( + "float_attributes", + DataTypes.MAP(DataTypes.STRING(), DataTypes.FLOAT().copy(false))) + .field( + "double_attributes", + DataTypes.MAP(DataTypes.STRING(), DataTypes.DOUBLE().copy(false))) + .field("items", DataTypes.ARRAY(NUMBER_TYPE.copy(false))) + .field("attributes", DataTypes.MAP(DataTypes.STRING(), NUMBER_TYPE.copy(false))) + .build(); + @RegisterExtension static final FlussClusterExtension CLUSTER = FlussClusterExtension.builder() @@ -114,63 +147,165 @@ class KafkaJsonProduceITCase { .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") .build(); - @Test - void testNestedJsonRoundTrip() throws Exception { - TableDescriptor descriptor = - descriptor( - Schema.newBuilder() - .column( - "customer", - DataTypes.ROW( - DataTypes.FIELD( - "name", - DataTypes.STRING().copy(false)))) - .column( - "items", - DataTypes.ARRAY(DataTypes.INT().copy(false))) - .column( - "attributes", - DataTypes.MAP( - DataTypes.STRING(), - DataTypes.ROW( - DataTypes.FIELD( - "count", DataTypes.INT())))) - .build()) - .build(); + @ParameterizedTest + @CsvSource({"flat", "nested"}) + void testComplexJsonRoundTripPreservesNumericValues(String layout) throws Exception { + TablePath path = TablePath.of(DATABASE, "json_complex_" + layout); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { - TablePath path = TablePath.of(DATABASE, "json_complex"); - create(admin, path, descriptor); + create(admin, path, descriptor(COMPLEX_TYPE, layout, true, false, 1)); try (KafkaProducer producer = producer()) { assertThat(producer.partitionsFor(path.toString())).hasSize(1); - assertThatThrownBy( - () -> - producer.send( - record( - path, - "{\"customer\":{\"name\":\"Alice\",\"extra\":1}}")) - .get(30, TimeUnit.SECONDS)) - .hasCauseInstanceOf(InvalidRecordException.class); + String[] negativeZeros = {"-0", "-0.0", "-0e0"}; + for (int index = 0; index < negativeZeros.length; index++) { + assertThat( + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + KEY, + bytes( + complexJson( + index, + negativeZeros[index])))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(index); + } + read( + connection, + path, + 0, + negativeZeros.length, + (row, index) -> { + assertThat(row.getBytes(0)).isEqualTo(KEY); + InternalRow value = + layout.equals("nested") + ? row.getRow(1, COMPLEX_TYPE.getFieldCount()) + : row; + int offset = layout.equals("nested") ? 0 : 1; + assertThat(value.getInt(offset)).isEqualTo(index); + InternalRow customer = + value.getRow(offset + 1, CUSTOMER_TYPE.getFieldCount()); + assertThat(customer.getString(0).toString()).isEqualTo("Alice"); + // Reserved system names are ordinary declared business fields below + // root. + for (int field = 1; field < CUSTOMER_TYPE.getFieldCount(); field++) { + assertThat(customer.getInt(field)).isEqualTo(field); + } + assertNumbers(value.getRow(offset + 2, NUMBER_TYPE.getFieldCount())); + assertThat(value.getArray(offset + 3).size()).isEqualTo(2); + assertNegativeZero(value.getArray(offset + 3).getFloat(0)); + assertThat(value.getArray(offset + 3).getFloat(1)) + .isEqualTo(1.0000001F); + assertThat(value.getArray(offset + 4).size()).isEqualTo(1); + assertNegativeZero(value.getArray(offset + 4).getDouble(0)); + InternalMap floats = value.getMap(offset + 5); + assertThat(floats.size()).isEqualTo(2); + assertThat(floats.keyArray().getString(0).toString()).isEqualTo("zero"); + assertNegativeZero(floats.valueArray().getFloat(0)); + assertThat(floats.keyArray().getString(1).toString()) + .isEqualTo("rounded"); + assertThat(floats.valueArray().getFloat(1)).isEqualTo(1.0000001F); + InternalMap doubles = value.getMap(offset + 6); + assertThat(doubles.size()).isEqualTo(1); + assertNegativeZero(doubles.valueArray().getDouble(0)); + assertThat(value.getArray(offset + 7).size()).isEqualTo(1); + assertNumbers( + value.getArray(offset + 7) + .getRow(0, NUMBER_TYPE.getFieldCount())); + InternalMap attributes = value.getMap(offset + 8); + assertThat(attributes.size()).isEqualTo(1); + assertThat(attributes.keyArray().getString(0).toString()) + .isEqualTo("dynamic"); + assertNumbers( + attributes.valueArray().getRow(0, NUMBER_TYPE.getFieldCount())); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testInvalidComplexPartitionIsAtomicAndDoesNotBlockOtherPartitions( + String layout, boolean withRescue) throws Exception { + TablePath path = + TablePath.of(DATABASE, "json_complex_rejections_" + layout + "_" + withRescue); + List invalidValues = + Arrays.asList( + "{\"id\":1,\"customer\":{\"name\":\"Alice\",\"unknown\":1}}", + "{\"id\":1,\"customer\":{}}", + "{\"id\":1,\"customer\":{\"name\":null}}", + "{\"id\":1,\"customer\":{\"name\":1}}", + "{\"id\":1,\"customer\":[]}", + "{\"id\":1,\"float_values\":[1,null]}", + "{\"id\":1,\"double_values\":[1,\"bad\"]}", + "{\"id\":1,\"float_values\":{}}", + "{\"id\":1,\"float_attributes\":[]}", + "{\"id\":1,\"double_attributes\":{\"bad\":null}}", + "{\"id\":1,\"items\":[null]}", + "{\"id\":1,\"items\":[{}]}", + "{\"id\":1,\"attributes\":{\"bad\":null}}", + "{\"id\":1,\"attributes\":{\"bad\":{}}}", + // The escaped surrogate must not be encoded as '?' and overwrite that key. + "{\"id\":1,\"float_attributes\":{\"?\":1,\"\\ud800\":2}}", + "{\"id\":1,\"double_attributes\":{\"?\":1,\"\\udc00\":2}}"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(COMPLEX_TYPE, layout, true, withRescue, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port()); + KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(2); + for (int index = 0; index < invalidValues.size(); index++) { + String name = index % 2 == 0 ? path.toString() : path.getTableName(); + ProduceResponse response = + produce( + socket, + name, + MemoryRecords.withRecords( + Compression.NONE, + record(complexJson(1000, "-0")), + record(invalidValues.get(index))), + MemoryRecords.withRecords( + Compression.NONE, record(complexJson(index, "-0e0")))); + PartitionProduceResponse rejected = partition(response, name, 0); + assertThat(rejected.errorCode()) + .as( + "rejection for %s with rescue=%s", + invalidValues.get(index), withRescue) + .isEqualTo(Errors.INVALID_RECORD.code()); + assertThat(rejected.baseOffset()).isEqualTo(-1L); + PartitionProduceResponse accepted = partition(response, name, 1); + assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(accepted.baseOffset()).isEqualTo(index); + } assertThat( producer.send( - record( - path, - "{\"customer\":{\"name\":\"Alice\"},\"items\":[1,2],\"attributes\":{\"dynamic\":{\"count\":3}}}")) + new ProducerRecord<>( + path.toString(), + 0, + KEY, + bytes(complexJson(-1, "-0.0")))) .get(30, TimeUnit.SECONDS) .offset()) .isZero(); read( connection, path, - row -> { - assertThat(row.getRow(0, 1).getString(0).toString()).isEqualTo("Alice"); - assertThat(row.getArray(1).size()).isEqualTo(2); - assertThat(row.getArray(1).getInt(1)).isEqualTo(2); - assertThat(row.getMap(2).keyArray().getString(0).toString()) - .isEqualTo("dynamic"); - assertThat(row.getMap(2).valueArray().getRow(0, 1).getInt(0)) - .isEqualTo(3); - }); + 0, + 1, + (row, index) -> + assertThat(valueId(row, layout, COMPLEX_TYPE)).isEqualTo(-1)); + read( + connection, + path, + 1, + invalidValues.size(), + (row, index) -> + assertThat(valueId(row, layout, COMPLEX_TYPE)).isEqualTo(index)); } finally { admin.dropTable(path, true).get(); } @@ -615,6 +750,58 @@ private static void assertMetadata( assertThat(metadata.getInt(2)).isEqualTo(partition); } + private static void assertNegativeZero(float value) { + assertThat(Float.floatToRawIntBits(value)).isEqualTo(Float.floatToRawIntBits(-0.0F)); + } + + private static void assertNegativeZero(double value) { + assertThat(Double.doubleToRawLongBits(value)).isEqualTo(Double.doubleToRawLongBits(-0.0D)); + } + + private static void assertNumbers(InternalRow row) { + assertNegativeZero(row.getFloat(0)); + assertNegativeZero(row.getDouble(1)); + assertThat(row.getFloat(2)).isEqualTo(1.0000001F); + assertThat(row.getDecimal(3, 30, 10).toBigDecimal()) + .isEqualByComparingTo(new BigDecimal("12345678901234567890.1234567890")); + } + + private static String complexJson(int id, String negativeZero) { + String numbers = + "{\"float_value\":" + + negativeZero + + ",\"double_value\":" + + negativeZero + + ",\"rounded_float\":1.0000000596046448," + + "\"amount\":12345678901234567890.1234567890}"; + return "{\"id\":" + + id + + ",\"customer\":{\"name\":\"Alice\",\"_kafka_key\":1,\"_kafka_value\":2," + + "\"_kafka_headers\":3,\"_kafka_metadata\":4,\"_kafka_rescue\":5}," + + "\"measurements\":" + + numbers + + ",\"float_values\":[" + + negativeZero + + ",1.0000000596046448],\"double_values\":[" + + negativeZero + + "],\"float_attributes\":{\"zero\":" + + negativeZero + + ",\"rounded\":1.0000000596046448},\"double_attributes\":{\"zero\":" + + negativeZero + + "},\"items\":[" + + numbers + + "],\"attributes\":{\"dynamic\":" + + numbers + + "}}"; + } + + private static int valueId(InternalRow row, String layout, RowType valueType) { + return layout.equals("nested") + ? row.getRow(1, valueType.getFieldCount()).getInt(0) + : row.getInt(1); + + } + private static int valueId(InternalRow row, String layout) { return layout.equals("nested") ? row.getRow(1, 1).getInt(0) : row.getInt(1); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index a8288620d7e..4fb9c01474b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -367,17 +367,59 @@ public void testJsonAdmissionReflectsAuthoritativeSchemaAndProperties() { .build()); assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) .containsEntry("json_topic", Errors.INVALID_TOPIC_EXCEPTION); - service.putTable( - "fluss.json_topic", - 140L, - TableDescriptor.builder(flat) - .schema( - Schema.newBuilder() - .column("items", DataTypes.ARRAY(DataTypes.INT())) - .build()) - .build()); - assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) - .containsEntry("json_topic", Errors.INVALID_TOPIC_EXCEPTION); + RowType business = + DataTypes.ROW( + DataTypes.FIELD("items", DataTypes.ARRAY(DataTypes.INT())), + DataTypes.FIELD( + "attributes", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW(DataTypes.FIELD("count", DataTypes.INT()))))); + for (boolean nested : new boolean[] {false, true}) { + Schema complexSchema = + nested + ? Schema.newBuilder().column("_kafka_value", business).build() + : Schema.newBuilder().fromRowType(business).build(); + TableDescriptor complex = + TableDescriptor.builder(flat) + .schema(complexSchema) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + nested ? "nested" : "flat") + .build(); + service.putTable("fluss.json_topic", 140L, complex); + assertThat( + handle(service, namedTopicRequest("json_topic"), (short) 11) + .data() + .topics() + .find("json_topic") + .errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat( + handle(service, allTopicsRequest((short) 11), (short) 11) + .data() + .topics() + .find("fluss.json_topic") + .errorCode()) + .isEqualTo(Errors.NONE.code()); + + RowType invalidBusiness = + DataTypes.ROW( + DataTypes.FIELD( + "items", DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()))); + Schema invalidSchema = + nested + ? Schema.newBuilder().column("_kafka_value", invalidBusiness).build() + : Schema.newBuilder().fromRowType(invalidBusiness).build(); + service.putTable( + "fluss.json_topic", + 140L, + TableDescriptor.builder(complex).schema(invalidSchema).build()); + assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) + .containsEntry("json_topic", Errors.INVALID_TOPIC_EXCEPTION); + assertThat(handle(service, allTopicsRequest((short) 11), (short) 11).errors()) + .containsEntry("fluss.json_topic", Errors.INVALID_TOPIC_EXCEPTION); + } service.putTable( "fluss.json_topic", 140L, diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index b3c356eb00a..23525036512 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -1037,7 +1037,7 @@ void testJsonConversionRejectsWholeBadPartitionAndPreservesOtherPartitions(boole @ParameterizedTest @ValueSource(booleans = {false, true}) - void testUnsupportedJsonSchemaIsInvalidTopicEvenForNullValues(boolean nested) throws Exception { + void testInvalidJsonSchemaIsInvalidTopicEvenForNullValues(boolean nested) throws Exception { TestingProduceService service = new TestingProduceService(); Schema schema = nested @@ -1046,10 +1046,13 @@ void testUnsupportedJsonSchemaIsInvalidTopicEvenForNullValues(boolean nested) th "_kafka_value", DataTypes.ROW( DataTypes.FIELD( - "id", DataTypes.ARRAY(DataTypes.INT())))) + "id", + DataTypes.MAP( + DataTypes.INT(), + DataTypes.STRING())))) .build() : Schema.newBuilder() - .column("id", DataTypes.ARRAY(DataTypes.INT())) + .column("id", DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())) .build(); service.pendingMetadata = CompletableFuture.completedFuture( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java index ff2a18c5344..924efddf4ad 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java @@ -17,10 +17,14 @@ package org.apache.fluss.kafka.format.json; -import org.apache.fluss.kafka.schema.KafkaFieldProjection; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; import org.apache.fluss.row.GenericArray; import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; @@ -35,8 +39,10 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.stream.Stream; @@ -46,21 +52,22 @@ /** Tests recursive JSON conversion, nested validation and resource bounds. */ class JsonKafkaComplexTypesTest { - @Test - void testDecodesRecursiveRowArrayAndMapTypes() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDecodesRecursiveRowArrayAndMapTypes(boolean nested) { RowType rowType = complexRowType(); Object[] values = - decoder(rowType) - .decode( - bytes( - "{" - + "\"customer\":{\"name\":\"Alice\"," - + "\"address\":{\"city\":\"Hangzhou\"}}," - + "\"items\":[{\"sku\":\"A-100\"," - + "\"quantity\":2}]," - + "\"attributes\":{\"scores\":[7,null,9]}" - + "}")); + decodeBusinessFields( + rowType, + "{" + + "\"customer\":{\"name\":\"Alice\"," + + "\"address\":{\"city\":\"Hangzhou\"}}," + + "\"items\":[{\"sku\":\"A-100\"," + + "\"quantity\":2}]," + + "\"attributes\":{\"scores\":[7,null,9]}" + + "}", + nested); GenericRow customer = (GenericRow) values[0]; assertThat(customer.getString(0).toString()).isEqualTo("Alice"); @@ -146,7 +153,7 @@ void testRejectsNonStringMapKeyBeforeDecodingRecords() { @Test void testEnforcesSchemaJsonAndContainerResourceBounds() { DataType nestedType = DataTypes.STRING(); - for (int i = 0; i <= JsonToFlussConverters.MAX_SCHEMA_NESTING_DEPTH; i++) { + for (int i = 0; i <= KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; i++) { nestedType = DataTypes.ARRAY(nestedType); } final DataType tooDeepType = nestedType; @@ -170,8 +177,7 @@ void testEnforcesSchemaJsonAndContainerResourceBounds() { "value", DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING())))); assertThatThrownBy(() -> complexDecoder.decode(bytes(nestedJson.toString()))) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("not valid strict UTF-8 JSON"); + .isInstanceOf(KafkaRecordEncodingException.class); StringBuilder oversizedArray = new StringBuilder("{\"values\":["); for (int i = 0; i <= JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; i++) { @@ -230,8 +236,9 @@ private static Stream oversizedContainers() { "$[\"value\"][\"nested\"]")); } - @Test - void testAcceptsLimitForEachSiblingContainer() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testAcceptsLimitForEachSiblingContainer(boolean nested) { String array = containerAtLimit(false); String object = containerAtLimit(true); RowType rowType = @@ -244,22 +251,22 @@ void testAcceptsLimitForEachSiblingContainer() { DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))), DataTypes.FIELD("rows", DataTypes.ARRAY(rowTypeAtLimit()))); Object[] values = - decoder(rowType) - .decode( - bytes( - "{\"arrays\":[" - + array - + "," - + array - + "],\"maps\":[" - + object - + "," - + object - + "],\"rows\":[" - + object - + "," - + object - + "]}")); + decodeBusinessFields( + rowType, + "{\"arrays\":[" + + array + + "," + + array + + "],\"maps\":[" + + object + + "," + + object + + "],\"rows\":[" + + object + + "," + + object + + "]}", + nested); int limit = JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; for (int i = 0; i < 2; i++) { @@ -356,14 +363,273 @@ void testBoundsAndEscapesMapKeysInErrorPaths() { .doesNotContain("\\u0..."); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDecodesTheSharedDepthBoundaryIncludingEmptyRows(boolean nested) { + for (boolean emptyRow : new boolean[] {false, true}) { + DataType type = emptyRow ? new RowType(Collections.emptyList()) : DataTypes.INT(); + String json = emptyRow ? "{}" : "7"; + Object expected = emptyRow ? new GenericRow(0) : 7; + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + switch (depth % 3) { + case 0: + type = DataTypes.ROW(DataTypes.FIELD("child", type)); + json = "{\"child\":" + json + "}"; + expected = GenericRow.of(expected); + break; + case 1: + type = DataTypes.ARRAY(type); + json = "[" + json + "]"; + expected = new GenericArray(new Object[] {expected}); + break; + default: + type = DataTypes.MAP(DataTypes.STRING(), type); + json = "{\"entry\":" + json + "}"; + expected = + new GenericMap( + Collections.singletonMap( + BinaryString.fromString("entry"), expected)); + } + } + KafkaTableSchema.validateJsonType(type); + RowType business = DataTypes.ROW(DataTypes.FIELD("value", type)); + // An empty ROW at business depth 64 is JSON container depth 66, including the root. + assertThat(decodeBusinessFields(business, "{\"value\":" + json + "}", nested)) + .containsExactly(expected); + + DataType tooDeep = DataTypes.ARRAY(type); + assertThatThrownBy(() -> KafkaTableSchema.validateJsonType(tooDeep)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("maximum nesting depth of 64"); + assertThatThrownBy( + () -> decoder(DataTypes.ROW(DataTypes.FIELD("value", tooDeep)), nested)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("maximum nesting depth of 64"); + String invalidJson = + "{\"value\":" + + (emptyRow + ? json.replace("{}", "{\"extra\":{}}") + : json.replace("7", "[7]")) + + "}"; + assertThatThrownBy(() -> decoder(business, nested).decode(bytes(invalidJson))) + .isInstanceOf(KafkaRecordEncodingException.class); + } + } + + @ParameterizedTest + @MethodSource("floatingPointZeroTokens") + void testPreservesFloatingPointZeroInEveryContainer(boolean nested, String token) { + for (Object value : numericContainerValues(DataTypes.FLOAT(), token, nested)) { + assertThat(Float.floatToRawIntBits((Float) value)) + .isEqualTo(Float.floatToRawIntBits(-0.0F)); + } + for (Object value : numericContainerValues(DataTypes.DOUBLE(), token, nested)) { + assertThat(Double.doubleToRawLongBits((Double) value)) + .isEqualTo(Double.doubleToRawLongBits(-0.0D)); + } + } + + private static Stream floatingPointZeroTokens() { + return Stream.of(false, true) + .flatMap( + nested -> + Stream.of("-0", "-0.0", "-0e0") + .map(token -> Arguments.of(nested, token))); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testPreservesOriginalNumericRoundingAndExactDecimalsInEveryContainer(boolean nested) { + for (String sign : new String[] {"", "-"}) { + float direction = sign.isEmpty() ? 1.0F : -1.0F; + assertThat( + numericContainerValues( + DataTypes.FLOAT(), sign + "1.0000000596046448", nested)) + .containsOnly(direction * 1.0000001F); + assertThat( + numericContainerValues( + DataTypes.FLOAT(), sign + "1.0000000596046447", nested)) + .containsOnly(direction); + assertThat( + numericContainerValues( + DataTypes.DOUBLE(), + sign + + "1.00000000000000011102230246251565404236316680908203126", + nested)) + .containsOnly(direction * 1.0000000000000002D); + assertThat( + numericContainerValues( + DataTypes.DOUBLE(), + sign + + "1.00000000000000011102230246251565404236316680908203124", + nested)) + .containsOnly((double) direction); + String amount = sign + "12345678901234567890.123456789012345678"; + for (Object value : numericContainerValues(DataTypes.DECIMAL(38, 18), amount, nested)) { + assertThat(((Decimal) value).toBigDecimal()).isEqualTo(new BigDecimal(amount)); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMapKeysRejectUnpairedSurrogatesBeforeEncodingOrColliding(boolean nested) { + RowType business = + DataTypes.ROW( + DataTypes.FIELD( + "value", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))); + JsonKafkaFieldDecoder decoder = decoder(business, nested); + for (String key : + new String[] { + "\\ud800", "\\udc00", "a\\ud800b", "\\ud800\\ud800", "\\udc00\\ud800" + }) { + for (String object : + new String[] {"{\"?\":1,\"" + key + "\":2}", "{\"" + key + "\":2,\"?\":1}"}) { + assertThatThrownBy(() -> decoder.decode(bytes("{\"value\":" + object + "}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$[\"value\"]") + .hasMessageContaining("surrogate"); + } + } + GenericMap values = + (GenericMap) + decodeBusinessFields( + business, + "{\"value\":{\"?\":1,\"\\ud83d\\ude00\":2,\"\\\\ud800\":3}}", + nested)[0]; + assertThat(values.size()).isEqualTo(3); + assertThat(values.get(BinaryString.fromString("?"))).isEqualTo(1); + assertThat(values.get(BinaryString.fromString("😀"))).isEqualTo(2); + assertThat(values.get(BinaryString.fromString("\\ud800"))).isEqualTo(3); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDeclaredReservedNamesAreOrdinaryFieldsBelowTheJsonRoot(boolean nested) { + String[] names = { + "_kafka_value", "_kafka_key", "_kafka_headers", "_kafka_metadata", "_kafka_rescue" + }; + DataField[] fields = new DataField[names.length]; + StringBuilder object = new StringBuilder("{"); + for (int i = 0; i < names.length; i++) { + fields[i] = DataTypes.FIELD(names[i], DataTypes.INT().copy(false)); + if (i > 0) { + object.append(','); + } + object.append('"').append(names[i]).append("\":").append(i); + } + object.append('}'); + RowType inner = DataTypes.ROW(fields); + RowType business = + DataTypes.ROW( + DataTypes.FIELD("row", inner), + DataTypes.FIELD("array", DataTypes.ARRAY(inner)), + DataTypes.FIELD("map", DataTypes.MAP(DataTypes.STRING(), inner))); + Object[] values = + decodeBusinessFields( + business, + "{\"row\":" + + object + + ",\"array\":[" + + object + + "],\"map\":{\"_kafka_value\":" + + object + + "}}", + nested); + GenericRow expected = GenericRow.of(0, 1, 2, 3, 4); + assertThat(values[0]).isEqualTo(expected); + assertThat(((GenericArray) values[1]).getRow(0, names.length)).isEqualTo(expected); + assertThat(((GenericMap) values[2]).get(BinaryString.fromString("_kafka_value"))) + .isEqualTo(expected); + } + + @Test + void testNestedNullabilityAppliesToRowsArrayElementsAndMapValues() { + RowType business = + DataTypes.ROW( + DataTypes.FIELD( + "row", + DataTypes.ROW( + DataTypes.FIELD("required", DataTypes.INT().copy(false)))), + DataTypes.FIELD("array", DataTypes.ARRAY(DataTypes.INT().copy(false))), + DataTypes.FIELD( + "map", + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT().copy(false)))); + assertFailure(business, "{\"row\":{}}", "$[\"row\"][\"required\"]", "missing or null"); + assertFailure( + business, + "{\"row\":{\"required\":null}}", + "$[\"row\"][\"required\"]", + "missing or null"); + assertFailure(business, "{\"array\":[1,null]}", "$[\"array\"][1]", "missing or null"); + assertFailure( + business, "{\"map\":{\"entry\":null}}", "$[\"map\"][\"entry\"]", "missing or null"); + for (boolean nested : new boolean[] {false, true}) { + assertThat( + decodeBusinessFields( + business, "{\"row\":null,\"array\":null,\"map\":null}", nested)) + .containsExactly(null, null, null); + } + } + + private static Object[] numericContainerValues(DataType type, String token, boolean nested) { + RowType business = + DataTypes.ROW( + DataTypes.FIELD("row", DataTypes.ROW(DataTypes.FIELD("value", type))), + DataTypes.FIELD("array", DataTypes.ARRAY(type)), + DataTypes.FIELD("map", DataTypes.MAP(DataTypes.STRING(), type))); + Object[] values = + decodeBusinessFields( + business, + "{\"row\":{\"value\":" + + token + + "},\"array\":[" + + token + + "],\"map\":{\"entry\":" + + token + + "}}", + nested); + return new Object[] { + ((GenericRow) values[0]).getField(0), + ((GenericArray) values[1]).toObjectArray()[0], + ((GenericMap) values[2]).get(BinaryString.fromString("entry")) + }; + } + + private static Object[] decodeBusinessFields(RowType rowType, String json, boolean nested) { + Object[] values = decoder(rowType, nested).decode(bytes(json)); + if (!nested) { + return values; + } + GenericRow row = (GenericRow) values[0]; + Object[] fields = new Object[row.getFieldCount()]; + for (int i = 0; i < fields.length; i++) { + fields[i] = row.getField(i); + } + return fields; + } + private static void assertFailure(RowType rowType, String json, String path, String reason) { - assertThatThrownBy(() -> decoder(rowType).decode(bytes(json))) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining(path) - .hasMessageContaining(reason); + for (boolean nested : new boolean[] {false, true}) { + assertThatThrownBy(() -> decoder(rowType, nested).decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining(path) + .hasMessageContaining(reason); + } } private static JsonKafkaFieldDecoder decoder(RowType rowType) { + return decoder(rowType, false); + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType, boolean nested) { + if (nested) { + return new JsonKafkaFieldDecoder( + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, rowType)), + Collections.singletonList(0)), + ValueLayout.NESTED); + } List positions = new ArrayList<>(); for (int i = 0; i < rowType.getFieldCount(); i++) { positions.add(i); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java index c7273ca7401..90b67058b41 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java @@ -23,6 +23,9 @@ import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.TimestampLtz; import org.apache.fluss.row.TimestampNtz; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; @@ -46,12 +49,13 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.util.Collections; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Verifies strict scalar conversion without transport or Arrow dependencies. */ +/** Verifies strict schema-aware conversion without transport or Arrow dependencies. */ class JsonToFlussConvertersTest { private static final ObjectMapper MAPPER = new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); @@ -235,6 +239,14 @@ void testRejectsUnpairedSurrogates(String escaped) { .hasMessageContaining("$.field") .hasMessageContaining("surrogate"); } + assertThatThrownBy( + () -> + convert( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + "{\"?\":1,\"" + escaped + "\":2}")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("surrogate"); } @Test @@ -367,18 +379,20 @@ private static Stream invalidScalars() { } @Test - void testComplexTypesRemainUnsupportedAtThisStage() { - for (DataType type : - new DataType[] { - DataTypes.ARRAY(DataTypes.INT()), - DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), - DataTypes.ROW(DataTypes.INT()) - }) { + void testComplexTypesUseThePublicSchemaContract() throws Exception { + DataType array = DataTypes.ARRAY(DataTypes.INT()); + DataType map = DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()); + DataType row = DataTypes.ROW(DataTypes.FIELD("value", DataTypes.INT())); + for (DataType type : new DataType[] {array, map, row}) { KafkaTableSchema.validateJsonType(type); - assertThatThrownBy(() -> JsonToFlussConverters.create(type)) - .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("does not support"); + assertThat(JsonToFlussConverters.create(type)).isNotNull(); } + assertThat(convert(array, "[1,null]")).isEqualTo(new GenericArray(new Object[] {1, null})); + assertThat(convert(map, "{\"key\":1}")) + .isEqualTo( + new GenericMap( + Collections.singletonMap(BinaryString.fromString("key"), 1))); + assertThat(convert(row, "{\"value\":1}")).isEqualTo(GenericRow.of(1)); } @ParameterizedTest @@ -419,9 +433,7 @@ void testSchemaDepthUsesTheSharedBusinessFieldLimit() { // The top-level business field starts at zero; each child consumes one level. DataType atLimit = type; KafkaTableSchema.validateJsonType(atLimit); - assertThatThrownBy(() -> JsonToFlussConverters.create(atLimit)) - .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("does not support"); + assertThat(JsonToFlussConverters.create(atLimit)).isNotNull(); DataType tooDeep = DataTypes.ROW(atLimit); String message = diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java index bf72ebcca07..7657a7ff7d3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java @@ -187,7 +187,7 @@ void testNestedJsonRejectsReservedRootBusinessNames(String name) { @ParameterizedTest @ValueSource(booleans = {false, true}) - void testComplexBusinessFieldsRemainUnavailableUntilRecursiveConversion(boolean nested) { + void testComplexBusinessFieldsUseTheSharedContractInBothLayouts(boolean nested) { for (DataType type : Arrays.asList( DataTypes.ARRAY(DataTypes.INT()), @@ -207,12 +207,9 @@ void testComplexBusinessFieldsRemainUnavailableUntilRecursiveConversion(boolean ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), nested ? "nested" : "flat") .build(); - // The persisted contract is valid, but this gateway stage must report INVALID_TOPIC - // before accepting records for a decoder that it does not yet implement. - assertThat(KafkaTableSchema.resolve(table).valueConversionType()) + assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); + assertThat(resolver.resolve(table).valueConversionType()) .isEqualTo(KafkaValueConversionType.PARSE_JSON); - assertThatThrownBy(() -> resolver.resolve(table)) - .isInstanceOf(KafkaTopicSchemaException.class); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 38b8c1128e3..889439ded7c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -44,6 +44,7 @@ import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; import org.apache.fluss.utils.CloseableIterator; import org.junit.jupiter.api.Test; @@ -541,14 +542,93 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() throws Exception .hasMessageContaining("empty"); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testComplexJsonFailureDiscardsOutputAndCanBeReused(boolean nested) throws Exception { + RowType business = + DataTypes.ROW( + DataTypes.FIELD( + "customer", + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING().copy(false)))), + DataTypes.FIELD("items", DataTypes.ARRAY(DataTypes.INT().copy(false))), + DataTypes.FIELD( + "attributes", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW(DataTypes.FIELD("count", DataTypes.INT()))))); + Schema schema = + nested + ? Schema.newBuilder().column("_kafka_value", business).build() + : Schema.newBuilder().fromRowType(business).build(); + TableInfo table = + table( + descriptor(schema, false) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + nested ? "nested" : "flat") + .build()); + Record valid = + new Record( + 1L, + null, + bytes( + "{\"customer\":{\"name\":\"Alice\"},\"items\":[1,2]," + + "\"attributes\":{\"?\":{\"count\":3}}}"), + Collections.emptyList()); + for (String invalid : + Arrays.asList( + "{\"customer\":{}}", + "{\"customer\":{\"name\":\"Alice\",\"unknown\":1}}", + "{\"items\":[1,null]}", + "{\"attributes\":[]}", + "{\"attributes\":{\"?\":{\"count\":1},\"\\ud800\":{\"count\":2}}}")) { + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + Arrays.asList( + valid, + new Record( + 2L, + null, + bytes(invalid), + Collections.emptyList())), + table))); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + read( + transcoder.transcode(Collections.singletonList(valid), table), + table, + 1, + physical -> { + InternalRow row = nested ? physical.getRow(0, 3) : physical; + assertThat(row.getRow(0, 1).getString(0).toString()).isEqualTo("Alice"); + assertThat(row.getArray(1).size()).isEqualTo(2); + assertThat(row.getArray(1).getInt(1)).isEqualTo(2); + assertThat(row.getMap(2).keyArray().getString(0).toString()).isEqualTo("?"); + assertThat(row.getMap(2).valueArray().getRow(0, 1).getInt(0)).isEqualTo(3); + }); + } + } + @Test - void testUnsupportedJsonBusinessTypesFailDuringPreparation() { + void testInvalidJsonMapKeysFailDuringPreparation() { TableInfo table = table( TableDescriptor.builder() .schema( Schema.newBuilder() - .column("body", DataTypes.ARRAY(DataTypes.INT())) + .column( + "body", + DataTypes.MAP( + DataTypes.INT(), + DataTypes.STRING())) .build()) .distributedBy(1) .logFormat(LogFormat.ARROW) @@ -559,7 +639,7 @@ void testUnsupportedJsonBusinessTypesFailDuringPreparation() { .build()); assertThatThrownBy(() -> transcoder.prepare(table)) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("ARRAY"); + .hasMessageContaining("only supports STRING map keys"); for (byte[] value : new byte[][] {null, bytes("{}")}) { assertThatThrownBy( () -> @@ -573,7 +653,7 @@ void testUnsupportedJsonBusinessTypesFailDuringPreparation() { Collections.emptyList())), table)) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("ARRAY"); + .hasMessageContaining("only supports STRING map keys"); } } From 4771eeba4d9b6d6f9a8ea834cb9e1731ccf48fc0 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 01:09:58 +0800 Subject: [PATCH 59/69] [test] Preserve complex JSON contracts after metadata migration Adapt complex transcode tests to the partition-aware prepared plan and verify metadata readback and atomic recovery in both JSON layouts. Cover signed underflow and minimum subnormal rounding in ROW/ARRAY/MAP. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 84/84 --- .../fluss/kafka/KafkaJsonProduceITCase.java | 35 +++++++++++++------ .../json/JsonKafkaComplexTypesTest.java | 31 ++++++++++++++++ .../ArrowKafkaRecordTranscoderTest.java | 18 ++++++---- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index cdb2c1222b8..e8848bcc3b9 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -149,20 +149,22 @@ class KafkaJsonProduceITCase { @ParameterizedTest @CsvSource({"flat", "nested"}) - void testComplexJsonRoundTripPreservesNumericValues(String layout) throws Exception { + void testComplexJsonRoundTripPreservesNumericValuesAndMetadata(String layout) throws Exception { TablePath path = TablePath.of(DATABASE, "json_complex_" + layout); + int metadataPosition = layout.equals("nested") ? 3 : COMPLEX_TYPE.getFieldCount() + 2; try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { - create(admin, path, descriptor(COMPLEX_TYPE, layout, true, false, 1)); + create(admin, path, descriptor(COMPLEX_TYPE, layout, true, false, 2)); try (KafkaProducer producer = producer()) { - assertThat(producer.partitionsFor(path.toString())).hasSize(1); + assertThat(producer.partitionsFor(path.toString())).hasSize(2); String[] negativeZeros = {"-0", "-0.0", "-0e0"}; for (int index = 0; index < negativeZeros.length; index++) { assertThat( producer.send( new ProducerRecord<>( path.toString(), - 0, + 1, + 123L + index, KEY, bytes( complexJson( @@ -175,10 +177,12 @@ void testComplexJsonRoundTripPreservesNumericValues(String layout) throws Except read( connection, path, - 0, + 1, negativeZeros.length, (row, index) -> { assertThat(row.getBytes(0)).isEqualTo(KEY); + assertHeaders(row.getMap(metadataPosition - 1), false); + assertMetadata(row, metadataPosition, 123L + index, 0, 1); InternalRow value = layout.equals("nested") ? row.getRow(1, COMPLEX_TYPE.getFieldCount()) @@ -233,6 +237,7 @@ void testInvalidComplexPartitionIsAtomicAndDoesNotBlockOtherPartitions( String layout, boolean withRescue) throws Exception { TablePath path = TablePath.of(DATABASE, "json_complex_rejections_" + layout + "_" + withRescue); + int metadataPosition = layout.equals("nested") ? 3 : COMPLEX_TYPE.getFieldCount() + 2; List invalidValues = Arrays.asList( "{\"id\":1,\"customer\":{\"name\":\"Alice\",\"unknown\":1}}", @@ -287,6 +292,7 @@ void testInvalidComplexPartitionIsAtomicAndDoesNotBlockOtherPartitions( new ProducerRecord<>( path.toString(), 0, + 789L, KEY, bytes(complexJson(-1, "-0.0")))) .get(30, TimeUnit.SECONDS) @@ -297,15 +303,25 @@ void testInvalidComplexPartitionIsAtomicAndDoesNotBlockOtherPartitions( path, 0, 1, - (row, index) -> - assertThat(valueId(row, layout, COMPLEX_TYPE)).isEqualTo(-1)); + (row, index) -> { + assertThat(valueId(row, layout, COMPLEX_TYPE)).isEqualTo(-1); + assertMetadata(row, metadataPosition, 789L, 0, 0); + if (withRescue) { + assertThat(row.isNullAt(metadataPosition + 1)).isTrue(); + } + }); read( connection, path, 1, invalidValues.size(), - (row, index) -> - assertThat(valueId(row, layout, COMPLEX_TYPE)).isEqualTo(index)); + (row, index) -> { + assertThat(valueId(row, layout, COMPLEX_TYPE)).isEqualTo(index); + assertMetadata(row, metadataPosition, 123L, 0, 1); + if (withRescue) { + assertThat(row.isNullAt(metadataPosition + 1)).isTrue(); + } + }); } finally { admin.dropTable(path, true).get(); } @@ -799,7 +815,6 @@ private static int valueId(InternalRow row, String layout, RowType valueType) { return layout.equals("nested") ? row.getRow(1, valueType.getFieldCount()).getInt(0) : row.getInt(1); - } private static int valueId(InternalRow row, String layout) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java index 924efddf4ad..fe09fc9a1b3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java @@ -437,6 +437,37 @@ private static Stream floatingPointZeroTokens() { .map(token -> Arguments.of(nested, token))); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFloatingPointUnderflowPreservesSignAndRoundsOnceInEveryContainer(boolean nested) { + // The last two pairs straddle half of the smallest subnormal for each target. + // Parsing FLOAT through DOUBLE first would round both tokens to the same midpoint. + String[][] tokens = { + {"1e-1000", "1e-1000"}, + {"7.006492321624085e-46", "2.4703282292062327e-324"}, + {"7.006492321624086e-46", "2.4703282292062328e-324"} + }; + for (boolean negative : new boolean[] {false, true}) { + String sign = negative ? "-" : ""; + for (int i = 0; i < tokens.length; i++) { + int magnitude = i == tokens.length - 1 ? 1 : 0; + String floatToken = sign + tokens[i][0]; + for (Object value : numericContainerValues(DataTypes.FLOAT(), floatToken, nested)) { + assertThat(Float.floatToRawIntBits((Float) value)) + .as("FLOAT %s, nested=%s", floatToken, nested) + .isEqualTo((negative ? Integer.MIN_VALUE : 0) | magnitude); + } + String doubleToken = sign + tokens[i][1]; + for (Object value : + numericContainerValues(DataTypes.DOUBLE(), doubleToken, nested)) { + assertThat(Double.doubleToRawLongBits((Double) value)) + .as("DOUBLE %s, nested=%s", doubleToken, nested) + .isEqualTo((negative ? Long.MIN_VALUE : 0L) | magnitude); + } + } + } + } + @ParameterizedTest @ValueSource(booleans = {false, true}) void testPreservesOriginalNumericRoundingAndExactDecimalsInEveryContainer(boolean nested) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 889439ded7c..62895122151 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -557,22 +557,26 @@ void testComplexJsonFailureDiscardsOutputAndCanBeReused(boolean nested) throws E DataTypes.MAP( DataTypes.STRING(), DataTypes.ROW(DataTypes.FIELD("count", DataTypes.INT()))))); - Schema schema = + Schema.Builder schema = nested - ? Schema.newBuilder().column("_kafka_value", business).build() - : Schema.newBuilder().fromRowType(business).build(); + ? Schema.newBuilder().column("_kafka_value", business) + : Schema.newBuilder().fromRowType(business); + schema.column("_kafka_metadata", metadataType(true)); TableInfo table = table( - descriptor(schema, false) + descriptor(schema.build(), false) .property( ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") .property( ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, nested ? "nested" : "flat") .build()); + KafkaTopicWritePlan plan = transcoder.prepare(table); + int partitionId = 3; Record valid = new Record( 1L, + TimestampType.CREATE_TIME, null, bytes( "{\"customer\":{\"name\":\"Alice\"},\"items\":[1,2]," @@ -591,6 +595,7 @@ void testComplexJsonFailureDiscardsOutputAndCanBeReused(boolean nested) throws E () -> output.set( transcoder.transcode( + partitionId, Arrays.asList( valid, new Record( @@ -598,12 +603,12 @@ void testComplexJsonFailureDiscardsOutputAndCanBeReused(boolean nested) throws E null, bytes(invalid), Collections.emptyList())), - table))); + plan))); assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); assertThat(failure.getSuppressed()).isEmpty(); assertThat(output).hasNullValue(); read( - transcoder.transcode(Collections.singletonList(valid), table), + transcoder.transcode(partitionId, Collections.singletonList(valid), plan), table, 1, physical -> { @@ -613,6 +618,7 @@ void testComplexJsonFailureDiscardsOutputAndCanBeReused(boolean nested) throws E assertThat(row.getArray(1).getInt(1)).isEqualTo(2); assertThat(row.getMap(2).keyArray().getString(0).toString()).isEqualTo("?"); assertThat(row.getMap(2).valueArray().getRow(0, 1).getInt(0)).isEqualTo(3); + assertMetadata(physical, nested ? 1 : 3, 1L, 0, partitionId); }); } } From ea3d60204a22aa5fdd7d739f7dc30a7550b78f17 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 21:40:36 +0800 Subject: [PATCH 60/69] [test] Verify complex JSON null persistence and constraints Extend nullable and required business-value Produce coverage to ROW, ARRAY and MAP in both layouts, including native offsets, system fields and partition-atomic rejection. Distinguish Kafka null from JSON null and verify parent and child nullability at the decoder boundary. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 193/193 --- .../fluss/kafka/KafkaJsonProduceITCase.java | 112 ++++++++++++++++-- .../json/JsonKafkaComplexTypesTest.java | 44 +++++++ 2 files changed, 144 insertions(+), 12 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index e8848bcc3b9..a1e0e2ba933 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -38,6 +38,7 @@ import org.apache.fluss.row.InternalMap; import org.apache.fluss.row.InternalRow; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; @@ -406,14 +407,23 @@ void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( } @ParameterizedTest - @ValueSource(strings = {"flat", "nested"}) - void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout) + @CsvSource({ + "flat,scalar", + "nested,scalar", + "flat,row", + "nested,row", + "flat,array", + "nested,array", + "flat,map", + "nested,map" + }) + void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout, String fieldType) throws Exception { - TablePath path = TablePath.of(DATABASE, "json_null_values_" + layout); + TablePath path = TablePath.of(DATABASE, "json_null_values_" + layout + "_" + fieldType); // Null Kafka values populate nullable flat fields or the nullable nested value container. RowType valueType = RowType.builder() - .field("id", DataTypes.INT()) + .field("id", nullValueBusinessType(fieldType).copy(layout.equals("flat"))) .field("label", DataTypes.STRING()) .build(); Header[] headers = { @@ -421,12 +431,22 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout) new RecordHeader("source", new byte[] {1, 2}), new RecordHeader("nullable", null) }; - SimpleRecord[] records = new SimpleRecord[5]; + // Nullable flat fields also accept explicit JSON null and missing fields. + SimpleRecord[] records = new SimpleRecord[layout.equals("flat") ? 7 : 5]; for (int index = 0; index < records.length; index++) { String json = index == 2 || index == 4 - ? "{\"id\":" + index + ",\"label\":\"v" + index + "\"}" + ? "{\"id\":" + + nullValueBusinessJson(fieldType, index) + + ",\"label\":\"v" + + index + + "\"}" : null; + if (index == 5) { + json = "{\"id\":null,\"label\":null}"; + } else if (index == 6) { + json = "{}"; + } records[index] = new SimpleRecord( (index + 1) * 10L, @@ -436,7 +456,8 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout) } MemoryRecords[] batches = { MemoryRecords.withRecords(Compression.NONE, records[0], records[1]), - MemoryRecords.withRecords(Compression.NONE, records[2], records[3], records[4]) + MemoryRecords.withRecords( + Compression.NONE, Arrays.copyOfRange(records, 2, records.length)) }; try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { @@ -470,7 +491,7 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout) InternalRow value = layout.equals("nested") ? row.getRow(1, 2) : row; int start = layout.equals("nested") ? 0 : 1; - assertThat(value.getInt(start)).isEqualTo(index); + assertNullValueBusinessId(value, start, fieldType, index); assertThat(value.getString(start + 1).toString()) .isEqualTo("v" + index); } else { @@ -497,10 +518,24 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout) } @ParameterizedTest - @ValueSource(strings = {"flat", "nested"}) - void testNotNullBusinessFieldsAreRejectedAtTableCreation(String layout) throws Exception { - TablePath path = TablePath.of(DATABASE, "json_required_value_" + layout); - RowType valueType = RowType.builder(false).field("id", DataTypes.INT().copy(false)).build(); + @CsvSource({ + "flat,scalar", + "nested,scalar", + "flat,row", + "nested,row", + "flat,array", + "nested,array", + "flat,map", + "nested,map" + }) + void testNotNullBusinessFieldsAreRejectedAtTableCreation(String layout, String fieldType) + throws Exception { + TablePath path = + TablePath.of(DATABASE, "json_required_value_" + layout + "_" + fieldType); + RowType valueType = + RowType.builder(false) + .field("id", nullValueBusinessType(fieldType).copy(false)) + .build(); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { assertThatThrownBy( @@ -855,6 +890,59 @@ private static void assertHeaders(InternalMap headers, boolean populated) { assertThat(values.get("nullable")).isNull(); } + private static DataType nullValueBusinessType(String fieldType) { + switch (fieldType) { + case "scalar": + return DataTypes.INT(); + case "row": + return DataTypes.ROW(DataTypes.FIELD("value", DataTypes.INT().copy(false))); + case "array": + return DataTypes.ARRAY(DataTypes.INT().copy(false)); + case "map": + return DataTypes.MAP(DataTypes.STRING(), DataTypes.INT().copy(false)); + default: + throw new IllegalArgumentException("Unexpected business field type: " + fieldType); + } + } + + private static String nullValueBusinessJson(String fieldType, int id) { + switch (fieldType) { + case "scalar": + return Integer.toString(id); + case "row": + case "map": + return "{\"value\":" + id + "}"; + case "array": + return "[" + id + "]"; + default: + throw new IllegalArgumentException("Unexpected business field type: " + fieldType); + } + } + + private static void assertNullValueBusinessId( + InternalRow row, int position, String fieldType, int id) { + switch (fieldType) { + case "scalar": + assertThat(row.getInt(position)).isEqualTo(id); + break; + case "row": + assertThat(row.getRow(position, 1).getInt(0)).isEqualTo(id); + break; + case "array": + assertThat(row.getArray(position).size()).isEqualTo(1); + assertThat(row.getArray(position).getInt(0)).isEqualTo(id); + break; + case "map": + InternalMap map = row.getMap(position); + assertThat(map.size()).isEqualTo(1); + assertThat(map.keyArray().getString(0).toString()).isEqualTo("value"); + assertThat(map.valueArray().getInt(0)).isEqualTo(id); + break; + default: + throw new IllegalArgumentException("Unexpected business field type: " + fieldType); + } + } + private static String scalarJson(int id, boolean explicitNull) { // Both layouts consume this same root object; nested does not require a value wrapper. return "{\"id\":" diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java index fe09fc9a1b3..d5fca808a86 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java @@ -603,6 +603,50 @@ void testNestedNullabilityAppliesToRowsArrayElementsAndMapValues() { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testNullValueRespectsComplexContainerAndParentNullability(boolean nested) { + DataType[] containers = { + DataTypes.ROW(DataTypes.FIELD("required", DataTypes.INT().copy(false))), + DataTypes.ARRAY(DataTypes.INT().copy(false)), + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT().copy(false)) + }; + for (DataType container : containers) { + RowType business = DataTypes.ROW(DataTypes.FIELD("value", container)); + assertThat(decoder(business, nested).decode(null)).containsExactly((Object) null); + assertThat(decodeBusinessFields(business, "{}", nested)).containsExactly((Object) null); + assertThat(decodeBusinessFields(business, "{\"value\":null}", nested)) + .containsExactly((Object) null); + assertThatThrownBy(() -> decoder(business, nested).decode(bytes("null"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("object root"); + + RowType requiredContainer = + DataTypes.ROW(DataTypes.FIELD("value", container.copy(false))); + for (String json : new String[] {"{}", "{\"value\":null}"}) { + assertThatThrownBy(() -> decoder(requiredContainer, nested).decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$[\"value\"]") + .hasMessageContaining("missing or null"); + } + if (nested) { + // An absent parent does not require values for its non-nullable children. + assertThat(decoder(requiredContainer, true).decode(null)) + .containsExactly((Object) null); + RowType requiredParent = (RowType) business.copy(false); + assertThatThrownBy(() -> decoder(requiredParent, true).decode(null)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL field '_kafka_value'"); + assertThat(decodeBusinessFields(requiredParent, "{}", true)) + .containsExactly((Object) null); + } else { + assertThatThrownBy(() -> decoder(requiredContainer, false).decode(null)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL field 'value'"); + } + } + } + private static Object[] numericContainerValues(DataType type, String token, boolean nested) { RowType business = DataTypes.ROW( From a7146482daa4c2e8c284733a04b99c60c3f8b73a Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 22:10:38 +0800 Subject: [PATCH 61/69] [test] Align complex JSON tests with nullable schema contract Update complex fixture types and malformed payload cases to match the nullable business-field contract in PR11. Keep non-null rejection coverage at table creation. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 73/73 --- .../fluss/kafka/KafkaJsonProduceITCase.java | 53 +++++++++---------- .../fluss/kafka/KafkaMetadataHandlerTest.java | 2 +- .../json/JsonKafkaComplexTypesTest.java | 4 +- .../ArrowKafkaRecordTranscoderTest.java | 14 ++--- 4 files changed, 35 insertions(+), 38 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index a1e0e2ba933..732bd619bff 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -109,14 +109,14 @@ class KafkaJsonProduceITCase { private static final RowType NUMBER_TYPE = RowType.builder() - .field("float_value", DataTypes.FLOAT().copy(false)) - .field("double_value", DataTypes.DOUBLE().copy(false)) - .field("rounded_float", DataTypes.FLOAT().copy(false)) - .field("amount", DataTypes.DECIMAL(30, 10).copy(false)) + .field("float_value", DataTypes.FLOAT()) + .field("double_value", DataTypes.DOUBLE()) + .field("rounded_float", DataTypes.FLOAT()) + .field("amount", DataTypes.DECIMAL(30, 10)) .build(); private static final RowType CUSTOMER_TYPE = RowType.builder() - .field("name", DataTypes.STRING().copy(false)) + .field("name", DataTypes.STRING()) .field(KafkaTableSchema.KEY_COLUMN, DataTypes.INT()) .field(KafkaTableSchema.VALUE_COLUMN, DataTypes.INT()) .field(KafkaTableSchema.HEADERS_COLUMN, DataTypes.INT()) @@ -125,19 +125,17 @@ class KafkaJsonProduceITCase { .build(); private static final RowType COMPLEX_TYPE = RowType.builder() - .field("id", DataTypes.INT().copy(false)) + .field("id", DataTypes.INT()) .field("customer", CUSTOMER_TYPE) .field("measurements", NUMBER_TYPE) - .field("float_values", DataTypes.ARRAY(DataTypes.FLOAT().copy(false))) - .field("double_values", DataTypes.ARRAY(DataTypes.DOUBLE().copy(false))) - .field( - "float_attributes", - DataTypes.MAP(DataTypes.STRING(), DataTypes.FLOAT().copy(false))) + .field("float_values", DataTypes.ARRAY(DataTypes.FLOAT())) + .field("double_values", DataTypes.ARRAY(DataTypes.DOUBLE())) + .field("float_attributes", DataTypes.MAP(DataTypes.STRING(), DataTypes.FLOAT())) .field( "double_attributes", - DataTypes.MAP(DataTypes.STRING(), DataTypes.DOUBLE().copy(false))) - .field("items", DataTypes.ARRAY(NUMBER_TYPE.copy(false))) - .field("attributes", DataTypes.MAP(DataTypes.STRING(), NUMBER_TYPE.copy(false))) + DataTypes.MAP(DataTypes.STRING(), DataTypes.DOUBLE())) + .field("items", DataTypes.ARRAY(NUMBER_TYPE)) + .field("attributes", DataTypes.MAP(DataTypes.STRING(), NUMBER_TYPE)) .build(); @RegisterExtension @@ -242,19 +240,19 @@ void testInvalidComplexPartitionIsAtomicAndDoesNotBlockOtherPartitions( List invalidValues = Arrays.asList( "{\"id\":1,\"customer\":{\"name\":\"Alice\",\"unknown\":1}}", - "{\"id\":1,\"customer\":{}}", - "{\"id\":1,\"customer\":{\"name\":null}}", + "{\"id\":1,\"customer\":{\"name\":[]}}", + "{\"id\":1,\"customer\":{\"name\":true}}", "{\"id\":1,\"customer\":{\"name\":1}}", "{\"id\":1,\"customer\":[]}", - "{\"id\":1,\"float_values\":[1,null]}", + "{\"id\":1,\"float_values\":[1,\"bad\"]}", "{\"id\":1,\"double_values\":[1,\"bad\"]}", "{\"id\":1,\"float_values\":{}}", "{\"id\":1,\"float_attributes\":[]}", - "{\"id\":1,\"double_attributes\":{\"bad\":null}}", - "{\"id\":1,\"items\":[null]}", - "{\"id\":1,\"items\":[{}]}", - "{\"id\":1,\"attributes\":{\"bad\":null}}", - "{\"id\":1,\"attributes\":{\"bad\":{}}}", + "{\"id\":1,\"double_attributes\":{\"bad\":\"bad\"}}", + "{\"id\":1,\"items\":[{\"float_value\":\"bad\"}]}", + "{\"id\":1,\"items\":[{\"unknown\":1}]}", + "{\"id\":1,\"attributes\":{\"bad\":{\"amount\":\"bad\"}}}", + "{\"id\":1,\"attributes\":{\"bad\":{\"unknown\":1}}}", // The escaped surrogate must not be encoded as '?' and overwrite that key. "{\"id\":1,\"float_attributes\":{\"?\":1,\"\\ud800\":2}}", "{\"id\":1,\"double_attributes\":{\"?\":1,\"\\udc00\":2}}"); @@ -423,7 +421,7 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout, S // Null Kafka values populate nullable flat fields or the nullable nested value container. RowType valueType = RowType.builder() - .field("id", nullValueBusinessType(fieldType).copy(layout.equals("flat"))) + .field("id", nullValueBusinessType(fieldType).copy(true)) .field("label", DataTypes.STRING()) .build(); Header[] headers = { @@ -530,8 +528,7 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout, S }) void testNotNullBusinessFieldsAreRejectedAtTableCreation(String layout, String fieldType) throws Exception { - TablePath path = - TablePath.of(DATABASE, "json_required_value_" + layout + "_" + fieldType); + TablePath path = TablePath.of(DATABASE, "json_required_value_" + layout + "_" + fieldType); RowType valueType = RowType.builder(false) .field("id", nullValueBusinessType(fieldType).copy(false)) @@ -895,11 +892,11 @@ private static DataType nullValueBusinessType(String fieldType) { case "scalar": return DataTypes.INT(); case "row": - return DataTypes.ROW(DataTypes.FIELD("value", DataTypes.INT().copy(false))); + return DataTypes.ROW(DataTypes.FIELD("value", DataTypes.INT())); case "array": - return DataTypes.ARRAY(DataTypes.INT().copy(false)); + return DataTypes.ARRAY(DataTypes.INT()); case "map": - return DataTypes.MAP(DataTypes.STRING(), DataTypes.INT().copy(false)); + return DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()); default: throw new IllegalArgumentException("Unexpected business field type: " + fieldType); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 4fb9c01474b..3b452fb28e0 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -384,7 +384,7 @@ public void testJsonAdmissionReflectsAuthoritativeSchemaAndProperties() { TableDescriptor.builder(flat) .schema(complexSchema) .property( - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), nested ? "nested" : "flat") .build(); service.putTable("fluss.json_topic", 140L, complex); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java index d5fca808a86..6de4094616f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java @@ -17,12 +17,12 @@ package org.apache.fluss.kafka.format.json; +import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; import org.apache.fluss.metadata.KafkaFieldProjection; import org.apache.fluss.metadata.KafkaTableSchema; -import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.Decimal; import org.apache.fluss.row.GenericArray; @@ -703,7 +703,7 @@ private static JsonKafkaFieldDecoder decoder(RowType rowType, boolean nested) { new KafkaFieldProjection( DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, rowType)), Collections.singletonList(0)), - ValueLayout.NESTED); + KafkaValueConversionLayout.NESTED); } List positions = new ArrayList<>(); for (int i = 0; i < rowType.getFieldCount(); i++) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 62895122151..7dd03050925 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -549,9 +549,8 @@ void testComplexJsonFailureDiscardsOutputAndCanBeReused(boolean nested) throws E DataTypes.ROW( DataTypes.FIELD( "customer", - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING().copy(false)))), - DataTypes.FIELD("items", DataTypes.ARRAY(DataTypes.INT().copy(false))), + DataTypes.ROW(DataTypes.FIELD("name", DataTypes.STRING()))), + DataTypes.FIELD("items", DataTypes.ARRAY(DataTypes.INT())), DataTypes.FIELD( "attributes", DataTypes.MAP( @@ -566,9 +565,10 @@ void testComplexJsonFailureDiscardsOutputAndCanBeReused(boolean nested) throws E table( descriptor(schema.build(), false) .property( - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + "parse_json") .property( - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), nested ? "nested" : "flat") .build()); KafkaTopicWritePlan plan = transcoder.prepare(table); @@ -584,9 +584,9 @@ void testComplexJsonFailureDiscardsOutputAndCanBeReused(boolean nested) throws E Collections.emptyList()); for (String invalid : Arrays.asList( - "{\"customer\":{}}", + "{\"customer\":{\"name\":1}}", "{\"customer\":{\"name\":\"Alice\",\"unknown\":1}}", - "{\"items\":[1,null]}", + "{\"items\":[1,\"bad\"]}", "{\"attributes\":[]}", "{\"attributes\":{\"?\":{\"count\":1},\"\\ud800\":{\"count\":2}}}")) { AtomicReference output = new AtomicReference<>(); From f7d8a9bc83ac15ac3bb18f2a632134781d9d7467 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 11 Sep 2026 16:03:18 +0800 Subject: [PATCH 62/69] [kafka] Preserve unknown JSON fields in an optional rescue column Collect root and nested unknown JSON fields in a nullable STRING value column while keeping known-field validation strict. Parse rescued values with the bounded streaming reader and retain streamed Arrow writes. Cover native readback, rescue boundaries and scalar parsing regressions. Co-Authored-By: Codex AI-Model: gpt-6-astra AI-Contributed/Feature: 256/256 AI-Contributed/UT: 419/419 --- .../format/json/JsonKafkaRescueTest.java | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java new file mode 100644 index 00000000000..4e8fe094c22 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.kafka.schema.KafkaFieldProjection; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests optional unknown-field rescue while preserving strict type validation. */ +class JsonKafkaRescueTest { + + @Test + void testRescuesUnknownRootAndNestedFieldsAsJsonString() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.BIGINT().copy(false)), + DataTypes.FIELD( + "customer", + DataTypes.ROW(DataTypes.FIELD("name", DataTypes.STRING()))), + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.ROW(DataTypes.FIELD("sku", DataTypes.STRING())))), + DataTypes.FIELD( + "attributes", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW(DataTypes.FIELD("known", DataTypes.INT())))), + DataTypes.FIELD("kafka_rescue", DataTypes.STRING())); + JsonKafkaFieldDecoder decoder = rescueDecoder(rowType, "kafka_rescue"); + + Object[] values = + decoder.decode( + bytes( + "{" + + "\"id\":1," + + "\"customer\":{\"name\":\"Alice\",\"extra_nested\":2}," + + "\"items\":[{\"sku\":\"A-1\",\"extra_item\":3}]," + + "\"attributes\":{\"dynamic\":{\"known\":1," + + "\"extra_map_value\":4}}," + + "\"extra\":\"unexpected\"," + + "\"kafka_rescue\":null" + + "}")); + + assertThat(values[0]).isEqualTo(1L); + assertThat(((GenericRow) values[1]).getString(0).toString()).isEqualTo("Alice"); + assertThat(((GenericArray) values[2]).getRow(0, 1).getString(0).toString()) + .isEqualTo("A-1"); + GenericMap attributes = (GenericMap) values[3]; + assertThat(((GenericRow) attributes.get(BinaryString.fromString("dynamic"))).getInt(0)) + .isEqualTo(1); + assertThat(((BinaryString) values[4]).toString()) + .isEqualTo( + "{\"extra\":\"unexpected\"," + + "\"customer\":{\"extra_nested\":2}," + + "\"items\":[{\"extra_item\":3}]," + + "\"attributes\":{\"dynamic\":{\"extra_map_value\":4}}}"); + + assertThat( + decoder.decode( + bytes( + "{\"id\":1,\"customer\":null,\"items\":null," + + "\"attributes\":null}"))[4]) + .isNull(); + } + + @Test + void testRejectsExplicitNonNullRescueColumnAndKeepsTypeChecksStrict() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.BIGINT()), + DataTypes.FIELD("kafka_rescue", DataTypes.STRING())); + JsonKafkaFieldDecoder decoder = rescueDecoder(rowType, "kafka_rescue"); + + assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":1,\"kafka_rescue\":\"supplied\"}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("configured rescue column is reserved"); + assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":\"not-a-number\"}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("expected a JSON integer"); + } + + @Test + void testRescueArrayPreservesOriginalElementPositions() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT())))), + DataTypes.FIELD("rescue", DataTypes.STRING())); + Object[] values = + rescueDecoder(rowType, "rescue") + .decode(bytes("{\"items\":[{\"id\":1},{\"id\":2,\"extra\":3}]}")); + assertThat(((BinaryString) values[1]).toString()) + .isEqualTo("{\"items\":[null,{\"extra\":3}]}"); + assertThat(((GenericArray) values[0]).getRow(1, 1).getInt(0)).isEqualTo(2); + } + + @Test + void testRescuesRootObjectAtLimitAndRejectsExcessFieldBeforeReadingItsValue() { + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW(DataTypes.FIELD("rescue", DataTypes.STRING())), "rescue"); + String object = containerAtLimit(true); + assertThat(((BinaryString) decoder.decode(bytes(object))[0]).toString()).isEqualTo(object); + + String excessField = object.substring(0, object.length() - 1) + ",\"excess\":{"; + assertThatThrownBy(() -> decoder.decode(bytes(excessField))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining( + "maximum field count of " + JsonToFlussConverters.MAX_CONTAINER_ELEMENTS); + } + + @ParameterizedTest(name = "{index}: {0}") + @MethodSource("rescuedContainers") + void testBoundsUnknownContainersBeforeReadingExcessChild( + String description, String prefix, String suffix, String path, boolean object) { + RowType nestedRow = DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT())); + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("details", nestedRow), + DataTypes.FIELD("items", DataTypes.ARRAY(nestedRow)), + DataTypes.FIELD( + "attributes", DataTypes.MAP(DataTypes.STRING(), nestedRow)), + DataTypes.FIELD("rescue", DataTypes.STRING())), + "rescue"); + String container = containerAtLimit(object); + String json = prefix + container + suffix; + assertThat(((BinaryString) decoder.decode(bytes(json))[3]).toString()).isEqualTo(json); + + // An unfinished excess child distinguishes the container limit from a syntax error. + String excessChild = + prefix + + container.substring(0, container.length() - 1) + + (object ? ",\"excess\":{" : ",["); + assertThatThrownBy(() -> decoder.decode(bytes(excessChild))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining(path) + .hasMessageContaining( + "container size exceeds " + JsonToFlussConverters.MAX_CONTAINER_ELEMENTS); + } + + private static Stream rescuedContainers() { + return Stream.of(false, true) + .flatMap( + object -> + Stream.of( + Arguments.of( + "unknown root " + (object ? "object" : "array"), + "{\"extra\":", + "}", + "$[\"extra\"]", + object), + Arguments.of( + "unknown subtree " + (object ? "object" : "array"), + "{\"extra\":{\"nested\":", + "}}", + "$[\"extra\"][\"nested\"]", + object), + Arguments.of( + "inside ROW " + (object ? "object" : "array"), + "{\"details\":{\"extra\":", + "}}", + "$[\"details\"][\"extra\"]", + object), + Arguments.of( + "inside ARRAY " + (object ? "object" : "array"), + "{\"items\":[{\"extra\":", + "}]}", + "$[\"items\"][0][\"extra\"]", + object), + Arguments.of( + "inside MAP " + (object ? "object" : "array"), + "{\"attributes\":{\"dynamic\":{\"extra\":", + "}}}", + "$[\"attributes\"][\"dynamic\"][\"extra\"]", + object))); + } + + @ParameterizedTest + @ValueSource( + strings = {"-0", "-0.0", "-0e3", "-0e-3", "1.0000000596046448", "-1.0000000596046448"}) + void testRescuePreservesRootFloatToken(String number) { + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("value", DataTypes.FLOAT()), + DataTypes.FIELD("rescue", DataTypes.STRING())), + "rescue"); + Object[] values = decoder.decode(bytes("{\"extra\":1,\"value\":" + number + "}")); + assertThat(Float.floatToRawIntBits((Float) values[0])) + .isEqualTo(Float.floatToRawIntBits(Float.parseFloat(number))); + assertThat(((BinaryString) values[1]).toString()).isEqualTo("{\"extra\":1}"); + } + + @ParameterizedTest + @ValueSource(strings = {"[", "{"}) + void testRescueRejectsContainersForKnownScalarsBeforeReadingTheirContents(String container) { + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("value", DataTypes.STRING()), + DataTypes.FIELD("rescue", DataTypes.STRING())), + "rescue"); + assertThatThrownBy(() -> decoder.decode(bytes("{\"extra\":1,\"value\":" + container))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$[\"value\"]") + .hasMessageContaining("expected a JSON scalar"); + } + + private static String containerAtLimit(boolean object) { + StringBuilder json = new StringBuilder(object ? "{" : "["); + for (int i = 0; i < JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; i++) { + if (i > 0) { + json.append(','); + } + if (object) { + json.append('"').append('k').append(i).append("\":"); + } + json.append('0'); + } + return json.append(object ? '}' : ']').toString(); + } + + private static JsonKafkaFieldDecoder rescueDecoder(RowType rowType, String rescueColumn) { + List positions = new ArrayList<>(); + for (int i = 0; i < rowType.getFieldCount(); i++) { + positions.add(i); + } + return new JsonKafkaFieldDecoder( + new KafkaFieldProjection(rowType, positions), rescueColumn); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } +} From 7c191b2de04b760defc613a11b032c5190e3c057 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 20 Sep 2026 15:57:32 +0800 Subject: [PATCH 63/69] [kafka] Reject lossy JSON values before rescue encoding Reject unpaired Unicode surrogates in JSON names and strings before UTF-8 conversion, including nested objects and MAP keys. Reject numeric overflow that Jackson would serialize as a quoted Infinity in rescue. Preserve exact numbers and valid Unicode, with decoder and native Produce readback regression coverage. Co-Authored-By: Codex AI-Model: gpt-6-astra AI-Contributed/Feature: 50/50 AI-Contributed/UT: 126/126 --- .../format/json/JsonKafkaRescueTest.java | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java index 4e8fe094c22..f7604656dd0 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java @@ -23,6 +23,9 @@ import org.apache.fluss.row.GenericArray; import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; @@ -32,6 +35,8 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -244,6 +249,101 @@ void testRescueRejectsContainersForKnownScalarsBeforeReadingTheirContents(String .hasMessageContaining("expected a JSON scalar"); } + @ParameterizedTest + @ValueSource(strings = {"\\uD800", "\\uDC00", "a\\uD800b", "\\uD800\\uD800", "\\uDC00\\uD800"}) + void testRejectsUnpairedSurrogatesBeforeRescueEncoding(String escaped) { + JsonKafkaFieldDecoder decoder = fidelityDecoder(); + for (String json : + new String[] { + "{\"" + escaped + "\":1,\"?\":2}", + "{\"extra\":\"" + escaped + "\"}", + "{\"extra\":{\"" + escaped + "\":1,\"?\":2}}", + "{\"extra\":[{\"key\":\"" + escaped + "\"}]}", + "{\"details\":{\"" + escaped + "\":1,\"?\":2}}", + "{\"items\":[{\"extra\":\"" + escaped + "\"}]}", + "{\"attributes\":{\"" + escaped + "\":{\"known\":1},\"?\":{\"known\":2}}}", + "{\"attributes\":{\"dynamic\":{\"extra\":\"" + escaped + "\"}}}" + }) { + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .as(json) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("unpaired UTF-16 surrogate"); + } + } + + @ParameterizedTest + @ValueSource(strings = {"1e400", "-1e400", "1e10000", "-1e10000", "1.7976931348623159e308"}) + void testRejectsOverflowingNumbersBeforeRescueEncoding(String number) { + JsonKafkaFieldDecoder decoder = fidelityDecoder(); + for (String json : + new String[] { + "{\"extra\":" + number + "}", + "{\"extra\":{\"nested\":" + number + "}}", + "{\"extra\":[" + number + "]}", + "{\"details\":{\"extra\":" + number + "}}", + "{\"items\":[{\"extra\":" + number + "}]}", + "{\"attributes\":{\"dynamic\":{\"extra\":" + number + "}}}" + }) { + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .as(json) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("non-finite or overflowing number"); + } + } + + @Test + void testRescuePreservesValidUnicodeAndExactNumbers() throws Exception { + String largeInteger = BigInteger.TEN.pow(400).subtract(BigInteger.ONE).toString(); + String decimal = "12345678901234567890.1234567890123456789"; + Object[] values = + fidelityDecoder() + .decode( + bytes( + "{\"\\uD83D\\uDE00\":\"\\uD83D\\uDE00\"," + + "\"extra\":{\"integer\":" + + largeInteger + + ",\"decimal\":" + + decimal + + ",\"small\":1e-400,\"large\":1e308," + + "\"text\":\"Infinity\",\"literal\":\"\\\\uD800\"}," + + "\"details\":{\"known\":7,\"\\uD83D\\uDE00\":\"ok\"}," + + "\"items\":[{\"known\":8,\"extra\":\"\\uD83D\\uDE00\"}]," + + "\"attributes\":{\"\\uD83D\\uDE00\":{\"known\":9,\"extra\":1}}}")); + ObjectMapper mapper = + new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + JsonNode rescued = mapper.readTree(((BinaryString) values[3]).toString()); + assertThat(rescued.get("😀").textValue()).isEqualTo("😀"); + JsonNode extra = rescued.get("extra"); + assertThat(extra.get("integer").bigIntegerValue()).isEqualTo(new BigInteger(largeInteger)); + assertThat(extra.get("decimal").decimalValue()) + .isEqualByComparingTo(new BigDecimal(decimal)); + assertThat(extra.get("small").decimalValue()) + .isEqualByComparingTo(new BigDecimal("1e-400")); + assertThat(extra.get("large").decimalValue()).isEqualByComparingTo(new BigDecimal("1e308")); + assertThat(extra.get("text").textValue()).isEqualTo("Infinity"); + assertThat(extra.get("literal").textValue()).isEqualTo("\\uD800"); + assertThat(rescued.get("details").get("😀").textValue()).isEqualTo("ok"); + assertThat(rescued.get("items").get(0).get("extra").textValue()).isEqualTo("😀"); + assertThat(rescued.get("attributes").get("😀").get("extra").intValue()).isEqualTo(1); + assertThat(((GenericRow) values[0]).getInt(0)).isEqualTo(7); + assertThat(((GenericArray) values[1]).getRow(0, 1).getInt(0)).isEqualTo(8); + assertThat( + ((GenericRow) ((GenericMap) values[2]).get(BinaryString.fromString("😀"))) + .getInt(0)) + .isEqualTo(9); + } + + private static JsonKafkaFieldDecoder fidelityDecoder() { + RowType nestedRow = DataTypes.ROW(DataTypes.FIELD("known", DataTypes.INT())); + return rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("details", nestedRow), + DataTypes.FIELD("items", DataTypes.ARRAY(nestedRow)), + DataTypes.FIELD("attributes", DataTypes.MAP(DataTypes.STRING(), nestedRow)), + DataTypes.FIELD("rescue", DataTypes.STRING())), + "rescue"); + } + private static String containerAtLimit(boolean object) { StringBuilder json = new StringBuilder(object ? "{" : "["); for (int i = 0; i < JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; i++) { From c0faa88047bf3e665831b3eec73bec44e987245f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:50:11 +0800 Subject: [PATCH 64/69] [kafka] Rescue nullable JSON type errors without hiding validation failures Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 262/262 AI-Contributed/UT: 213/213 --- .../fluss/kafka/format/json/JsonRescue.java | 100 ++++++++++++ .../format/json/JsonToFlussConverter.java | 10 +- .../format/json/JsonKafkaRescueTest.java | 144 +++++++++++++++++- 3 files changed, 247 insertions(+), 7 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonRescue.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonRescue.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonRescue.java new file mode 100644 index 00000000000..74c47315f7b --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonRescue.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeFactory; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; + +import javax.annotation.Nullable; + +import java.util.Iterator; +import java.util.Map; +import java.util.function.Consumer; + +/** Collects rescue values by their structural location, independently of bounded error paths. */ +final class JsonRescue { + private JsonRescue() {} + + @Nullable + static Consumer field(@Nullable Consumer parent, String name) { + return parent == null + ? null + : value -> { + ObjectNode result = JsonNodeFactory.instance.objectNode(); + result.set(name, value); + parent.accept(result); + }; + } + + /** Accumulates all rescued array elements in one array, avoiding quadratic copying. */ + static final class ArrayCollector { + private final Consumer parent; + private final int size; + @Nullable private ArrayNode result; + + ArrayCollector(Consumer parent, int size) { + this.parent = parent; + this.size = size; + } + + Consumer element(int index) { + return value -> { + if (result == null) { + result = JsonNodeFactory.instance.arrayNode(size); + for (int i = 0; i < size; i++) { + result.addNull(); + } + } + result.set(index, merge(result.get(index), value)); + }; + } + + void finish() { + if (result != null) { + parent.accept(result); + } + } + } + + static JsonNode merge(JsonNode first, JsonNode second) { + if (first == null || first.isNull()) { + return second; + } + if (second == null || second.isNull()) { + return first; + } + if (first.isObject() && second.isObject()) { + Iterator> fields = second.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + ((ObjectNode) first) + .set(field.getKey(), merge(first.get(field.getKey()), field.getValue())); + } + return first; + } + if (first.isArray() && second.isArray()) { + for (int i = 0; i < second.size(); i++) { + ((ArrayNode) first).set(i, merge(first.get(i), second.get(i))); + } + return first; + } + return second; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java index e1bbf8ae52b..e827d81a3d5 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java @@ -22,11 +22,19 @@ import javax.annotation.Nullable; +import java.util.function.Consumer; + /** Converts one JSON node into the internal representation of one Fluss field. */ @Internal interface JsonToFlussConverter { /** Converts the node, which is null when the JSON property is missing. */ @Nullable - Object convert(@Nullable JsonNode node, String path); + default Object convert(@Nullable JsonNode node, String path) { + return convert(node, path, null); + } + + /** Converts a node, optionally collecting recoverable nullable-field conversion failures. */ + @Nullable + Object convert(@Nullable JsonNode node, String path, @Nullable Consumer rescue); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java index f7604656dd0..d8f5ad88e52 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java @@ -26,6 +26,7 @@ import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; @@ -44,10 +45,141 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; -/** Tests optional unknown-field rescue while preserving strict type validation. */ +/** Tests structural rescue while keeping NOT NULL and resource violations fatal. */ class JsonKafkaRescueTest { + @Test + void testUnexpectedConversionFailureIsNotRescued() { + JsonNode broken = mock(JsonNode.class); + IllegalStateException failure = new IllegalStateException("unexpected conversion failure"); + when(broken.getNodeType()).thenReturn(JsonNodeType.BOOLEAN); + when(broken.booleanValue()).thenThrow(failure); + assertThatThrownBy( + () -> + JsonToFlussConverters.create(DataTypes.BOOLEAN()) + .convert( + broken, + "$", + rescued -> { + throw new AssertionError("Unexpected rescue"); + })) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasCause(failure); + } + + @Test + void testNullableFailuresMergeWithUnknownFieldsAtStructuralLocations() throws Exception { + RowType nested = DataTypes.ROW(DataTypes.FIELD("known", DataTypes.INT())); + RowType schema = + DataTypes.ROW( + DataTypes.FIELD("details", nested), + DataTypes.FIELD("items", DataTypes.ARRAY(nested)), + DataTypes.FIELD("attributes", DataTypes.MAP(DataTypes.STRING(), nested)), + DataTypes.FIELD("rescue", DataTypes.STRING())); + String longKey = new String(new char[512]).replace('\0', 'x') + ".[]"; + Object[] values = + rescueDecoder(schema, "rescue") + .decode( + bytes( + "{\"details\":{\"known\":\"bad\",\"extra\":1}," + + "\"items\":[{\"known\":2},{\"known\":\"bad\",\"extra\":3}]," + + "\"attributes\":{\"" + + longKey + + "\":{\"known\":false,\"extra\":4}}}")); + assertThat(((GenericRow) values[0]).isNullAt(0)).isTrue(); + GenericArray array = (GenericArray) values[1]; + assertThat(array.getRow(0, 1).getInt(0)).isEqualTo(2); + assertThat(array.getRow(1, 1).isNullAt(0)).isTrue(); + assertThat( + ((GenericRow) + ((GenericMap) values[2]) + .get(BinaryString.fromString(longKey))) + .isNullAt(0)) + .isTrue(); + JsonNode rescued = new ObjectMapper().readTree(values[3].toString()); + assertThat(rescued.get("details").get("known").textValue()).isEqualTo("bad"); + assertThat(rescued.get("details").get("extra").intValue()).isEqualTo(1); + assertThat(rescued.get("items").get(0).isNull()).isTrue(); + assertThat(rescued.get("items").get(1).get("known").textValue()).isEqualTo("bad"); + assertThat(rescued.get("items").get(1).get("extra").intValue()).isEqualTo(3); + assertThat(rescued.get("attributes").get(longKey).get("known").booleanValue()).isFalse(); + assertThat(rescued.get("attributes").get(longKey).get("extra").intValue()).isEqualTo(4); + } + + @Test + void testNullableParentNeverRescuesRequiredDescendantFailures() { + RowType required = DataTypes.ROW(DataTypes.FIELD("required", DataTypes.INT().copy(false))); + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("row", required), + DataTypes.FIELD("array", DataTypes.ARRAY(required)), + DataTypes.FIELD("map", DataTypes.MAP(DataTypes.STRING(), required)), + DataTypes.FIELD("rescue", DataTypes.STRING())), + "rescue"); + for (String json : + new String[] { + "{\"row\":{\"required\":\"bad\"}}", + "{\"row\":{}}", + "{\"row\":{\"required\":null}}", + "{\"array\":[{\"required\":\"bad\"}]}", + "{\"map\":{\"x\":{\"required\":\"bad\"}}}" + }) { + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("required"); + } + } + + @Test + void testScalarFailuresAndWrongContainersPreserveOriginalValues() throws Exception { + RowType schema = + DataTypes.ROW( + DataTypes.FIELD("integer", DataTypes.INT()), + DataTypes.FIELD("decimal", DataTypes.DECIMAL(3, 1)), + DataTypes.FIELD("float", DataTypes.FLOAT()), + DataTypes.FIELD("date", DataTypes.DATE()), + DataTypes.FIELD("bytes", DataTypes.BYTES()), + DataTypes.FIELD("string", DataTypes.STRING()), + DataTypes.FIELD("rescue", DataTypes.STRING())); + String json = + "{\"integer\":2147483648,\"decimal\":123.45,\"float\":1e40," + + "\"date\":\"invalid\",\"bytes\":\"%%%\",\"string\":{\"original\":[1,2]}}"; + Object[] values = rescueDecoder(schema, "rescue").decode(bytes(json)); + for (int i = 0; i < 6; i++) { + assertThat(values[i]).isNull(); + } + ObjectMapper mapper = + new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + assertThat(mapper.readTree(values[6].toString())).isEqualTo(mapper.readTree(json)); + assertThatThrownBy(() -> rescueDecoder(schema, null).decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class); + } + + @Test + void testNullableWrongContainerStillEnforcesResourceLimits() { + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("value", DataTypes.INT()), + DataTypes.FIELD("rescue", DataTypes.STRING())), + "rescue"); + String atLimit = containerAtLimit(false); + assertThat(decoder.decode(bytes("{\"value\":" + atLimit + "}"))[0]).isNull(); + assertThatThrownBy( + () -> + decoder.decode( + bytes( + "{\"value\":" + + atLimit.substring(0, atLimit.length() - 1) + + ",0]}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("container size exceeds"); + } + @Test void testRescuesUnknownRootAndNestedFieldsAsJsonString() { RowType rowType = @@ -104,7 +236,7 @@ void testRescuesUnknownRootAndNestedFieldsAsJsonString() { } @Test - void testRejectsExplicitNonNullRescueColumnAndKeepsTypeChecksStrict() { + void testRejectsExplicitNonNullRescueColumnAndRescuesNullableTypeMismatch() { RowType rowType = DataTypes.ROW( DataTypes.FIELD("id", DataTypes.BIGINT()), @@ -114,9 +246,9 @@ void testRejectsExplicitNonNullRescueColumnAndKeepsTypeChecksStrict() { assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":1,\"kafka_rescue\":\"supplied\"}"))) .isInstanceOf(KafkaRecordEncodingException.class) .hasMessageContaining("configured rescue column is reserved"); - assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":\"not-a-number\"}"))) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("expected a JSON integer"); + Object[] values = decoder.decode(bytes("{\"id\":\"not-a-number\"}")); + assertThat(values[0]).isNull(); + assertThat(values[1].toString()).isEqualTo("{\"id\":\"not-a-number\"}"); } @Test @@ -240,7 +372,7 @@ void testRescueRejectsContainersForKnownScalarsBeforeReadingTheirContents(String JsonKafkaFieldDecoder decoder = rescueDecoder( DataTypes.ROW( - DataTypes.FIELD("value", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.STRING().copy(false)), DataTypes.FIELD("rescue", DataTypes.STRING())), "rescue"); assertThatThrownBy(() -> decoder.decode(bytes("{\"extra\":1,\"value\":" + container))) From 1ce2df990cd09d4500f8235aa1eccb2c08ce63ac Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 21:32:42 +0800 Subject: [PATCH 65/69] [kafka] Keep rescue limited to unknown JSON fields Reject known-field conversion and nullability errors even with rescue configured, including nullable fields and nested ROW/ARRAY/MAP values. Collect payload members matching the rescue column name under that name, including explicit null, without assigning the generated column directly. Retain bounded parsing and Unicode/numeric validation. Verify partition atomicity for 17 invalid inputs alongside successful sibling partitions, then read native rows after recovery. Defer Core-dependent fixed names and flat/nested layout contracts. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 275/275 AI-Contributed/UT: 473/473 --- .../fluss/kafka/format/json/JsonRescue.java | 100 ------- .../format/json/JsonToFlussConverter.java | 10 +- .../format/json/JsonKafkaRescueTest.java | 263 +++++++++++------- 3 files changed, 165 insertions(+), 208 deletions(-) delete mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonRescue.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonRescue.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonRescue.java deleted file mode 100644 index 74c47315f7b..00000000000 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonRescue.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka.format.json; - -import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; -import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeFactory; -import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; - -import javax.annotation.Nullable; - -import java.util.Iterator; -import java.util.Map; -import java.util.function.Consumer; - -/** Collects rescue values by their structural location, independently of bounded error paths. */ -final class JsonRescue { - private JsonRescue() {} - - @Nullable - static Consumer field(@Nullable Consumer parent, String name) { - return parent == null - ? null - : value -> { - ObjectNode result = JsonNodeFactory.instance.objectNode(); - result.set(name, value); - parent.accept(result); - }; - } - - /** Accumulates all rescued array elements in one array, avoiding quadratic copying. */ - static final class ArrayCollector { - private final Consumer parent; - private final int size; - @Nullable private ArrayNode result; - - ArrayCollector(Consumer parent, int size) { - this.parent = parent; - this.size = size; - } - - Consumer element(int index) { - return value -> { - if (result == null) { - result = JsonNodeFactory.instance.arrayNode(size); - for (int i = 0; i < size; i++) { - result.addNull(); - } - } - result.set(index, merge(result.get(index), value)); - }; - } - - void finish() { - if (result != null) { - parent.accept(result); - } - } - } - - static JsonNode merge(JsonNode first, JsonNode second) { - if (first == null || first.isNull()) { - return second; - } - if (second == null || second.isNull()) { - return first; - } - if (first.isObject() && second.isObject()) { - Iterator> fields = second.fields(); - while (fields.hasNext()) { - Map.Entry field = fields.next(); - ((ObjectNode) first) - .set(field.getKey(), merge(first.get(field.getKey()), field.getValue())); - } - return first; - } - if (first.isArray() && second.isArray()) { - for (int i = 0; i < second.size(); i++) { - ((ArrayNode) first).set(i, merge(first.get(i), second.get(i))); - } - return first; - } - return second; - } -} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java index e827d81a3d5..e1bbf8ae52b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java @@ -22,19 +22,11 @@ import javax.annotation.Nullable; -import java.util.function.Consumer; - /** Converts one JSON node into the internal representation of one Fluss field. */ @Internal interface JsonToFlussConverter { /** Converts the node, which is null when the JSON property is missing. */ @Nullable - default Object convert(@Nullable JsonNode node, String path) { - return convert(node, path, null); - } - - /** Converts a node, optionally collecting recoverable nullable-field conversion failures. */ - @Nullable - Object convert(@Nullable JsonNode node, String path, @Nullable Consumer rescue); + Object convert(@Nullable JsonNode node, String path); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java index d8f5ad88e52..bda54b1b9b1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java @@ -27,6 +27,7 @@ import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; @@ -48,7 +49,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -/** Tests structural rescue while keeping NOT NULL and resource violations fatal. */ +/** Tests unknown-field rescue while keeping known-field and resource violations fatal. */ class JsonKafkaRescueTest { @Test @@ -60,124 +61,149 @@ void testUnexpectedConversionFailureIsNotRescued() { assertThatThrownBy( () -> JsonToFlussConverters.create(DataTypes.BOOLEAN()) - .convert( - broken, - "$", - rescued -> { - throw new AssertionError("Unexpected rescue"); - })) + .convert(broken, "$")) .isInstanceOf(KafkaRecordEncodingException.class) .hasCause(failure); } - @Test - void testNullableFailuresMergeWithUnknownFieldsAtStructuralLocations() throws Exception { - RowType nested = DataTypes.ROW(DataTypes.FIELD("known", DataTypes.INT())); - RowType schema = - DataTypes.ROW( - DataTypes.FIELD("details", nested), - DataTypes.FIELD("items", DataTypes.ARRAY(nested)), - DataTypes.FIELD("attributes", DataTypes.MAP(DataTypes.STRING(), nested)), - DataTypes.FIELD("rescue", DataTypes.STRING())); - String longKey = new String(new char[512]).replace('\0', 'x') + ".[]"; - Object[] values = - rescueDecoder(schema, "rescue") - .decode( - bytes( - "{\"details\":{\"known\":\"bad\",\"extra\":1}," - + "\"items\":[{\"known\":2},{\"known\":\"bad\",\"extra\":3}]," - + "\"attributes\":{\"" - + longKey - + "\":{\"known\":false,\"extra\":4}}}")); - assertThat(((GenericRow) values[0]).isNullAt(0)).isTrue(); - GenericArray array = (GenericArray) values[1]; - assertThat(array.getRow(0, 1).getInt(0)).isEqualTo(2); - assertThat(array.getRow(1, 1).isNullAt(0)).isTrue(); - assertThat( - ((GenericRow) - ((GenericMap) values[2]) - .get(BinaryString.fromString(longKey))) - .isNullAt(0)) - .isTrue(); - JsonNode rescued = new ObjectMapper().readTree(values[3].toString()); - assertThat(rescued.get("details").get("known").textValue()).isEqualTo("bad"); - assertThat(rescued.get("details").get("extra").intValue()).isEqualTo(1); - assertThat(rescued.get("items").get(0).isNull()).isTrue(); - assertThat(rescued.get("items").get(1).get("known").textValue()).isEqualTo("bad"); - assertThat(rescued.get("items").get(1).get("extra").intValue()).isEqualTo(3); - assertThat(rescued.get("attributes").get(longKey).get("known").booleanValue()).isFalse(); - assertThat(rescued.get("attributes").get(longKey).get("extra").intValue()).isEqualTo(4); + @ParameterizedTest + @MethodSource("invalidKnownValues") + void testKnownFailuresRemainFatalAcrossNullableAndRequiredContainers( + DataType dataType, String jsonValue, String reason) { + for (boolean nullable : new boolean[] {true, false}) { + assertRejectedInAllContainers(dataType.copy(nullable), jsonValue, reason); + } } - @Test - void testNullableParentNeverRescuesRequiredDescendantFailures() { - RowType required = DataTypes.ROW(DataTypes.FIELD("required", DataTypes.INT().copy(false))); + private static Stream invalidKnownValues() { + return Stream.of( + Arguments.of(DataTypes.BOOLEAN(), "1", "expected a JSON boolean"), + Arguments.of(DataTypes.TINYINT(), "128", "integer overflow"), + Arguments.of(DataTypes.SMALLINT(), "32768", "integer overflow"), + Arguments.of(DataTypes.INT(), "2147483648", "integer overflow"), + Arguments.of(DataTypes.INT(), "1.5", "expected a JSON integer"), + Arguments.of(DataTypes.INT(), "\"bad\"", "expected a JSON integer"), + Arguments.of(DataTypes.BIGINT(), "9223372036854775808", "integer overflow"), + Arguments.of(DataTypes.DECIMAL(3, 1), "123.4", "decimal precision exceeds target"), + Arguments.of(DataTypes.DECIMAL(3, 1), "1.23", "decimal scale exceeds target"), + Arguments.of(DataTypes.FLOAT(), "1e40", "non-finite or overflowing number"), + Arguments.of(DataTypes.DOUBLE(), "1e400", "non-finite or overflowing number"), + Arguments.of(DataTypes.DATE(), "\"invalid\"", "invalid ISO-8601 date"), + Arguments.of(DataTypes.TIME(3), "\"invalid\"", "invalid ISO-8601 local time"), + Arguments.of( + DataTypes.TIMESTAMP(3), "\"invalid\"", "invalid ISO-8601 local timestamp"), + Arguments.of( + DataTypes.TIMESTAMP_LTZ(3), "\"invalid\"", "invalid ISO-8601 timestamp"), + Arguments.of(DataTypes.CHAR(1), "\"ab\"", "character length exceeds target"), + Arguments.of(DataTypes.BINARY(2), "\"YQ==\"", "decoded binary length must equal 2"), + Arguments.of(DataTypes.BYTES(), "\"%%%\"", "invalid Base64 string"), + Arguments.of(DataTypes.STRING(), "1", "expected a JSON string"), + Arguments.of(DataTypes.STRING(), "{}", "expected a JSON"), + Arguments.of(DataTypes.STRING(), "[]", "expected a JSON"), + Arguments.of( + DataTypes.ROW(DataTypes.FIELD("n", DataTypes.INT())), + "[]", + "expected a JSON object"), + Arguments.of(DataTypes.ARRAY(DataTypes.INT()), "{}", "expected a JSON array"), + Arguments.of( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + "[]", + "expected a JSON object")); + } + + private static void assertRejectedInAllContainers( + DataType dataType, String value, String reason) { JsonKafkaFieldDecoder decoder = rescueDecoder( DataTypes.ROW( - DataTypes.FIELD("row", required), - DataTypes.FIELD("array", DataTypes.ARRAY(required)), - DataTypes.FIELD("map", DataTypes.MAP(DataTypes.STRING(), required)), + DataTypes.FIELD("value", dataType), + DataTypes.FIELD( + "row", DataTypes.ROW(DataTypes.FIELD("value", dataType))), + DataTypes.FIELD("items", DataTypes.ARRAY(dataType)), + DataTypes.FIELD( + "attributes", DataTypes.MAP(DataTypes.STRING(), dataType)), DataTypes.FIELD("rescue", DataTypes.STRING())), "rescue"); - for (String json : - new String[] { - "{\"row\":{\"required\":\"bad\"}}", - "{\"row\":{}}", - "{\"row\":{\"required\":null}}", - "{\"array\":[{\"required\":\"bad\"}]}", - "{\"map\":{\"x\":{\"required\":\"bad\"}}}" - }) { + String[][] cases = { + {"\"value\":" + value, "$[\"value\"]"}, + {"\"row\":{\"value\":" + value + ",\"extra\":2}", "$[\"row\"][\"value\"]"}, + {"\"items\":[" + value + "]", "$[\"items\"][0]"}, + {"\"attributes\":{\"dynamic\":" + value + "}", "$[\"attributes\"][\"dynamic\"]"} + }; + for (String[] entry : cases) { + String json = "{\"extra\":1," + entry[0] + "}"; assertThatThrownBy(() -> decoder.decode(bytes(json))) + .as("%s: %s", dataType, json) .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("required"); + .hasMessageContaining(entry[1]) + .hasMessageContaining(reason); } } @Test - void testScalarFailuresAndWrongContainersPreserveOriginalValues() throws Exception { - RowType schema = - DataTypes.ROW( - DataTypes.FIELD("integer", DataTypes.INT()), - DataTypes.FIELD("decimal", DataTypes.DECIMAL(3, 1)), - DataTypes.FIELD("float", DataTypes.FLOAT()), - DataTypes.FIELD("date", DataTypes.DATE()), - DataTypes.FIELD("bytes", DataTypes.BYTES()), - DataTypes.FIELD("string", DataTypes.STRING()), - DataTypes.FIELD("rescue", DataTypes.STRING())); - String json = - "{\"integer\":2147483648,\"decimal\":123.45,\"float\":1e40," - + "\"date\":\"invalid\",\"bytes\":\"%%%\",\"string\":{\"original\":[1,2]}}"; - Object[] values = rescueDecoder(schema, "rescue").decode(bytes(json)); - for (int i = 0; i < 6; i++) { - assertThat(values[i]).isNull(); + void testNullableParentNeverRescuesRequiredDescendantFailures() { + assertRejectedInAllContainers( + DataTypes.INT().copy(false), "null", "field is missing or null"); + RowType required = DataTypes.ROW(DataTypes.FIELD("required", DataTypes.INT().copy(false))); + for (String value : new String[] {"{\"required\":\"bad\"}", "{}", "{\"required\":null}"}) { + assertRejectedInAllContainers(required, value, "required"); } - ObjectMapper mapper = - new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); - assertThat(mapper.readTree(values[6].toString())).isEqualTo(mapper.readTree(json)); - assertThatThrownBy(() -> rescueDecoder(schema, null).decode(bytes(json))) + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("required", DataTypes.INT().copy(false)), + DataTypes.FIELD("rescue", DataTypes.STRING())), + "rescue"); + assertThatThrownBy(() -> decoder.decode(bytes("{\"extra\":1}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("field is missing or null"); + } + + @ParameterizedTest + @ValueSource( + strings = { + "{", + "[]", + "{\"extra\":1,\"extra\":2}", + "{\"rescue\":null,\"rescue\":1}", + "{\"extra\":{\"x\":1,\"x\":2}}", + "{} {}", + "{\"rescue\":\"\\uD800\"}", + "{\"rescue\":1e400}" + }) + void testInvalidJsonCannotBeRescued(String json) { + assertThatThrownBy(() -> fidelityDecoder().decode(bytes(json))) .isInstanceOf(KafkaRecordEncodingException.class); } @Test - void testNullableWrongContainerStillEnforcesResourceLimits() { + void testRescueNameStillEnforcesResourceLimitsAndUtf8() { JsonKafkaFieldDecoder decoder = rescueDecoder( - DataTypes.ROW( - DataTypes.FIELD("value", DataTypes.INT()), - DataTypes.FIELD("rescue", DataTypes.STRING())), - "rescue"); + DataTypes.ROW(DataTypes.FIELD("rescue", DataTypes.STRING())), "rescue"); String atLimit = containerAtLimit(false); - assertThat(decoder.decode(bytes("{\"value\":" + atLimit + "}"))[0]).isNull(); + assertThat(decoder.decode(bytes("{\"rescue\":" + atLimit + "}"))[0].toString()) + .isEqualTo("{\"rescue\":" + atLimit + "}"); assertThatThrownBy( () -> decoder.decode( bytes( - "{\"value\":" + "{\"rescue\":" + atLimit.substring(0, atLimit.length() - 1) + ",0]}"))) .isInstanceOf(KafkaRecordEncodingException.class) .hasMessageContaining("container size exceeds"); + String nested = "0"; + for (int i = 0; i < JsonKafkaFieldDecoder.MAX_JSON_NESTING_DEPTH; i++) { + nested = "[" + nested + "]"; + } + byte[] tooDeep = bytes("{\"rescue\":" + nested + "}"); + assertThatThrownBy(() -> decoder.decode(tooDeep)) + .isInstanceOf(KafkaRecordEncodingException.class); + byte[] invalidUtf8 = bytes("{\"rescue\":\"x\"}"); + invalidUtf8[invalidUtf8.length - 3] = (byte) 0xff; + assertThatThrownBy(() -> decoder.decode(invalidUtf8)) + .isInstanceOf(KafkaRecordEncodingException.class); } @Test @@ -222,7 +248,7 @@ void testRescuesUnknownRootAndNestedFieldsAsJsonString() { .isEqualTo(1); assertThat(((BinaryString) values[4]).toString()) .isEqualTo( - "{\"extra\":\"unexpected\"," + "{\"extra\":\"unexpected\",\"kafka_rescue\":null," + "\"customer\":{\"extra_nested\":2}," + "\"items\":[{\"extra_item\":3}]," + "\"attributes\":{\"dynamic\":{\"extra_map_value\":4}}}"); @@ -235,20 +261,59 @@ void testRescuesUnknownRootAndNestedFieldsAsJsonString() { .isNull(); } + @ParameterizedTest + @ValueSource( + strings = { + "null", + "\"payload\"", + "7", + "false", + "[1,{\"x\":2}]", + "{\"rescue\":\"nested\",\"x\":1}" + }) + void testPayloadRescueNameIsPreservedAsJsonMember(String value) { + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT()), + DataTypes.FIELD("rescue", DataTypes.STRING())), + "rescue"); + Object[] decoded = decoder.decode(bytes("{\"id\":1,\"rescue\":" + value + ",\"extra\":2}")); + assertThat(decoded[0]).isEqualTo(1); + assertThat(decoded[1].toString()).isEqualTo("{\"rescue\":" + value + ",\"extra\":2}"); + assertThat(decoder.decode(bytes("{\"id\":1}"))[1]).isNull(); + assertThat(decoder.decode(bytes("{\"rescue\":null}"))[1].toString()) + .isEqualTo("{\"rescue\":null}"); + } + @Test - void testRejectsExplicitNonNullRescueColumnAndRescuesNullableTypeMismatch() { - RowType rowType = + void testRescueNameIsConfigurationDrivenAndNestedBusinessFieldsKeepTheirMeaning() { + RowType schema = DataTypes.ROW( - DataTypes.FIELD("id", DataTypes.BIGINT()), - DataTypes.FIELD("kafka_rescue", DataTypes.STRING())); - JsonKafkaFieldDecoder decoder = rescueDecoder(rowType, "kafka_rescue"); - - assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":1,\"kafka_rescue\":\"supplied\"}"))) + DataTypes.FIELD("_kafka_key", DataTypes.STRING()), + DataTypes.FIELD("_kafka_headers", DataTypes.STRING()), + DataTypes.FIELD("_kafka_metadata", DataTypes.STRING()), + DataTypes.FIELD("_kafka_value", DataTypes.STRING()), + DataTypes.FIELD( + "row", DataTypes.ROW(DataTypes.FIELD("rescue", DataTypes.INT()))), + DataTypes.FIELD("rescue", DataTypes.STRING())); + String json = + "{\"_kafka_key\":\"k\",\"_kafka_headers\":\"h\",\"_kafka_metadata\":\"m\"," + + "\"_kafka_value\":\"v\",\"row\":{\"rescue\":7,\"extra\":8}}"; + Object[] values = rescueDecoder(schema, "rescue").decode(bytes(json)); + for (int i = 0; i < 4; i++) { + assertThat(values[i].toString()).isEqualTo(new String[] {"k", "h", "m", "v"}[i]); + } + assertThat(((GenericRow) values[4]).getInt(0)).isEqualTo(7); + assertThat(values[5].toString()).isEqualTo("{\"row\":{\"extra\":8}}"); + assertThatThrownBy(() -> rescueDecoder(schema, null).decode(bytes(json))) .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("configured rescue column is reserved"); - Object[] values = decoder.decode(bytes("{\"id\":\"not-a-number\"}")); - assertThat(values[0]).isNull(); - assertThat(values[1].toString()).isEqualTo("{\"id\":\"not-a-number\"}"); + .hasMessageContaining("unknown field"); + assertThat( + rescueDecoder(schema, null) + .decode(bytes("{\"rescue\":\"business\"}"))[5] + .toString()) + .isEqualTo("business"); } @Test @@ -372,7 +437,7 @@ void testRescueRejectsContainersForKnownScalarsBeforeReadingTheirContents(String JsonKafkaFieldDecoder decoder = rescueDecoder( DataTypes.ROW( - DataTypes.FIELD("value", DataTypes.STRING().copy(false)), + DataTypes.FIELD("value", DataTypes.STRING()), DataTypes.FIELD("rescue", DataTypes.STRING())), "rescue"); assertThatThrownBy(() -> decoder.decode(bytes("{\"extra\":1,\"value\":" + container))) From 8eb4d4426c5a37c9224baa7d9076f1e45193e66b Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 23:10:28 +0800 Subject: [PATCH 66/69] [kafka] Align rescue with the fixed Kafka table contract Collect unknown JSON fields through the schema-aware bounded parser when _kafka_rescue is declared. Preserve nested structure and typed numeric conversion while keeping known-field and resource violations fatal. Route the full schema through decoder creation and map rescue output to its physical column independently of business projections. Cover both layouts, reordered system columns, strict failures and native readback. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 253/253 AI-Contributed/UT: 673/673 --- .../fluss/kafka/format/KafkaFieldDecoder.java | 4 + .../kafka/format/KafkaFormatFactory.java | 6 + .../format/KafkaFormatFactoryRegistry.java | 12 +- .../format/json/JsonKafkaFieldDecoder.java | 196 +++++++++- .../format/json/JsonKafkaFormatFactory.java | 6 + .../format/json/JsonToFlussConverters.java | 2 +- .../schema/KafkaTopicSchemaResolver.java | 2 +- .../transcode/ArrowKafkaRecordTranscoder.java | 6 +- .../kafka/transcode/KafkaRowAssembler.java | 20 +- .../kafka/transcode/KafkaTopicWritePlan.java | 6 +- .../fluss/kafka/KafkaJsonProduceITCase.java | 278 ++++++++++++-- .../GatewayKafkaProduceBackendTest.java | 28 +- .../KafkaFormatFactoryRegistryTest.java | 68 +++- .../format/json/JsonKafkaRescueTest.java | 344 ++++++++++++------ 14 files changed, 794 insertions(+), 184 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java index b70c85f17d4..6b22c566e14 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java @@ -28,6 +28,10 @@ public interface KafkaFieldDecoder { /** * Decodes Kafka value bytes into internal Fluss field values. * + *

A JSON decoder created from a table schema with a declared {@code _kafka_rescue} appends + * that column's nullable STRING value after the ordered value projection. Binary/string + * decoders return only the value projection. + * *

The input is borrowed and must not be modified. Returned values may borrow its storage; * callers must encode them synchronously before the input can be reused. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java index 02b3dc2846d..6fcf4cd8bac 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java @@ -21,6 +21,7 @@ import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; import javax.annotation.Nullable; @@ -34,6 +35,11 @@ public interface KafkaFormatFactory { /** Creates and validates a decoder for the projection. */ KafkaFieldDecoder createDecoder(KafkaFieldProjection projection); + /** Creates a decoder from the validated fixed-column contract. */ + default KafkaFieldDecoder createDecoder(KafkaTableSchema schema) { + return createDecoder(schema.valueProjection(), schema.valueConversionLayout()); + } + /** Creates a decoder with the JSON layout, or no layout for binary/string values. */ default KafkaFieldDecoder createDecoder( KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java index 29339483bd0..7288bb61f7b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java @@ -23,6 +23,7 @@ import org.apache.fluss.kafka.format.json.JsonKafkaFormatFactory; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; import javax.annotation.Nullable; @@ -54,12 +55,21 @@ public KafkaFieldDecoder createDecoder( KafkaValueConversionType format, KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { + return factory(format).createDecoder(projection, layout); + } + + /** Creates a decoder with fixed rescue-column support from the validated table schema. */ + public KafkaFieldDecoder createDecoder(KafkaTableSchema schema) { + return factory(schema.valueConversionType()).createDecoder(schema); + } + + private KafkaFormatFactory factory(KafkaValueConversionType format) { KafkaFormatFactory factory = factories.get(format); if (factory == null) { throw new KafkaTopicSchemaException( "No Kafka format factory is registered for '" + format + "'."); } - return factory.createDecoder(projection, layout); + return factory; } private void register(KafkaFormatFactory factory) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java index 9ea06f5ee08..37094fb55d1 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java @@ -24,6 +24,7 @@ import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; import org.apache.fluss.metadata.KafkaFieldProjection; import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.GenericRow; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonParser; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonToken; @@ -38,6 +39,7 @@ import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.MapType; import org.apache.fluss.types.RowType; @@ -51,6 +53,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -63,7 +66,10 @@ public final class JsonKafkaFieldDecoder implements KafkaFieldDecoder { static final int MAX_CONTAINER_ELEMENTS = JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; private static final ObjectMapper OBJECT_MAPPER = createObjectMapper(); + private static final DataType JSON_STRING_TYPE = DataTypes.STRING(); + private final KafkaFieldProjection projection; + private final boolean rescueEnabled; private final @Nullable RowType nestedType; private final Map projectedFields = new HashMap<>(); private final JsonToFlussConverter[] converters; @@ -76,6 +82,22 @@ public JsonKafkaFieldDecoder(KafkaFieldProjection projection) { /** Creates a decoder whose nested physical ROW represents the JSON object root. */ public JsonKafkaFieldDecoder( KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { + this(projection, layout, false); + } + + /** Creates a decoder with rescue enabled only by the validated fixed system column. */ + public JsonKafkaFieldDecoder(KafkaTableSchema schema) { + this( + schema.valueProjection(), + schema.valueConversionLayout(), + schema.rescuePosition() >= 0); + } + + private JsonKafkaFieldDecoder( + KafkaFieldProjection projection, + @Nullable KafkaValueConversionLayout layout, + boolean rescueEnabled) { + this.rescueEnabled = rescueEnabled; if (layout == KafkaValueConversionLayout.NESTED) { if (projection.size() != 1 || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) @@ -124,7 +146,10 @@ public Object[] decode(@Nullable byte[] bytes) { throw new KafkaRecordEncodingException( "Kafka JSON record value must have an object root."); } - Object[] values = new Object[projection.size()]; + Object[] values = + new Object[projection.size() + (rescueEnabled && nestedType == null ? 1 : 0)]; + ObjectNode rescuedFields = rescueEnabled ? OBJECT_MAPPER.createObjectNode() : null; + JsonNode[] nestedRescues = rescueEnabled ? new JsonNode[projection.size()] : null; boolean[] present = new boolean[projection.size()]; int fieldCount = 0; while (parser.nextToken() != JsonToken.END_OBJECT) { @@ -136,17 +161,28 @@ public Object[] decode(@Nullable byte[] bytes) { } String fieldName = parser.currentName(); String path = JsonPath.field(JsonPath.ROOT, fieldName); - if (KafkaTableSchema.isReservedName(fieldName)) { - throw new KafkaRecordEncodingException( - "Invalid Kafka record value at " + path + ": reserved field."); - } + validateUnicode(fieldName, path); Integer position = projectedFields.get(fieldName); - if (position == null) { - throw new KafkaRecordEncodingException( - "Invalid Kafka record value at " + path + ": unknown field."); + if (position == null || KafkaTableSchema.isReservedName(fieldName)) { + if (!rescueEnabled) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + (KafkaTableSchema.isReservedName(fieldName) + ? ": reserved field." + : ": unknown field.")); + } + parser.nextToken(); + rescuedFields.set(fieldName, readValue(parser, null, path)); + continue; } parser.nextToken(); JsonNode value = readValue(parser, projection.dataTypeAt(position), path); + if (rescueEnabled) { + nestedRescues[position] = + extractNestedUnknownFields( + value, projection.dataTypeAt(position), path); + } values[position] = converters[position].convert(value, path); present[position] = true; } @@ -161,7 +197,24 @@ public Object[] decode(@Nullable byte[] bytes) { null, JsonPath.field(JsonPath.ROOT, projection.nameAt(i))); } } - return nestedType == null ? values : new Object[] {GenericRow.of(values)}; + Object[] decoded = values; + if (nestedType != null) { + decoded = new Object[rescueEnabled ? 2 : 1]; + decoded[0] = GenericRow.of(values); + } + if (rescueEnabled) { + // Root members retain input order; nested unknowns follow projection order. + for (int i = 0; i < projection.size(); i++) { + if (nestedRescues[i] != null) { + rescuedFields.set(projection.nameAt(i), nestedRescues[i]); + } + } + decoded[decoded.length - 1] = + rescuedFields.isEmpty() + ? null + : BinaryString.fromString(rescuedFields.toString()); + } + return decoded; } catch (KafkaRecordEncodingException e) { throw e; } catch (IOException | RuntimeException e) { @@ -170,8 +223,11 @@ public Object[] decode(@Nullable byte[] bytes) { } } - private static JsonNode readValue(JsonParser parser, DataType dataType, String path) + private JsonNode readValue(JsonParser parser, @Nullable DataType dataType, String path) throws IOException { + if (dataType == null) { + return readBoundedTree(parser, null, path); + } if (parser.currentToken() == JsonToken.START_OBJECT || parser.currentToken() == JsonToken.START_ARRAY) { switch (dataType.getTypeRoot()) { @@ -200,18 +256,19 @@ private static JsonNode readValue(JsonParser parser, DataType dataType, String p return readBoundedTree(parser, dataType, path); } - private static JsonNode readBoundedTree(JsonParser parser, DataType dataType, String path) + private JsonNode readBoundedTree(JsonParser parser, @Nullable DataType dataType, String path) throws IOException { // Bound every container while consuming tokens, before allocating excess children. // Read each child with its schema so numeric tokens retain their original precision // and floating-point sign. The parser also bounds depth and rejects duplicate fields. switch (parser.currentToken()) { case START_ARRAY: - if (!(dataType instanceof ArrayType)) { + if (dataType != null && !(dataType instanceof ArrayType)) { throw new KafkaRecordEncodingException( "Invalid Kafka record value at " + path + ": expected a JSON object."); } - DataType elementType = ((ArrayType) dataType).getElementType(); + DataType elementType = + dataType == null ? null : ((ArrayType) dataType).getElementType(); ArrayNode array = OBJECT_MAPPER.createArrayNode(); while (parser.nextToken() != JsonToken.END_ARRAY) { checkContainerSize(array.size() + 1, path); @@ -234,11 +291,12 @@ private static JsonNode readBoundedTree(JsonParser parser, DataType dataType, St checkContainerSize(object.size() + 1, path); String fieldName = parser.currentName(); String fieldPath = JsonPath.field(path, fieldName); + validateUnicode(fieldName, fieldPath); DataType fieldType = dataType instanceof MapType ? ((MapType) dataType).getValueType() : rowFields.get(fieldName); - if (fieldType == null) { + if (fieldType == null && dataType != null && !rescueEnabled) { throw new KafkaRecordEncodingException( "Invalid Kafka record value at " + fieldPath + ": unknown field."); } @@ -247,10 +305,27 @@ private static JsonNode readBoundedTree(JsonParser parser, DataType dataType, St } return object; default: - return OBJECT_MAPPER.readTree(parser); + JsonNode value = OBJECT_MAPPER.readTree(parser); + if (value.isTextual()) { + validateUnicode(value.textValue(), path); + } + // Jackson can produce a DoubleNode on overflow even in BigDecimal mode. + // Never turn a rescued numeric token into the JSON string "Infinity". + if ((value.isDouble() || value.isFloat()) + && !Double.isFinite(value.doubleValue())) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + ": non-finite or overflowing number."); + } + return value; } } + private static void validateUnicode(String value, String path) { + JsonToFlussConverters.validateUnicode(value, path, JSON_STRING_TYPE); + } + private static void checkContainerSize(int size, String path) { if (size > MAX_CONTAINER_ELEMENTS) { throw new KafkaRecordEncodingException( @@ -262,15 +337,102 @@ private static void checkContainerSize(int size, String path) { } } + private static JsonNode extractNestedUnknownFields( + JsonNode node, DataType dataType, String path) { + if (node == null || node.isNull()) { + return null; + } + if (dataType instanceof RowType && node.isObject()) { + return extractRowUnknownFields((ObjectNode) node, (RowType) dataType, path); + } + if (dataType instanceof ArrayType && node.isArray()) { + return extractArrayUnknownFields((ArrayNode) node, (ArrayType) dataType, path); + } + if (dataType instanceof MapType && node.isObject()) { + return extractMapValueUnknownFields((ObjectNode) node, (MapType) dataType, path); + } + return null; + } + + private static JsonNode extractRowUnknownFields(ObjectNode node, RowType rowType, String path) { + checkContainerSize(node.size(), path); + ObjectNode rescuedFields = OBJECT_MAPPER.createObjectNode(); + List fieldsToRemove = new ArrayList<>(); + Iterator> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + int fieldPosition = rowType.getFieldIndex(field.getKey()); + if (fieldPosition < 0) { + rescuedFields.set(field.getKey(), field.getValue()); + fieldsToRemove.add(field.getKey()); + } else { + JsonNode nestedRescue = + extractNestedUnknownFields( + field.getValue(), + rowType.getTypeAt(fieldPosition), + JsonPath.field(path, field.getKey())); + if (nestedRescue != null) { + rescuedFields.set(field.getKey(), nestedRescue); + } + } + } + for (String fieldName : fieldsToRemove) { + node.remove(fieldName); + } + return rescuedFields.isEmpty() ? null : rescuedFields; + } + + private static JsonNode extractArrayUnknownFields( + ArrayNode node, ArrayType arrayType, String path) { + checkContainerSize(node.size(), path); + ArrayNode rescuedElements = OBJECT_MAPPER.createArrayNode(); + boolean hasRescuedElement = false; + int elementPosition = 0; + for (JsonNode element : node) { + JsonNode rescuedElement = + extractNestedUnknownFields( + element, + arrayType.getElementType(), + JsonPath.index(path, elementPosition)); + if (rescuedElement == null) { + rescuedElements.addNull(); + } else { + rescuedElements.add(rescuedElement); + hasRescuedElement = true; + } + elementPosition++; + } + return hasRescuedElement ? rescuedElements : null; + } + + private static JsonNode extractMapValueUnknownFields( + ObjectNode node, MapType mapType, String path) { + checkContainerSize(node.size(), path); + ObjectNode rescuedValues = OBJECT_MAPPER.createObjectNode(); + Iterator> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + JsonNode rescuedValue = + extractNestedUnknownFields( + field.getValue(), + mapType.getValueType(), + JsonPath.field(path, field.getKey())); + if (rescuedValue != null) { + rescuedValues.set(field.getKey(), rescuedValue); + } + } + return rescuedValues.isEmpty() ? null : rescuedValues; + } + private Object[] nullValues() { if (nestedType != null) { if (!nestedType.isNullable()) { throw new KafkaRecordEncodingException( "Kafka null value cannot populate NOT NULL field '_kafka_value'."); } - return new Object[] {null}; + return new Object[rescueEnabled ? 2 : 1]; } - Object[] values = new Object[projection.size()]; + Object[] values = new Object[projection.size() + (rescueEnabled ? 1 : 0)]; for (int i = 0; i < projection.size(); i++) { DataType dataType = projection.dataTypeAt(i); if (!dataType.isNullable()) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java index a09be5218d4..7a9628966b7 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java @@ -23,6 +23,7 @@ import org.apache.fluss.kafka.format.KafkaFieldDecoder; import org.apache.fluss.kafka.format.KafkaFormatFactory; import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; import javax.annotation.Nullable; @@ -40,6 +41,11 @@ public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { return new JsonKafkaFieldDecoder(projection); } + @Override + public KafkaFieldDecoder createDecoder(KafkaTableSchema schema) { + return new JsonKafkaFieldDecoder(schema); + } + @Override public KafkaFieldDecoder createDecoder( KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java index e030889a076..c5fce8b4d13 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java @@ -429,7 +429,7 @@ private static String textualValue(JsonNode node, String path, DataType dataType return value; } - private static void validateUnicode(String value, String path, DataType dataType) { + static void validateUnicode(String value, String path, DataType dataType) { // Jackson can retain unpaired JSON escapes in Java strings. Reject them before // BinaryString's UTF-8 encoding silently replaces them with a question mark. for (int i = 0; i < value.length(); i++) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index 6b84fe7c675..8d5465ca6a7 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -43,7 +43,7 @@ public KafkaTableSchema resolve(TableDescriptor table) { KafkaTableSchema schema = KafkaTableSchema.resolve(table); if (schema.valueConversionType() == KafkaValueConversionType.PARSE_JSON) { // Metadata and Produce must reject schemas beyond the installed decoder support. - new JsonKafkaFieldDecoder(schema.valueProjection(), schema.valueConversionLayout()); + new JsonKafkaFieldDecoder(schema); } return schema; } catch (InvalidTableException | IllegalArgumentException e) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 3f9bb7b53ef..e990e70c4e4 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -59,11 +59,7 @@ public BytesView transcode(int partitionId, List records, KafkaTopicWrit checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTableSchema schema = plan.topicSchema(); - KafkaFieldDecoder valueDecoder = - formats.createDecoder( - schema.valueConversionType(), - schema.valueProjection(), - schema.valueConversionLayout()); + KafkaFieldDecoder valueDecoder = formats.createDecoder(schema); KafkaRowAssembler assembler = plan.rowAssembler(); return arrowRecordEncoder.encodeStreaming( consumer -> { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index b5cb170b483..289b5bd7177 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; @@ -56,7 +57,13 @@ public KafkaRowAssembler(KafkaTableSchema topicSchema) { */ public GenericRow assemble(Object[] valueValues, Record record, int partitionId) { GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); - setProjectedFields(row, topicSchema.valueProjection(), valueValues); + boolean withRescue = + topicSchema.valueConversionType() == KafkaValueConversionType.PARSE_JSON + && topicSchema.rescuePosition() >= 0; + setProjectedFields(row, topicSchema.valueProjection(), valueValues, withRescue); + if (withRescue) { + row.setField(topicSchema.rescuePosition(), valueValues[valueValues.length - 1]); + } if (topicSchema.keyPosition() >= 0) { int position = topicSchema.keyPosition(); byte[] key = record.borrowedKey(); @@ -109,16 +116,17 @@ private static int timestampTypeId(TimestampType type) { } private static void setProjectedFields( - GenericRow row, KafkaFieldProjection projection, Object[] values) { - if (values.length != projection.size()) { + GenericRow row, KafkaFieldProjection projection, Object[] values, boolean withRescue) { + int expected = projection.size() + (withRescue ? 1 : 0); + if (values.length != expected) { throw new IllegalArgumentException( "Kafka decoder returned " + values.length - + " fields for a projection of " - + projection.size() + + " fields but the value/rescue contract requires " + + expected + "."); } - for (int i = 0; i < values.length; i++) { + for (int i = 0; i < projection.size(); i++) { Object value = values[i]; if (value == null && !projection.dataTypeAt(i).isNullable()) { throw new KafkaRecordEncodingException( diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java index 4f8c5d90312..1a494192660 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -38,11 +38,7 @@ public final class KafkaTopicWritePlan { this.tableInfo = checkNotNull(tableInfo); this.topicSchema = checkNotNull(topicSchema); // Validate supported value formats and projections when compiling the plan. - new KafkaFormatFactoryRegistry() - .createDecoder( - topicSchema.valueConversionType(), - topicSchema.valueProjection(), - topicSchema.valueConversionLayout()); + new KafkaFormatFactoryRegistry().createDecoder(topicSchema); this.rowAssembler = new KafkaRowAssembler(topicSchema); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 732bd619bff..3b04b5294cc 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -26,6 +26,7 @@ import org.apache.fluss.cluster.ServerNode; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.KafkaTableSchema; @@ -35,9 +36,12 @@ import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.InternalMap; import org.apache.fluss.row.InternalRow; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; @@ -238,24 +242,32 @@ void testInvalidComplexPartitionIsAtomicAndDoesNotBlockOtherPartitions( TablePath.of(DATABASE, "json_complex_rejections_" + layout + "_" + withRescue); int metadataPosition = layout.equals("nested") ? 3 : COMPLEX_TYPE.getFieldCount() + 2; List invalidValues = - Arrays.asList( - "{\"id\":1,\"customer\":{\"name\":\"Alice\",\"unknown\":1}}", - "{\"id\":1,\"customer\":{\"name\":[]}}", - "{\"id\":1,\"customer\":{\"name\":true}}", - "{\"id\":1,\"customer\":{\"name\":1}}", - "{\"id\":1,\"customer\":[]}", - "{\"id\":1,\"float_values\":[1,\"bad\"]}", - "{\"id\":1,\"double_values\":[1,\"bad\"]}", - "{\"id\":1,\"float_values\":{}}", - "{\"id\":1,\"float_attributes\":[]}", - "{\"id\":1,\"double_attributes\":{\"bad\":\"bad\"}}", - "{\"id\":1,\"items\":[{\"float_value\":\"bad\"}]}", - "{\"id\":1,\"items\":[{\"unknown\":1}]}", - "{\"id\":1,\"attributes\":{\"bad\":{\"amount\":\"bad\"}}}", - "{\"id\":1,\"attributes\":{\"bad\":{\"unknown\":1}}}", - // The escaped surrogate must not be encoded as '?' and overwrite that key. - "{\"id\":1,\"float_attributes\":{\"?\":1,\"\\ud800\":2}}", - "{\"id\":1,\"double_attributes\":{\"?\":1,\"\\udc00\":2}}"); + new ArrayList<>( + Arrays.asList( + "{\"id\":1,\"customer\":{\"name\":\"Alice\",\"unknown\":1}}", + "{\"id\":1,\"customer\":{\"name\":[]}}", + "{\"id\":1,\"customer\":{\"name\":true}}", + "{\"id\":1,\"customer\":{\"name\":1}}", + "{\"id\":1,\"customer\":[]}", + "{\"id\":1,\"float_values\":[1,\"bad\"]}", + "{\"id\":1,\"double_values\":[1,\"bad\"]}", + "{\"id\":1,\"float_values\":{}}", + "{\"id\":1,\"float_attributes\":[]}", + "{\"id\":1,\"double_attributes\":{\"bad\":\"bad\"}}", + "{\"id\":1,\"items\":[{\"float_value\":\"bad\"}]}", + "{\"id\":1,\"items\":[{\"unknown\":1}]}", + "{\"id\":1,\"attributes\":{\"bad\":{\"amount\":\"bad\"}}}", + "{\"id\":1,\"attributes\":{\"bad\":{\"unknown\":1}}}", + // The escaped surrogate must not be encoded as '?' and overwrite + // that key. + "{\"id\":1,\"float_attributes\":{\"?\":1,\"\\ud800\":2}}", + "{\"id\":1,\"double_attributes\":{\"?\":1,\"\\udc00\":2}}")); + if (withRescue) { + invalidValues.removeIf(value -> value.contains("\"unknown\"")); + } + invalidValues.add("{\"id\":1,\"measurements\":{\"float_value\":1e40}}"); + invalidValues.add( + "{\"id\":1,\"measurements\":{\"float_value\":0,\"double_value\":0,\"rounded_float\":0,\"amount\":1.12345678901}}"); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { create(admin, path, descriptor(COMPLEX_TYPE, layout, true, withRescue, 2)); @@ -559,24 +571,43 @@ void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( "{", "null", "{\"id\":\"bad\"}", - "{\"id\":1,\"unknown\":2}", + "{\"id\":[]}", "{\"id\":1} {}", "{\"id\":1,\"id\":2}")); - for (String reserved : - new String[] { - KafkaTableSchema.KEY_COLUMN, - KafkaTableSchema.HEADERS_COLUMN, - KafkaTableSchema.METADATA_COLUMN, - KafkaTableSchema.RESCUE_COLUMN, - KafkaTableSchema.VALUE_COLUMN - }) { - // This also rejects the erroneous nested payload wrapper {"_kafka_value":{...}}. - String value = - reserved.equals(KafkaTableSchema.METADATA_COLUMN) - ? "{\"timestamp\":0,\"timestamp_type\":1,\"partition\":99}" - : "{\"id\":2}"; - invalidValues.add("{\"id\":1,\"" + reserved + "\":" + value + "}"); + if (!withRescue) { + invalidValues.add("{\"id\":1,\"unknown\":2}"); + for (String reserved : + new String[] { + KafkaTableSchema.KEY_COLUMN, + KafkaTableSchema.HEADERS_COLUMN, + KafkaTableSchema.METADATA_COLUMN, + KafkaTableSchema.RESCUE_COLUMN, + KafkaTableSchema.VALUE_COLUMN + }) { + // This also rejects the erroneous nested payload wrapper {"_kafka_value":{...}}. + String value = + reserved.equals(KafkaTableSchema.METADATA_COLUMN) + ? "{\"timestamp\":0,\"timestamp_type\":1,\"partition\":99}" + : "{\"id\":2}"; + invalidValues.add("{\"id\":1,\"" + reserved + "\":" + value + "}"); + } + } + invalidValues.add("{\"id\":2147483648}"); + invalidValues.add("{\"id\":1,\"_kafka_rescue\":\"\\ud800\"}"); + invalidValues.add("{\"id\":1,\"extra\":{\"x\":1,\"x\":2}}"); + StringBuilder oversized = new StringBuilder("{\"id\":1,\"extra\":["); + for (int i = 0; i < 10000; i++) { + if (i > 0) { + oversized.append(','); + } + oversized.append('0'); } + invalidValues.add(oversized.append(",0]}").toString()); + String deep = "0"; + for (int i = 0; i < 67; i++) { + deep = "[" + deep + "]"; + } + invalidValues.add("{\"id\":1,\"extra\":" + deep + "}"); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { create(admin, path, descriptor(valueType, layout, true, withRescue, 2)); @@ -610,7 +641,7 @@ void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); assertThat(accepted.baseOffset()).isEqualTo(index); } - // Even when the schema declares rescue, unsupported capture must not lose data. + // Known-field and JSON validation failures remain atomic with rescue enabled. // A later successful write must be the first actual append to the rejected bucket. assertThat( producer.send( @@ -798,6 +829,185 @@ private static void assertMetadata( assertThat(metadata.getInt(2)).isEqualTo(partition); } + @ParameterizedTest + @CsvSource({ + "flat,false,false", + "flat,false,true", + "flat,true,false", + "flat,true,true", + "nested,false,false", + "nested,false,true", + "nested,true,false", + "nested,true,true" + }) + void testRescueRoundTripWithReorderedSystemColumns( + String layout, boolean withSystemColumns, boolean rescueFirst) throws Exception { + TablePath path = + TablePath.of( + DATABASE, + "json_rescue_" + layout + "_" + withSystemColumns + "_" + rescueFirst); + TableDescriptor descriptor = + reorderedRescueDescriptor(layout, withSystemColumns, rescueFirst); + KafkaTableSchema contract = KafkaTableSchema.resolve(descriptor); + ObjectMapper mapper = new ObjectMapper(); + List payloads = new ArrayList<>(); + List expected = new ArrayList<>(); + String extra = "{\"values\":[1,null,2]}"; + String[] rescueValues = {"null", "123", "\"text\"", "{\"a\":[1,null]}"}; + String[] zeros = {"-0", "-0.0", "-0e0", "-0"}; + for (int i = 0; i < rescueValues.length; i++) { + String json = + complexJson(i, zeros[i]) + .replace( + "\"amount\":12345678901234567890.1234567890}", + "\"amount\":12345678901234567890.1234567890,\"extra\":" + + extra + + "}"); + String collisions = + "\"_kafka_key\":{\"id\":2},\"_kafka_headers\":null," + + "\"_kafka_metadata\":{\"id\":2},\"_kafka_value\":{\"id\":2}," + + "\"_kafka_rescue\":" + + rescueValues[i]; + payloads.add(json.substring(0, json.length() - 1) + "," + collisions + "}"); + expected.add( + mapper.readTree( + "{" + + collisions + + ",\"measurements\":{\"extra\":" + + extra + + "},\"items\":[{\"extra\":" + + extra + + "}],\"attributes\":{\"dynamic\":{\"extra\":" + + extra + + "}}}")); + } + payloads.add(complexJson(4, "-0e0")); + expected.add(null); + // These were rejection cases in the inherited no-capture implementation. + payloads.add("{\"id\":1,\"unknown\":2}"); + expected.add(mapper.readTree("{\"unknown\":2}")); + payloads.add("{\"id\":1,\"customer\":{\"name\":\"Alice\",\"unknown\":1}}"); + expected.add(mapper.readTree("{\"customer\":{\"unknown\":1}}")); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor); + try (KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(1); + for (int i = 0; i < payloads.size(); i++) { + assertThat( + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + 123L, + KEY, + bytes(payloads.get(i)), + Arrays.asList( + new RecordHeader( + "source", + new byte[] {1, 2}), + new RecordHeader( + "nullable", null)))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(i); + } + read( + connection, + path, + 0, + payloads.size(), + (physical, index) -> { + InternalRow value = jsonBusinessRow(physical, contract); + assertThat(value.getInt(0)).isEqualTo(index < 5 ? index : 1); + if (index < 5) { + InternalRow customer = + value.getRow(1, CUSTOMER_TYPE.getFieldCount()); + assertThat(customer.getString(0).toString()).isEqualTo("Alice"); + for (int field = 1; + field < CUSTOMER_TYPE.getFieldCount(); + field++) { + assertThat(customer.getInt(field)).isEqualTo(field); + } + assertNumbers(value.getRow(2, NUMBER_TYPE.getFieldCount())); + assertNumbers( + value.getArray(7).getRow(0, NUMBER_TYPE.getFieldCount())); + assertNumbers( + value.getMap(8) + .valueArray() + .getRow(0, NUMBER_TYPE.getFieldCount())); + } + if (expected.get(index) == null) { + assertThat(physical.isNullAt(contract.rescuePosition())).isTrue(); + } else { + try { + assertThat( + mapper.readTree( + physical.getString( + contract + .rescuePosition()) + .toString())) + .isEqualTo(expected.get(index)); + } catch (java.io.IOException e) { + throw new AssertionError(e); + } + } + if (withSystemColumns) { + assertThat(physical.getBytes(contract.keyPosition())) + .isEqualTo(KEY); + assertHeaders(physical.getMap(contract.headersPosition()), true); + } + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + private static InternalRow jsonBusinessRow(InternalRow row, KafkaTableSchema contract) { + if (contract.valueConversionLayout() == KafkaValueConversionLayout.NESTED) { + return row.getRow( + contract.valueProjection().positionAt(0), COMPLEX_TYPE.getFieldCount()); + } + GenericRow business = new GenericRow(COMPLEX_TYPE.getFieldCount()); + for (int i = 0; i < business.getFieldCount(); i++) { + business.setField( + i, + InternalRow.createFieldGetter( + COMPLEX_TYPE.getTypeAt(i), + contract.valueProjection().positionAt(i)) + .getFieldOrNull(row)); + } + return business; + } + + private static TableDescriptor reorderedRescueDescriptor( + String layout, boolean withSystemColumns, boolean rescueFirst) { + List columns = + new ArrayList<>( + descriptor(COMPLEX_TYPE, layout, withSystemColumns, false, 1) + .getSchema() + .getColumns()); + Schema.Builder schema = Schema.newBuilder(); + int rescuePosition = rescueFirst ? 0 : columns.size() / 2 + columns.size() % 2; + for (int i = 0; i <= columns.size(); i++) { + if (i == rescuePosition) { + schema.column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()); + } + if (i < columns.size()) { + schema.column(columns.get(i).getName(), columns.get(i).getDataType()); + } + } + return TableDescriptor.builder() + .schema(schema.build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), layout) + .build(); + } + private static void assertNegativeZero(float value) { assertThat(Float.floatToRawIntBits(value)).isEqualTo(Float.floatToRawIntBits(-0.0F)); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index 23525036512..39603e33a01 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -983,11 +983,11 @@ void testLegacyMetadataOffsetsRejectedAndJsonNullValuesPersisted(boolean nested) @ParameterizedTest @ValueSource(booleans = {false, true}) - void testJsonConversionRejectsWholeBadPartitionAndPreservesOtherPartitions(boolean nested) - throws Exception { + void testJsonConversionHonorsRescueAndPartitionAtomicity(boolean nested) throws Exception { List invalidValues = - new ArrayList<>( - Arrays.asList("{", "[]", "{\"id\":\"bad\"}", "{\"id\":1,\"unknown\":2}")); + new ArrayList<>(Arrays.asList("{", "[]", "{\"id\":\"bad\"}", "{} {}")); + List rescuableValues = + new ArrayList<>(Collections.singletonList("{\"id\":1,\"unknown\":2}")); for (String reserved : Arrays.asList( "_kafka_key", @@ -995,11 +995,18 @@ void testJsonConversionRejectsWholeBadPartitionAndPreservesOtherPartitions(boole "_kafka_metadata", "_kafka_rescue", "_kafka_value")) { - invalidValues.add("{\"id\":1,\"" + reserved + "\":null}"); + rescuableValues.add("{\"id\":1,\"" + reserved + "\":null}"); } + invalidValues.addAll(rescuableValues); for (boolean rescue : new boolean[] {false, true}) { for (String invalid : invalidValues) { + boolean captured = rescue && rescuableValues.contains(invalid); TestingProduceService service = new TestingProduceService(); + if (captured) { + ProduceLogResponse response = success(0); + response.addAllBucketsResps(success(1).getBucketsRespsList()); + service.pendingAppend = CompletableFuture.completedFuture(response); + } service.pendingMetadata = CompletableFuture.completedFuture( metadata(false) @@ -1024,12 +1031,17 @@ void testJsonConversionRejectsWholeBadPartitionAndPreservesOtherPartitions(boole .join(); assertErrors( result, - Errors.INVALID_RECORD, + captured ? Errors.NONE : Errors.INVALID_RECORD, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); assertThat(service.appends).hasSize(1); - assertThat(service.append.getBucketsReqsList()).hasSize(1); - assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + assertThat(service.append.getBucketsReqsList()).hasSize(captured ? 2 : 1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()) + .isEqualTo(captured ? 0 : 1); + if (captured) { + assertThat(result.topics().get(0).partitions().get(0).baseOffset()) + .isEqualTo(17L); + } assertThat(result.topics().get(0).partitions().get(1).baseOffset()).isEqualTo(17L); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java index 77f5b445b24..fdc4f7a2568 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java @@ -20,8 +20,10 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.kafka.transcode.KafkaRowAssembler; import org.apache.fluss.metadata.KafkaFieldProjection; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; @@ -35,6 +37,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import java.nio.charset.StandardCharsets; @@ -114,8 +117,7 @@ void testFactoriesUseThePublicValueProjection(boolean string) { .build(); KafkaTableSchema schema = KafkaTableSchema.resolve(table); assertThat(schema.valueProjection().positions()).containsExactly(1); - KafkaFieldDecoder decoder = - formats.createDecoder(schema.valueConversionType(), schema.valueProjection()); + KafkaFieldDecoder decoder = formats.createDecoder(schema); byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); assertThat(decoder.decode(payload)) .containsExactly(string ? BinaryString.fromString("payload") : payload); @@ -207,6 +209,68 @@ void testJsonLayoutDeterminesPhysicalValueContainer(boolean nested) { .hasMessageContaining("reserved field"); } + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testSchemaFactoryAndAssemblerKeepRescueSeparateFromPhysicalOrder( + String layout, boolean first) { + Schema.Builder schema = Schema.newBuilder(); + if (first) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + schema.column("_kafka_key", DataTypes.BYTES()); + if (!first) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + if (layout.equals("nested")) { + schema.column("_kafka_value", DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT()))); + } else { + schema.column("id", DataTypes.INT()); + } + KafkaTableSchema contract = + KafkaTableSchema.resolve( + TableDescriptor.builder() + .schema(schema.build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + layout) + .build()); + KafkaFieldDecoder decoder = formats.createDecoder(contract); + KafkaRowAssembler assembler = new KafkaRowAssembler(contract); + Object[] values = decoder.decode("{\"id\":7,\"extra\":2}".getBytes(StandardCharsets.UTF_8)); + assertThat(values).hasSize(2); + byte[] key = {(byte) 0xff}; + Record record = new Record(123L, key, null, Collections.emptyList()); + InternalRow row = assembler.assemble(values, record, 7); + assertThat(row.getString(contract.rescuePosition()).toString()).isEqualTo("{\"extra\":2}"); + assertThat(row.getBytes(contract.keyPosition())).isEqualTo(key); + int valuePosition = contract.valueProjection().positionAt(0); + assertThat( + layout.equals("nested") + ? row.getRow(valuePosition, 1).getInt(0) + : row.getInt(valuePosition)) + .isEqualTo(7); + assertThat( + assembler + .assemble( + decoder.decode( + "{\"id\":8}".getBytes(StandardCharsets.UTF_8)), + record, + 7) + .isNullAt(contract.rescuePosition())) + .isTrue(); + for (Object[] invalid : new Object[][] {new Object[1], new Object[3]}) { + assertThatThrownBy(() -> assembler.assemble(invalid, record, 7)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("contract requires 2"); + } + } + private static KafkaFieldProjection projection(DataType type) { return new KafkaFieldProjection( DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type)), diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java index bda54b1b9b1..64e47526291 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java @@ -17,9 +17,15 @@ package org.apache.fluss.kafka.format.json; -import org.apache.fluss.kafka.schema.KafkaFieldProjection; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; import org.apache.fluss.row.GenericArray; import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; @@ -27,6 +33,7 @@ import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; +import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; @@ -41,6 +48,8 @@ import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Stream; @@ -68,11 +77,9 @@ void testUnexpectedConversionFailureIsNotRescued() { @ParameterizedTest @MethodSource("invalidKnownValues") - void testKnownFailuresRemainFatalAcrossNullableAndRequiredContainers( + void testKnownFailuresRemainFatalAcrossNullableContainers( DataType dataType, String jsonValue, String reason) { - for (boolean nullable : new boolean[] {true, false}) { - assertRejectedInAllContainers(dataType.copy(nullable), jsonValue, reason); - } + assertRejectedInAllContainers(dataType.copy(true), jsonValue, reason); } private static Stream invalidKnownValues() { @@ -113,50 +120,48 @@ private static Stream invalidKnownValues() { private static void assertRejectedInAllContainers( DataType dataType, String value, String reason) { - JsonKafkaFieldDecoder decoder = - rescueDecoder( - DataTypes.ROW( - DataTypes.FIELD("value", dataType), - DataTypes.FIELD( - "row", DataTypes.ROW(DataTypes.FIELD("value", dataType))), - DataTypes.FIELD("items", DataTypes.ARRAY(dataType)), - DataTypes.FIELD( - "attributes", DataTypes.MAP(DataTypes.STRING(), dataType)), - DataTypes.FIELD("rescue", DataTypes.STRING())), - "rescue"); - String[][] cases = { - {"\"value\":" + value, "$[\"value\"]"}, - {"\"row\":{\"value\":" + value + ",\"extra\":2}", "$[\"row\"][\"value\"]"}, - {"\"items\":[" + value + "]", "$[\"items\"][0]"}, - {"\"attributes\":{\"dynamic\":" + value + "}", "$[\"attributes\"][\"dynamic\"]"} - }; - for (String[] entry : cases) { - String json = "{\"extra\":1," + entry[0] + "}"; - assertThatThrownBy(() -> decoder.decode(bytes(json))) - .as("%s: %s", dataType, json) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining(entry[1]) - .hasMessageContaining(reason); + for (boolean nested : new boolean[] {false, true}) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("value", dataType), + DataTypes.FIELD( + "row", + DataTypes.ROW(DataTypes.FIELD("value", dataType))), + DataTypes.FIELD("items", DataTypes.ARRAY(dataType)), + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.STRING(), dataType)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())), + nested); + String[][] cases = { + {"\"value\":" + value, "$[\"value\"]"}, + {"\"row\":{\"value\":" + value + ",\"extra\":2}", "$[\"row\"][\"value\"]"}, + {"\"items\":[" + value + "]", "$[\"items\"][0]"}, + {"\"attributes\":{\"dynamic\":" + value + "}", "$[\"attributes\"][\"dynamic\"]"} + }; + for (String[] entry : cases) { + String json = "{\"extra\":1," + entry[0] + "}"; + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .as("%s: %s", dataType, json) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining(entry[1]) + .hasMessageContaining(reason); + } } } @Test - void testNullableParentNeverRescuesRequiredDescendantFailures() { - assertRejectedInAllContainers( - DataTypes.INT().copy(false), "null", "field is missing or null"); - RowType required = DataTypes.ROW(DataTypes.FIELD("required", DataTypes.INT().copy(false))); - for (String value : new String[] {"{\"required\":\"bad\"}", "{}", "{\"required\":null}"}) { - assertRejectedInAllContainers(required, value, "required"); + void testNotNullBusinessTypesAreRejectedBeforeRescue() { + RowType required = + DataTypes.ROW( + DataTypes.FIELD("value", DataTypes.INT().copy(false)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + for (boolean nested : new boolean[] {false, true}) { + assertThatThrownBy(() -> decoder(required, nested)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("nullable fields"); } - JsonKafkaFieldDecoder decoder = - rescueDecoder( - DataTypes.ROW( - DataTypes.FIELD("required", DataTypes.INT().copy(false)), - DataTypes.FIELD("rescue", DataTypes.STRING())), - "rescue"); - assertThatThrownBy(() -> decoder.decode(bytes("{\"extra\":1}"))) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("field is missing or null"); } @ParameterizedTest @@ -165,11 +170,11 @@ void testNullableParentNeverRescuesRequiredDescendantFailures() { "{", "[]", "{\"extra\":1,\"extra\":2}", - "{\"rescue\":null,\"rescue\":1}", + "{\"_kafka_rescue\":null,\"_kafka_rescue\":1}", "{\"extra\":{\"x\":1,\"x\":2}}", "{} {}", - "{\"rescue\":\"\\uD800\"}", - "{\"rescue\":1e400}" + "{\"_kafka_rescue\":\"\\uD800\"}", + "{\"_kafka_rescue\":1e400}" }) void testInvalidJsonCannotBeRescued(String json) { assertThatThrownBy(() -> fidelityDecoder().decode(bytes(json))) @@ -179,16 +184,15 @@ void testInvalidJsonCannotBeRescued(String json) { @Test void testRescueNameStillEnforcesResourceLimitsAndUtf8() { JsonKafkaFieldDecoder decoder = - rescueDecoder( - DataTypes.ROW(DataTypes.FIELD("rescue", DataTypes.STRING())), "rescue"); + rescueDecoder(DataTypes.ROW(DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); String atLimit = containerAtLimit(false); - assertThat(decoder.decode(bytes("{\"rescue\":" + atLimit + "}"))[0].toString()) - .isEqualTo("{\"rescue\":" + atLimit + "}"); + assertThat(decoder.decode(bytes("{\"_kafka_rescue\":" + atLimit + "}"))[0].toString()) + .isEqualTo("{\"_kafka_rescue\":" + atLimit + "}"); assertThatThrownBy( () -> decoder.decode( bytes( - "{\"rescue\":" + "{\"_kafka_rescue\":" + atLimit.substring(0, atLimit.length() - 1) + ",0]}"))) .isInstanceOf(KafkaRecordEncodingException.class) @@ -197,10 +201,10 @@ void testRescueNameStillEnforcesResourceLimitsAndUtf8() { for (int i = 0; i < JsonKafkaFieldDecoder.MAX_JSON_NESTING_DEPTH; i++) { nested = "[" + nested + "]"; } - byte[] tooDeep = bytes("{\"rescue\":" + nested + "}"); + byte[] tooDeep = bytes("{\"_kafka_rescue\":" + nested + "}"); assertThatThrownBy(() -> decoder.decode(tooDeep)) .isInstanceOf(KafkaRecordEncodingException.class); - byte[] invalidUtf8 = bytes("{\"rescue\":\"x\"}"); + byte[] invalidUtf8 = bytes("{\"_kafka_rescue\":\"x\"}"); invalidUtf8[invalidUtf8.length - 3] = (byte) 0xff; assertThatThrownBy(() -> decoder.decode(invalidUtf8)) .isInstanceOf(KafkaRecordEncodingException.class); @@ -210,7 +214,7 @@ void testRescueNameStillEnforcesResourceLimitsAndUtf8() { void testRescuesUnknownRootAndNestedFieldsAsJsonString() { RowType rowType = DataTypes.ROW( - DataTypes.FIELD("id", DataTypes.BIGINT().copy(false)), + DataTypes.FIELD("id", DataTypes.BIGINT()), DataTypes.FIELD( "customer", DataTypes.ROW(DataTypes.FIELD("name", DataTypes.STRING()))), @@ -223,8 +227,8 @@ void testRescuesUnknownRootAndNestedFieldsAsJsonString() { DataTypes.MAP( DataTypes.STRING(), DataTypes.ROW(DataTypes.FIELD("known", DataTypes.INT())))), - DataTypes.FIELD("kafka_rescue", DataTypes.STRING())); - JsonKafkaFieldDecoder decoder = rescueDecoder(rowType, "kafka_rescue"); + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + JsonKafkaFieldDecoder decoder = rescueDecoder(rowType); Object[] values = decoder.decode( @@ -236,7 +240,7 @@ void testRescuesUnknownRootAndNestedFieldsAsJsonString() { + "\"attributes\":{\"dynamic\":{\"known\":1," + "\"extra_map_value\":4}}," + "\"extra\":\"unexpected\"," - + "\"kafka_rescue\":null" + + "\"_kafka_rescue\":null" + "}")); assertThat(values[0]).isEqualTo(1L); @@ -248,7 +252,7 @@ void testRescuesUnknownRootAndNestedFieldsAsJsonString() { .isEqualTo(1); assertThat(((BinaryString) values[4]).toString()) .isEqualTo( - "{\"extra\":\"unexpected\",\"kafka_rescue\":null," + "{\"extra\":\"unexpected\",\"_kafka_rescue\":null," + "\"customer\":{\"extra_nested\":2}," + "\"items\":[{\"extra_item\":3}]," + "\"attributes\":{\"dynamic\":{\"extra_map_value\":4}}}"); @@ -269,53 +273,160 @@ void testRescuesUnknownRootAndNestedFieldsAsJsonString() { "7", "false", "[1,{\"x\":2}]", - "{\"rescue\":\"nested\",\"x\":1}" + "{\"_kafka_rescue\":\"nested\",\"x\":1}" }) void testPayloadRescueNameIsPreservedAsJsonMember(String value) { JsonKafkaFieldDecoder decoder = rescueDecoder( DataTypes.ROW( DataTypes.FIELD("id", DataTypes.INT()), - DataTypes.FIELD("rescue", DataTypes.STRING())), - "rescue"); - Object[] decoded = decoder.decode(bytes("{\"id\":1,\"rescue\":" + value + ",\"extra\":2}")); + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); + Object[] decoded = + decoder.decode(bytes("{\"id\":1,\"_kafka_rescue\":" + value + ",\"extra\":2}")); assertThat(decoded[0]).isEqualTo(1); - assertThat(decoded[1].toString()).isEqualTo("{\"rescue\":" + value + ",\"extra\":2}"); + assertThat(decoded[1].toString()) + .isEqualTo("{\"_kafka_rescue\":" + value + ",\"extra\":2}"); assertThat(decoder.decode(bytes("{\"id\":1}"))[1]).isNull(); - assertThat(decoder.decode(bytes("{\"rescue\":null}"))[1].toString()) - .isEqualTo("{\"rescue\":null}"); + assertThat(decoder.decode(bytes("{\"_kafka_rescue\":null}"))[1].toString()) + .isEqualTo("{\"_kafka_rescue\":null}"); } - @Test - void testRescueNameIsConfigurationDrivenAndNestedBusinessFieldsKeepTheirMeaning() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFixedReservedNamesAndDeepBusinessNames(boolean nested) { + String[] names = { + "_kafka_key", "_kafka_headers", "_kafka_metadata", "_kafka_rescue", "_kafka_value" + }; + List fields = new ArrayList<>(); + for (String name : names) { + fields.add(DataTypes.FIELD(name, DataTypes.STRING())); + } RowType schema = DataTypes.ROW( - DataTypes.FIELD("_kafka_key", DataTypes.STRING()), - DataTypes.FIELD("_kafka_headers", DataTypes.STRING()), - DataTypes.FIELD("_kafka_metadata", DataTypes.STRING()), - DataTypes.FIELD("_kafka_value", DataTypes.STRING()), - DataTypes.FIELD( - "row", DataTypes.ROW(DataTypes.FIELD("rescue", DataTypes.INT()))), + DataTypes.FIELD("row", new RowType(fields)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + StringBuilder json = new StringBuilder("{\"row\":{"); + for (int i = 0; i < names.length; i++) { + if (i > 0) { + json.append(','); + } + json.append('"').append(names[i]).append("\":\"business\""); + } + json.append("}"); + for (String name : names) { + json.append(",\"").append(name).append("\":null"); + } + json.append('}'); + Object[] values = decoder(schema, nested).decode(bytes(json.toString())); + GenericRow business = businessRow(values, nested); + GenericRow row = (GenericRow) business.getField(0); + for (int i = 0; i < names.length; i++) { + assertThat(row.getString(i).toString()).isEqualTo("business"); + } + assertThat(values[values.length - 1].toString()) + .isEqualTo( + "{\"_kafka_key\":null,\"_kafka_headers\":null,\"_kafka_metadata\":null,\"_kafka_rescue\":null,\"_kafka_value\":null}"); + RowType noRescue = + DataTypes.ROW( + DataTypes.FIELD("row", new RowType(fields)), DataTypes.FIELD("rescue", DataTypes.STRING())); - String json = - "{\"_kafka_key\":\"k\",\"_kafka_headers\":\"h\",\"_kafka_metadata\":\"m\"," - + "\"_kafka_value\":\"v\",\"row\":{\"rescue\":7,\"extra\":8}}"; - Object[] values = rescueDecoder(schema, "rescue").decode(bytes(json)); - for (int i = 0; i < 4; i++) { - assertThat(values[i].toString()).isEqualTo(new String[] {"k", "h", "m", "v"}[i]); + for (String name : names) { + assertThatThrownBy( + () -> + decoder(noRescue, nested) + .decode(bytes("{\"" + name + "\":null}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("reserved field"); } - assertThat(((GenericRow) values[4]).getInt(0)).isEqualTo(7); - assertThat(values[5].toString()).isEqualTo("{\"row\":{\"extra\":8}}"); - assertThatThrownBy(() -> rescueDecoder(schema, null).decode(bytes(json))) + assertThatThrownBy(() -> decoder(noRescue, nested).decode(bytes("{\"unknown\":1}"))) .isInstanceOf(KafkaRecordEncodingException.class) .hasMessageContaining("unknown field"); - assertThat( - rescueDecoder(schema, null) - .decode(bytes("{\"rescue\":\"business\"}"))[5] - .toString()) + Object[] plain = decoder(noRescue, nested).decode(bytes("{\"rescue\":\"business\"}")); + assertThat((nested ? ((GenericRow) plain[0]).getField(1) : plain[1]).toString()) .isEqualTo("business"); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRescueKeepsTypedNestedNumbersAndUnknownStructure(boolean nested) { + RowType numbers = + DataTypes.ROW( + DataTypes.FIELD("f", DataTypes.FLOAT()), + DataTypes.FIELD("d", DataTypes.DOUBLE()), + DataTypes.FIELD("n", DataTypes.DECIMAL(38, 18))); + RowType type = + DataTypes.ROW( + DataTypes.FIELD("row", numbers), + DataTypes.FIELD("array", DataTypes.ARRAY(numbers)), + DataTypes.FIELD("map", DataTypes.MAP(DataTypes.STRING(), numbers)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + String decimal = "12345678901234567890.123456789012345678"; + for (String token : + new String[] {"-0", "-0.0", "-0e0", "1.0000000596046448", "-1.0000000596046448"}) { + String row = + "{\"f\":" + + token + + ",\"d\":" + + token + + ",\"n\":" + + decimal + + ",\"extra\":{\"a\":[1,null,2]}}"; + Object[] decoded = + decoder(type, nested) + .decode( + bytes( + "{\"row\":" + + row + + ",\"array\":[null," + + row + + "],\"map\":{\"k\":" + + row + + "}}")); + GenericRow values = businessRow(decoded, nested); + GenericRow[] rows = { + (GenericRow) values.getField(0), + (GenericRow) ((GenericArray) values.getField(1)).getRow(1, 3), + (GenericRow) ((GenericMap) values.getField(2)).get(BinaryString.fromString("k")) + }; + for (GenericRow number : rows) { + assertThat(Float.floatToRawIntBits(number.getFloat(0))) + .isEqualTo(Float.floatToRawIntBits(Float.parseFloat(token))); + assertThat(Double.doubleToRawLongBits(number.getDouble(1))) + .isEqualTo(Double.doubleToRawLongBits(Double.parseDouble(token))); + assertThat(((Decimal) number.getField(2)).toBigDecimal()) + .isEqualTo(new BigDecimal(decimal)); + } + assertThat(decoded[decoded.length - 1].toString()) + .isEqualTo( + "{\"row\":{\"extra\":{\"a\":[1,null,2]}},\"array\":[null,{\"extra\":{\"a\":[1,null,2]}}],\"map\":{\"k\":{\"extra\":{\"a\":[1,null,2]}}}}"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRescueSharesTheMaximumBusinessDepthIncludingEmptyRow(boolean nested) { + DataType type = new RowType(Collections.emptyList()); + String json = "{}"; + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + type = DataTypes.ARRAY(type); + json = "[" + json + "]"; + } + RowType fields = + DataTypes.ROW( + DataTypes.FIELD("value", type), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + Object[] decoded = + decoder(fields, nested).decode(bytes("{\"value\":" + json + ",\"extra\":1}")); + assertThat(decoded[decoded.length - 1].toString()).isEqualTo("{\"extra\":1}"); + assertThat(decoder(fields, nested).decode(bytes("{}"))[1]).isNull(); + } + + private static GenericRow businessRow(Object[] values, boolean nested) { + return nested + ? (GenericRow) values[0] + : GenericRow.of(Arrays.copyOf(values, values.length - 1)); + } + @Test void testRescueArrayPreservesOriginalElementPositions() { RowType rowType = @@ -324,9 +435,9 @@ void testRescueArrayPreservesOriginalElementPositions() { "items", DataTypes.ARRAY( DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT())))), - DataTypes.FIELD("rescue", DataTypes.STRING())); + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); Object[] values = - rescueDecoder(rowType, "rescue") + rescueDecoder(rowType) .decode(bytes("{\"items\":[{\"id\":1},{\"id\":2,\"extra\":3}]}")); assertThat(((BinaryString) values[1]).toString()) .isEqualTo("{\"items\":[null,{\"extra\":3}]}"); @@ -336,8 +447,7 @@ void testRescueArrayPreservesOriginalElementPositions() { @Test void testRescuesRootObjectAtLimitAndRejectsExcessFieldBeforeReadingItsValue() { JsonKafkaFieldDecoder decoder = - rescueDecoder( - DataTypes.ROW(DataTypes.FIELD("rescue", DataTypes.STRING())), "rescue"); + rescueDecoder(DataTypes.ROW(DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); String object = containerAtLimit(true); assertThat(((BinaryString) decoder.decode(bytes(object))[0]).toString()).isEqualTo(object); @@ -360,8 +470,7 @@ void testBoundsUnknownContainersBeforeReadingExcessChild( DataTypes.FIELD("items", DataTypes.ARRAY(nestedRow)), DataTypes.FIELD( "attributes", DataTypes.MAP(DataTypes.STRING(), nestedRow)), - DataTypes.FIELD("rescue", DataTypes.STRING())), - "rescue"); + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); String container = containerAtLimit(object); String json = prefix + container + suffix; assertThat(((BinaryString) decoder.decode(bytes(json))[3]).toString()).isEqualTo(json); @@ -423,8 +532,7 @@ void testRescuePreservesRootFloatToken(String number) { rescueDecoder( DataTypes.ROW( DataTypes.FIELD("value", DataTypes.FLOAT()), - DataTypes.FIELD("rescue", DataTypes.STRING())), - "rescue"); + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); Object[] values = decoder.decode(bytes("{\"extra\":1,\"value\":" + number + "}")); assertThat(Float.floatToRawIntBits((Float) values[0])) .isEqualTo(Float.floatToRawIntBits(Float.parseFloat(number))); @@ -438,8 +546,7 @@ void testRescueRejectsContainersForKnownScalarsBeforeReadingTheirContents(String rescueDecoder( DataTypes.ROW( DataTypes.FIELD("value", DataTypes.STRING()), - DataTypes.FIELD("rescue", DataTypes.STRING())), - "rescue"); + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); assertThatThrownBy(() -> decoder.decode(bytes("{\"extra\":1,\"value\":" + container))) .isInstanceOf(KafkaRecordEncodingException.class) .hasMessageContaining("$[\"value\"]") @@ -537,8 +644,7 @@ private static JsonKafkaFieldDecoder fidelityDecoder() { DataTypes.FIELD("details", nestedRow), DataTypes.FIELD("items", DataTypes.ARRAY(nestedRow)), DataTypes.FIELD("attributes", DataTypes.MAP(DataTypes.STRING(), nestedRow)), - DataTypes.FIELD("rescue", DataTypes.STRING())), - "rescue"); + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); } private static String containerAtLimit(boolean object) { @@ -555,13 +661,43 @@ private static String containerAtLimit(boolean object) { return json.append(object ? '}' : ']').toString(); } - private static JsonKafkaFieldDecoder rescueDecoder(RowType rowType, String rescueColumn) { - List positions = new ArrayList<>(); - for (int i = 0; i < rowType.getFieldCount(); i++) { - positions.add(i); + private static JsonKafkaFieldDecoder rescueDecoder(RowType rowType) { + return decoder(rowType, false); + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType, boolean nested) { + Schema.Builder schema = Schema.newBuilder(); + List business = new ArrayList<>(); + for (DataField field : rowType.getFields()) { + if (!KafkaTableSchema.RESCUE_COLUMN.equals(field.getName())) { + business.add(field); + } + } + if (nested) { + schema.column(KafkaTableSchema.VALUE_COLUMN, new RowType(business)); + } else { + for (DataField field : business) { + schema.column(field.getName(), field.getType()); + } + } + int rescue = rowType.getFieldIndex(KafkaTableSchema.RESCUE_COLUMN); + if (rescue >= 0) { + schema.column(KafkaTableSchema.RESCUE_COLUMN, rowType.getTypeAt(rescue)); } return new JsonKafkaFieldDecoder( - new KafkaFieldProjection(rowType, positions), rescueColumn); + KafkaTableSchema.resolve( + TableDescriptor.builder() + .schema(schema.build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + nested ? "nested" : "flat") + .build())); } private static byte[] bytes(String value) { From 691d9c97c65448499c142523d131086bb316a6a1 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 10:11:37 +0800 Subject: [PATCH 67/69] [test] Verify rescue with three-field Kafka metadata Verify that forged JSON metadata is collected without replacing the input record timestamp, batch timestamp type or requested partition. Cover both layouts, reordered rescue columns, absent timestamps, NOT NULL failures and atomic recovery through native readback. Retain typed nested floating-point underflow and subnormal rounding when unknown fields are collected. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 115/115 --- .../fluss/kafka/KafkaJsonProduceITCase.java | 53 ++++++++++++------- .../KafkaFormatFactoryRegistryTest.java | 22 ++++++-- .../format/json/JsonKafkaRescueTest.java | 12 ++++- 3 files changed, 64 insertions(+), 23 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 3b04b5294cc..56d9a8b3ea0 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -72,7 +72,6 @@ import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.ValueSource; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -99,6 +98,8 @@ class KafkaJsonProduceITCase { private static final String DATABASE = "fluss"; private static final byte[] KEY = new byte[] {(byte) 0xff, 1}; + private static final String FORGED_METADATA = + "\"_kafka_metadata\":{\"timestamp\":0,\"timestamp_type\":-1,\"partition\":99}"; private static final RowType SCALAR_TYPE = RowType.builder() .field("id", DataTypes.INT()) @@ -592,6 +593,7 @@ void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( invalidValues.add("{\"id\":1,\"" + reserved + "\":" + value + "}"); } } + invalidValues.add("{" + FORGED_METADATA + ",\"id\":\"bad\"}"); invalidValues.add("{\"id\":2147483648}"); invalidValues.add("{\"id\":1,\"_kafka_rescue\":\"\\ud800\"}"); invalidValues.add("{\"id\":1,\"extra\":{\"x\":1,\"x\":2}}"); @@ -679,9 +681,10 @@ void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( } @ParameterizedTest - @ValueSource(strings = {"flat", "nested"}) - void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout) throws Exception { - TablePath path = TablePath.of(DATABASE, "json_metadata_" + layout); + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout, boolean withRescue) + throws Exception { + TablePath path = TablePath.of(DATABASE, "json_metadata_" + layout + "_" + withRescue); RowType valueType = RowType.builder().field("id", DataTypes.INT()).build(); // Magic v2 batches carry CREATE_TIME or LOG_APPEND_TIME even without a timestamp. TimestampType[] types = { @@ -693,7 +696,7 @@ void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout) throws Ex long[] expectedTimestamps = {0L, 456L, 123L, 123L, -1L, -1L, -1L, -1L}; try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { - create(admin, path, descriptor(valueType, layout, true, false, 2)); + create(admin, path, descriptor(valueType, layout, true, withRescue, 2)); ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); try (Socket socket = new Socket(node.host(), node.port())) { for (int batch = 0; batch < types.length; batch++) { @@ -707,11 +710,11 @@ void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout) throws Ex new SimpleRecord( firstTimestamp, KEY, - bytes("{\"id\":" + (batch * 2) + "}")), + bytes(idJson(batch * 2, withRescue))), new SimpleRecord( secondTimestamp, KEY, - bytes("{\"id\":" + (batch * 2 + 1) + "}"))); + bytes(idJson(batch * 2 + 1, withRescue)))); String topic = batch % 2 == 0 ? path.toString() : path.getTableName(); ProduceResponse response = produce(socket, topic, records, records); for (int partition = 0; partition < 2; partition++) { @@ -735,6 +738,10 @@ void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout) throws Ex expectedTimestamps[index], (index / 2) % 2, expectedPartition); + if (withRescue) { + assertThat(row.getString(4).toString()) + .isEqualTo("{" + FORGED_METADATA + "}"); + } }); } } finally { @@ -744,13 +751,15 @@ void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout) throws Ex } @ParameterizedTest - @ValueSource(strings = {"flat", "nested"}) - void testMissingMetadataTimestampPersistsAsNull(String layout) throws Exception { - TablePath path = TablePath.of(DATABASE, "json_required_timestamp_" + layout); + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testMissingMetadataTimestampPersistsAsNull(String layout, boolean withRescue) + throws Exception { + TablePath path = + TablePath.of(DATABASE, "json_required_timestamp_" + layout + "_" + withRescue); RowType valueType = RowType.builder().field("id", DataTypes.INT()).build(); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { - create(admin, path, descriptor(valueType, layout, true, false, 2)); + create(admin, path, descriptor(valueType, layout, true, withRescue, 2)); ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); try (Socket socket = new Socket(node.host(), node.port()); KafkaProducer producer = producer()) { @@ -761,11 +770,11 @@ void testMissingMetadataTimestampPersistsAsNull(String layout) throws Exception timestampedRecords( TimestampType.CREATE_TIME, RecordBatch.NO_TIMESTAMP, - record("{\"id\":1000}"), + record(idJson(1000, withRescue)), new SimpleRecord( RecordBatch.NO_TIMESTAMP, KEY, - bytes("{\"id\":1001}"))), + bytes(idJson(1001, withRescue)))), MemoryRecords.withRecords(Compression.NONE, record("{\"id\":1}"))); PartitionProduceResponse missingTimestamp = partition(response, path.toString(), 0); assertThat(missingTimestamp.errorCode()).isEqualTo(Errors.NONE.code()); @@ -816,6 +825,10 @@ void testMissingMetadataTimestampPersistsAsNull(String layout) throws Exception } } + private static String idJson(int id, boolean forgedMetadata) { + return "{\"id\":" + id + (forgedMetadata ? "," + FORGED_METADATA : "") + "}"; + } + private static void assertMetadata( InternalRow row, int position, long timestamp, int timestampType, int partition) { assertThat(row.isNullAt(position)).isFalse(); @@ -865,7 +878,8 @@ void testRescueRoundTripWithReorderedSystemColumns( + "}"); String collisions = "\"_kafka_key\":{\"id\":2},\"_kafka_headers\":null," - + "\"_kafka_metadata\":{\"id\":2},\"_kafka_value\":{\"id\":2}," + + FORGED_METADATA + + ",\"_kafka_value\":{\"id\":2}," + "\"_kafka_rescue\":" + rescueValues[i]; payloads.add(json.substring(0, json.length() - 1) + "," + collisions + "}"); @@ -892,13 +906,13 @@ void testRescueRoundTripWithReorderedSystemColumns( Admin admin = connection.getAdmin()) { create(admin, path, descriptor); try (KafkaProducer producer = producer()) { - assertThat(producer.partitionsFor(path.toString())).hasSize(1); + assertThat(producer.partitionsFor(path.toString())).hasSize(2); for (int i = 0; i < payloads.size(); i++) { assertThat( producer.send( new ProducerRecord<>( path.toString(), - 0, + 1, 123L, KEY, bytes(payloads.get(i)), @@ -915,7 +929,7 @@ void testRescueRoundTripWithReorderedSystemColumns( read( connection, path, - 0, + 1, payloads.size(), (physical, index) -> { InternalRow value = jsonBusinessRow(physical, contract); @@ -956,6 +970,7 @@ void testRescueRoundTripWithReorderedSystemColumns( assertThat(physical.getBytes(contract.keyPosition())) .isEqualTo(KEY); assertHeaders(physical.getMap(contract.headersPosition()), true); + assertMetadata(physical, contract.metadataPosition(), 123L, 0, 1); } }); } finally { @@ -985,7 +1000,7 @@ private static TableDescriptor reorderedRescueDescriptor( String layout, boolean withSystemColumns, boolean rescueFirst) { List columns = new ArrayList<>( - descriptor(COMPLEX_TYPE, layout, withSystemColumns, false, 1) + descriptor(COMPLEX_TYPE, layout, withSystemColumns, false, 2) .getSchema() .getColumns()); Schema.Builder schema = Schema.newBuilder(); @@ -1000,7 +1015,7 @@ private static TableDescriptor reorderedRescueDescriptor( } return TableDescriptor.builder() .schema(schema.build()) - .distributedBy(1) + .distributedBy(2) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java index fdc4f7a2568..ed28b097cc8 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java @@ -21,6 +21,7 @@ import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; import org.apache.fluss.kafka.transcode.KafkaRowAssembler; @@ -218,6 +219,13 @@ void testSchemaFactoryAndAssemblerKeepRescueSeparateFromPhysicalOrder( schema.column("_kafka_rescue", DataTypes.STRING()); } schema.column("_kafka_key", DataTypes.BYTES()); + schema.column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))) + .copy(false)); if (!first) { schema.column("_kafka_rescue", DataTypes.STRING()); } @@ -242,13 +250,21 @@ void testSchemaFactoryAndAssemblerKeepRescueSeparateFromPhysicalOrder( .build()); KafkaFieldDecoder decoder = formats.createDecoder(contract); KafkaRowAssembler assembler = new KafkaRowAssembler(contract); - Object[] values = decoder.decode("{\"id\":7,\"extra\":2}".getBytes(StandardCharsets.UTF_8)); + String rescued = "{\"extra\":2,\"_kafka_metadata\":{\"timestamp\":0,\"partition\":99}}"; + Object[] values = + decoder.decode( + ("{\"id\":7," + rescued.substring(1)).getBytes(StandardCharsets.UTF_8)); assertThat(values).hasSize(2); byte[] key = {(byte) 0xff}; - Record record = new Record(123L, key, null, Collections.emptyList()); + Record record = + new Record(123L, TimestampType.LOG_APPEND_TIME, key, null, Collections.emptyList()); InternalRow row = assembler.assemble(values, record, 7); - assertThat(row.getString(contract.rescuePosition()).toString()).isEqualTo("{\"extra\":2}"); + assertThat(row.getString(contract.rescuePosition()).toString()).isEqualTo(rescued); assertThat(row.getBytes(contract.keyPosition())).isEqualTo(key); + InternalRow metadata = row.getRow(contract.metadataPosition(), 3); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(metadata.getInt(1)).isEqualTo(1); + assertThat(metadata.getInt(2)).isEqualTo(7); int valuePosition = contract.valueProjection().positionAt(0); assertThat( layout.equals("nested") diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java index 64e47526291..741b1a8387a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java @@ -362,7 +362,17 @@ void testRescueKeepsTypedNestedNumbersAndUnknownStructure(boolean nested) { DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); String decimal = "12345678901234567890.123456789012345678"; for (String token : - new String[] {"-0", "-0.0", "-0e0", "1.0000000596046448", "-1.0000000596046448"}) { + new String[] { + "-0", + "-0.0", + "-0e0", + "1.0000000596046448", + "-1.0000000596046448", + "-1e-100", + "-1e-400", + "7.006492321624085e-46", + "-7.006492321624085e-46" + }) { String row = "{\"f\":" + token From b51254ed84acb2e430d1ce24ab76fc20ca639e62 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 22:00:39 +0800 Subject: [PATCH 68/69] [test] Verify rescue with persisted null Kafka values Cover flat and nested JSON null-value persistence together with populated rescue, continuous native offsets, physical system columns, and nullable missing fields. Verify that rescue cannot hide a required null-value failure or enable collection through a differently named STRING column. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 99/99 --- .../fluss/kafka/KafkaJsonProduceITCase.java | 49 +++++++++++++------ .../KafkaFormatFactoryRegistryTest.java | 8 +++ .../format/json/JsonKafkaRescueTest.java | 18 +++++++ 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java index 56d9a8b3ea0..8f4f753cf1a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -454,9 +454,11 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout, S + "\"}" : null; if (index == 5) { - json = "{\"id\":null,\"label\":null}"; + json = "{\"id\":null,\"label\":null,\"extra\":{\"index\":5}}"; } else if (index == 6) { - json = "{}"; + json = "{\"extra\":{\"index\":6}}"; + } else if (index == 2) { + json = "{\"extra\":{\"index\":2}," + json.substring(1); } records[index] = new SimpleRecord( @@ -519,7 +521,13 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout, S (index + 1) * 10L, 0, expectedPartition); - assertThat(row.isNullAt(headersPosition + 2)).isTrue(); + if (index == 2 || index == 5 || index == 6) { + assertThat(row.getString(headersPosition + 2).toString()) + .isEqualTo("{\"extra\":{\"index\":" + index + "}}"); + } else { + // A following null or known-only value must not reuse rescue. + assertThat(row.isNullAt(headersPosition + 2)).isTrue(); + } }); } } finally { @@ -530,18 +538,29 @@ void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout, S @ParameterizedTest @CsvSource({ - "flat,scalar", - "nested,scalar", - "flat,row", - "nested,row", - "flat,array", - "nested,array", - "flat,map", - "nested,map" + "flat,scalar,false", + "nested,scalar,false", + "flat,row,false", + "nested,row,false", + "flat,array,false", + "nested,array,false", + "flat,map,false", + "nested,map,false", + "flat,scalar,true", + "nested,scalar,true", + "flat,row,true", + "nested,row,true", + "flat,array,true", + "nested,array,true", + "flat,map,true", + "nested,map,true" }) - void testNotNullBusinessFieldsAreRejectedAtTableCreation(String layout, String fieldType) - throws Exception { - TablePath path = TablePath.of(DATABASE, "json_required_value_" + layout + "_" + fieldType); + void testNotNullBusinessFieldsAreRejectedAtTableCreation( + String layout, String fieldType, boolean withRescue) throws Exception { + TablePath path = + TablePath.of( + DATABASE, + "json_required_value_" + layout + "_" + fieldType + "_" + withRescue); RowType valueType = RowType.builder(false) .field("id", nullValueBusinessType(fieldType).copy(false)) @@ -553,7 +572,7 @@ void testNotNullBusinessFieldsAreRejectedAtTableCreation(String layout, String f create( admin, path, - descriptor(valueType, layout, true, false, 2))) + descriptor(valueType, layout, true, withRescue, 2))) .hasCauseInstanceOf(InvalidTableException.class) .hasStackTraceContaining("nullable fields"); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java index ed28b097cc8..6a68455d7b6 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java @@ -271,6 +271,14 @@ void testSchemaFactoryAndAssemblerKeepRescueSeparateFromPhysicalOrder( ? row.getRow(valuePosition, 1).getInt(0) : row.getInt(valuePosition)) .isEqualTo(7); + InternalRow nullRow = assembler.assemble(decoder.decode(null), record, 7); + assertThat(nullRow.isNullAt(valuePosition)).isTrue(); + assertThat(nullRow.isNullAt(contract.rescuePosition())).isTrue(); + assertThat(nullRow.getBytes(contract.keyPosition())).isEqualTo(key); + InternalRow nullMetadata = nullRow.getRow(contract.metadataPosition(), 3); + assertThat(nullMetadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(nullMetadata.getInt(1)).isEqualTo(1); + assertThat(nullMetadata.getInt(2)).isEqualTo(7); assertThat( assembler .assemble( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java index 741b1a8387a..5163b4cae76 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java @@ -61,6 +61,24 @@ /** Tests unknown-field rescue while keeping known-field and resource violations fatal. */ class JsonKafkaRescueTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testOtherStringColumnDoesNotEnableRescue(boolean nested) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW(DataTypes.FIELD("custom_rescue", DataTypes.STRING())), + nested); + Object[] values = decoder.decode(bytes("{\"custom_rescue\":\"business\"}")); + assertThat(values).hasSize(1); + assertThat(nested ? ((GenericRow) values[0]).getString(0).toString() : values[0].toString()) + .isEqualTo("business"); + for (String json : + new String[] {"{\"extra\":1}", "{\"_kafka_rescue\":1}", "{\"_kafka_key\":1}"}) { + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class); + } + } + @Test void testUnexpectedConversionFailureIsNotRescued() { JsonNode broken = mock(JsonNode.class); From 697bb68f28f804cdbc0485d8a663f9770086e2ea Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 19 Oct 2026 22:36:15 +0800 Subject: [PATCH 69/69] [test] Keep Kafka metadata nullable in schema fixture Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 4/4 --- .../fluss/kafka/format/KafkaFormatFactoryRegistryTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java index 6a68455d7b6..5d4cfd0f960 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java @@ -222,10 +222,9 @@ void testSchemaFactoryAndAssemblerKeepRescueSeparateFromPhysicalOrder( schema.column( "_kafka_metadata", DataTypes.ROW( - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), - DataTypes.FIELD("partition", DataTypes.INT().copy(false))) - .copy(false)); + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))); if (!first) { schema.column("_kafka_rescue", DataTypes.STRING()); }