From 41ecc50487d9b8193ad34c9f21eb032c31fe4749 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 8 Sep 2026 11:31:07 +0800 Subject: [PATCH 01/48] [kafka] Serve ApiVersions from registered capabilities Route requests through the dispatcher and advertise only implemented APIs. Return version-aware errors for unsupported APIs and invalid requests. Validated with mvn -o -pl fluss-kafka verify (23 unit tests and 1 IT). Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 313/313 AI-Contributed/UT: 137/137 --- .../fluss/kafka/KafkaProtocolPlugin.java | 3 +- .../fluss/kafka/KafkaRequestHandler.java | 232 ++---------------- .../api/versions/ApiVersionsHandler.java | 78 ++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 137 ++++++++--- 4 files changed, 207 insertions(+), 243 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 2bd2f5d81bc..9a9e2e9ff6e 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -66,7 +66,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " + service.getClass().getSimpleName()); } - TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(gateway); + return new KafkaRequestHandler(); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 73555093ff0..2df16a8bd7f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,27 +17,24 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import org.apache.kafka.common.message.ApiVersionsResponseData; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.requests.AbstractRequest; -import org.apache.kafka.common.requests.AbstractResponse; -import org.apache.kafka.common.requests.ApiVersionsResponse; - -/** Kafka protocol implementation for request handler. */ +/** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { - // TODO: we may need a new abstraction between TabletService and ReplicaManager to avoid - // affecting Fluss protocol when supporting compatibility with Kafka. - private final TabletServerGateway gateway; + private final KafkaRequestDispatcher dispatcher; - public KafkaRequestHandler(TabletServerGateway gateway) { - this.gateway = gateway; + /** Creates a Kafka request handler with the implemented server capabilities. */ + public KafkaRequestHandler() { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ApiVersionsHandler(registry)); + registry.freeze(); + this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @Override @@ -47,200 +44,15 @@ public RequestType requestType() { @Override public void processRequest(KafkaRequest request) { - // See kafka.server.KafkaApis#handle - switch (request.apiKey()) { - case API_VERSIONS: - handleApiVersionsRequest(request); - break; - case METADATA: - handleMetadataRequest(request); - break; - case PRODUCE: - handleProducerRequest(request); - break; - case FIND_COORDINATOR: - handleFindCoordinatorRequest(request); - break; - case LIST_OFFSETS: - handleListOffsetRequest(request); - break; - case OFFSET_FETCH: - handleOffsetFetchRequest(request); - break; - case OFFSET_COMMIT: - handleOffsetCommitRequest(request); - break; - case FETCH: - handleFetchRequest(request); - break; - case JOIN_GROUP: - handleJoinGroupRequest(request); - break; - case SYNC_GROUP: - handleSyncGroupRequest(request); - break; - case HEARTBEAT: - handleHeartbeatRequest(request); - break; - case LEAVE_GROUP: - handleLeaveGroupRequest(request); - break; - case DESCRIBE_GROUPS: - handleDescribeGroupsRequest(request); - break; - case LIST_GROUPS: - handleListGroupsRequest(request); - break; - case DELETE_GROUPS: - handleDeleteGroupsRequest(request); - break; - case SASL_HANDSHAKE: - handleSaslHandshakeRequest(request); - break; - case SASL_AUTHENTICATE: - handleSaslAuthenticateRequest(request); - break; - case CREATE_TOPICS: - handleCreateTopicsRequest(request); - break; - case INIT_PRODUCER_ID: - handleInitProducerIdRequest(request); - break; - case ADD_PARTITIONS_TO_TXN: - handleAddPartitionsToTxnRequest(request); - break; - case ADD_OFFSETS_TO_TXN: - handleAddOffsetsToTxnRequest(request); - break; - case TXN_OFFSET_COMMIT: - handleTxnOffsetCommitRequest(request); - break; - case END_TXN: - handleEndTxnRequest(request); - break; - case WRITE_TXN_MARKERS: - handleWriteTxnMarkersRequest(request); - break; - case DESCRIBE_CONFIGS: - handleDescribeConfigsRequest(request); - break; - case ALTER_CONFIGS: - handleAlterConfigsRequest(request); - break; - case DELETE_TOPICS: - handleDeleteTopicsRequest(request); - break; - case DELETE_RECORDS: - handleDeleteRecordsRequest(request); - break; - case OFFSET_DELETE: - handleOffsetDeleteRequest(request); - break; - case CREATE_PARTITIONS: - handleCreatePartitionsRequest(request); - break; - case DESCRIBE_CLUSTER: - handleDescribeClusterRequest(request); - break; - default: - handleUnsupportedRequest(request); - } - } - - private void handleUnsupportedRequest(KafkaRequest request) { - String message = String.format("Unsupported request with api key %s", request.apiKey()); - AbstractRequest abstractRequest = request.request(); - AbstractResponse response = - abstractRequest.getErrorResponse(new UnsupportedOperationException(message)); - request.complete(response); - } - - void handleApiVersionsRequest(KafkaRequest request) { - short apiVersion = request.apiVersion(); - if (!ApiKeys.API_VERSIONS.isVersionSupported(apiVersion)) { - request.fail(Errors.UNSUPPORTED_VERSION.exception()); - return; - } - ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (ApiKeys apiKey : ApiKeys.values()) { - if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { - ApiVersionsResponseData.ApiVersion apiVersionData = - new ApiVersionsResponseData.ApiVersion() - .setApiKey(apiKey.id) - .setMinVersion(apiKey.oldestVersion()) - .setMaxVersion(apiKey.latestVersion()); - if (apiKey.equals(ApiKeys.METADATA)) { - // Not support TopicId - short v = apiKey.latestVersion() > 11 ? 11 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } else if (apiKey.equals(ApiKeys.FETCH)) { - // Not support TopicId - short v = apiKey.latestVersion() > 12 ? 12 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } - data.apiKeys().add(apiVersionData); - } - } - request.complete(new ApiVersionsResponse(data)); + dispatcher + .dispatch(request) + .whenComplete( + (response, failure) -> { + if (failure == null) { + request.complete(response); + } else { + request.fail(failure); + } + }); } - - void handleProducerRequest(KafkaRequest request) {} - - void handleMetadataRequest(KafkaRequest request) {} - - void handleFindCoordinatorRequest(KafkaRequest request) {} - - void handleListOffsetRequest(KafkaRequest request) {} - - void handleOffsetFetchRequest(KafkaRequest request) {} - - void handleOffsetCommitRequest(KafkaRequest request) {} - - void handleFetchRequest(KafkaRequest request) {} - - void handleJoinGroupRequest(KafkaRequest request) {} - - void handleSyncGroupRequest(KafkaRequest request) {} - - void handleHeartbeatRequest(KafkaRequest request) {} - - void handleLeaveGroupRequest(KafkaRequest request) {} - - void handleDescribeGroupsRequest(KafkaRequest request) {} - - void handleListGroupsRequest(KafkaRequest request) {} - - void handleDeleteGroupsRequest(KafkaRequest request) {} - - void handleSaslHandshakeRequest(KafkaRequest request) {} - - void handleSaslAuthenticateRequest(KafkaRequest request) {} - - void handleCreateTopicsRequest(KafkaRequest request) {} - - void handleInitProducerIdRequest(KafkaRequest request) {} - - void handleAddPartitionsToTxnRequest(KafkaRequest request) {} - - void handleAddOffsetsToTxnRequest(KafkaRequest request) {} - - void handleTxnOffsetCommitRequest(KafkaRequest request) {} - - void handleEndTxnRequest(KafkaRequest request) {} - - void handleWriteTxnMarkersRequest(KafkaRequest request) {} - - void handleDescribeConfigsRequest(KafkaRequest request) {} - - void handleAlterConfigsRequest(KafkaRequest request) {} - - void handleDeleteTopicsRequest(KafkaRequest request) {} - - void handleDeleteRecordsRequest(KafkaRequest request) {} - - void handleOffsetDeleteRequest(KafkaRequest request) {} - - void handleCreatePartitionsRequest(KafkaRequest request) {} - - void handleDescribeClusterRequest(KafkaRequest request) {} } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java new file mode 100644 index 00000000000..c38d7bc6cb2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements ApiVersions from the capabilities actually registered on this server. */ +@Internal +public final class ApiVersionsHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + true); + + private final KafkaApiRegistry registry; + + /** Creates an ApiVersions handler backed by the server capability registry. */ + public ApiVersionsHandler(KafkaApiRegistry registry) { + this.registry = checkNotNull(registry); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + if (!request.isValid()) { + return CompletableFuture.completedFuture( + request.getErrorResponse(Errors.INVALID_REQUEST.exception())); + } + ApiVersionsResponseData data = new ApiVersionsResponseData(); + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + data.apiKeys() + .add( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(spec.apiKey().id) + .setMinVersion(spec.minVersion()) + .setMaxVersion(spec.maxVersion())); + } + return CompletableFuture.completedFuture(new ApiVersionsResponse(data)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index bf8e23b4356..b0f25149e43 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,24 +17,32 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; /** Tests for {@link KafkaRequestHandler}. */ public class KafkaRequestHandlerTest { @@ -53,7 +61,7 @@ public void testKafkaApiVersionsNotSupported() { new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), apiVersionsRequest, ctx); - handler.handleApiVersionsRequest(request); + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); Map errorCounts = response.errorCounts(); @@ -61,44 +69,111 @@ public void testKafkaApiVersionsNotSupported() { assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); } + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + public void testKafkaApiVersionsRequest(short version) { + KafkaRequestHandler handler = createKafkaRequestHandler(); + ApiVersionsResponse response = requestApiVersions(handler, version); + + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + } + + private static ApiVersionsResponse requestApiVersions( + KafkaRequestHandler handler, short version) { + ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build(version); + ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + KafkaRequest request = + newRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), + apiVersionsRequest, + ctx); + handler.processRequest(request); + + return parseApiVersionsResponse(request); + } + + private static ApiVersionsResponse parseApiVersionsResponse(KafkaRequest request) { + return (ApiVersionsResponse) parseResponse(request); + } + + private static void assertSuccessfulResponseDefaults(ApiVersionsResponse response) { + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1)); + assertThat(response.data().throttleTimeMs()).isZero(); + assertThat(response.data().supportedFeatures()).isEmpty(); + assertThat(response.data().finalizedFeaturesEpoch()).isEqualTo(-1L); + assertThat(response.data().finalizedFeatures()).isEmpty(); + assertThat(response.data().zkMigrationReady()).isFalse(); + } + + private static void assertBrokerCapabilities(ApiVersionsResponse response) { + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion())); + } + @Test - public void testKafkaApiVersionsRequest() { + public void testInvalidApiVersionsRequest() { KafkaRequestHandler handler = createKafkaRequestHandler(); short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + ApiVersionsRequest requestBody = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData() + .setClientSoftwareName("invalid client name") + .setClientSoftwareVersion("1.0"), + latestVersion, + latestVersion) + .build(latestVersion); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, latestVersion, new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, - ctx); - handler.handleApiVersionsRequest(request); + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.NONE)); - response.data() - .apiKeys() - .forEach( - apiVersion -> { - if (ApiKeys.METADATA.id == apiVersion.apiKey()) { - assertThat((short) 11) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else if (ApiKeys.FETCH.id == apiVersion.apiKey()) { - assertThat((short) 12) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else { - ApiKeys apiKeys = ApiKeys.forId(apiVersion.apiKey()); - assertThat(apiVersion.minVersion()) - .isEqualTo(apiKeys.oldestVersion()); - assertThat(apiVersion.maxVersion()) - .isEqualTo(apiKeys.latestVersion()); - } - }); + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + } + + @Test + public void testUnregisteredApiIsNotRouted() { + KafkaRequestHandler handler = createKafkaRequestHandler(); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + CreateTopicsRequestData requestData = + new CreateTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + new CreateTopicsRequestData.CreatableTopicCollection( + Collections.singletonList( + new CreateTopicsRequestData.CreatableTopic() + .setName("topic") + .setNumPartitions(1) + .setReplicationFactor((short) 1)) + .iterator())); + CreateTopicsRequest requestBody = + new CreateTopicsRequest.Builder(requestData).build(version); + KafkaRequest request = + newRequest( + ApiKeys.CREATE_TOPICS, + version, + new RequestHeader(ApiKeys.CREATE_TOPICS, version, "client-id", 0), + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); + + CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request); + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); } private static KafkaRequest newRequest( @@ -134,6 +209,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(new TestingTabletGatewayService()); + return new KafkaRequestHandler(); } } From 4317d87922b826751d2a23283e8f3eb5e3044a0f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 17:28:33 +0800 Subject: [PATCH 02/48] [kafka] Cover ApiVersions wire negotiation and capability ranges Exercise future-version decoding, v0 error serialization, correlation IDs, unknown tagged fields, buffer release, and renegotiation on one connection. Verify registry-defined ranges, hidden APIs, ordering, and invalid client software fields across the supported ApiVersions schemas. Validated with Java 11 reactor clean verify: 37 unit tests and 2 integration tests passed, with Checkstyle, Spotless, RAT, and git diff checks. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 238/238 --- .../fluss/kafka/KafkaRequestHandlerTest.java | 149 ++++++++++++++---- .../api/versions/ApiVersionsHandlerTest.java | 89 +++++++++++ 2 files changed, 205 insertions(+), 33 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index b0f25149e43..40fdab04cc4 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,15 +17,21 @@ package org.apache.fluss.kafka; +import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; @@ -33,12 +39,15 @@ import org.apache.kafka.common.requests.CreateTopicsRequest; import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import java.nio.ByteBuffer; import java.util.Collections; -import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; @@ -48,25 +57,49 @@ public class KafkaRequestHandlerTest { @Test - public void testKafkaApiVersionsNotSupported() { + public void testUnsupportedWireVersionFallsBackAndAllowsRenegotiation() { KafkaRequestHandler handler = createKafkaRequestHandler(); - short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); - KafkaRequest request = - newRequest( - ApiKeys.API_VERSIONS, - (short) (latestVersion + 1), // unsupported version - new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, - ctx); - handler.processRequest(request); + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + short unsupportedVersion = (short) (ApiKeys.API_VERSIONS.latestVersion() + 1); + try { + // An unknown request version must be rejected without parsing its body schema. + processWireRequest( + handler, + requests, + channel, + unsupportedVersion, + new ApiVersionsRequest.Builder().build((short) 4), + 41); + ApiVersionsResponse unsupportedResponse = parseWireResponse(channel, (short) 0, 41); + assertThat(unsupportedResponse.data().errorCode()) + .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + assertBrokerCapabilities(unsupportedResponse); - ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); + ApiVersionsRequestData data = + new ApiVersionsRequestData() + .setClientSoftwareName("test-client") + .setClientSoftwareVersion("1.0"); + data.unknownTaggedFields().add(new RawTaggedField(42, new byte[] {1, 2, 3})); + processWireRequest( + handler, + requests, + channel, + (short) 4, + new ApiVersionsRequest(data, (short) 4), + 42); + ApiVersionsResponse response = parseWireResponse(channel, (short) 4, 42); + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + assertThat(channel.isActive()).isTrue(); + assertThat(requests.requestsCount()).isZero(); + Object extraResponse = channel.readOutbound(); + assertThat(extraResponse).isNull(); + } finally { + channel.finishAndReleaseAll(); + } } @ParameterizedTest @@ -112,30 +145,32 @@ private static void assertSuccessfulResponseDefaults(ApiVersionsResponse respons private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) - .containsExactly( - tuple( - ApiKeys.API_VERSIONS.id, - ApiKeys.API_VERSIONS.oldestVersion(), - ApiKeys.API_VERSIONS.latestVersion())); + .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } - @Test - public void testInvalidApiVersionsRequest() { + @ParameterizedTest + @CsvSource({ + "3, invalid client name, 1.0", + "3, client, ''", + "4, invalid client name, 1.0", + "4, client, ''" + }) + public void testInvalidApiVersionsRequest( + short version, String softwareName, String softwareVersion) { KafkaRequestHandler handler = createKafkaRequestHandler(); - short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); ApiVersionsRequest requestBody = new ApiVersionsRequest.Builder( new ApiVersionsRequestData() - .setClientSoftwareName("invalid client name") - .setClientSoftwareVersion("1.0"), - latestVersion, - latestVersion) - .build(latestVersion); + .setClientSoftwareName(softwareName) + .setClientSoftwareVersion(softwareVersion), + version, + version) + .build(version); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, - latestVersion, - new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), requestBody, new TestingChannelHandlerContext()); @@ -143,6 +178,7 @@ public void testInvalidApiVersionsRequest() { ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + assertThat(response.data().apiKeys()).isEmpty(); } @Test @@ -176,6 +212,53 @@ public void testUnregisteredApiIsNotRouted() { assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); } + private static void processWireRequest( + KafkaRequestHandler handler, + RequestChannel requests, + EmbeddedChannel channel, + short version, + ApiVersionsRequest body, + int correlationId) { + RequestHeader header = + new RequestHeader(ApiKeys.API_VERSIONS, version, "test-client", correlationId); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + channel.writeInbound(buffer); + KafkaRequest request = (KafkaRequest) requests.pollRequest(1000); + assertThat(request).isNotNull(); + try { + assertThat(request.apiVersion()).isEqualTo(version); + handler.processRequest(request); + assertThat(request.future()).isDone(); + channel.runPendingTasks(); + assertThat(buffer.refCnt()).isZero(); + } finally { + request.releaseBuffer(); + } + } + + private static ApiVersionsResponse parseWireResponse( + EmbeddedChannel channel, short version, int correlationId) { + ByteBuf buffer = channel.readOutbound(); + assertThat(buffer).isNotNull(); + try { + ByteBuffer bytes = buffer.nioBuffer(); + assertThat(ResponseHeader.parse(bytes, (short) 0).correlationId()) + .isEqualTo(correlationId); + ApiVersionsResponseData data = + new ApiVersionsResponseData(new ByteBufferAccessor(bytes), version); + assertThat(bytes.hasRemaining()).isFalse(); + return new ApiVersionsResponse(data); + } finally { + buffer.release(); + } + } + private static KafkaRequest newRequest( ApiKeys apiKey, short apiVersion, diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java new file mode 100644 index 00000000000..31269af56c5 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +/** Tests registry-backed capability advertisement by {@link ApiVersionsHandler}. */ +class ApiVersionsHandlerTest { + + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { + KafkaApiRegistry registry = new KafkaApiRegistry(); + ApiVersionsHandler handler = new ApiVersionsHandler(registry); + registry.register(handler); + registry.register( + new TestingApiHandler( + new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true))); + registry.register( + new TestingApiHandler( + new KafkaApiSpec(ApiKeys.FETCH, (short) 0, (short) 3, false))); + registry.freeze(); + + ApiVersionsResponse response = + (ApiVersionsResponse) + handler.handle(null, new ApiVersionsRequest.Builder().build(version)) + .join(); + + assertThat(response.data().errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple(ApiKeys.METADATA.id, (short) 2, (short) 7), + tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); + } + + private static final class TestingApiHandler implements KafkaApiHandler { + private final KafkaApiSpec spec; + + private TestingApiHandler(KafkaApiSpec spec) { + this.spec = spec; + } + + @Override + public KafkaApiSpec apiSpec() { + return spec; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, AbstractRequest request) { + throw new AssertionError( + "Capability advertisement must not invoke other API handlers."); + } + } +} From 229fc9434f705b4df9a894a5743c72166679cc45 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 13 Oct 2026 09:43:06 +0800 Subject: [PATCH 03/48] [kafka] Adapt ApiVersions to immutable API registry Construct the registry once with all handlers and defer ApiVersions registry access until request handling. Advertised capabilities and dispatch continue to share the same immutable registry without a separate capability list. Update coverage to verify lazy access and handler identity across versions. Validated with Java 11 reactor clean verify: 44 unit and 2 integration cases passed in both Maven test executions, including inherited error-mapping and idle regressions. Kafka main and test sources also compile with --release 8. Spotless, Checkstyle, RAT, and diff checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 28/28 AI-Contributed/UT: 33/33 --- .../fluss/kafka/KafkaRequestHandler.java | 13 ++++++-- .../api/versions/ApiVersionsHandler.java | 15 ++++++--- .../api/versions/ApiVersionsHandlerTest.java | 33 ++++++++++++++----- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 2df16a8bd7f..b3b555a8f9a 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -24,16 +24,19 @@ import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; +import java.util.Collections; + /** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { + private final KafkaApiRegistry registry; private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the implemented server capabilities. */ public KafkaRequestHandler() { - KafkaApiRegistry registry = new KafkaApiRegistry(); - registry.register(new ApiVersionsHandler(registry)); - registry.freeze(); + this.registry = + new KafkaApiRegistry( + Collections.singletonList(new ApiVersionsHandler(this::apiRegistry))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @@ -55,4 +58,8 @@ public void processRequest(KafkaRequest request) { } }); } + + private KafkaApiRegistry apiRegistry() { + return registry; + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java index c38d7bc6cb2..82e3559c5f4 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -31,6 +31,7 @@ import org.apache.kafka.common.requests.ApiVersionsResponse; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -45,11 +46,15 @@ public final class ApiVersionsHandler implements KafkaApiHandler registrySupplier; - /** Creates an ApiVersions handler backed by the server capability registry. */ - public ApiVersionsHandler(KafkaApiRegistry registry) { - this.registry = checkNotNull(registry); + /** + * Creates an ApiVersions handler backed by the same immutable registry used for dispatch. The + * supplier is read only when handling requests, so this handler can be included in the + * registry's construction snapshot. + */ + public ApiVersionsHandler(Supplier registrySupplier) { + this.registrySupplier = checkNotNull(registrySupplier); } @Override @@ -65,7 +70,7 @@ public CompletableFuture handle( request.getErrorResponse(Errors.INVALID_REQUEST.exception())); } ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + for (KafkaApiSpec spec : registrySupplier.get().advertisedApiSpecs()) { data.apiKeys() .add( new ApiVersionsResponseData.ApiVersion() diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java index 31269af56c5..0492ee3572f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -32,7 +32,9 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.util.Arrays; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; @@ -43,16 +45,29 @@ class ApiVersionsHandlerTest { @ParameterizedTest @ValueSource(shorts = {0, 1, 2, 3, 4}) void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { - KafkaApiRegistry registry = new KafkaApiRegistry(); - ApiVersionsHandler handler = new ApiVersionsHandler(registry); - registry.register(handler); - registry.register( + AtomicReference registryReference = new AtomicReference<>(); + ApiVersionsHandler handler = + new ApiVersionsHandler( + () -> { + assertThat(registryReference.get()).isNotNull(); + return registryReference.get(); + }); + TestingApiHandler metadataHandler = new TestingApiHandler( - new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true))); - registry.register( - new TestingApiHandler( - new KafkaApiSpec(ApiKeys.FETCH, (short) 0, (short) 3, false))); - registry.freeze(); + new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true)); + KafkaApiRegistry registry = + new KafkaApiRegistry( + Arrays.asList( + handler, + metadataHandler, + new TestingApiHandler( + new KafkaApiSpec( + ApiKeys.FETCH, (short) 0, (short) 3, false)))); + registryReference.set(registry); + + assertThat(registry.lookup(ApiKeys.API_VERSIONS)).isSameAs(handler); + assertThat(registry.lookup(ApiKeys.METADATA)).isSameAs(metadataHandler); + assertThat(registry.lookup(ApiKeys.FETCH)).isNull(); ApiVersionsResponse response = (ApiVersionsResponse) From 10bf516b6a04c579079b6e69b3788ac633041525 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 13 Oct 2026 18:06:21 +0800 Subject: [PATCH 04/48] [kafka] Adapt version fallback regression to PR02 handler Use the dispatcher-backed handler constructor in the fallback test added by PR01. Preserve its unsupported-version, listener, response, and buffer checks. Validated with Java 11 reactor clean verify: 39 unit and 2 integration cases passed in both Maven test executions. Kafka main and test sources compile with --release 8; Spotless, Checkstyle, RAT, and diff checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 3/3 --- .../java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index f4a255b013d..dda2e52bde5 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -17,7 +17,6 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; @@ -71,7 +70,7 @@ public void testUnsupportedApiVersionsRequestReturnsVersionZeroErrorResponse() { assertThat(parsedRequest.hasUnsupportedRequestVersion()).isTrue(); assertThat(parsedRequest.version()).isZero(); - new KafkaRequestHandler(new TestingTabletGatewayService()).processRequest(request); + new KafkaRequestHandler().processRequest(request); channel.runPendingTasks(); responseBuffer = channel.readOutbound(); From 062047def41ff5fe2d89b6c43b0b355e3772de1f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 13 Oct 2026 21:00:25 +0800 Subject: [PATCH 05/48] [kafka] Clarify uninitialized API registry failures Check the deferred registry before reading advertised capabilities so an initialization-order failure reports an explicit diagnostic message. Validated with Java 11 reactor tests: 16 ApiVersions and request-handler cases passed, along with Spotless, Checkstyle, RAT, and diff checks. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 4/4 AI-Contributed/UT: 0/0 --- .../apache/fluss/kafka/api/versions/ApiVersionsHandler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java index 82e3559c5f4..5cac0c3cca5 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -69,8 +69,10 @@ public CompletableFuture handle( return CompletableFuture.completedFuture( request.getErrorResponse(Errors.INVALID_REQUEST.exception())); } + KafkaApiRegistry registry = + checkNotNull(registrySupplier.get(), "Kafka API registry is not initialized."); ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (KafkaApiSpec spec : registrySupplier.get().advertisedApiSpecs()) { + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { data.apiKeys() .add( new ApiVersionsResponseData.ApiVersion() From 34964400fef3a407a9d05a4d35bfd9e71c5d6394 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 15:28:16 +0800 Subject: [PATCH 06/48] [kafka] Define DDL table mapping for Kafka compatibility Extract topic identity and the raw/string table mapping contract before Metadata. Validate table kinds, field projections and metadata columns independently of request handling and record decoding. Validation: Java 11, mvn -o -pl fluss-kafka clean verify (47 unit tests and 2 integration tests); Checkstyle, Spotless and RAT passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 769/769 AI-Contributed/UT: 480/480 --- fluss-kafka/pom.xml | 7 + .../fluss/kafka/format/KafkaDataFormat.java | 73 ++++ .../fluss/kafka/mapping/KafkaTopicMapper.java | 73 ++++ .../kafka/schema/KafkaFieldProjection.java | 123 +++++++ .../fluss/kafka/schema/KafkaTopicSchema.java | 150 ++++++++ .../schema/KafkaTopicSchemaException.java | 30 ++ .../schema/KafkaTopicSchemaResolver.java | 313 +++++++++++++++++ .../kafka/mapping/KafkaTopicMapperTest.java | 62 ++++ .../kafka/schema/KafkaTableMappingITCase.java | 88 +++++ .../schema/KafkaTopicSchemaResolverTest.java | 330 ++++++++++++++++++ 10 files changed, 1249 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index f48a94ab9e9..b77695848ca 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,6 +64,13 @@ + + org.apache.curator + curator-test + ${curator.version} + test + + org.apache.fluss fluss-test-utils diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java new file mode 100644 index 00000000000..ad0039b7e78 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import java.util.Locale; + +/** Supported interpretations of Kafka record key and value bytes. */ +@Internal +public enum KafkaDataFormat { + RAW("raw"), + STRING("string"); + + /** Fluss table custom property controlling the record key format. */ + public static final String KEY_FORMAT_CONFIG = "kafka.key.format"; + + /** Fluss table custom property controlling the record value format. */ + public static final String VALUE_FORMAT_CONFIG = "kafka.value.format"; + + /** Fluss fields populated from the Kafka record key. */ + public static final String KEY_FIELDS_CONFIG = "kafka.key.fields"; + + /** Strategy for deriving fields populated from the Kafka record value. */ + public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; + + /** Fluss column populated from the Kafka record timestamp. */ + public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; + + /** Fluss column populated from the Kafka record headers. */ + public static final String HEADERS_COLUMN_CONFIG = "kafka.metadata.headers.column"; + + private final String value; + + KafkaDataFormat(String value) { + this.value = value; + } + + /** Parses a table custom property value. */ + public static KafkaDataFormat parse(String value) { + if (value == null) { + throw new IllegalArgumentException("Kafka data format must not be null."); + } + String normalized = value.trim().toLowerCase(Locale.ROOT); + for (KafkaDataFormat format : values()) { + if (format.value.equals(normalized)) { + return format; + } + } + throw new IllegalArgumentException( + "Unsupported Kafka data format '" + value + "'. Expected raw or string."); + } + + /** Returns the persisted table property value. */ + public String value() { + return value; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java new file mode 100644 index 00000000000..9f8ed63f543 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.Topic; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +@Internal +public final class KafkaTopicMapper { + + // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. + private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + + private final String databaseName; + + /** Creates a topic mapper for one Fluss database. */ + public KafkaTopicMapper(String databaseName) { + this.databaseName = checkNotNull(databaseName); + } + + /** Maps a Kafka topic name to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + Topic.validate(topicName); + return TablePath.of(databaseName, topicName); + } + + /** Returns whether a table belongs to this database and has a valid Kafka topic name. */ + public boolean isMappedTable(TablePath tablePath) { + return databaseName.equals(tablePath.getDatabaseName()) + && Topic.isValid(tablePath.getTableName()); + } + + /** Maps a Fluss table ID to a stable Kafka topic ID. */ + public Uuid toTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(TOPIC_ID_NAMESPACE, tableId); + } + + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ + public boolean isMappedTopicId(Uuid topicId) { + return topicId != null + && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && topicId.getLeastSignificantBits() >= 0L; + } + + /** Extracts the Fluss table ID encoded in a Kafka topic ID. */ + public long toTableId(Uuid topicId) { + checkArgument(isMappedTopicId(topicId), "Topic ID %s is not a Fluss topic ID.", topicId); + return topicId.getLeastSignificantBits(); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java new file mode 100644 index 00000000000..b1715758a9d --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Ordered physical Fluss fields populated by one Kafka record component. */ +@Internal +public final class KafkaFieldProjection { + + private final List positions; + private final List names; + private final List dataTypes; + + /** Creates a projection from physical row positions. */ + public KafkaFieldProjection(RowType rowType, List positions) { + checkNotNull(rowType); + checkNotNull(positions); + List positionCopy = new ArrayList<>(positions.size()); + List projectedNames = new ArrayList<>(positions.size()); + List projectedTypes = new ArrayList<>(positions.size()); + for (Integer position : positions) { + checkArgument( + position != null && position >= 0 && position < rowType.getFieldCount(), + "Invalid Kafka field projection position %s.", + position); + positionCopy.add(position); + projectedNames.add(rowType.getFieldNames().get(position)); + projectedTypes.add(rowType.getTypeAt(position)); + } + this.positions = Collections.unmodifiableList(positionCopy); + this.names = Collections.unmodifiableList(projectedNames); + this.dataTypes = Collections.unmodifiableList(projectedTypes); + } + + /** Returns the number of projected fields. */ + public int size() { + return positions.size(); + } + + /** Returns whether this projection owns no fields. */ + public boolean isEmpty() { + return positions.isEmpty(); + } + + /** Returns the physical row position at the projection position. */ + public int positionAt(int projectionPosition) { + return positions.get(projectionPosition); + } + + /** Returns the physical field name at the projection position. */ + public String nameAt(int projectionPosition) { + return names.get(projectionPosition); + } + + /** Returns the physical data type at the projection position. */ + public DataType dataTypeAt(int projectionPosition) { + return dataTypes.get(projectionPosition); + } + + /** Returns the projected physical positions. */ + public List positions() { + return positions; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaFieldProjection)) { + return false; + } + KafkaFieldProjection that = (KafkaFieldProjection) obj; + return Objects.equals(positions, that.positions) + && Objects.equals(names, that.names) + && Objects.equals(dataTypes, that.dataTypes); + } + + @Override + public int hashCode() { + return Objects.hash(positions, names, dataTypes); + } + + @Override + public String toString() { + return "KafkaFieldProjection{" + + "positions=" + + positions + + ", " + + "names=" + + names + + ", " + + "dataTypes=" + + dataTypes + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java new file mode 100644 index 00000000000..64263e69286 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.types.RowType; + +import javax.annotation.Nullable; + +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Resolved Kafka key, value, and metadata mapping for one Fluss table schema. */ +@Internal +public final class KafkaTopicSchema { + + private final RowType rowType; + private final @Nullable KafkaDataFormat keyFormat; + private final KafkaFieldProjection keyProjection; + private final KafkaDataFormat valueFormat; + private final KafkaFieldProjection valueProjection; + private final int timestampPosition; + private final int headersPosition; + + /** Creates a resolved Kafka topic schema. */ + public KafkaTopicSchema( + RowType rowType, + @Nullable KafkaDataFormat keyFormat, + KafkaFieldProjection keyProjection, + KafkaDataFormat valueFormat, + KafkaFieldProjection valueProjection, + int timestampPosition, + int headersPosition) { + this.rowType = checkNotNull(rowType); + this.keyFormat = keyFormat; + this.keyProjection = checkNotNull(keyProjection); + this.valueFormat = checkNotNull(valueFormat); + this.valueProjection = checkNotNull(valueProjection); + this.timestampPosition = timestampPosition; + this.headersPosition = headersPosition; + } + + /** Returns the physical Fluss row type. */ + public RowType rowType() { + return rowType; + } + + /** Returns the key format, or null when the Kafka key is not mapped. */ + public @Nullable KafkaDataFormat keyFormat() { + return keyFormat; + } + + /** Returns the key field projection. */ + public KafkaFieldProjection keyProjection() { + return keyProjection; + } + + /** Returns the value format. */ + public KafkaDataFormat valueFormat() { + return valueFormat; + } + + /** Returns the value field projection. */ + public KafkaFieldProjection valueProjection() { + return valueProjection; + } + + /** Returns the timestamp physical position, or -1 when it is not mapped. */ + public int timestampPosition() { + return timestampPosition; + } + + /** Returns the headers physical position, or -1 when they are not mapped. */ + public int headersPosition() { + return headersPosition; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaTopicSchema)) { + return false; + } + KafkaTopicSchema that = (KafkaTopicSchema) obj; + return Objects.equals(rowType, that.rowType) + && Objects.equals(keyFormat, that.keyFormat) + && Objects.equals(keyProjection, that.keyProjection) + && Objects.equals(valueFormat, that.valueFormat) + && Objects.equals(valueProjection, that.valueProjection) + && Objects.equals(timestampPosition, that.timestampPosition) + && Objects.equals(headersPosition, that.headersPosition); + } + + @Override + public int hashCode() { + return Objects.hash( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + @Override + public String toString() { + return "KafkaTopicSchema{" + + "rowType=" + + rowType + + ", " + + "keyFormat=" + + keyFormat + + ", " + + "keyProjection=" + + keyProjection + + ", " + + "valueFormat=" + + valueFormat + + ", " + + "valueProjection=" + + valueProjection + + ", " + + "timestampPosition=" + + timestampPosition + + ", " + + "headersPosition=" + + headersPosition + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java new file mode 100644 index 00000000000..0e6a294c4d5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that a Fluss table does not define a valid Kafka record mapping contract. */ +@Internal +public final class KafkaTopicSchemaException extends IllegalArgumentException { + + /** Creates a Kafka topic schema exception. */ + public KafkaTopicSchemaException(String message) { + super(message); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java new file mode 100644 index 00000000000..840ce8a4f58 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** Resolves and validates the Kafka record mapping stored in Fluss table custom properties. */ +@Internal +public final class KafkaTopicSchemaResolver { + + private static final String INCLUDE_ALL = "ALL"; + private static final String INCLUDE_EXCEPT_KEY = "EXCEPT_KEY"; + + private static final Set SUPPORTED_PROPERTIES = + new HashSet<>( + Arrays.asList( + KafkaDataFormat.KEY_FORMAT_CONFIG, + KafkaDataFormat.KEY_FIELDS_CONFIG, + KafkaDataFormat.VALUE_FORMAT_CONFIG, + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, + KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + + /** Resolves one table's Kafka record mapping contract. */ + public KafkaTopicSchema resolve(TableDescriptor table) { + validateTableKind(table); + RowType rowType = table.getSchema().getRowType(); + Map properties = table.getCustomProperties(); + + for (String property : properties.keySet()) { + if (property.startsWith("kafka.") && !SUPPORTED_PROPERTIES.contains(property)) { + throw invalid("Unsupported Kafka table property '" + property + "'."); + } + } + + int timestampPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG)); + int headersPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + if (timestampPosition >= 0 && timestampPosition == headersPosition) { + throw invalid("Kafka timestamp and headers cannot map to the same Fluss column."); + } + validateMetadataColumns(rowType, timestampPosition, headersPosition); + + String keyFormatValue = properties.get(KafkaDataFormat.KEY_FORMAT_CONFIG); + KafkaDataFormat keyFormat = keyFormatValue == null ? null : parseFormat(keyFormatValue); + List keyPositions = + resolveKeyPositions( + rowType, keyFormat, properties.get(KafkaDataFormat.KEY_FIELDS_CONFIG)); + for (Integer keyPosition : keyPositions) { + if (keyPosition == timestampPosition || keyPosition == headersPosition) { + throw invalid( + "Kafka key field '" + + rowType.getFieldNames().get(keyPosition) + + "' cannot be a Kafka metadata column."); + } + } + KafkaFieldProjection keyProjection = new KafkaFieldProjection(rowType, keyPositions); + + String valueFormatValue = properties.get(KafkaDataFormat.VALUE_FORMAT_CONFIG); + if (valueFormatValue == null) { + throw invalid( + "Missing required table property '" + + KafkaDataFormat.VALUE_FORMAT_CONFIG + + "'."); + } + KafkaDataFormat valueFormat = parseFormat(valueFormatValue); + String fieldsInclude = + normalizeFieldsInclude(properties.get(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG)); + if (!keyPositions.isEmpty() && INCLUDE_ALL.equals(fieldsInclude)) { + throw invalid( + "Mapping Kafka key fields requires " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + "=EXCEPT_KEY."); + } + + Set metadataPositions = new HashSet<>(); + if (timestampPosition >= 0) { + metadataPositions.add(timestampPosition); + } + if (headersPosition >= 0) { + metadataPositions.add(headersPosition); + } + List valuePositions = + resolveValuePositions(rowType, fieldsInclude, keyPositions, metadataPositions); + KafkaFieldProjection valueProjection = new KafkaFieldProjection(rowType, valuePositions); + if (valueProjection.isEmpty()) { + throw invalid("Kafka value projection must contain at least one Fluss column."); + } + + validateSingleFieldFormat(keyFormat, keyProjection, "key"); + validateSingleFieldFormat(valueFormat, valueProjection, "value"); + return new KafkaTopicSchema( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + private static void validateTableKind(TableDescriptor table) { + if (table.hasPrimaryKey()) { + throw invalid("Kafka topic table must be a log table."); + } + if (table.isPartitioned()) { + throw invalid("Partitioned Fluss tables are not supported."); + } + if (new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() + != LogFormat.ARROW) { + throw invalid("Kafka topic table must use the Arrow log format."); + } + } + + private static List resolveKeyPositions( + RowType rowType, KafkaDataFormat keyFormat, String keyFieldsValue) { + if (keyFormat == null) { + if (keyFieldsValue != null) { + throw invalid( + KafkaDataFormat.KEY_FIELDS_CONFIG + + " requires " + + KafkaDataFormat.KEY_FORMAT_CONFIG + + "."); + } + return Collections.emptyList(); + } + if (keyFieldsValue == null || keyFieldsValue.trim().isEmpty()) { + throw invalid( + "Missing required table property '" + KafkaDataFormat.KEY_FIELDS_CONFIG + "'."); + } + String[] fieldNames = keyFieldsValue.split(",", -1); + List positions = new ArrayList<>(fieldNames.length); + Set uniquePositions = new HashSet<>(); + for (String fieldNameValue : fieldNames) { + String fieldName = fieldNameValue.trim(); + if (fieldName.isEmpty()) { + throw invalid("Kafka key field names must not be empty."); + } + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka key field '" + fieldName + "' does not exist."); + } + if (!uniquePositions.add(position)) { + throw invalid("Duplicate Kafka key field '" + fieldName + "'."); + } + positions.add(position); + } + return positions; + } + + private static String normalizeFieldsInclude(String value) { + String normalized = value == null ? INCLUDE_ALL : value.trim().toUpperCase(Locale.ROOT); + if (!INCLUDE_ALL.equals(normalized) && !INCLUDE_EXCEPT_KEY.equals(normalized)) { + throw invalid( + "Invalid " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + " '" + + value + + "'. Expected ALL or EXCEPT_KEY."); + } + return normalized; + } + + private static List resolveValuePositions( + RowType rowType, + String fieldsInclude, + List keyPositions, + Set metadataPositions) { + Set excludedKeyPositions = + INCLUDE_EXCEPT_KEY.equals(fieldsInclude) + ? new HashSet<>(keyPositions) + : Collections.emptySet(); + List positions = new ArrayList<>(); + for (int position = 0; position < rowType.getFieldCount(); position++) { + if (!metadataPositions.contains(position) && !excludedKeyPositions.contains(position)) { + positions.add(position); + } + } + return positions; + } + + private static int resolveOptionalPosition(RowType rowType, String fieldNameValue) { + if (fieldNameValue == null) { + return -1; + } + if (fieldNameValue.trim().isEmpty()) { + throw invalid("Kafka metadata column name must not be empty."); + } + String fieldName = fieldNameValue.trim(); + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka metadata column '" + fieldName + "' does not exist."); + } + return position; + } + + private static void validateMetadataColumns( + RowType rowType, int timestampPosition, int headersPosition) { + if (timestampPosition >= 0) { + if (!(rowType.getTypeAt(timestampPosition) instanceof LocalZonedTimestampType) + || rowType.getTypeAt(timestampPosition).isNullable() + || ((LocalZonedTimestampType) rowType.getTypeAt(timestampPosition)) + .getPrecision() + != 3) { + throw invalid("Kafka timestamp column must be TIMESTAMP_LTZ(3) NOT NULL."); + } + } + if (headersPosition >= 0) { + validateHeadersType(rowType.getTypeAt(headersPosition)); + } + } + + private static void validateHeadersType(org.apache.fluss.types.DataType dataType) { + if (!(dataType instanceof ArrayType) || !dataType.isNullable()) { + throw invalid( + "Kafka headers column must be nullable " + + "ARRAY>."); + } + ArrayType arrayType = (ArrayType) dataType; + if (!(arrayType.getElementType() instanceof RowType)) { + throw invalid("Kafka headers elements must be rows."); + } + RowType headerType = (RowType) arrayType.getElementType(); + if (!headerType.getFieldNames().equals(Arrays.asList("name", "value")) + || !(headerType.getTypeAt(0) instanceof StringType) + || !(headerType.getTypeAt(1) instanceof BytesType) + || !headerType.getTypeAt(1).isNullable()) { + throw invalid("Kafka headers elements must be ROW."); + } + } + + private static void validateSingleFieldFormat( + KafkaDataFormat format, KafkaFieldProjection projection, String component) { + if (format == null) { + return; + } + if (format == KafkaDataFormat.RAW || format == KafkaDataFormat.STRING) { + if (projection.size() != 1) { + throw invalid( + "Kafka " + + component + + " format " + + format.value() + + " requires exactly one Fluss field."); + } + boolean validType = + format == KafkaDataFormat.RAW + ? projection.dataTypeAt(0) instanceof BytesType + : projection.dataTypeAt(0) instanceof StringType; + if (!validType) { + throw invalid( + "Kafka " + + component + + " field '" + + projection.nameAt(0) + + "' must be " + + (format == KafkaDataFormat.RAW ? "BYTES" : "STRING") + + " for format " + + format.value() + + "."); + } + } + } + + private static KafkaDataFormat parseFormat(String value) { + try { + return KafkaDataFormat.parse(value); + } catch (IllegalArgumentException e) { + throw invalid(e.getMessage()); + } + } + + private static KafkaTopicSchemaException invalid(String message) { + return new KafkaTopicSchemaException(message); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java new file mode 100644 index 00000000000..20c32ded941 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link KafkaTopicMapper}. */ +public class KafkaTopicMapperTest { + + @Test + public void testTopicNameAndIdMapping() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + + assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + Uuid topicId = mapper.toTopicId(123L); + assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); + assertThat(mapper.isMappedTopicId(topicId)).isTrue(); + assertThat(mapper.toTableId(topicId)).isEqualTo(123L); + + Uuid firstTableTopicId = mapper.toTopicId(0L); + assertThat(firstTableTopicId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(mapper.isMappedTopicId(firstTableTopicId)).isTrue(); + assertThat(mapper.toTableId(firstTableTopicId)).isZero(); + } + + @Test + public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue(); + assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse(); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath("invalid topic")) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); + assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java new file mode 100644 index 00000000000..0c92b8a7300 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies Kafka mapping properties survive the native Fluss table creation path. */ +public class KafkaTableMappingITCase { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder().setNumOfTabletServers(1).build(); + + @Test + public void testResolveMappingFromCreatedTableMetadata() throws Exception { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("event_key", DataTypes.STRING()) + .column("event_body", DataTypes.BYTES()) + .build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .build(); + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl"); + TablePath tablePath = mapper.toTablePath("events"); + long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor); + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + MetadataRequest request = new MetadataRequest(); + request.addTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()); + PbTableMetadata metadata = + FLUSS_CLUSTER_EXTENSION + .newCoordinatorClient() + .metadata(request) + .get() + .getTableMetadatasList() + .get(0); + TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); + + assertThat(metadata.getTableId()).isEqualTo(tableId); + assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId); + assertThat(metadata.getBucketMetadatasList()).hasSize(2); + assertThat(persisted.getCustomProperties()) + .containsAllEntriesOf(descriptor.getCustomProperties()); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java new file mode 100644 index 00000000000..4c698b5ad6b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the DDL contract independently of Kafka request handling and record decoding. */ +public class KafkaTopicSchemaResolverTest { + + private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); + + @Test + public void testRawMappingSurvivesTableMetadataSerialization() { + Schema schema = + Schema.newBuilder() + .column("message", DataTypes.BYTES()) + .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("attributes", headersType()) + .column("message_key", DataTypes.BYTES()) + .build(); + TableDescriptor table = + table(schema, "raw") + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .build(); + KafkaTopicSchema mapping = + resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes())); + + assertThat(mapping.rowType()).isEqualTo(schema.getRowType()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(3); + assertThat(mapping.keyProjection().nameAt(0)).isEqualTo("message_key"); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0)).isEqualTo(DataTypes.BYTES()); + assertThat(mapping.timestampPosition()).isEqualTo(1); + assertThat(mapping.headersPosition()).isEqualTo(2); + assertThat(mapping).isEqualTo(resolver.resolve(table)); + assertThatThrownBy(() -> mapping.keyProjection().positions().add(1)) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void testValueOnlyStringAndNullableColumns() { + for (boolean nullable : new boolean[] {true, false}) { + KafkaTopicSchema mapping = + resolver.resolve( + table( + Schema.newBuilder() + .column( + "body", + DataTypes.STRING().copy(nullable)) + .build(), + " STRING ") + .build()); + assertThat(mapping.keyFormat()).isNull(); + assertThat(mapping.keyProjection().isEmpty()).isTrue(); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0).isNullable()).isEqualTo(nullable); + assertThat(mapping.timestampPosition()).isEqualTo(-1); + assertThat(mapping.headersPosition()).isEqualTo(-1); + } + } + + @Test + public void testMixedFormatsAndMetadataAreOptional() { + KafkaTopicSchema mapping = + resolver.resolve( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, " except_key ") + .build()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + } + + @Test + public void testRejectsUnsupportedTableKinds() { + Schema primaryKeySchema = + Schema.newBuilder() + .column("id", DataTypes.STRING().copy(false)) + .primaryKey("id") + .build(); + assertInvalid(table(primaryKeySchema, "string"), "must be a log table"); + assertInvalid(keyValueTable().partitionedBy("id"), "Partitioned Fluss tables"); + assertInvalid(valueTable().logFormat(LogFormat.INDEXED), "Arrow log format"); + } + + @Test + public void testRequiresExplicitValueFormat() { + assertInvalid( + TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(1) + .customProperty("fluss.value.format", "raw"), + KafkaDataFormat.VALUE_FORMAT_CONFIG); + } + + @ParameterizedTest + @ValueSource(strings = {"json", "avro", ""}) + public void testRejectsUnavailableFormats(String format) { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + } + + @Test + public void testRejectsUnsupportedKafkaOptions() { + assertInvalid( + valueTable().customProperty("kafka.value.rescue-column", "body"), + "Unsupported Kafka table property"); + assertInvalid( + valueTable().customProperty("kafka.key.field", "body"), + "Unsupported Kafka table property"); + assertThat(resolver.resolve(valueTable().customProperty("owner", "team").build())) + .isNotNull(); + } + + @Test + public void testKeyFormatAndFieldMustBeSpecifiedTogether() { + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires kafka.key.format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"), + "kafka.key.fields"); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "missing", "id,id", "id,", ",id", "id,,body"}) + public void testRejectsInvalidKeyFields(String fields) { + assertThatThrownBy( + () -> + resolver.resolve( + keyValueTable() + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty( + KafkaDataFormat.KEY_FIELDS_CONFIG, fields) + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + "EXCEPT_KEY") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class); + } + + @Test + public void testRejectsAmbiguousAndEmptyValueProjections() { + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "bad"), + "Expected ALL or EXCEPT_KEY"); + assertInvalid( + valueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + assertInvalid(keyValueTable(), "exactly one Fluss field"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id,body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + } + + @Test + public void testRejectsWrongPhysicalTypes() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"), + "must be STRING"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "must be BYTES"); + } + + @Test + public void testRejectsWrongTimestampTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + DataTypes.TIMESTAMP_LTZ(3), + DataTypes.TIMESTAMP_LTZ(6).copy(false) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), + "TIMESTAMP_LTZ(3) NOT NULL"); + } + } + + @Test + public void testRejectsWrongHeaderTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + headersType().copy(false), + DataTypes.ARRAY(DataTypes.STRING()), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("key", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES().copy(false)))) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("attributes", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes"), + "headers"); + } + } + + @Test + public void testRejectsConflictingOrMissingMetadataColumns() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "missing"), + "does not exist"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, " "), + "must not be empty"); + TableDescriptor.Builder table = + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "ts"), + "same Fluss column"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "ts"), + "metadata column"); + } + + private void assertInvalid(TableDescriptor.Builder table, String message) { + assertThatThrownBy(() -> resolver.resolve(table.build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining(message); + } + + private static TableDescriptor.Builder keyValueTable() { + return table( + Schema.newBuilder() + .column("id", DataTypes.STRING()) + .column("body", DataTypes.BYTES()) + .build(), + "raw"); + } + + private static TableDescriptor.Builder valueTable() { + return table(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "raw"); + } + + private static TableDescriptor.Builder table(Schema schema, String valueFormat) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat); + } + + private static DataType headersType() { + return DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))); + } +} From 3252d38e1b7f28fb717234486bc363b5a64b39cc Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 11:39:47 +0800 Subject: [PATCH 07/48] [kafka] Route qualified topics and streamline field projections Resolve database.table topic names directly to Fluss table paths and return qualified names for metadata. Validate name boundaries and cover same-named tables in different databases through native metadata round trips. Read projected field names directly from DataField to avoid rebuilding the complete field-name list for every projected column. Validated on Java 11: 67 Kafka unit tests and 2 integration tests passed, along with Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 52/52 AI-Contributed/UT: 167/167 --- .../fluss/kafka/mapping/KafkaTopicMapper.java | 50 ++++++--- .../kafka/schema/KafkaFieldProjection.java | 2 +- .../kafka/mapping/KafkaTopicMapperTest.java | 101 ++++++++++++++++-- .../kafka/schema/KafkaTableMappingITCase.java | 66 ++++++++---- 4 files changed, 172 insertions(+), 47 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index 9f8ed63f543..f8b5744f71d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -21,35 +21,57 @@ import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.internals.Topic; import static org.apache.fluss.utils.Preconditions.checkArgument; -import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +/** Maps fully qualified Kafka topic identities to Fluss tables. */ @Internal public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; - private final String databaseName; - - /** Creates a topic mapper for one Fluss database. */ - public KafkaTopicMapper(String databaseName) { - this.databaseName = checkNotNull(databaseName); + /** Maps a Kafka topic in database.table form to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + if (!isValidTopic(topicName)) { + throw new InvalidTopicException( + "Kafka topic must be a valid database.table name: " + topicName); + } + int separator = topicName.indexOf('.'); + return TablePath.of(topicName.substring(0, separator), topicName.substring(separator + 1)); } - /** Maps a Kafka topic name to its Fluss table path. */ - public TablePath toTablePath(String topicName) { - Topic.validate(topicName); - return TablePath.of(databaseName, topicName); + /** Returns the fully qualified Kafka name of a representable Fluss table. */ + public String toTopicName(TablePath tablePath) { + if (!isMappedTable(tablePath)) { + throw new InvalidTopicException( + "Fluss table cannot be represented as a Kafka database.table name: " + + tablePath); + } + return tablePath.toString(); } - /** Returns whether a table belongs to this database and has a valid Kafka topic name. */ + /** Returns whether a Fluss user table has a valid, fully qualified Kafka topic name. */ public boolean isMappedTable(TablePath tablePath) { - return databaseName.equals(tablePath.getDatabaseName()) - && Topic.isValid(tablePath.getTableName()); + return tablePath != null && tablePath.isValid() && isValidTopic(tablePath.toString()); + } + + /** Returns whether a name uniquely represents a user table in a Fluss database. */ + public static boolean isValidTopic(String topicName) { + if (topicName == null || !Topic.isValid(topicName)) { + return false; + } + int separator = topicName.indexOf('.'); + if (separator <= 0 || separator != topicName.lastIndexOf('.')) { + return false; + } + String database = topicName.substring(0, separator); + String table = topicName.substring(separator + 1); + return TablePath.of(database, table).isValid() + && TablePath.validatePrefix(database) == null + && TablePath.validatePrefix(table) == null; } /** Maps a Fluss table ID to a stable Kafka topic ID. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java index b1715758a9d..b6833475a20 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java @@ -50,7 +50,7 @@ public KafkaFieldProjection(RowType rowType, List positions) { "Invalid Kafka field projection position %s.", position); positionCopy.add(position); - projectedNames.add(rowType.getFieldNames().get(position)); + projectedNames.add(rowType.getFields().get(position).getName()); projectedTypes.add(rowType.getTypeAt(position)); } this.positions = Collections.unmodifiableList(positionCopy); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index 20c32ded941..c5d4a582a88 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -22,6 +22,12 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InvalidTopicException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -29,11 +35,88 @@ /** Tests for {@link KafkaTopicMapper}. */ public class KafkaTopicMapperTest { + private final KafkaTopicMapper mapper = new KafkaTopicMapper(); + + @Test + public void testQualifiedNamesAcrossDatabases() { + for (String database : Arrays.asList("sales", "archive", "sales_1-2026")) { + TablePath tablePath = TablePath.of(database, "orders_1-2026"); + String topicName = database + ".orders_1-2026"; + assertThat(mapper.toTablePath(topicName)).isEqualTo(tablePath); + assertThat(mapper.toTopicName(tablePath)).isEqualTo(topicName); + assertThat(mapper.isMappedTable(tablePath)).isTrue(); + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isTrue(); + } + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource( + strings = { + "orders", + ".orders", + "sales.", + "sales.orders.extra", + "sales..orders", + "sales.bad name", + " sales.orders", + "sales.orders ", + "sales/orders", + "__system.orders", + "sales.__internal", + "数据库.orders", + "sales.订单" + }) + public void testRejectsInvalidTopicNames(String topicName) { + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(topicName)) + .isInstanceOf(InvalidTopicException.class); + } + @Test - public void testTopicNameAndIdMapping() { - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + public void testNameLengthLimits() { + for (TablePath valid : + Arrays.asList( + TablePath.of(repeat('a', 200), "orders"), + TablePath.of("sales", repeat('a', 200)), + TablePath.of(repeat('a', 124), repeat('b', 124)))) { + assertThat(mapper.toTablePath(mapper.toTopicName(valid))).isEqualTo(valid); + } + for (TablePath invalid : + Arrays.asList( + TablePath.of(repeat('a', 201), "orders"), + TablePath.of("sales", repeat('a', 201)), + TablePath.of(repeat('a', 124), repeat('b', 125)))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(invalid.toString())) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } + + @Test + public void testRejectsUnrepresentableTablePaths() { + for (TablePath invalid : + Arrays.asList( + null, + TablePath.of(null, "orders"), + TablePath.of("sales", null), + TablePath.of("", "orders"), + TablePath.of("sales", ""), + TablePath.of("sales.region", "orders"), + TablePath.of("sales", "orders.v1"), + TablePath.of("__system", "orders"), + TablePath.of("sales", "__internal"), + TablePath.of("sales", "bad name"))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } - assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + @Test + public void testTopicIdMapping() { Uuid topicId = mapper.toTopicId(123L); assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); assertThat(mapper.isMappedTopicId(topicId)).isTrue(); @@ -46,17 +129,15 @@ public void testTopicNameAndIdMapping() { } @Test - public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() { - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); - assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue(); - assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse(); - assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse(); - assertThatThrownBy(() -> mapper.toTablePath("invalid topic")) - .isInstanceOf(InvalidTopicException.class); + public void testRejectsInvalidTopicIds() { assertThatThrownBy(() -> mapper.toTopicId(-1L)) .isInstanceOf(IllegalArgumentException.class); assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) .isInstanceOf(IllegalArgumentException.class); } + + private static String repeat(char character, int count) { + return String.join("", Collections.nCopies(count, String.valueOf(character))); + } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 0c92b8a7300..2ded7bd17f6 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -31,6 +31,11 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; import static org.assertj.core.api.Assertions.assertThat; @@ -42,7 +47,7 @@ public class KafkaTableMappingITCase { FlussClusterExtension.builder().setNumOfTabletServers(1).build(); @Test - public void testResolveMappingFromCreatedTableMetadata() throws Exception { + public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Exception { TableDescriptor descriptor = TableDescriptor.builder() .schema( @@ -57,32 +62,49 @@ public void testResolveMappingFromCreatedTableMetadata() throws Exception { .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl"); - TablePath tablePath = mapper.toTablePath("events"); - long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor); - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + KafkaTopicMapper mapper = new KafkaTopicMapper(); + Map tableIds = new LinkedHashMap<>(); MetadataRequest request = new MetadataRequest(); - request.addTablePath() - .setDatabaseName(tablePath.getDatabaseName()) - .setTableName(tablePath.getTableName()); - PbTableMetadata metadata = + for (String database : Arrays.asList("kafka_ddl", "kafka_ddl_archive")) { + TablePath tablePath = mapper.toTablePath(database + ".events"); + assertThat(tablePath).isEqualTo(TablePath.of(database, "events")); + tableIds.put(tablePath, createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor)); + request.addTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()); + } + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + List metadatas = FLUSS_CLUSTER_EXTENSION .newCoordinatorClient() .metadata(request) .get() - .getTableMetadatasList() - .get(0); - TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); + .getTableMetadatasList(); + assertThat(metadatas).hasSize(2); + assertThat(metadatas) + .extracting(PbTableMetadata::getTableId) + .containsExactlyInAnyOrderElementsOf(tableIds.values()) + .doesNotHaveDuplicates(); + for (PbTableMetadata metadata : metadatas) { + TablePath tablePath = + TablePath.of( + metadata.getTablePath().getDatabaseName(), + metadata.getTablePath().getTableName()); + TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); - assertThat(metadata.getTableId()).isEqualTo(tableId); - assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId); - assertThat(metadata.getBucketMetadatasList()).hasSize(2); - assertThat(persisted.getCustomProperties()) - .containsAllEntriesOf(descriptor.getCustomProperties()); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(1); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(metadata.getTableId()).isEqualTo(tableIds.get(tablePath)); + assertThat(mapper.toTopicName(tablePath)) + .isIn("kafka_ddl.events", "kafka_ddl_archive.events"); + assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))) + .isEqualTo(tableIds.get(tablePath)); + assertThat(metadata.getBucketMetadatasList()).hasSize(2); + assertThat(persisted.getCustomProperties()) + .containsAllEntriesOf(descriptor.getCustomProperties()); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + } } } From 1275b338946fb4aabd78af1533b026129502563e Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:28:20 +0800 Subject: [PATCH 08/48] [kafka] Define Kafka timestamp metadata as TIMESTAMP(3) Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 16/16 AI-Contributed/UT: 33/33 --- .../fluss/kafka/format/KafkaDataFormat.java | 5 +++- .../schema/KafkaTopicSchemaResolver.java | 11 +++----- .../kafka/schema/KafkaTableMappingITCase.java | 5 ++++ .../schema/KafkaTopicSchemaResolverTest.java | 28 ++++++++++++++++--- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java index ad0039b7e78..df5d871907c 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java @@ -39,7 +39,10 @@ public enum KafkaDataFormat { /** Strategy for deriving fields populated from the Kafka record value. */ public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; - /** Fluss column populated from the Kafka record timestamp. */ + /** + * TIMESTAMP(3) column populated from the Kafka record epoch milliseconds as UTC calendar + * values. A missing Kafka timestamp (-1) maps to null and requires a nullable column. + */ public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; /** Fluss column populated from the Kafka record headers. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index 840ce8a4f58..8959c420d15 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -25,9 +25,9 @@ import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BytesType; -import org.apache.fluss.types.LocalZonedTimestampType; import org.apache.fluss.types.RowType; import org.apache.fluss.types.StringType; +import org.apache.fluss.types.TimestampType; import java.util.ArrayList; import java.util.Arrays; @@ -234,12 +234,9 @@ private static int resolveOptionalPosition(RowType rowType, String fieldNameValu private static void validateMetadataColumns( RowType rowType, int timestampPosition, int headersPosition) { if (timestampPosition >= 0) { - if (!(rowType.getTypeAt(timestampPosition) instanceof LocalZonedTimestampType) - || rowType.getTypeAt(timestampPosition).isNullable() - || ((LocalZonedTimestampType) rowType.getTypeAt(timestampPosition)) - .getPrecision() - != 3) { - throw invalid("Kafka timestamp column must be TIMESTAMP_LTZ(3) NOT NULL."); + if (!(rowType.getTypeAt(timestampPosition) instanceof TimestampType) + || ((TimestampType) rowType.getTypeAt(timestampPosition)).getPrecision() != 3) { + throw invalid("Kafka timestamp column must be TIMESTAMP(3)."); } } if (headersPosition >= 0) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 2ded7bd17f6..2f3e75bdd3d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -54,12 +54,14 @@ public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Excep Schema.newBuilder() .column("event_key", DataTypes.STRING()) .column("event_body", DataTypes.BYTES()) + .column("event_time", DataTypes.TIMESTAMP(3)) .build()) .distributedBy(2) .logFormat(LogFormat.ARROW) .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "event_time") .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); KafkaTopicMapper mapper = new KafkaTopicMapper(); @@ -105,6 +107,9 @@ public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Excep assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); assertThat(mapping.valueProjection().positions()).containsExactly(1); assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.timestampPosition()).isEqualTo(2); + assertThat(persisted.getSchema().getRowType().getTypeAt(2)) + .isEqualTo(DataTypes.TIMESTAMP(3)); } } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 4c698b5ad6b..66c13a4711a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -41,7 +41,7 @@ public void testRawMappingSurvivesTableMetadataSerialization() { Schema schema = Schema.newBuilder() .column("message", DataTypes.BYTES()) - .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) .column("attributes", headersType()) .column("message_key", DataTypes.BYTES()) .build(); @@ -219,13 +219,33 @@ public void testRejectsWrongPhysicalTypes() { "must be BYTES"); } + @Test + public void testAcceptsTimestampWithEitherNullability() { + for (boolean nullable : new boolean[] {true, false}) { + TableDescriptor descriptor = + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", DataTypes.TIMESTAMP(3).copy(nullable)) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts") + .build(); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(descriptor); + assertThat(mapping.timestampPosition()).isEqualTo(1); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + } + } + @Test public void testRejectsWrongTimestampTypes() { for (DataType type : new DataType[] { DataTypes.STRING(), DataTypes.TIMESTAMP_LTZ(3), - DataTypes.TIMESTAMP_LTZ(6).copy(false) + DataTypes.TIMESTAMP_LTZ(3).copy(false), + DataTypes.TIMESTAMP(2), + DataTypes.TIMESTAMP(6).copy(false) }) { assertInvalid( table( @@ -235,7 +255,7 @@ public void testRejectsWrongTimestampTypes() { .build(), "raw") .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), - "TIMESTAMP_LTZ(3) NOT NULL"); + "TIMESTAMP(3)"); } } @@ -279,7 +299,7 @@ public void testRejectsConflictingOrMissingMetadataColumns() { table( Schema.newBuilder() .column("body", DataTypes.BYTES()) - .column("ts", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("ts", DataTypes.TIMESTAMP(3).copy(false)) .build(), "raw") .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); From dea30c267b3b5d5b14ebaf5134a3ee34fbb2de43 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 11 Oct 2026 23:16:34 +0800 Subject: [PATCH 09/48] [test] Wait for table readiness in snapshot lease validation Wait for the newly created log table to become ready before acquiring or releasing snapshot leases. This prevents the coordinator's asynchronous table-ID registration from masking the expected NonPrimaryKeyTableException. Validated on Java 11 with the same-worktree reactor: FlussAdminITCase#testKvSnapshotLeaseForLogTable passed, along with Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 1/1 --- .../java/org/apache/fluss/client/admin/FlussAdminITCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index e67faf84b24..b39739c6b01 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -2831,6 +2831,7 @@ void testKvSnapshotLeaseForLogTable() throws Exception { TableDescriptor logTableDescriptor = TableDescriptor.builder().schema(logSchema).distributedBy(3).build(); long tableId = createTable(logTablePath, logTableDescriptor, true); + FLUSS_CLUSTER_EXTENSION.waitUntilTableReady(tableId); // Create a KvSnapshotLease KvSnapshotLease lease = admin.createKvSnapshotLease("test-lease-log", 60000); From 6e9dc555cd6e2db25e3def9a800d1966995e3179 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 15:55:33 +0800 Subject: [PATCH 10/48] [kafka] Align DDL table mapping with the FIP contract Register table.kafka options and validate the declared fixed-column mapping through a shared schema contract. Support binary, string and flat/nested JSON values with optional key, headers and rescue columns. Validate CREATE and ALTER before persistence, and fence schema publication with the table version and coordinator epoch so concurrent schema and property changes cannot bypass the contract. Resolve bare topic aliases through kafka.default.database and stop automatically creating that database. Keep real offset persistence deferred and reject Kafka enablement for tables that declare offset metadata until that capability is available. Validation: 146 targeted tests passed on Java 11; Checkstyle, Spotless, RAT and git diff checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 1131/1131 AI-Contributed/UT: 1451/1451 --- .../apache/fluss/config/ConfigOptions.java | 39 +- .../apache/fluss/config/FlussConfigUtils.java | 4 + .../org/apache/fluss/config/TableConfig.java | 5 + .../fluss/metadata}/KafkaFieldProjection.java | 2 +- .../fluss/metadata/KafkaTableSchema.java | 391 ++++++++++++++++ .../fluss/metadata/KafkaTableSchemaTest.java | 437 ++++++++++++++++++ .../flink/utils/FlinkConversionsTest.java | 45 ++ .../fluss/kafka/format/KafkaDataFormat.java | 76 --- .../fluss/kafka/mapping/KafkaTopicMapper.java | 49 +- .../fluss/kafka/schema/KafkaTopicSchema.java | 150 ------ .../schema/KafkaTopicSchemaResolver.java | 292 +----------- .../apache/fluss/kafka/KafkaConfigsTest.java | 8 +- .../kafka/mapping/KafkaTopicMapperTest.java | 34 +- .../kafka/schema/KafkaTableMappingITCase.java | 410 +++++++++++++--- .../schema/KafkaTopicSchemaResolverTest.java | 335 ++------------ .../server/coordinator/CoordinatorServer.java | 8 - .../coordinator/CoordinatorService.java | 5 +- .../server/coordinator/MetadataManager.java | 36 +- .../utils/TableDescriptorValidation.java | 7 + .../fluss/server/zk/ZooKeeperClient.java | 30 ++ .../KafkaTableSchemaAlterTest.java | 182 ++++++++ website/docs/maintenance/configuration.md | 37 +- 22 files changed, 1672 insertions(+), 910 deletions(-) rename {fluss-kafka/src/main/java/org/apache/fluss/kafka/schema => fluss-common/src/main/java/org/apache/fluss/metadata}/KafkaFieldProjection.java (99%) create mode 100644 fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java create mode 100644 fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java delete mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java delete mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 3e86eb77579..667e1af5f68 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2269,6 +2269,38 @@ public class ConfigOptions { + "Downstream consumers must be upgraded to Fluss v1.0+ before enabling this option, " + "as older versions cannot parse the extended batch format."); + public static final ConfigOption TABLE_KAFKA_ENABLED = + key("table.kafka.enabled") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether the table is exposed as a Kafka topic. Enabling validates the " + + "existing schema and conversion options without adding columns or defaults."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_TYPE = + key("table.kafka.convert.value.type") + .stringType() + .noDefaultValue() + .withDescription( + "Required when Kafka compatibility is enabled: binary, string, or parse_json. " + + "Binary and string target the declared _kafka_value column."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_LAYOUT = + key("table.kafka.convert.value.layout") + .stringType() + .noDefaultValue() + .withDescription( + "JSON layout: flat (the read-time default for parse_json) or nested. " + + "Nested targets _kafka_value ROW. Not allowed for binary or string."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_KEY_TYPE = + key("table.kafka.convert.key.type") + .stringType() + .noDefaultValue() + .withDescription( + "Optional Kafka key conversion type. Only binary is supported and requires " + + "_kafka_key BYTES. Declaring that column implies binary without persisting a default."); + // ------------------------------------------------------------------------ // ConfigOptions for Kv // ------------------------------------------------------------------------ @@ -2905,12 +2937,13 @@ public class ConfigOptions { .withDescription( "The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma."); - public static final ConfigOption KAFKA_DATABASE = - key("kafka.database") + public static final ConfigOption KAFKA_DEFAULT_DATABASE = + key("kafka.default.database") .stringType() .defaultValue("kafka") .withDescription( - "The database for fluss kafka. The default database is `kafka`."); + "Default database for unqualified Kafka topic names. Fully qualified names may " + + "access other databases. Databases must already exist; this option does not create them."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index 37d5cc9f88e..415103c46ea 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -47,6 +47,10 @@ public class FlussConfigUtils { CLIENT_OPTIONS = extractConfigOptions("client."); ALTERABLE_TABLE_OPTIONS = Arrays.asList( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), ConfigOptions.TABLE_DATALAKE_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_MODE.key(), diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index ed2c111c335..b1bc64dfc83 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -54,6 +54,11 @@ public TableConfig(Configuration config) { this.config = config; } + /** Whether the table explicitly enables Kafka compatibility. */ + public boolean isKafkaEnabled() { + return config.get(ConfigOptions.TABLE_KAFKA_ENABLED); + } + /** Gets the replication factor of the table. */ public int getReplicationFactor() { return config.get(ConfigOptions.TABLE_REPLICATION_FACTOR); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java similarity index 99% rename from fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java rename to fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java index b6833475a20..f8946107020 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.fluss.kafka.schema; +package org.apache.fluss.metadata; import org.apache.fluss.annotation.Internal; import org.apache.fluss.types.DataType; diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java new file mode 100644 index 00000000000..7c040f15871 --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; + +/** + * Immutable Kafka record mapping, shared by native DDL validation and Kafka request handling. + * Resolving a contract never changes the schema or materializes configuration defaults. + */ +@Internal +public final class KafkaTableSchema { + + public static final String KEY_COLUMN = "_kafka_key"; + public static final String VALUE_COLUMN = "_kafka_value"; + public static final String HEADERS_COLUMN = "_kafka_headers"; + public static final String METADATA_COLUMN = "_kafka_metadata"; + public static final String RESCUE_COLUMN = "_kafka_rescue"; + + private static final Set RESERVED_NAMES = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + KEY_COLUMN, + VALUE_COLUMN, + HEADERS_COLUMN, + METADATA_COLUMN, + RESCUE_COLUMN))); + private static final Set OPTIONS = + new HashSet<>( + Arrays.asList( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key())); + + /** Supported value conversion contracts; decoders are provided by the Kafka module. */ + public enum ValueType { + BINARY, + STRING, + PARSE_JSON + } + + /** JSON business fields either occupy the table row or the fixed value ROW column. */ + public enum ValueLayout { + FLAT, + NESTED + } + + private final RowType rowType; + private final ValueType valueType; + private final @Nullable ValueLayout valueLayout; + private final KafkaFieldProjection valueProjection; + private final int keyPosition; + private final int headersPosition; + private final int rescuePosition; + + private KafkaTableSchema( + RowType rowType, + ValueType valueType, + @Nullable ValueLayout valueLayout, + List valuePositions, + int keyPosition, + int headersPosition, + int rescuePosition) { + this.rowType = rowType; + this.valueType = valueType; + this.valueLayout = valueLayout; + this.valueProjection = new KafkaFieldProjection(rowType, valuePositions); + this.keyPosition = keyPosition; + this.headersPosition = headersPosition; + this.rescuePosition = rescuePosition; + } + + /** Returns only the persisted opt-in flag; schema compatibility does not imply enablement. */ + public static boolean isEnabled(TableDescriptor table) { + return new TableConfig(Configuration.fromMap(table.getProperties())).isKafkaEnabled(); + } + + /** Validates the Kafka contract only when the final table state enables it. */ + public static void validate(TableDescriptor table) { + if (isEnabled(table)) { + resolve(table); + } + } + + /** Checks whether the canonical topic name satisfies Fluss and Kafka name restrictions. */ + public static boolean isValidTablePath(TablePath path) { + // Fluss identifiers are already restricted to Kafka's ASCII character set, without dots. + return path != null + && path.isValid() + && TablePath.validatePrefix(path.getDatabaseName()) == null + && TablePath.validatePrefix(path.getTableName()) == null + && path.toString().length() <= 249; + } + + /** Validates the canonical topic name before persisting an enabled table. */ + public static void validateTablePath(TablePath path, TableDescriptor table) { + if (isEnabled(table) && !isValidTablePath(path)) { + throw invalid("Table path cannot be represented as a Kafka topic: " + path); + } + } + + /** Returns whether a JSON member at the value mapping level has a reserved name. */ + public static boolean isReservedName(String name) { + return RESERVED_NAMES.contains(name); + } + + /** Resolves an explicitly enabled table or rejects its invalid contract. */ + public static KafkaTableSchema resolve(TableDescriptor table) { + if (!isEnabled(table)) { + throw invalid("Kafka compatibility is disabled; table.kafka.enabled must be true."); + } + validateOptions(table); + if (table.hasPrimaryKey() + || table.isPartitioned() + || new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() + != LogFormat.ARROW) { + throw invalid( + "Kafka requires a non-partitioned Arrow log table without a primary key."); + } + Configuration options = Configuration.fromMap(table.getProperties()); + String value = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE); + if (value == null) { + throw invalid("Missing required table property 'table.kafka.convert.value.type'."); + } + ValueType valueType; + try { + valueType = ValueType.valueOf(value.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw invalid( + "Unsupported table.kafka.convert.value.type '" + + value + + "'. Expected binary, string or parse_json."); + } + String layout = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT); + ValueLayout valueLayout = null; + if (valueType == ValueType.PARSE_JSON) { + try { + valueLayout = + layout == null + ? ValueLayout.FLAT + : ValueLayout.valueOf(layout.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw invalid("Unsupported table.kafka.convert.value.layout '" + layout + "'."); + } + } else if (layout != null) { + throw invalid("table.kafka.convert.value.layout is only supported for parse_json."); + } + + RowType rowType = table.getSchema().getRowType(); + int keyPosition = rowType.getFieldIndex(KEY_COLUMN); + if (keyPosition >= 0 && !(rowType.getTypeAt(keyPosition) instanceof BytesType)) { + throw invalid("_kafka_key must be BYTES."); + } + String keyType = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE); + if (keyType != null && (!"binary".equalsIgnoreCase(keyType.trim()) || keyPosition < 0)) { + throw invalid( + "table.kafka.convert.key.type only supports binary and requires _kafka_key BYTES."); + } + int headersPosition = rowType.getFieldIndex(HEADERS_COLUMN); + if (headersPosition >= 0) { + DataType headers = rowType.getTypeAt(headersPosition); + if (!(headers instanceof MapType) + || !(((MapType) headers).getKeyType() instanceof StringType) + || !(((MapType) headers).getValueType() instanceof BytesType)) { + throw invalid("_kafka_headers must be MAP."); + } + } + int rescuePosition = rowType.getFieldIndex(RESCUE_COLUMN); + if (rescuePosition >= 0 + && (!(rowType.getTypeAt(rescuePosition) instanceof StringType) + || !rowType.getTypeAt(rescuePosition).isNullable())) { + throw invalid("_kafka_rescue must be nullable STRING."); + } + int metadataPosition = rowType.getFieldIndex(METADATA_COLUMN); + if (metadataPosition >= 0) { + validateMetadataType(rowType.getTypeAt(metadataPosition)); + throw invalid( + "_kafka_metadata requires native append support for persisting actual per-row offsets; " + + "this capability is not available yet."); + } + + List valuePositions = new ArrayList<>(); + if (valueLayout == ValueLayout.FLAT) { + if (rowType.getFieldIndex(VALUE_COLUMN) >= 0) { + throw invalid( + "_kafka_value is reserved for binary, string or nested JSON; " + + "it cannot be a flat JSON business field."); + } + for (int i = 0; i < rowType.getFieldCount(); i++) { + if (!isReservedName(rowType.getFieldNames().get(i))) { + valuePositions.add(i); + } + } + } else { + int valuePosition = rowType.getFieldIndex(VALUE_COLUMN); + if (valuePosition < 0) { + throw invalid("Missing required value column _kafka_value."); + } + DataType type = rowType.getTypeAt(valuePosition); + boolean valid = + valueType == ValueType.BINARY + ? type instanceof BytesType + : valueType == ValueType.STRING + ? type instanceof StringType + : type instanceof RowType; + if (!valid) { + throw invalid( + "_kafka_value must be " + + (valueType == ValueType.BINARY + ? "BYTES" + : valueType == ValueType.STRING ? "STRING" : "ROW") + + " for " + + valueType + + "."); + } + for (String name : rowType.getFieldNames()) { + if (!isReservedName(name)) { + throw invalid( + "Column '" + + name + + "' has no value mapping; this conversion targets _kafka_value only."); + } + } + if (type instanceof RowType) { + for (String name : ((RowType) type).getFieldNames()) { + if (isReservedName(name)) { + throw invalid( + "Reserved name '" + + name + + "' cannot be a nested JSON business field."); + } + } + } + valuePositions.add(valuePosition); + } + return new KafkaTableSchema( + rowType, + valueType, + valueLayout, + valuePositions, + keyPosition, + headersPosition, + rescuePosition); + } + + private static void validateOptions(TableDescriptor table) { + for (String name : table.getProperties().keySet()) { + if (name.startsWith("table.kafka.") && !OPTIONS.contains(name)) { + throw invalid("Unsupported Kafka table property '" + name + "'."); + } + } + for (String name : table.getCustomProperties().keySet()) { + if (name.startsWith("table.kafka.") + || name.startsWith("kafka.") + || name.startsWith("fluss.")) { + throw invalid( + "Unsupported Kafka custom property '" + + name + + "'. Use registered table.kafka.* properties and fixed system columns."); + } + } + } + + private static void validateMetadataType(DataType type) { + String message = + "_kafka_metadata must be ROW."; + if (!(type instanceof RowType)) { + throw invalid(message); + } + RowType row = (RowType) type; + if (!row.getFieldNames() + .equals(Arrays.asList("timestamp", "timestamp_type", "partition", "offset")) + || !row.getTypeAt(0).copy(true).equals(DataTypes.TIMESTAMP_LTZ(3)) + || !row.getTypeAt(1).copy(true).equals(DataTypes.INT()) + || !row.getTypeAt(2).copy(true).equals(DataTypes.INT()) + || !row.getTypeAt(3).copy(true).equals(DataTypes.BIGINT())) { + throw invalid(message); + } + } + + /** Returns the unchanged physical schema, including declared nullability. */ + public RowType rowType() { + return rowType; + } + + /** Returns the value conversion type. */ + public ValueType valueType() { + return valueType; + } + + /** Returns the JSON layout, or null for binary/string. */ + public @Nullable ValueLayout valueLayout() { + return valueLayout; + } + + /** Returns physical value targets; nested JSON projects its single ROW container. */ + public KafkaFieldProjection valueProjection() { + return valueProjection; + } + + /** Returns the key column position, or -1 when keys are not stored. */ + public int keyPosition() { + return keyPosition; + } + + /** + * Returns the headers column position, or -1 when headers are not stored. Decoding maps names + * to binary values, with the last occurrence winning for duplicate header names. + */ + public int headersPosition() { + return headersPosition; + } + + /** + * Returns the rescue column position, or -1 when absent. JSON decoders collect unknown and + * reserved members here; binary/string decoders write null to this optional column. + */ + public int rescuePosition() { + return rescuePosition; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof KafkaTableSchema)) { + return false; + } + KafkaTableSchema that = (KafkaTableSchema) other; + return rowType.equals(that.rowType) + && valueType == that.valueType + && valueLayout == that.valueLayout + && valueProjection.equals(that.valueProjection) + && keyPosition == that.keyPosition + && headersPosition == that.headersPosition + && rescuePosition == that.rescuePosition; + } + + @Override + public int hashCode() { + return Objects.hash( + rowType, + valueType, + valueLayout, + valueProjection, + keyPosition, + headersPosition, + rescuePosition); + } + + private static InvalidTableException invalid(String message) { + return new InvalidTableException(message); + } +} diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java new file mode 100644 index 00000000000..e2b9c5be187 --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.FlussConfigUtils; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the shared DDL and Kafka conversion contract. */ +class KafkaTableSchemaTest { + + @Test + void testRegisteredAlterableOptionsAndReadTimeDefaults() { + for (String key : + Arrays.asList( + "table.kafka.enabled", + "table.kafka.convert.value.type", + "table.kafka.convert.value.layout", + "table.kafka.convert.key.type")) { + assertThat(FlussConfigUtils.TABLE_OPTIONS).containsKey(key); + assertThat(FlussConfigUtils.isAlterableTableOption(key)).isTrue(); + } + Configuration config = new Configuration(); + assertThat(new TableConfig(config).isKafkaEnabled()).isFalse(); + assertThat(config.toMap()).isEmpty(); + } + + @Test + void testExplicitOptInAndDisabledTables() { + TableDescriptor disabled = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + assertThat(KafkaTableSchema.isEnabled(disabled)).isFalse(); + assertThatCode(() -> KafkaTableSchema.validate(disabled)).doesNotThrowAnyException(); + assertThatThrownBy(() -> KafkaTableSchema.resolve(disabled)) + .hasMessageContaining("disabled"); + TableDescriptor arbitrary = + TableDescriptor.builder(disabled) + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .primaryKey("id") + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "future-format") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build(); + assertThatCode(() -> KafkaTableSchema.validate(arbitrary)).doesNotThrowAnyException(); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testFixedValueColumnWithOptionalSystemFields(String format) { + DataType valueType = format.equals("binary") ? DataTypes.BYTES() : DataTypes.STRING(); + for (int mask = 0; mask < 8; mask++) { + Schema.Builder schema = + Schema.newBuilder().column("_kafka_value", valueType.copy(false)); + if ((mask & 1) != 0) { + schema.column("_kafka_key", DataTypes.BYTES()); + } + if ((mask & 2) != 0) { + schema.column( + "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + } + if ((mask & 4) != 0) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + TableDescriptor table = enabled(schema.build(), format).build(); + byte[] before = table.toJsonBytes(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueProjection().positions()).containsExactly(0); + assertThat(resolved.valueProjection().dataTypeAt(0).isNullable()).isFalse(); + assertThat(resolved.valueLayout()).isNull(); + assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); + assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); + assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); + assertThat(table.toJsonBytes()).isEqualTo(before); + assertThat(table.getProperties()) + .doesNotContainKeys( + "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); + assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) + .isEqualTo(resolved); + } + } + + @Test + void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { + RowType nested = + DataTypes.ROW( + DataTypes.FIELD("_kafka_key", DataTypes.STRING()), + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())))); + TableDescriptor table = + enabled( + Schema.newBuilder() + .column("id", DataTypes.BIGINT().copy(false)) + .column("_kafka_rescue", DataTypes.STRING()) + .column("payload", nested) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column("custom_rescue", DataTypes.STRING()) + .build(), + "parse_json") + .build(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueType()).isEqualTo(ValueType.PARSE_JSON); + assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.FLAT); + assertThat(resolved.valueProjection().positions()).containsExactly(0, 2, 5); + assertThat(resolved.valueProjection().dataTypeAt(1)).isEqualTo(nested); + assertThat(resolved.rescuePosition()).isEqualTo(1); + assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); + TableDescriptor noRescue = + enabled( + Schema.newBuilder() + .column("custom_rescue", DataTypes.STRING()) + .build(), + "parse_json") + .build(); + assertThat(KafkaTableSchema.resolve(noRescue).rescuePosition()).isEqualTo(-1); + } + + @Test + void testNestedJsonTargetsOneRow() { + RowType row = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.STRING()), + DataTypes.FIELD( + "details", + DataTypes.ROW( + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())))); + TableDescriptor table = + enabled( + Schema.newBuilder() + .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_value", row) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.NESTED); + assertThat(resolved.valueProjection().positions()).containsExactly(1); + assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(row); + assertThat(resolved.rescuePosition()).isZero(); + } + + @ParameterizedTest + @ValueSource( + strings = { + "_kafka_key", + "_kafka_value", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue" + }) + void testReservedNamesAtValueMappingLevel(String name) { + assertThat(KafkaTableSchema.isReservedName(name)).isTrue(); + TableDescriptor nested = + enabled( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD(name, DataTypes.STRING()))) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(); + assertInvalid(nested, "Reserved name"); + } + + @ParameterizedTest + @ValueSource(strings = {"raw", "json", "schema_id_prefix", "schema_latest", "avro", ""}) + void testRejectsUnsupportedValueTypes(String type) { + assertInvalid(enabled(valueSchema(), type).build(), "table.kafka.convert.value.type"); + } + + @Test + void testRejectsMissingValueTypeAndTargetConflicts() { + assertInvalid( + TableDescriptor.builder() + .schema(valueSchema()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .build(), + "Missing required"); + assertInvalid( + enabled(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "binary") + .build(), + "Missing required value column"); + assertInvalid(enabled(valueSchema(), "string").build(), "must be STRING"); + assertInvalid( + enabled(valueSchema(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + "must be ROW"); + assertInvalid(enabled(valueSchema(), "parse_json").build(), "flat JSON"); + assertInvalid( + enabled( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("unused", DataTypes.STRING()) + .build(), + "binary") + .build(), + "no value mapping"); + assertInvalid( + enabled( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.STRING()))) + .column("unused", DataTypes.STRING()) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + "no value mapping"); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNonJsonRejectsExplicitLayout(String type) { + assertInvalid( + enabled(valueSchema(), type) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "flat") + .build(), + "only supported for parse_json"); + } + + @Test + void testInvalidLayoutAndKeyConversion() { + assertInvalid( + enabled(valueSchema(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "unknown") + .build(), + "value.layout"); + assertInvalid( + enabled(valueSchema(), "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "binary") + .build(), + "requires _kafka_key"); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .build(); + assertInvalid( + enabled(schema, "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "string") + .build(), + "only supports binary"); + assertThat( + KafkaTableSchema.resolve( + enabled(schema, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + "binary") + .build()) + .keyPosition()) + .isEqualTo(1); + } + + @Test + void testSystemColumnTypes() { + for (DataType invalid : Arrays.asList(DataTypes.STRING(), DataTypes.BINARY(8))) { + assertInvalid(withSystem("_kafka_key", invalid), "_kafka_key must be BYTES"); + } + for (DataType invalid : + Arrays.asList( + DataTypes.BYTES(), + DataTypes.ARRAY(DataTypes.STRING()), + DataTypes.MAP(DataTypes.INT(), DataTypes.BYTES()), + DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()))) { + assertInvalid(withSystem("_kafka_headers", invalid), "MAP"); + } + assertInvalid(withSystem("_kafka_rescue", DataTypes.INT()), "nullable STRING"); + assertInvalid( + withSystem("_kafka_rescue", DataTypes.STRING().copy(false)), "nullable STRING"); + } + + @Test + void testMetadataStructureAndUnavailableOffsetPersistence() { + RowType metadata = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT())); + assertInvalid(withSystem("_kafka_metadata", metadata), "persisting actual per-row offsets"); + assertInvalid(withSystem("_kafka_metadata", DataTypes.STRING()), "must be ROW"); + assertInvalid( + withSystem( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))), + "TIMESTAMP_LTZ(3)"); + assertInvalid( + withSystem( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("offset", DataTypes.BIGINT()), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)))), + "must be ROW"); + } + + @Test + void testUnknownAndPrototypeOptions() { + for (String key : + Arrays.asList("table.kafka.unknown", "table.kafka.convert.rescue-column")) { + assertInvalid( + enabled(valueSchema(), "binary").property(key, "anything").build(), + "Unsupported Kafka table property"); + } + for (String key : + Arrays.asList( + "kafka.value.format", + "kafka.key.fields", + "kafka.metadata.headers.column", + "fluss.value.format", + "table.kafka.convert.value.type")) { + assertInvalid( + enabled(valueSchema(), "binary").customProperty(key, "raw").build(), + "Unsupported Kafka custom property"); + } + assertThat( + KafkaTableSchema.resolve( + enabled(valueSchema(), "binary") + .customProperty("owner", "team") + .build())) + .isNotNull(); + } + + @Test + void testTableKindsAndCanonicalNameLimits() { + Schema keyed = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES().copy(false)) + .primaryKey("_kafka_value") + .build(); + assertInvalid(enabled(keyed, "binary").build(), "without a primary key"); + assertInvalid( + enabled(valueSchema(), "binary").partitionedBy("_kafka_value").build(), + "non-partitioned"); + assertInvalid( + enabled(valueSchema(), "binary").logFormat(LogFormat.INDEXED).build(), "Arrow"); + String database = String.join("", Collections.nCopies(124, "d")); + String tableName = String.join("", Collections.nCopies(124, "t")); + TableDescriptor table = enabled(valueSchema(), "binary").build(); + assertThatCode( + () -> + KafkaTableSchema.validateTablePath( + TablePath.of(database, tableName), table)) + .doesNotThrowAnyException(); + assertThatThrownBy( + () -> + KafkaTableSchema.validateTablePath( + TablePath.of(database, tableName + "t"), table)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("Kafka topic"); + } + + private static TableDescriptor withSystem(String name, DataType type) { + return enabled( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column(name, type) + .build(), + "binary") + .build(); + } + + private static Schema valueSchema() { + return Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build(); + } + + private static TableDescriptor.Builder enabled(Schema schema, String type) { + return TableDescriptor.builder() + .schema(schema) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, type); + } + + private static void assertInvalid(TableDescriptor table, String message) { + assertThatThrownBy(() -> KafkaTableSchema.resolve(table)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining(message); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java index 67f843a7276..56e990d8cd5 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.flink.catalog.TestSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -75,6 +76,50 @@ /** Test for {@link FlinkConversions}. */ public class FlinkConversionsTest { + @Test + void testKafkaDdlUsesRegisteredPropertiesAndFixedColumns() { + ResolvedSchema schema = + new ResolvedSchema( + Arrays.asList( + Column.physical( + "_kafka_value", + org.apache.flink.table.api.DataTypes.BYTES()), + Column.physical( + "_kafka_key", org.apache.flink.table.api.DataTypes.BYTES()), + Column.physical( + "_kafka_headers", + org.apache.flink.table.api.DataTypes.MAP( + org.apache.flink.table.api.DataTypes.STRING(), + org.apache.flink.table.api.DataTypes.BYTES())), + Column.physical( + "_kafka_rescue", + org.apache.flink.table.api.DataTypes.STRING())), + Collections.emptyList(), + null); + Map options = new HashMap<>(); + options.put("table.kafka.enabled", "true"); + options.put("table.kafka.convert.value.type", "binary"); + CatalogTable table = + CatalogTable.of( + Schema.newBuilder().fromResolvedSchema(schema).build(), + "Kafka DDL", + Collections.emptyList(), + options); + TableDescriptor descriptor = + FlinkConversions.toFlussTable(new ResolvedCatalogTable(table, schema)); + assertThat(descriptor.getProperties()) + .containsAllEntriesOf(options) + .doesNotContainKeys( + "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); + assertThat(descriptor.getCustomProperties()) + .doesNotContainKeys("table.kafka.enabled", "table.kafka.convert.value.type"); + KafkaTableSchema mapping = KafkaTableSchema.resolve(descriptor); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.keyPosition()).isEqualTo(1); + assertThat(mapping.headersPosition()).isEqualTo(2); + assertThat(mapping.rescuePosition()).isEqualTo(3); + } + @Test void testConvertBucketCountChange() { assertThat( diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java deleted file mode 100644 index df5d871907c..00000000000 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka.format; - -import org.apache.fluss.annotation.Internal; - -import java.util.Locale; - -/** Supported interpretations of Kafka record key and value bytes. */ -@Internal -public enum KafkaDataFormat { - RAW("raw"), - STRING("string"); - - /** Fluss table custom property controlling the record key format. */ - public static final String KEY_FORMAT_CONFIG = "kafka.key.format"; - - /** Fluss table custom property controlling the record value format. */ - public static final String VALUE_FORMAT_CONFIG = "kafka.value.format"; - - /** Fluss fields populated from the Kafka record key. */ - public static final String KEY_FIELDS_CONFIG = "kafka.key.fields"; - - /** Strategy for deriving fields populated from the Kafka record value. */ - public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; - - /** - * TIMESTAMP(3) column populated from the Kafka record epoch milliseconds as UTC calendar - * values. A missing Kafka timestamp (-1) maps to null and requires a nullable column. - */ - public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; - - /** Fluss column populated from the Kafka record headers. */ - public static final String HEADERS_COLUMN_CONFIG = "kafka.metadata.headers.column"; - - private final String value; - - KafkaDataFormat(String value) { - this.value = value; - } - - /** Parses a table custom property value. */ - public static KafkaDataFormat parse(String value) { - if (value == null) { - throw new IllegalArgumentException("Kafka data format must not be null."); - } - String normalized = value.trim().toLowerCase(Locale.ROOT); - for (KafkaDataFormat format : values()) { - if (format.value.equals(normalized)) { - return format; - } - } - throw new IllegalArgumentException( - "Unsupported Kafka data format '" + value + "'. Expected raw or string."); - } - - /** Returns the persisted table property value. */ - public String value() { - return value; - } -} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index f8b5744f71d..704d322f2e0 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -18,6 +18,9 @@ package org.apache.fluss.kafka.mapping; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; @@ -26,21 +29,43 @@ import static org.apache.fluss.utils.Preconditions.checkArgument; -/** Maps fully qualified Kafka topic identities to Fluss tables. */ +/** Maps canonical Kafka topic names and default-database aliases to Fluss tables. */ @Internal public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; - /** Maps a Kafka topic in database.table form to its Fluss table path. */ + private final String defaultDatabase; + + /** Creates a mapper using the default database configuration. */ + public KafkaTopicMapper() { + this(new Configuration()); + } + + /** Creates a mapper using the server's default database; it never creates databases. */ + public KafkaTopicMapper(Configuration config) { + this.defaultDatabase = config.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); + TablePath.validateDatabaseName(defaultDatabase); + } + + /** Maps a canonical topic or an unqualified alias to a table path, without fallback lookup. */ public TablePath toTablePath(String topicName) { if (!isValidTopic(topicName)) { throw new InvalidTopicException( - "Kafka topic must be a valid database.table name: " + topicName); + "Kafka topic must be a valid table or database.table name: " + topicName); } int separator = topicName.indexOf('.'); - return TablePath.of(topicName.substring(0, separator), topicName.substring(separator + 1)); + TablePath path = + separator < 0 + ? TablePath.of(defaultDatabase, topicName) + : TablePath.of( + topicName.substring(0, separator), + topicName.substring(separator + 1)); + if (!isMappedTable(path)) { + throw new InvalidTopicException("Canonical Kafka topic name is invalid: " + path); + } + return path; } /** Returns the fully qualified Kafka name of a representable Fluss table. */ @@ -55,7 +80,7 @@ public String toTopicName(TablePath tablePath) { /** Returns whether a Fluss user table has a valid, fully qualified Kafka topic name. */ public boolean isMappedTable(TablePath tablePath) { - return tablePath != null && tablePath.isValid() && isValidTopic(tablePath.toString()); + return KafkaTableSchema.isValidTablePath(tablePath); } /** Returns whether a name uniquely represents a user table in a Fluss database. */ @@ -64,14 +89,16 @@ public static boolean isValidTopic(String topicName) { return false; } int separator = topicName.indexOf('.'); - if (separator <= 0 || separator != topicName.lastIndexOf('.')) { + if (separator < 0) { + return TablePath.detectInvalidName(topicName) == null + && TablePath.validatePrefix(topicName) == null; + } + if (separator == 0 || separator != topicName.lastIndexOf('.')) { return false; } - String database = topicName.substring(0, separator); - String table = topicName.substring(separator + 1); - return TablePath.of(database, table).isValid() - && TablePath.validatePrefix(database) == null - && TablePath.validatePrefix(table) == null; + return KafkaTableSchema.isValidTablePath( + TablePath.of( + topicName.substring(0, separator), topicName.substring(separator + 1))); } /** Maps a Fluss table ID to a stable Kafka topic ID. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java deleted file mode 100644 index 64263e69286..00000000000 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka.schema; - -import org.apache.fluss.annotation.Internal; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.types.RowType; - -import javax.annotation.Nullable; - -import java.util.Objects; - -import static org.apache.fluss.utils.Preconditions.checkNotNull; - -/** Resolved Kafka key, value, and metadata mapping for one Fluss table schema. */ -@Internal -public final class KafkaTopicSchema { - - private final RowType rowType; - private final @Nullable KafkaDataFormat keyFormat; - private final KafkaFieldProjection keyProjection; - private final KafkaDataFormat valueFormat; - private final KafkaFieldProjection valueProjection; - private final int timestampPosition; - private final int headersPosition; - - /** Creates a resolved Kafka topic schema. */ - public KafkaTopicSchema( - RowType rowType, - @Nullable KafkaDataFormat keyFormat, - KafkaFieldProjection keyProjection, - KafkaDataFormat valueFormat, - KafkaFieldProjection valueProjection, - int timestampPosition, - int headersPosition) { - this.rowType = checkNotNull(rowType); - this.keyFormat = keyFormat; - this.keyProjection = checkNotNull(keyProjection); - this.valueFormat = checkNotNull(valueFormat); - this.valueProjection = checkNotNull(valueProjection); - this.timestampPosition = timestampPosition; - this.headersPosition = headersPosition; - } - - /** Returns the physical Fluss row type. */ - public RowType rowType() { - return rowType; - } - - /** Returns the key format, or null when the Kafka key is not mapped. */ - public @Nullable KafkaDataFormat keyFormat() { - return keyFormat; - } - - /** Returns the key field projection. */ - public KafkaFieldProjection keyProjection() { - return keyProjection; - } - - /** Returns the value format. */ - public KafkaDataFormat valueFormat() { - return valueFormat; - } - - /** Returns the value field projection. */ - public KafkaFieldProjection valueProjection() { - return valueProjection; - } - - /** Returns the timestamp physical position, or -1 when it is not mapped. */ - public int timestampPosition() { - return timestampPosition; - } - - /** Returns the headers physical position, or -1 when they are not mapped. */ - public int headersPosition() { - return headersPosition; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof KafkaTopicSchema)) { - return false; - } - KafkaTopicSchema that = (KafkaTopicSchema) obj; - return Objects.equals(rowType, that.rowType) - && Objects.equals(keyFormat, that.keyFormat) - && Objects.equals(keyProjection, that.keyProjection) - && Objects.equals(valueFormat, that.valueFormat) - && Objects.equals(valueProjection, that.valueProjection) - && Objects.equals(timestampPosition, that.timestampPosition) - && Objects.equals(headersPosition, that.headersPosition); - } - - @Override - public int hashCode() { - return Objects.hash( - rowType, - keyFormat, - keyProjection, - valueFormat, - valueProjection, - timestampPosition, - headersPosition); - } - - @Override - public String toString() { - return "KafkaTopicSchema{" - + "rowType=" - + rowType - + ", " - + "keyFormat=" - + keyFormat - + ", " - + "keyProjection=" - + keyProjection - + ", " - + "valueFormat=" - + valueFormat - + ", " - + "valueProjection=" - + valueProjection - + ", " - + "timestampPosition=" - + timestampPosition - + ", " - + "headersPosition=" - + headersPosition - + "}"; - } -} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index 8959c420d15..c02879e922a 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -18,293 +18,29 @@ package org.apache.fluss.kafka.schema; import org.apache.fluss.annotation.Internal; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.TableConfig; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.types.ArrayType; -import org.apache.fluss.types.BytesType; -import org.apache.fluss.types.RowType; -import org.apache.fluss.types.StringType; -import org.apache.fluss.types.TimestampType; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -/** Resolves and validates the Kafka record mapping stored in Fluss table custom properties. */ +/** Resolves the same persisted Kafka contract that native DDL validates. */ @Internal public final class KafkaTopicSchemaResolver { - private static final String INCLUDE_ALL = "ALL"; - private static final String INCLUDE_EXCEPT_KEY = "EXCEPT_KEY"; - - private static final Set SUPPORTED_PROPERTIES = - new HashSet<>( - Arrays.asList( - KafkaDataFormat.KEY_FORMAT_CONFIG, - KafkaDataFormat.KEY_FIELDS_CONFIG, - KafkaDataFormat.VALUE_FORMAT_CONFIG, - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, - KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, - KafkaDataFormat.HEADERS_COLUMN_CONFIG)); - - /** Resolves one table's Kafka record mapping contract. */ - public KafkaTopicSchema resolve(TableDescriptor table) { - validateTableKind(table); - RowType rowType = table.getSchema().getRowType(); - Map properties = table.getCustomProperties(); - - for (String property : properties.keySet()) { - if (property.startsWith("kafka.") && !SUPPORTED_PROPERTIES.contains(property)) { - throw invalid("Unsupported Kafka table property '" + property + "'."); - } - } - - int timestampPosition = - resolveOptionalPosition( - rowType, properties.get(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG)); - int headersPosition = - resolveOptionalPosition( - rowType, properties.get(KafkaDataFormat.HEADERS_COLUMN_CONFIG)); - if (timestampPosition >= 0 && timestampPosition == headersPosition) { - throw invalid("Kafka timestamp and headers cannot map to the same Fluss column."); - } - validateMetadataColumns(rowType, timestampPosition, headersPosition); - - String keyFormatValue = properties.get(KafkaDataFormat.KEY_FORMAT_CONFIG); - KafkaDataFormat keyFormat = keyFormatValue == null ? null : parseFormat(keyFormatValue); - List keyPositions = - resolveKeyPositions( - rowType, keyFormat, properties.get(KafkaDataFormat.KEY_FIELDS_CONFIG)); - for (Integer keyPosition : keyPositions) { - if (keyPosition == timestampPosition || keyPosition == headersPosition) { - throw invalid( - "Kafka key field '" - + rowType.getFieldNames().get(keyPosition) - + "' cannot be a Kafka metadata column."); - } - } - KafkaFieldProjection keyProjection = new KafkaFieldProjection(rowType, keyPositions); - - String valueFormatValue = properties.get(KafkaDataFormat.VALUE_FORMAT_CONFIG); - if (valueFormatValue == null) { - throw invalid( - "Missing required table property '" - + KafkaDataFormat.VALUE_FORMAT_CONFIG - + "'."); - } - KafkaDataFormat valueFormat = parseFormat(valueFormatValue); - String fieldsInclude = - normalizeFieldsInclude(properties.get(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG)); - if (!keyPositions.isEmpty() && INCLUDE_ALL.equals(fieldsInclude)) { - throw invalid( - "Mapping Kafka key fields requires " - + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG - + "=EXCEPT_KEY."); - } - - Set metadataPositions = new HashSet<>(); - if (timestampPosition >= 0) { - metadataPositions.add(timestampPosition); - } - if (headersPosition >= 0) { - metadataPositions.add(headersPosition); - } - List valuePositions = - resolveValuePositions(rowType, fieldsInclude, keyPositions, metadataPositions); - KafkaFieldProjection valueProjection = new KafkaFieldProjection(rowType, valuePositions); - if (valueProjection.isEmpty()) { - throw invalid("Kafka value projection must contain at least one Fluss column."); - } - - validateSingleFieldFormat(keyFormat, keyProjection, "key"); - validateSingleFieldFormat(valueFormat, valueProjection, "value"); - return new KafkaTopicSchema( - rowType, - keyFormat, - keyProjection, - valueFormat, - valueProjection, - timestampPosition, - headersPosition); - } - - private static void validateTableKind(TableDescriptor table) { - if (table.hasPrimaryKey()) { - throw invalid("Kafka topic table must be a log table."); - } - if (table.isPartitioned()) { - throw invalid("Partitioned Fluss tables are not supported."); - } - if (new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() - != LogFormat.ARROW) { - throw invalid("Kafka topic table must use the Arrow log format."); - } - } - - private static List resolveKeyPositions( - RowType rowType, KafkaDataFormat keyFormat, String keyFieldsValue) { - if (keyFormat == null) { - if (keyFieldsValue != null) { - throw invalid( - KafkaDataFormat.KEY_FIELDS_CONFIG - + " requires " - + KafkaDataFormat.KEY_FORMAT_CONFIG - + "."); - } - return Collections.emptyList(); - } - if (keyFieldsValue == null || keyFieldsValue.trim().isEmpty()) { - throw invalid( - "Missing required table property '" + KafkaDataFormat.KEY_FIELDS_CONFIG + "'."); - } - String[] fieldNames = keyFieldsValue.split(",", -1); - List positions = new ArrayList<>(fieldNames.length); - Set uniquePositions = new HashSet<>(); - for (String fieldNameValue : fieldNames) { - String fieldName = fieldNameValue.trim(); - if (fieldName.isEmpty()) { - throw invalid("Kafka key field names must not be empty."); - } - int position = rowType.getFieldIndex(fieldName); - if (position < 0) { - throw invalid("Kafka key field '" + fieldName + "' does not exist."); - } - if (!uniquePositions.add(position)) { - throw invalid("Duplicate Kafka key field '" + fieldName + "'."); - } - positions.add(position); - } - return positions; - } - - private static String normalizeFieldsInclude(String value) { - String normalized = value == null ? INCLUDE_ALL : value.trim().toUpperCase(Locale.ROOT); - if (!INCLUDE_ALL.equals(normalized) && !INCLUDE_EXCEPT_KEY.equals(normalized)) { - throw invalid( - "Invalid " - + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG - + " '" - + value - + "'. Expected ALL or EXCEPT_KEY."); - } - return normalized; - } - - private static List resolveValuePositions( - RowType rowType, - String fieldsInclude, - List keyPositions, - Set metadataPositions) { - Set excludedKeyPositions = - INCLUDE_EXCEPT_KEY.equals(fieldsInclude) - ? new HashSet<>(keyPositions) - : Collections.emptySet(); - List positions = new ArrayList<>(); - for (int position = 0; position < rowType.getFieldCount(); position++) { - if (!metadataPositions.contains(position) && !excludedKeyPositions.contains(position)) { - positions.add(position); - } - } - return positions; - } - - private static int resolveOptionalPosition(RowType rowType, String fieldNameValue) { - if (fieldNameValue == null) { - return -1; - } - if (fieldNameValue.trim().isEmpty()) { - throw invalid("Kafka metadata column name must not be empty."); - } - String fieldName = fieldNameValue.trim(); - int position = rowType.getFieldIndex(fieldName); - if (position < 0) { - throw invalid("Kafka metadata column '" + fieldName + "' does not exist."); - } - return position; - } - - private static void validateMetadataColumns( - RowType rowType, int timestampPosition, int headersPosition) { - if (timestampPosition >= 0) { - if (!(rowType.getTypeAt(timestampPosition) instanceof TimestampType) - || ((TimestampType) rowType.getTypeAt(timestampPosition)).getPrecision() != 3) { - throw invalid("Kafka timestamp column must be TIMESTAMP(3)."); - } - } - if (headersPosition >= 0) { - validateHeadersType(rowType.getTypeAt(headersPosition)); - } - } - - private static void validateHeadersType(org.apache.fluss.types.DataType dataType) { - if (!(dataType instanceof ArrayType) || !dataType.isNullable()) { - throw invalid( - "Kafka headers column must be nullable " - + "ARRAY>."); - } - ArrayType arrayType = (ArrayType) dataType; - if (!(arrayType.getElementType() instanceof RowType)) { - throw invalid("Kafka headers elements must be rows."); - } - RowType headerType = (RowType) arrayType.getElementType(); - if (!headerType.getFieldNames().equals(Arrays.asList("name", "value")) - || !(headerType.getTypeAt(0) instanceof StringType) - || !(headerType.getTypeAt(1) instanceof BytesType) - || !headerType.getTypeAt(1).isNullable()) { - throw invalid("Kafka headers elements must be ROW."); - } - } - - private static void validateSingleFieldFormat( - KafkaDataFormat format, KafkaFieldProjection projection, String component) { - if (format == null) { - return; - } - if (format == KafkaDataFormat.RAW || format == KafkaDataFormat.STRING) { - if (projection.size() != 1) { - throw invalid( - "Kafka " - + component - + " format " - + format.value() - + " requires exactly one Fluss field."); - } - boolean validType = - format == KafkaDataFormat.RAW - ? projection.dataTypeAt(0) instanceof BytesType - : projection.dataTypeAt(0) instanceof StringType; - if (!validType) { - throw invalid( - "Kafka " - + component - + " field '" - + projection.nameAt(0) - + "' must be " - + (format == KafkaDataFormat.RAW ? "BYTES" : "STRING") - + " for format " - + format.value() - + "."); - } - } - } - - private static KafkaDataFormat parseFormat(String value) { + /** Returns the table's explicit opt-in flag without inferring it from schema compatibility. */ + public boolean isEnabled(TableDescriptor table) { try { - return KafkaDataFormat.parse(value); + return KafkaTableSchema.isEnabled(table); } catch (IllegalArgumentException e) { - throw invalid(e.getMessage()); + throw new KafkaTopicSchemaException(e.getMessage()); } } - private static KafkaTopicSchemaException invalid(String message) { - return new KafkaTopicSchemaException(message); + /** Resolves one enabled table's contract; invalid enabled tables remain visible as errors. */ + public KafkaTableSchema resolve(TableDescriptor table) { + try { + return KafkaTableSchema.resolve(table); + } catch (InvalidTableException | IllegalArgumentException e) { + throw new KafkaTopicSchemaException(e.getMessage()); + } } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java index 7f502494f2e..55e8ba6fc57 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java @@ -36,13 +36,14 @@ public void testFromMap() throws Exception { Map map = new HashMap<>(); map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DATABASE.key(), "fluss"); + map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "fluss"); Configuration configuration = Configuration.fromMap(map); assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("fluss"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) + .isEqualTo("fluss"); } @Test @@ -51,6 +52,7 @@ public void testFromDefault() throws Exception { assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isFalse(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Collections.singletonList("KAFKA")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("kafka"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) + .isEqualTo("kafka"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index c5d4a582a88..7ef7e281244 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -17,6 +17,8 @@ package org.apache.fluss.kafka.mapping; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; @@ -49,11 +51,41 @@ public void testQualifiedNamesAcrossDatabases() { } } + @Test + void testDefaultDatabaseAliases() { + assertThat(mapper.toTablePath("orders")).isEqualTo(mapper.toTablePath("kafka.orders")); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); + KafkaTopicMapper custom = new KafkaTopicMapper(config); + assertThat(custom.toTablePath("orders")).isEqualTo(TablePath.of("sales", "orders")); + assertThat(custom.toTablePath("warehouse.orders")) + .isEqualTo(TablePath.of("warehouse", "orders")); + assertThat(custom.toTopicName(custom.toTablePath("orders"))).isEqualTo("sales.orders"); + config.setString("kafka.database", "legacy"); + assertThat(new KafkaTopicMapper(config).toTablePath("orders")) + .isEqualTo(TablePath.of("sales", "orders")); + } + + @Test + void testInvalidDefaultDatabaseAndCanonicalAliasLength() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "bad.name"); + assertThatThrownBy(() -> new KafkaTopicMapper(config)).hasMessageContaining("invalid"); + config.set( + ConfigOptions.KAFKA_DEFAULT_DATABASE, + String.join("", Collections.nCopies(128, "d"))); + KafkaTopicMapper longDatabase = new KafkaTopicMapper(config); + assertThatThrownBy( + () -> + longDatabase.toTablePath( + String.join("", Collections.nCopies(128, "t")))) + .isInstanceOf(InvalidTopicException.class); + } + @ParameterizedTest @NullAndEmptySource @ValueSource( strings = { - "orders", ".orders", "sales.", "sales.orders.extra", diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 2f3e75bdd3d..45744dbc87a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -4,7 +4,7 @@ * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -17,99 +17,371 @@ package org.apache.fluss.kafka.schema; -import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.InvalidConfigException; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.rpc.messages.MetadataRequest; -import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.gateway.CoordinatorGateway; +import org.apache.fluss.rpc.messages.AlterTableRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbAddColumn; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.json.DataTypeJsonSerde; +import org.apache.fluss.utils.json.JsonSerdeUtils; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; +import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateDatabaseRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newDatabaseExistsRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newGetTableInfoRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newTableExistsRequest; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Verifies Kafka mapping properties survive the native Fluss table creation path. */ -public class KafkaTableMappingITCase { +/** Verifies Kafka DDL contracts through real coordinator RPC and persisted metadata. */ +class KafkaTableMappingITCase { + + private static final Configuration CONFIG = new Configuration(); + + static { + CONFIG.set(ConfigOptions.KAFKA_ENABLED, true); + CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + CONFIG.setString("kafka.database", "legacy_database"); + } @RegisterExtension - public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = - FlussClusterExtension.builder().setNumOfTabletServers(1).build(); + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder().setNumOfTabletServers(1).setClusterConf(CONFIG).build(); + + private CoordinatorGateway gateway; + + @BeforeEach + void setup() { + gateway = CLUSTER.newCoordinatorClient(); + } @Test - public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Exception { + void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() throws Exception { + for (String database : Arrays.asList("missing_default", "kafka", "legacy_database")) { + assertThat(gateway.databaseExists(newDatabaseExistsRequest(database)).get().isExists()) + .isFalse(); + } + assertThat( + gateway.databaseExists(newDatabaseExistsRequest(DEFAULT_DATABASE)) + .get() + .isExists()) + .isTrue(); + KafkaTopicMapper mapper = new KafkaTopicMapper(CONFIG); + TablePath sales = mapper.toTablePath("sales.orders"); + TablePath warehouse = mapper.toTablePath("warehouse.orders"); + long salesId = createTable(CLUSTER, sales, enabled().build()); + long warehouseId = createTable(CLUSTER, warehouse, enabled().build()); + assertThat(salesId).isNotEqualTo(warehouseId); + assertThat(mapper.toTablePath("orders")) + .isEqualTo(TablePath.of("missing_default", "orders")); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + mapper.toTablePath("orders"), + enabled().build(), + false)) + .get()) + .hasCauseInstanceOf(DatabaseNotExistException.class); + assertThat( + gateway.databaseExists(newDatabaseExistsRequest("missing_default")) + .get() + .isExists()) + .isFalse(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); + KafkaTopicMapper aliasMapper = new KafkaTopicMapper(config); + assertThat(info(aliasMapper.toTablePath("orders")).getTableId()) + .isEqualTo(info(sales).getTableId()); + assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); + } + + @Test + void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { + TablePath path = TablePath.of("ddl", "roundtrip"); + Schema schema = + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES()) + .column( + "payload", + DataTypes.ROW(DataTypes.FIELD("nested", DataTypes.STRING()))) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .build(); TableDescriptor descriptor = - TableDescriptor.builder() + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build(); + createTable(CLUSTER, path, descriptor); + TableDescriptor stored = stored(path); + assertThat(stored.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(stored)).isEqualTo(kafkaProperties(descriptor)); + assertThat(new KafkaTopicSchemaResolver().resolve(stored).valueProjection().positions()) + .containsExactly(1); + } + + @Test + void testEnableDisableAndFailedPropertyAlterAreAtomic() throws Exception { + TablePath path = TablePath.of("ddl", "enable_disable"); + TableDescriptor disabled = + enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + createTable(CLUSTER, path, disabled); + TableDescriptor before = stored(path); + assertThat(KafkaTableSchema.isEnabled(before)).isFalse(); + set(path, Collections.singletonMap("table.kafka.enabled", "true")); + TableDescriptor enabled = stored(path); + assertThat(KafkaTableSchema.isEnabled(enabled)).isTrue(); + Map expected = new HashMap<>(before.getProperties()); + expected.put("table.kafka.enabled", "true"); + assertThat(enabled.getProperties()).isEqualTo(expected); + assertThat(enabled.getSchema()).isEqualTo(before.getSchema()); + Map invalid = new HashMap<>(); + invalid.put("table.kafka.convert.value.type", "string"); + invalid.put("table.kafka.convert.key.type", "binary"); + assertThatThrownBy(() -> set(path, invalid)) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(enabled); + assertThatThrownBy(() -> reset(path, "table.kafka.convert.value.type")) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(enabled); + reset(path, "table.kafka.enabled"); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + assertThat(stored(path).getProperties()).doesNotContainKey("table.kafka.enabled"); + // Disabled tables can stage incompatible Kafka conversion properties without Kafka + // validation. + set(path, Collections.singletonMap("table.kafka.convert.value.type", "string")); + TableDescriptor staged = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(staged); + } + + @Test + void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exception { + TablePath path = TablePath.of("ddl", "schema_change"); + createTable(CLUSTER, path, enabled().build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> addColumn(path, "_kafka_headers", DataTypes.STRING())) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(info(path).getTableJson()).isEqualTo(before.getTableJson()); + assertThat(info(path).getSchemaId()).isEqualTo(before.getSchemaId()); + addColumn(path, "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + assertThat(KafkaTableSchema.resolve(stored(path)).headersPosition()).isEqualTo(1); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + TableDescriptor withHeaders = stored(path); + assertThatThrownBy(() -> addColumn(path, "_kafka_rescue", DataTypes.INT())) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(withHeaders); + set(path, Collections.singletonMap("table.kafka.enabled", "false")); + addColumn(path, "unmapped_business_field", DataTypes.INT()); + assertThat(stored(path).getSchema().getRowType().getFieldNames()) + .contains("unmapped_business_field"); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + } + + @Test + void testCreateRejectsInvalidContractWithoutPublishingMetadata() throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl_invalid", true)).get(); + TablePath path = TablePath.of("ddl_invalid", "bad"); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled() + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE, + "string") + .build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + // Non-null added columns are rejected by generic schema evolution first, so exercise + // Kafka-specific rescue nullability through CREATE, where such columns are allowed. + TableDescriptor nonNullableRescue = + enabled() .schema( Schema.newBuilder() - .column("event_key", DataTypes.STRING()) - .column("event_body", DataTypes.BYTES()) - .column("event_time", DataTypes.TIMESTAMP(3)) + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_rescue", DataTypes.STRING().copy(false)) .build()) - .distributedBy(2) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "event_time") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); - KafkaTopicMapper mapper = new KafkaTopicMapper(); - Map tableIds = new LinkedHashMap<>(); - MetadataRequest request = new MetadataRequest(); - for (String database : Arrays.asList("kafka_ddl", "kafka_ddl_archive")) { - TablePath tablePath = mapper.toTablePath(database + ".events"); - assertThat(tablePath).isEqualTo(TablePath.of(database, "events")); - tableIds.put(tablePath, createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor)); - request.addTablePath() - .setDatabaseName(tablePath.getDatabaseName()) - .setTableName(tablePath.getTableName()); - } - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); - List metadatas = - FLUSS_CLUSTER_EXTENSION - .newCoordinatorClient() - .metadata(request) - .get() - .getTableMetadatasList(); - assertThat(metadatas).hasSize(2); - assertThat(metadatas) - .extracting(PbTableMetadata::getTableId) - .containsExactlyInAnyOrderElementsOf(tableIds.values()) - .doesNotHaveDuplicates(); - for (PbTableMetadata metadata : metadatas) { - TablePath tablePath = - TablePath.of( - metadata.getTablePath().getDatabaseName(), - metadata.getTablePath().getTableName()); - TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); - - assertThat(metadata.getTableId()).isEqualTo(tableIds.get(tablePath)); - assertThat(mapper.toTopicName(tablePath)) - .isIn("kafka_ddl.events", "kafka_ddl_archive.events"); - assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))) - .isEqualTo(tableIds.get(tablePath)); - assertThat(metadata.getBucketMetadatasList()).hasSize(2); - assertThat(persisted.getCustomProperties()) - .containsAllEntriesOf(descriptor.getCustomProperties()); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(1); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); - assertThat(mapping.timestampPosition()).isEqualTo(2); - assertThat(persisted.getSchema().getRowType().getTypeAt(2)) - .isEqualTo(DataTypes.TIMESTAMP(3)); - } + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, nonNullableRescue, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("nullable STRING"); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled() + .property( + "table.kafka.convert.rescue-column", + "rescue") + .build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidConfigException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + createTable(CLUSTER, path, enabled().build()); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isTrue(); + } + + @Test + void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { + TablePath path = TablePath.of("ddl", "offset_capability"); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))) + .build(); + createTable( + CLUSTER, + path, + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + TableDescriptor before = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("actual per-row offsets"); + assertThat(stored(path)).isEqualTo(before); + } + + @Test + void testCanonicalNameCheckedOnCreateAndEnable() throws Exception { + String database = String.join("", Collections.nCopies(128, "d")); + TablePath path = TablePath.of(database, String.join("", Collections.nCopies(128, "t"))); + gateway.createDatabase(newCreateDatabaseRequest(database, true)).get(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, enabled().build(), false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("Kafka topic"); + createTable( + CLUSTER, + path, + enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build()); + TableDescriptor before = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(before); + } + + private GetTableInfoResponse info(TablePath path) throws Exception { + return gateway.getTableInfo(newGetTableInfoRequest(path)).get(); + } + + private TableDescriptor stored(TablePath path) throws Exception { + return TableDescriptor.fromJsonBytes(info(path).getTableJson()); + } + + private void set(TablePath path, Map properties) throws Exception { + gateway.alterTable( + newAlterTableRequest( + path, + properties, + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private void reset(TablePath path, String property) throws Exception { + gateway.alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList(property), + Collections.emptyList(), + false)) + .get(); + } + + private void addColumn(TablePath path, String name, DataType type) throws Exception { + PbAddColumn column = + new PbAddColumn() + .setColumnName(name) + .setDataTypeJson( + JsonSerdeUtils.writeValueAsBytes(type, DataTypeJsonSerde.INSTANCE)) + .setColumnPositionType(0); + AlterTableRequest request = + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.emptyList(), + Collections.singletonList(column), + false); + gateway.alterTable(request).get(); + } + + private static Map kafkaProperties(TableDescriptor table) { + Map properties = new HashMap<>(); + table.getProperties() + .forEach( + (key, value) -> { + if (key.startsWith("table.kafka.")) { + properties.put(key, value); + } + }); + return properties; + } + + private static TableDescriptor.Builder enabled() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 66c13a4711a..b7e4174421c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -4,7 +4,7 @@ * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -17,334 +17,61 @@ package org.apache.fluss.kafka.schema; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Tests the DDL contract independently of Kafka request handling and record decoding. */ -public class KafkaTopicSchemaResolverTest { - +/** Tests the Kafka adapter's use of the shared DDL contract. */ +class KafkaTopicSchemaResolverTest { private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); @Test - public void testRawMappingSurvivesTableMetadataSerialization() { - Schema schema = - Schema.newBuilder() - .column("message", DataTypes.BYTES()) - .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) - .column("attributes", headersType()) - .column("message_key", DataTypes.BYTES()) - .build(); - TableDescriptor table = - table(schema, "raw") - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") - .build(); - KafkaTopicSchema mapping = - resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes())); - - assertThat(mapping.rowType()).isEqualTo(schema.getRowType()); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.RAW); - assertThat(mapping.keyProjection().positions()).containsExactly(3); - assertThat(mapping.keyProjection().nameAt(0)).isEqualTo("message_key"); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - assertThat(mapping.valueProjection().dataTypeAt(0)).isEqualTo(DataTypes.BYTES()); - assertThat(mapping.timestampPosition()).isEqualTo(1); - assertThat(mapping.headersPosition()).isEqualTo(2); - assertThat(mapping).isEqualTo(resolver.resolve(table)); - assertThatThrownBy(() -> mapping.keyProjection().positions().add(1)) - .isInstanceOf(UnsupportedOperationException.class); - } - - @Test - public void testValueOnlyStringAndNullableColumns() { - for (boolean nullable : new boolean[] {true, false}) { - KafkaTopicSchema mapping = - resolver.resolve( - table( - Schema.newBuilder() - .column( - "body", - DataTypes.STRING().copy(nullable)) - .build(), - " STRING ") - .build()); - assertThat(mapping.keyFormat()).isNull(); - assertThat(mapping.keyProjection().isEmpty()).isTrue(); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - assertThat(mapping.valueProjection().dataTypeAt(0).isNullable()).isEqualTo(nullable); - assertThat(mapping.timestampPosition()).isEqualTo(-1); - assertThat(mapping.headersPosition()).isEqualTo(-1); - } + void testEnabledContractUsesSharedValidation() { + TableDescriptor table = table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true).build(); + assertThat(resolver.isEnabled(table)).isTrue(); + assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); + assertThat(resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) + .isEqualTo(resolver.resolve(table)); } @Test - public void testMixedFormatsAndMetadataAreOptional() { - KafkaTopicSchema mapping = - resolver.resolve( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty( - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, " except_key ") - .build()); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.valueProjection().positions()).containsExactly(1); + void testCompatibleSchemaDoesNotEnableTable() { + assertThat(resolver.isEnabled(table().build())).isFalse(); + assertThatThrownBy(() -> resolver.resolve(table().build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); } @Test - public void testRejectsUnsupportedTableKinds() { - Schema primaryKeySchema = - Schema.newBuilder() - .column("id", DataTypes.STRING().copy(false)) - .primaryKey("id") + void testInvalidEnabledContractIsNotTreatedAsDisabled() { + TableDescriptor table = + table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "schema_latest") .build(); - assertInvalid(table(primaryKeySchema, "string"), "must be a log table"); - assertInvalid(keyValueTable().partitionedBy("id"), "Partitioned Fluss tables"); - assertInvalid(valueTable().logFormat(LogFormat.INDEXED), "Arrow log format"); - } - - @Test - public void testRequiresExplicitValueFormat() { - assertInvalid( - TableDescriptor.builder() - .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) - .distributedBy(1) - .customProperty("fluss.value.format", "raw"), - KafkaDataFormat.VALUE_FORMAT_CONFIG); - } - - @ParameterizedTest - @ValueSource(strings = {"json", "avro", ""}) - public void testRejectsUnavailableFormats(String format) { - assertInvalid( - valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format), - "Unsupported Kafka data format"); - assertInvalid( - keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format), - "Unsupported Kafka data format"); - } - - @Test - public void testRejectsUnsupportedKafkaOptions() { - assertInvalid( - valueTable().customProperty("kafka.value.rescue-column", "body"), - "Unsupported Kafka table property"); - assertInvalid( - valueTable().customProperty("kafka.key.field", "body"), - "Unsupported Kafka table property"); - assertThat(resolver.resolve(valueTable().customProperty("owner", "team").build())) - .isNotNull(); - } - - @Test - public void testKeyFormatAndFieldMustBeSpecifiedTogether() { - assertInvalid( - keyValueTable().customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), - "requires kafka.key.format"); - assertInvalid( - keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"), - "kafka.key.fields"); - } - - @ParameterizedTest - @ValueSource(strings = {"", " ", "missing", "id,id", "id,", ",id", "id,,body"}) - public void testRejectsInvalidKeyFields(String fields) { - assertThatThrownBy( - () -> - resolver.resolve( - keyValueTable() - .customProperty( - KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty( - KafkaDataFormat.KEY_FIELDS_CONFIG, fields) - .customProperty( - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, - "EXCEPT_KEY") - .build())) - .isInstanceOf(KafkaTopicSchemaException.class); - } - - @Test - public void testRejectsAmbiguousAndEmptyValueProjections() { - assertInvalid( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), - "requires"); - assertInvalid( - valueTable().customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "bad"), - "Expected ALL or EXCEPT_KEY"); - assertInvalid( - valueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "body") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), - "at least one Fluss column"); - assertInvalid(keyValueTable(), "exactly one Fluss field"); - assertInvalid( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id,body") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), - "at least one Fluss column"); - } - - @Test - public void testRejectsWrongPhysicalTypes() { - assertInvalid( - valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"), - "must be STRING"); - assertInvalid( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), - "must be BYTES"); - } - - @Test - public void testAcceptsTimestampWithEitherNullability() { - for (boolean nullable : new boolean[] {true, false}) { - TableDescriptor descriptor = - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("ts", DataTypes.TIMESTAMP(3).copy(nullable)) - .build(), - "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts") - .build(); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(descriptor); - assertThat(mapping.timestampPosition()).isEqualTo(1); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - } - } - - @Test - public void testRejectsWrongTimestampTypes() { - for (DataType type : - new DataType[] { - DataTypes.STRING(), - DataTypes.TIMESTAMP_LTZ(3), - DataTypes.TIMESTAMP_LTZ(3).copy(false), - DataTypes.TIMESTAMP(2), - DataTypes.TIMESTAMP(6).copy(false) - }) { - assertInvalid( - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("ts", type) - .build(), - "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), - "TIMESTAMP(3)"); - } - } - - @Test - public void testRejectsWrongHeaderTypes() { - for (DataType type : - new DataType[] { - DataTypes.STRING(), - headersType().copy(false), - DataTypes.ARRAY(DataTypes.STRING()), - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("key", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES()))), - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES().copy(false)))) - }) { - assertInvalid( - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("attributes", type) - .build(), - "raw") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes"), - "headers"); - } + assertThat(resolver.isEnabled(table)).isTrue(); + assertThatThrownBy(() -> resolver.resolve(table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("schema_latest"); } @Test - public void testRejectsConflictingOrMissingMetadataColumns() { - assertInvalid( - valueTable().customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "missing"), - "does not exist"); - assertInvalid( - valueTable().customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, " "), - "must not be empty"); - TableDescriptor.Builder table = - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("ts", DataTypes.TIMESTAMP(3).copy(false)) - .build(), - "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); - assertInvalid( - TableDescriptor.builder(table.build()) - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "ts"), - "same Fluss column"); - assertInvalid( - TableDescriptor.builder(table.build()) - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "ts"), - "metadata column"); - } - - private void assertInvalid(TableDescriptor.Builder table, String message) { - assertThatThrownBy(() -> resolver.resolve(table.build())) + void testMalformedPersistedEnableFlagIsAnExplicitContractError() { + TableDescriptor table = table().property("table.kafka.enabled", "maybe").build(); + assertThatThrownBy(() -> resolver.isEnabled(table)) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining(message); + .hasMessageContaining("table.kafka.enabled"); } - private static TableDescriptor.Builder keyValueTable() { - return table( - Schema.newBuilder() - .column("id", DataTypes.STRING()) - .column("body", DataTypes.BYTES()) - .build(), - "raw"); - } - - private static TableDescriptor.Builder valueTable() { - return table(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "raw"); - } - - private static TableDescriptor.Builder table(Schema schema, String valueFormat) { + private static TableDescriptor.Builder table() { return TableDescriptor.builder() - .schema(schema) - .distributedBy(2) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat); - } - - private static DataType headersType() { - return DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES()))); + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index 2cdd2c5bf97..aff8bb78761 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -553,14 +553,6 @@ private void createDefaultDatabase() { metadataManager.createDatabase(DEFAULT_DATABASE, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' because no database exists.", DEFAULT_DATABASE); } - // create Kafka default database if Kafka is enabled. - if (conf.get(ConfigOptions.KAFKA_ENABLED)) { - String kafkaDB = conf.get(ConfigOptions.KAFKA_DATABASE); - if (!databases.contains(kafkaDB)) { - metadataManager.createDatabase(kafkaDB, DatabaseDescriptor.EMPTY, true); - LOG.info("Created default database '{}' for Kafka protocol.", kafkaDB); - } - } } /** diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index 6bed0ecd3f0..e8e09dab26d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -507,7 +507,7 @@ public CompletableFuture createTable(CreateTableRequest req // validate table descriptor before creating table in lake or fluss metadata, // to avoid orphaned lake tables when validation fails - metadataManager.validateTableDescriptor(tableDescriptor); + metadataManager.validateTableDescriptor(tablePath, tableDescriptor); // the distribution and bucket count must be set now //noinspection OptionalGetWithoutIsPresent @@ -615,7 +615,8 @@ public CompletableFuture alterTable(AlterTableRequest reques tablePath, alterSchemaChanges, request.isIgnoreIfNotExists(), - currentSession().getPrincipal()); + currentSession().getPrincipal(), + coordinatorZkVersionSupplier.get()); } if (hasConfigChanges) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 14b97c927eb..b1a5fae3d04 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -39,6 +39,7 @@ import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.DatabaseInfo; import org.apache.fluss.metadata.DatabaseSummary; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LakeTableUtil; import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -58,6 +59,7 @@ import org.apache.fluss.server.zk.data.PartitionRegistration; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.server.zk.data.ZkVersion; import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; import org.apache.fluss.utils.function.RunnableWithException; import org.apache.fluss.utils.function.ThrowingRunnable; @@ -131,6 +133,12 @@ public void validateTableDescriptor(TableDescriptor tableDescriptor) { lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat()); } + /** Validates a descriptor and its Kafka topic identity before persistence. */ + public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDescriptor) { + validateTableDescriptor(tableDescriptor); + KafkaTableSchema.validateTablePath(tablePath, tableDescriptor); + } + public void createDatabase( String databaseName, DatabaseDescriptor databaseDescriptor, boolean ignoreIfExists) throws DatabaseAlreadyExistException { @@ -458,8 +466,25 @@ public void alterTableSchema( boolean ignoreIfNotExists, FlussPrincipal flussPrincipal) throws TableNotExistException, TableNotPartitionedException { - try { + alterTableSchema( + tablePath, + schemaChanges, + ignoreIfNotExists, + flussPrincipal, + ZkVersion.MATCH_ANY_VERSION.getVersion()); + } + /** Alters schema with a table-version and coordinator-epoch fence against concurrent ALTER. */ + public void alterTableSchema( + TablePath tablePath, + List schemaChanges, + boolean ignoreIfNotExists, + FlussPrincipal flussPrincipal, + int coordinatorZkVersion) + throws TableNotExistException, TableNotPartitionedException { + try { + ZooKeeperClient.VersionedData versionedTable = + getTableRegistrationWithVersion(tablePath); TableInfo table = getTable(tablePath); TableDescriptor tableDescriptor = table.toTableDescriptor(); @@ -480,7 +505,12 @@ public void alterTableSchema( // Update Fluss schema (ZK) after Lake sync succeeds if (!newSchema.equals(table.getSchema())) { - zookeeperClient.registerSchema(tablePath, newSchema, table.getSchemaId() + 1); + zookeeperClient.registerSchemaAndBumpTableVersion( + tablePath, + newSchema, + table.getSchemaId() + 1, + versionedTable, + coordinatorZkVersion); } else { LOG.info( "Skipping schema evolution for table {} because the column(s) to add {} already exist.", @@ -852,7 +882,7 @@ private void doAlterTablePropertiesOnce( } // reuse the same validate logic with the createTable() method - validateTableDescriptor(newDescriptor); + validateTableDescriptor(tablePath, newDescriptor); beforeUpdate.accept(tableInfo, newDescriptor); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index de40fbf01a8..fafbbf9a6c1 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -32,6 +32,7 @@ import org.apache.fluss.metadata.ChangelogImage; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.DeleteBehavior; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.MergeEngineType; @@ -122,6 +123,8 @@ public static void validateTableDescriptor( } } + KafkaTableSchema.validate(tableDescriptor); + // check distribution checkDistribution(tableDescriptor, maxBucketNum); checkPrimaryKey(tableDescriptor); @@ -146,6 +149,10 @@ public static void validateTableDescriptor( /** Validates the schema after altering table columns. */ @Internal public static void validateAlterTableSchema(TableInfo table, Schema newSchema) { + TableDescriptor updated = + TableDescriptor.builder(table.toTableDescriptor()).schema(newSchema).build(); + KafkaTableSchema.validate(updated); + KafkaTableSchema.validateTablePath(table.getTablePath(), updated); checkSystemColumns(newSchema.getRowType()); if (table.getTableConfig() .getMergeEngineType() diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index 01f897cff29..a7a706c214e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -1315,6 +1315,36 @@ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) thro return schemaId; } + /** + * Publishes a schema and advances the table registration version in one fenced transaction. + * Property ALTER validates against a schema snapshot and compares the table version, so schema + * ALTER must participate in the same CAS to prevent enabling an incompatible Kafka contract. + */ + public void registerSchemaAndBumpTableVersion( + TablePath tablePath, + Schema schema, + int schemaId, + VersionedData expectedTable, + int expectedCoordinatorEpochZkVersion) + throws Exception { + List ops = new ArrayList<>(); + ops.add( + zkClient.transactionOp() + .setData() + .withVersion(expectedTable.zkVersion()) + .forPath( + TableZNode.path(tablePath), + TableZNode.encode(expectedTable.data()))); + ops.add( + zkClient.transactionOp() + .create() + .withMode(CreateMode.PERSISTENT) + .forPath( + SchemaZNode.path(tablePath, schemaId), SchemaZNode.encode(schema))); + zkClient.transaction() + .forOperations(wrapRequestsWithEpochCheck(ops, expectedCoordinatorEpochZkVersion)); + } + /** Get the specific schema by schema id in ZK metadata. */ public Optional getSchemaById(TablePath tablePath, int schemaId) throws Exception { Optional bytes = getOrEmpty(SchemaZNode.path(tablePath, schemaId)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java new file mode 100644 index 00000000000..e76dbda662d --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.coordinator; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.FlussRuntimeException; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableChange; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.entity.TablePropertyChanges; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.server.zk.data.ZkVersion; +import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; +import org.apache.fluss.testutils.common.AllCallbackWrapper; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Collections; +import java.util.function.BiConsumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Deterministic interleavings of schema ALTER and Kafka enablement using real ZooKeeper. */ +class KafkaTableSchemaAlterTest { + + @RegisterExtension + public static final AllCallbackWrapper ZK = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private ZooKeeperClient zk; + private MetadataManager manager; + private int epoch; + + @BeforeEach + void setup() throws Exception { + zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); + epoch = zk.fenceBecomeCoordinatorLeader("kafka-ddl").getCoordinatorEpochZkVersion(); + manager = + new MetadataManager( + zk, + new Configuration(), + new LakeCatalogDynamicLoader(new Configuration(), null, true)); + manager.createDatabase("ddl", DatabaseDescriptor.EMPTY, true); + } + + @Test + void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { + TablePath path = create("schema_wins"); + assertThatThrownBy( + () -> + enable( + path, + (oldTable, updated) -> + manager.alterTableSchema( + path, + Collections.singletonList( + TableChange.addColumn( + "unmapped", + DataTypes.STRING(), + null, + TableChange.ColumnPosition + .last())), + false, + null, + epoch))) + .isInstanceOf(FlussRuntimeException.class) + .hasMessageContaining("Concurrent metadata change"); + assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isFalse(); + assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) + .contains("unmapped"); + } + + @Test + void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { + TablePath path = create("enable_wins"); + ZooKeeperClient.VersionedData snapshot = + zk.getTableWithVersion(path).get(); + int schemaId = zk.getCurrentSchemaId(path); + Schema incompatible = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("unmapped", DataTypes.STRING()) + .build(); + // This candidate is valid for the disabled snapshot, but not after the competing enable. + KafkaTableSchema.validate( + TableDescriptor.builder(manager.getTable(path).toTableDescriptor()) + .schema(incompatible) + .build()); + enable(path, (oldTable, updated) -> {}); + assertThatThrownBy( + () -> + zk.registerSchemaAndBumpTableVersion( + path, incompatible, schemaId + 1, snapshot, epoch)) + .isInstanceOf(KeeperException.BadVersionException.class); + assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); + assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isTrue(); + assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) + .containsExactly("_kafka_value"); + } + + @Test + void testSchemaFailureDoesNotAdvanceTableVersion() throws Exception { + TablePath path = create("atomic_failure"); + ZooKeeperClient.VersionedData snapshot = + zk.getTableWithVersion(path).get(); + int schemaId = zk.getCurrentSchemaId(path); + Schema schema = manager.getTable(path).getSchema(); + assertThatThrownBy( + () -> + zk.registerSchemaAndBumpTableVersion( + path, schema, schemaId, snapshot, epoch)) + .isInstanceOf(KeeperException.NodeExistsException.class); + assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); + zk.fenceBecomeCoordinatorLeader("next-coordinator"); + assertThatThrownBy( + () -> + zk.registerSchemaAndBumpTableVersion( + path, schema, schemaId + 1, snapshot, epoch)) + .isInstanceOf(KeeperException.BadVersionException.class); + assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); + } + + private TablePath create(String name) { + TablePath path = TablePath.of("ddl", name); + TableDescriptor table = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build() + .withReplicationFactor(1); + manager.createTable(path, zk.getDefaultRemoteDataDir(), table, null, false); + return path; + } + + private void enable( + TablePath path, + BiConsumer beforeUpdate) { + TablePropertyChanges.Builder changes = new TablePropertyChanges.Builder(); + changes.setTableProperty("table.kafka.enabled", "true"); + manager.alterTableProperties( + path, + Collections.singletonList(TableChange.set("table.kafka.enabled", "true")), + changes.build(), + false, + null, + beforeUpdate, + (oldTable, updated) -> {}, + ZkVersion.MATCH_ANY_VERSION.getVersion()); + } +} diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index ce76beca9a1..491f7e83c6d 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -254,7 +254,42 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.database | String | kafka | The database for Fluss Kafka. The default database is `kafka`. | +| kafka.default.database | String | kafka | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. + +Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named +`sales.orders` maps to that table directly; `orders` maps to the same-named table in +`kafka.default.database`. Databases and tables must already exist. Reverse mapping always uses +the fully qualified name. All TabletServers must use the same default database. Changing it +redirects unqualified names, so migrate clients to fully qualified names before changing the +configuration. The prototype `kafka.database` option is no longer used. + +### Kafka table properties + +| Option | Type | Default | Description | +|---|---|---|---| +| table.kafka.enabled | Boolean | false | Explicitly expose a non-partitioned Arrow log table as a Kafka topic. CREATE and ALTER validate the contract before persisting it. | +| table.kafka.convert.value.type | String | (None) | Required when enabled: `binary`, `string`, or `parse_json`. | +| table.kafka.convert.value.layout | String | (None) | JSON uses `flat` when omitted; `nested` targets `_kafka_value ROW<...>`. Setting this property for binary/string is invalid. | +| table.kafka.convert.key.type | String | (None) | Only `binary` is supported, and requires `_kafka_key BYTES`. Declaring that column implies binary without adding a property. | + +Binary and string values require a user-declared `_kafka_value BYTES` or `_kafka_value STRING` +column respectively. Flat JSON targets business columns at the table level; nested JSON targets +the fields of `_kafka_value ROW<...>`. Both exclude reserved system names from business matching. +Optional `_kafka_key BYTES` and `_kafka_headers MAP` columns retain keys and headers. +Duplicate header names use the last value. Optional `_kafka_rescue STRING` must be nullable and +collects unknown or reserved JSON members; it requires no mapping property. Binary/string +conversion writes null to a declared rescue column. + +Enabling Kafka compatibility does not add columns or persist default Kafka properties. When +disabled, ordinary Fluss tables are not subject to Kafka-specific schema validation. Prototype +`kafka.key.*`, `kafka.value.*`, `kafka.metadata.*`, and `fluss.*` conversion properties are not +accepted on enabled tables. + +The `_kafka_metadata` contract is +`ROW`. +Enabling tables with this column is currently rejected until native append can assign and +persist each row's actual offset. Other optional system columns can be used independently. +These DDL contracts do not by themselves advertise Kafka Metadata or Produce API support. From 48c84f9a5324428ef22d77673066d9723dd8593a Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 17:15:35 +0800 Subject: [PATCH 11/48] [kafka] Validate recursive JSON schemas and default to fluss Default bare topic names to the native fluss database while preserving explicit database overrides, canonical names and no automatic Kafka database creation. Validate JSON business types recursively before CREATE, ALTER and enablement. Reject non-STRING map keys at every nesting level and schemas deeper than the supported limit. Expose the validation entry point and depth constant for subsequent decoders. Count business fields equally in flat and nested layouts, without charging a level for the physical _kafka_value ROW container. Keep disabled tables outside Kafka validation and preserve user schemas and explicit properties. Cover complex scalar/container types, depth boundaries, failed DDL state preservation and successful enablement with real RPC tests. Validation: Java 11 clean reactor verify passed 161 targeted cases, including snapshot lease and concurrent ALTER regressions. Checkstyle, Spotless, RAT and git diff checks passed. The five prior PR03 commits were rebased onto PR02 97d20b18549ec34f49e4e608392cdf1ec3a1b68a and checked with range-diff. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 92/92 AI-Contributed/UT: 292/292 --- .../apache/fluss/config/ConfigOptions.java | 2 +- .../fluss/metadata/KafkaTableSchema.java | 80 ++++++++ .../fluss/metadata/KafkaTableSchemaTest.java | 103 ++++++++++ .../apache/fluss/kafka/KafkaConfigsTest.java | 7 +- .../kafka/mapping/KafkaTopicMapperTest.java | 4 +- .../kafka/schema/KafkaTableMappingITCase.java | 178 ++++++++++++++++++ website/docs/maintenance/configuration.md | 10 +- 7 files changed, 377 insertions(+), 7 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 667e1af5f68..66f908d74d0 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2940,7 +2940,7 @@ public class ConfigOptions { public static final ConfigOption KAFKA_DEFAULT_DATABASE = key("kafka.default.database") .stringType() - .defaultValue("kafka") + .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " + "access other databases. Databases must already exist; this option does not create them."); diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java index 7c040f15871..990cec50416 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -22,6 +22,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BytesType; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; @@ -53,6 +54,9 @@ public final class KafkaTableSchema { public static final String METADATA_COLUMN = "_kafka_metadata"; public static final String RESCUE_COLUMN = "_kafka_rescue"; + /** Maximum recursive depth below each top-level JSON business field. */ + public static final int MAX_JSON_SCHEMA_NESTING_DEPTH = 64; + private static final Set RESERVED_NAMES = Collections.unmodifiableSet( new HashSet<>( @@ -271,6 +275,20 @@ public static KafkaTableSchema resolve(TableDescriptor table) { } valuePositions.add(valuePosition); } + if (valueType == ValueType.PARSE_JSON) { + for (int position : valuePositions) { + DataType type = rowType.getTypeAt(position); + if (valueLayout == ValueLayout.NESTED) { + // The physical value container represents the JSON object root, just like + // the table row in flat layout; it must not add a business nesting level. + for (DataType field : ((RowType) type).getChildren()) { + validateJsonType(field); + } + } else { + validateJsonType(type); + } + } + } return new KafkaTableSchema( rowType, valueType, @@ -281,6 +299,68 @@ public static KafkaTableSchema resolve(TableDescriptor table) { rescuePosition); } + /** + * Validates a top-level JSON business field without parser or protocol dependencies. DDL and + * decoder construction share this contract. Each business field starts at depth zero; ROW + * fields, ARRAY elements and MAP values each advance the depth by one. Nullability and scalar + * value constraints are enforced when decoding individual records. The JSON object root + * (including the physical nested _kafka_value ROW) is not counted. + */ + public static void validateJsonType(DataType type) { + validateJsonType(type, 0); + } + + private static void validateJsonType(DataType type, int depth) { + if (depth > MAX_JSON_SCHEMA_NESTING_DEPTH) { + throw invalid( + "Kafka JSON schema exceeds the maximum nesting depth of " + + MAX_JSON_SCHEMA_NESTING_DEPTH + + "."); + } + switch (type.getTypeRoot()) { + case CHAR: + case STRING: + case BOOLEAN: + case BINARY: + case BYTES: + case DECIMAL: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case FLOAT: + case DOUBLE: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return; + case ARRAY: + validateJsonType(((ArrayType) type).getElementType(), depth + 1); + return; + case MAP: + MapType map = (MapType) type; + if (!(map.getKeyType() instanceof StringType)) { + throw invalid( + "Kafka JSON format only supports STRING map keys, but found " + + type.asSummaryString() + + "."); + } + validateJsonType(map.getValueType(), depth + 1); + return; + case ROW: + for (DataType field : ((RowType) type).getChildren()) { + validateJsonType(field, depth + 1); + } + return; + default: + throw invalid( + "Kafka JSON format does not support Fluss type " + + type.asSummaryString() + + "."); + } + } + private static void validateOptions(TableDescriptor table) { for (String name : table.getProperties().keySet()) { if (name.startsWith("table.kafka.") && !OPTIONS.contains(name)) { diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index e2b9c5be187..809f13ac553 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -186,6 +186,109 @@ void testNestedJsonTargetsOneRow() { assertThat(resolved.rescuePosition()).isZero(); } + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonRejectsNonStringMapKeysRecursively(String layout) { + for (DataType key : Arrays.asList(DataTypes.INT(), DataTypes.CHAR(10), DataTypes.BYTES())) { + DataType invalidMap = DataTypes.MAP(key, DataTypes.STRING()); + for (DataType type : + Arrays.asList( + invalidMap, + DataTypes.ARRAY(invalidMap), + DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { + TableDescriptor table = jsonTable(layout, type); + assertInvalid(table, "only supports STRING map keys"); + assertThatCode( + () -> + KafkaTableSchema.validate( + TableDescriptor.builder(table) + .property( + ConfigOptions.TABLE_KAFKA_ENABLED, + false) + .build())) + .doesNotThrowAnyException(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonAcceptsAllScalarTypesInNestedContainers(String layout) { + for (DataType scalar : + Arrays.asList( + DataTypes.CHAR(10), + DataTypes.STRING(), + DataTypes.BOOLEAN(), + DataTypes.BINARY(4), + DataTypes.BYTES(), + DataTypes.DECIMAL(18, 2), + DataTypes.TINYINT(), + DataTypes.SMALLINT(), + DataTypes.INT(), + DataTypes.BIGINT(), + DataTypes.FLOAT(), + DataTypes.DOUBLE(), + DataTypes.DATE(), + DataTypes.TIME(3), + DataTypes.TIMESTAMP(9), + DataTypes.TIMESTAMP_LTZ(9))) { + DataType type = + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING().copy(false), + DataTypes.ROW( + DataTypes.FIELD("_kafka_key", scalar.copy(false))))); + TableDescriptor table = jsonTable(layout, type); + byte[] before = table.toJsonBytes(); + assertThatCode(() -> KafkaTableSchema.validate(table)).doesNotThrowAnyException(); + assertThat(table.toJsonBytes()).isEqualTo(before); + assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) + .isEqualTo(KafkaTableSchema.resolve(table)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonBusinessNestingDepthBoundary(String layout) { + // Exercise every recursive edge, including map values (keys do not add a level). + DataType boundary = DataTypes.STRING(); + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + if (depth % 3 == 0) { + boundary = DataTypes.ARRAY(boundary); + } else if (depth % 3 == 1) { + boundary = DataTypes.MAP(DataTypes.STRING(), boundary); + } else { + boundary = DataTypes.ROW(DataTypes.FIELD("child", boundary)); + } + } + TableDescriptor valid = jsonTable(layout, boundary); + assertThatCode(() -> KafkaTableSchema.validate(valid)).doesNotThrowAnyException(); + TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)); + assertInvalid(tooDeep, "maximum nesting depth of 64"); + assertThatCode( + () -> + KafkaTableSchema.validate( + TableDescriptor.builder(tooDeep) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build())) + .doesNotThrowAnyException(); + } + + private static TableDescriptor jsonTable(String layout, DataType businessType) { + Schema schema = + "flat".equals(layout) + ? Schema.newBuilder().column("payload", businessType).build() + : Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("payload", businessType))) + .build(); + return enabled(schema, "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout) + .build(); + } + @ParameterizedTest @ValueSource( strings = { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java index 55e8ba6fc57..17a8c6c0f9c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java @@ -36,14 +36,14 @@ public void testFromMap() throws Exception { Map map = new HashMap<>(); map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "fluss"); + map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "sales"); Configuration configuration = Configuration.fromMap(map); assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("fluss"); + .isEqualTo("sales"); } @Test @@ -53,6 +53,7 @@ public void testFromDefault() throws Exception { assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Collections.singletonList("KAFKA")); assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("kafka"); + .isEqualTo("fluss"); + assertThat(configuration.toMap()).doesNotContainKey("kafka.default.database"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index 7ef7e281244..f70c7ffca22 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -53,7 +53,9 @@ public void testQualifiedNamesAcrossDatabases() { @Test void testDefaultDatabaseAliases() { - assertThat(mapper.toTablePath("orders")).isEqualTo(mapper.toTablePath("kafka.orders")); + assertThat(mapper.toTablePath("orders")).isEqualTo(TablePath.of("fluss", "orders")); + assertThat(mapper.toTopicName(mapper.toTablePath("orders"))).isEqualTo("fluss.orders"); + assertThat(mapper.toTablePath("kafka.orders")).isEqualTo(TablePath.of("kafka", "orders")); Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); KafkaTopicMapper custom = new KafkaTopicMapper(config); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 45744dbc87a..76b9c0f773d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -41,6 +41,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; import java.util.Collections; @@ -121,6 +123,182 @@ void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() thro assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); } + @Test + void testDefaultAliasUsesExistingFlussDatabase() throws Exception { + KafkaTopicMapper mapper = new KafkaTopicMapper(); + TablePath path = mapper.toTablePath("default_alias"); + assertThat(path).isEqualTo(TablePath.of(DEFAULT_DATABASE, "default_alias")); + createTable(CLUSTER, path, enabled().build()); + assertThat(info(mapper.toTablePath("fluss.default_alias")).getTableId()) + .isEqualTo(info(path).getTableId()); + assertThat(mapper.toTopicName(path)).isEqualTo("fluss.default_alias"); + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonCreateAndEnableRejectUnsupportedBusinessSchema(String layout) throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl_json", true)).get(); + DataType invalidMap = DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()); + int index = 0; + for (DataType type : + Arrays.asList( + invalidMap, + DataTypes.ARRAY(invalidMap), + DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { + TablePath path = TablePath.of("ddl_json", layout + "_invalid_" + index++); + TableDescriptor descriptor = jsonTable(layout, type).build(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest(path, descriptor, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + + // The identical schema is legal for an ordinary, explicitly disabled Fluss table. + createTable( + CLUSTER, + path, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy( + () -> + set( + path, + Collections.singletonMap( + "table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, before); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonComplexSchemaRoundtripAndEnable(String layout) throws Exception { + TablePath path = TablePath.of("ddl_json", layout + "_complex"); + DataType type = + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW( + DataTypes.FIELD("amount", DataTypes.DECIMAL(18, 2)), + DataTypes.FIELD("time", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("_kafka_key", DataTypes.BYTES())))); + TableDescriptor descriptor = + jsonTable(layout, type).property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + createTable(CLUSTER, path, descriptor); + GetTableInfoResponse before = info(path); + set(path, Collections.singletonMap("table.kafka.enabled", "true")); + TableDescriptor actual = stored(path); + assertThat(actual.getSchema()).isEqualTo(descriptor.getSchema()); + assertThat(info(path).getSchemaId()).isEqualTo(before.getSchemaId()); + Map expected = new HashMap<>(kafkaProperties(descriptor)); + expected.put("table.kafka.enabled", "true"); + assertThat(kafkaProperties(actual)).isEqualTo(expected); + addColumn(path, "_kafka_rescue", DataTypes.STRING()); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + assertThat(stored(path).getSchema().getRowType().getTypeAt(0)) + .isEqualTo(descriptor.getSchema().getRowType().getTypeAt(0)); + assertThat(KafkaTableSchema.resolve(stored(path)).rescuePosition()).isEqualTo(1); + assertThat(kafkaProperties(stored(path))).isEqualTo(expected); + } + + @Test + void testJsonSchemaAlterRejectsUnsupportedNestedMapAtomically() throws Exception { + TablePath path = TablePath.of("ddl_json", "alter_business_type"); + createTable(CLUSTER, path, jsonTable("flat", DataTypes.STRING()).build()); + DataType invalid = + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())))); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> addColumn(path, "bad", invalid)) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, before); + addColumn( + path, "good", DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(DataTypes.INT()))); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + set(path, Collections.singletonMap("table.kafka.enabled", "false")); + addColumn(path, "ordinary", invalid); + GetTableInfoResponse disabled = info(path); + assertThat(stored(path).getSchema().getRowType().getFieldNames()).contains("ordinary"); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, disabled); + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonDepthValidationThroughDdl(String layout) throws Exception { + DataType boundary = DataTypes.STRING(); + for (int i = 0; i < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; i++) { + boundary = DataTypes.ARRAY(boundary); + } + TablePath valid = TablePath.of("ddl_json", layout + "_depth_boundary"); + TableDescriptor descriptor = jsonTable(layout, boundary).build(); + createTable(CLUSTER, valid, descriptor); + assertThat(stored(valid).getSchema()).isEqualTo(descriptor.getSchema()); + TablePath invalid = TablePath.of("ddl_json", layout + "_too_deep"); + TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)).build(); + assertThatThrownBy( + () -> + gateway.createTable(newCreateTableRequest(invalid, tooDeep, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("maximum nesting depth of 64"); + assertThat(gateway.tableExists(newTableExistsRequest(invalid)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(invalid)).isZero(); + createTable( + CLUSTER, + invalid, + TableDescriptor.builder(tooDeep) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(invalid); + assertThatThrownBy( + () -> set(invalid, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("maximum nesting depth of 64"); + assertUnchanged(invalid, before); + } + + private void assertUnchanged(TablePath path, GetTableInfoResponse before) throws Exception { + GetTableInfoResponse after = info(path); + assertThat(after.getTableJson()).isEqualTo(before.getTableJson()); + assertThat(after.getSchemaId()).isEqualTo(before.getSchemaId()); + } + + private static TableDescriptor.Builder jsonTable(String layout, DataType businessType) { + Schema schema = + "flat".equals(layout) + ? Schema.newBuilder().column("payload", businessType).build() + : Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("payload", businessType))) + .build(); + TableDescriptor.Builder builder = + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json"); + // Omitted flat layout must remain omitted through CREATE and ALTER. + if ("nested".equals(layout)) { + builder.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout); + } + return builder; + } + @Test void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { TablePath path = TablePath.of("ddl", "roundtrip"); diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 491f7e83c6d..f5fb15b670b 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -254,14 +254,14 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | kafka | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named `sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database`. Databases and tables must already exist. Reverse mapping always uses +`kafka.default.database` (by default, `fluss.orders`). Databases and tables must already exist. Reverse mapping always uses the fully qualified name. All TabletServers must use the same default database. Changing it redirects unqualified names, so migrate clients to fully qualified names before changing the configuration. The prototype `kafka.database` option is no longer used. @@ -278,6 +278,12 @@ configuration. The prototype `kafka.database` option is no longer used. Binary and string values require a user-declared `_kafka_value BYTES` or `_kafka_value STRING` column respectively. Flat JSON targets business columns at the table level; nested JSON targets the fields of `_kafka_value ROW<...>`. Both exclude reserved system names from business matching. +JSON business schemas support nested ROW, ARRAY and MAP values. Every MAP key must be STRING, +including maps inside other containers. CREATE and ALTER reject unsupported types or a nesting +depth above 64, counted from each top-level JSON business field at depth zero; ROW fields, +ARRAY elements and MAP values each add one level. The JSON root and the physical nested +`_kafka_value ROW` container add no level, so equivalent flat/nested schemas have the same limit. +Scalar value and nullability checks apply when decoding records. Optional `_kafka_key BYTES` and `_kafka_headers MAP` columns retain keys and headers. Duplicate header names use the last value. Optional `_kafka_rescue STRING` must be nullable and collects unknown or reserved JSON members; it requires no mapping property. Binary/string From e7673be5719768ad5a72d4711a6f184188beed71 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 10:54:36 +0800 Subject: [PATCH 12/48] [kafka] Enable metadata mapping and fix table change notifications Allow optional _kafka_metadata with timestamp, timestamp_type and partition, without a physical offset field. Cover optional fixed-column combinations, metadata validation and CREATE/ALTER behavior. Ignore unchanged table registrations when schema ALTER advances the ZK version. Preserve atomic concurrency fencing and notifications for real property and bucket changes, with deterministic regression coverage. Validation: Java 11 reactor verify passed 111 targeted test cases; Checkstyle, Spotless and git diff --check passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 57/57 AI-Contributed/UT: 265/265 --- .../fluss/metadata/KafkaTableSchema.java | 25 +++-- .../fluss/metadata/KafkaTableSchemaTest.java | 93 ++++++++++++++----- .../kafka/schema/KafkaTableMappingITCase.java | 79 +++++++++++++++- .../schema/KafkaTopicSchemaResolverTest.java | 20 +++- .../event/watcher/TableChangeWatcher.java | 21 +++-- .../event/watcher/TableChangeWatcherTest.java | 73 ++++++++++++++- website/docs/maintenance/configuration.md | 11 ++- 7 files changed, 277 insertions(+), 45 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java index 990cec50416..9ab16195fbd 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -93,6 +93,7 @@ public enum ValueLayout { private final KafkaFieldProjection valueProjection; private final int keyPosition; private final int headersPosition; + private final int metadataPosition; private final int rescuePosition; private KafkaTableSchema( @@ -102,6 +103,7 @@ private KafkaTableSchema( List valuePositions, int keyPosition, int headersPosition, + int metadataPosition, int rescuePosition) { this.rowType = rowType; this.valueType = valueType; @@ -109,6 +111,7 @@ private KafkaTableSchema( this.valueProjection = new KafkaFieldProjection(rowType, valuePositions); this.keyPosition = keyPosition; this.headersPosition = headersPosition; + this.metadataPosition = metadataPosition; this.rescuePosition = rescuePosition; } @@ -216,9 +219,6 @@ public static KafkaTableSchema resolve(TableDescriptor table) { int metadataPosition = rowType.getFieldIndex(METADATA_COLUMN); if (metadataPosition >= 0) { validateMetadataType(rowType.getTypeAt(metadataPosition)); - throw invalid( - "_kafka_metadata requires native append support for persisting actual per-row offsets; " - + "this capability is not available yet."); } List valuePositions = new ArrayList<>(); @@ -296,6 +296,7 @@ public static KafkaTableSchema resolve(TableDescriptor table) { valuePositions, keyPosition, headersPosition, + metadataPosition, rescuePosition); } @@ -382,17 +383,15 @@ private static void validateOptions(TableDescriptor table) { private static void validateMetadataType(DataType type) { String message = "_kafka_metadata must be ROW."; + + "timestamp_type INT, partition INT>."; if (!(type instanceof RowType)) { throw invalid(message); } RowType row = (RowType) type; - if (!row.getFieldNames() - .equals(Arrays.asList("timestamp", "timestamp_type", "partition", "offset")) + if (!row.getFieldNames().equals(Arrays.asList("timestamp", "timestamp_type", "partition")) || !row.getTypeAt(0).copy(true).equals(DataTypes.TIMESTAMP_LTZ(3)) || !row.getTypeAt(1).copy(true).equals(DataTypes.INT()) - || !row.getTypeAt(2).copy(true).equals(DataTypes.INT()) - || !row.getTypeAt(3).copy(true).equals(DataTypes.BIGINT())) { + || !row.getTypeAt(2).copy(true).equals(DataTypes.INT())) { throw invalid(message); } } @@ -430,6 +429,14 @@ public int headersPosition() { return headersPosition; } + /** + * Returns the metadata column position, or -1 when absent. The ROW stores the input record + * timestamp, batch timestamp type and requested partition; log offsets are not row fields. + */ + public int metadataPosition() { + return metadataPosition; + } + /** * Returns the rescue column position, or -1 when absent. JSON decoders collect unknown and * reserved members here; binary/string decoders write null to this optional column. @@ -450,6 +457,7 @@ public boolean equals(Object other) { && valueProjection.equals(that.valueProjection) && keyPosition == that.keyPosition && headersPosition == that.headersPosition + && metadataPosition == that.metadataPosition && rescuePosition == that.rescuePosition; } @@ -462,6 +470,7 @@ public int hashCode() { valueProjection, keyPosition, headersPosition, + metadataPosition, rescuePosition); } diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index 809f13ac553..00aacbd51e5 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -89,7 +89,7 @@ void testExplicitOptInAndDisabledTables() { @ValueSource(strings = {"binary", "string"}) void testFixedValueColumnWithOptionalSystemFields(String format) { DataType valueType = format.equals("binary") ? DataTypes.BYTES() : DataTypes.STRING(); - for (int mask = 0; mask < 8; mask++) { + for (int mask = 0; mask < 16; mask++) { Schema.Builder schema = Schema.newBuilder().column("_kafka_value", valueType.copy(false)); if ((mask & 1) != 0) { @@ -102,6 +102,9 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { if ((mask & 4) != 0) { schema.column("_kafka_rescue", DataTypes.STRING()); } + if ((mask & 8) != 0) { + schema.column("_kafka_metadata", metadataType()); + } TableDescriptor table = enabled(schema.build(), format).build(); byte[] before = table.toJsonBytes(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); @@ -111,6 +114,9 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); + assertThat(resolved.metadataPosition() >= 0).isEqualTo((mask & 8) != 0); + assertThat(resolved.metadataPosition()) + .isEqualTo(table.getSchema().getRowType().getFieldIndex("_kafka_metadata")); assertThat(table.toJsonBytes()).isEqualTo(before); assertThat(table.getProperties()) .doesNotContainKeys( @@ -134,6 +140,7 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { Schema.newBuilder() .column("id", DataTypes.BIGINT().copy(false)) .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_metadata", metadataType()) .column("payload", nested) .column("_kafka_key", DataTypes.BYTES()) .column( @@ -147,9 +154,10 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueType()).isEqualTo(ValueType.PARSE_JSON); assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.FLAT); - assertThat(resolved.valueProjection().positions()).containsExactly(0, 2, 5); + assertThat(resolved.valueProjection().positions()).containsExactly(0, 3, 6); assertThat(resolved.valueProjection().dataTypeAt(1)).isEqualTo(nested); assertThat(resolved.rescuePosition()).isEqualTo(1); + assertThat(resolved.metadataPosition()).isEqualTo(2); assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); TableDescriptor noRescue = enabled( @@ -174,6 +182,7 @@ void testNestedJsonTargetsOneRow() { enabled( Schema.newBuilder() .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_metadata", metadataType()) .column("_kafka_value", row) .build(), "parse_json") @@ -181,9 +190,10 @@ void testNestedJsonTargetsOneRow() { .build(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.NESTED); - assertThat(resolved.valueProjection().positions()).containsExactly(1); + assertThat(resolved.valueProjection().positions()).containsExactly(2); assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(row); assertThat(resolved.rescuePosition()).isZero(); + assertThat(resolved.metadataPosition()).isEqualTo(1); } @ParameterizedTest @@ -425,33 +435,63 @@ void testSystemColumnTypes() { } @Test - void testMetadataStructureAndUnavailableOffsetPersistence() { + void testMetadataPreservesDeclaredNullability() { RowType metadata = DataTypes.ROW( DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), - DataTypes.FIELD("partition", DataTypes.INT()), - DataTypes.FIELD("offset", DataTypes.BIGINT())); - assertInvalid(withSystem("_kafka_metadata", metadata), "persisting actual per-row offsets"); - assertInvalid(withSystem("_kafka_metadata", DataTypes.STRING()), "must be ROW"); - assertInvalid( - withSystem( - "_kafka_metadata", + DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))); + for (boolean nullable : new boolean[] {true, false}) { + DataType type = metadata.copy(nullable); + TableDescriptor table = withSystem("_kafka_metadata", type); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())).isEqualTo(type); + assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) + .isEqualTo(resolved); + } + } + + @Test + void testMetadataRejectsInvalidStructureAndOffsetField() { + for (DataType invalid : + Arrays.asList( + DataTypes.STRING(), DataTypes.ROW( - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), DataTypes.FIELD("partition", DataTypes.INT()), - DataTypes.FIELD("offset", DataTypes.BIGINT()))), - "TIMESTAMP_LTZ(3)"); - assertInvalid( - withSystem( - "_kafka_metadata", + DataTypes.FIELD("offset", DataTypes.BIGINT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(6)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.STRING()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.BIGINT())), DataTypes.ROW( - DataTypes.FIELD("offset", DataTypes.BIGINT()), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), DataTypes.FIELD("partition", DataTypes.INT()), - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)))), - "must be ROW"); + DataTypes.FIELD("timestamp_type", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("bucket", DataTypes.INT())))) { + assertInvalid( + withSystem("_kafka_metadata", invalid), + "ROW"); + } } @Test @@ -510,6 +550,13 @@ void testTableKindsAndCanonicalNameLimits() { .hasMessageContaining("Kafka topic"); } + private static RowType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + private static TableDescriptor withSystem(String name, DataType type) { return enabled( Schema.newBuilder() diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 76b9c0f773d..97e75987f89 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -444,9 +444,62 @@ void testCreateRejectsInvalidContractWithoutPublishingMetadata() throws Exceptio assertThat(KafkaTableSchema.isEnabled(stored(path))).isTrue(); } + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testMetadataCanBeCreatedAndEnabledWithoutOffset(String format) throws Exception { + Schema schema = + Schema.newBuilder() + .column("_kafka_metadata", metadataType()) + .column( + "_kafka_value", + "binary".equals(format) ? DataTypes.BYTES() : DataTypes.STRING()) + .build(); + TableDescriptor descriptor = + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .build(); + TablePath created = TablePath.of("ddl", "metadata_create_" + format); + createTable(CLUSTER, created, descriptor); + TableDescriptor persisted = stored(created); + assertThat(persisted.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); + assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); + + TablePath altered = TablePath.of("ddl", "metadata_enable_" + format); + createTable( + CLUSTER, + altered, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + int schemaId = info(altered).getSchemaId(); + set(altered, Collections.singletonMap("table.kafka.enabled", "true")); + persisted = stored(altered); + assertThat(info(altered).getSchemaId()).isEqualTo(schemaId); + assertThat(persisted.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); + assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); + } + @Test - void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { - TablePath path = TablePath.of("ddl", "offset_capability"); + void testMetadataCanBeAddedToEnabledTable() throws Exception { + TablePath path = TablePath.of("ddl", "metadata_add"); + createTable(CLUSTER, path, enabled().build()); + TableDescriptor before = stored(path); + addColumn(path, "_kafka_metadata", metadataType()); + TableDescriptor after = stored(path); + KafkaTableSchema resolved = KafkaTableSchema.resolve(after); + assertThat(resolved.metadataPosition()).isGreaterThanOrEqualTo(0); + assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())) + .isEqualTo(metadataType()); + assertThat(kafkaProperties(after)).isEqualTo(kafkaProperties(before)); + } + + @Test + void testMetadataOffsetFieldRejectedOnCreateAndEnable() throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl", true)).get(); + TablePath path = TablePath.of("ddl", "metadata_with_offset"); Schema schema = Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES()) @@ -458,6 +511,18 @@ void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { DataTypes.FIELD("partition", DataTypes.INT()), DataTypes.FIELD("offset", DataTypes.BIGINT()))) .build(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled().schema(schema).build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining( + "ROW"); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); createTable( CLUSTER, path, @@ -468,7 +533,8 @@ void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { TableDescriptor before = stored(path); assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("actual per-row offsets"); + .hasStackTraceContaining( + "ROW"); assertThat(stored(path)).isEqualTo(before); } @@ -495,6 +561,13 @@ path, enabled().build(), false)) assertThat(stored(path)).isEqualTo(before); } + private static DataType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + private GetTableInfoResponse info(TablePath path) throws Exception { return gateway.getTableInfo(newGetTableInfoRequest(path)).get(); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index b7e4174421c..8327c6fb60e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -34,9 +34,27 @@ class KafkaTopicSchemaResolverTest { @Test void testEnabledContractUsesSharedValidation() { - TableDescriptor table = table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true).build(); + TableDescriptor table = + table().schema( + Schema.newBuilder() + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()))) + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .build(); assertThat(resolver.isEnabled(table)).isTrue(); assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); + assertThat(resolver.resolve(table).metadataPosition()).isZero(); + assertThat(resolver.resolve(table).valueProjection().positions()).containsExactly(1); assertThat(resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) .isEqualTo(resolver.resolve(table)); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java index 5e05877b4be..f38a7f0afe4 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java @@ -17,6 +17,7 @@ package org.apache.fluss.server.coordinator.event.watcher; +import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.metadata.PhysicalTablePath; @@ -87,7 +88,8 @@ public void stop() { } /** A listener to monitor the changes of table nodes in zookeeper. */ - private final class TablePathChangeListener implements CuratorCacheListener { + @VisibleForTesting + final class TablePathChangeListener implements CuratorCacheListener { @Override public void event(Type type, ChildData oldData, ChildData newData) { @@ -150,13 +152,13 @@ public void event(Type type, ChildData oldData, ChildData newData) { if (tablePath == null) { break; } - // Distinguish between table creation and properties change. + // Distinguish between table creation and registration updates. // If oldData exists and contains valid table registration data, - // it's a properties change; otherwise, it's a table creation. + // compare its contents; otherwise, it's a table creation. if (oldData != null && oldData.getData() != null && oldData.getData().length > 0) { - processTableRegistrationChange(tablePath, newData); + processTableRegistrationChange(tablePath, oldData, newData); } else { processCreateTable(tablePath, newData); } @@ -276,9 +278,16 @@ private void processCreatePartition( tablePath, tableId, partitionId, partitionName, partitionAssignment)); } - private void processTableRegistrationChange(TablePath tablePath, ChildData newData) { + private void processTableRegistrationChange( + TablePath tablePath, ChildData oldData, ChildData newData) { + TableRegistration oldTable = TableZNode.decode(oldData.getData()); TableRegistration newTable = TableZNode.decode(newData.getData()); - eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); + // Schema ALTER bumps the table znode version to fence concurrent registration updates, + // even when its contents are unchanged. Compare decoded registrations so differences + // in JSON serialization do not trigger a registration change either. + if (!oldTable.equals(newTable)) { + eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); + } } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java index 28a4131ecd8..9a36280d891 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java @@ -42,11 +42,16 @@ import org.apache.fluss.server.entity.TablePropertyChanges; import org.apache.fluss.server.zk.NOPErrorHandler; import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperClient.VersionedData; import org.apache.fluss.server.zk.ZooKeeperExtension; import org.apache.fluss.server.zk.data.PartitionAssignment; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.server.zk.data.ZkData.TableZNode; import org.apache.fluss.server.zk.data.ZkVersion; +import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCacheListener; +import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.data.Stat; import org.apache.fluss.testutils.common.AllCallbackWrapper; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; @@ -56,9 +61,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -302,7 +311,7 @@ void testPartitionedTable() throws Exception { } @Test - void testSchemaChanges() { + void testSchemaChanges() throws Exception { // create tables, collect create table events List expectedEvents = new ArrayList<>(); for (int i = 0; i < 10; i++) { @@ -349,6 +358,8 @@ void testSchemaChanges() { } CreateTableEvent createTableEvent = (CreateTableEvent) coordinatorEvent; TableInfo tableInfo = createTableEvent.getTableInfo(); + VersionedData beforeAlter = + zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); metadataManager.alterTableSchema( tableInfo.getTablePath(), Collections.singletonList( @@ -359,6 +370,10 @@ void testSchemaChanges() { TableChange.ColumnPosition.last())), false, null); + VersionedData afterAlter = + zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); + assertThat(afterAlter.zkVersion()).isEqualTo(beforeAlter.zkVersion() + 1); + assertThat(afterAlter.data()).isEqualTo(beforeAlter.data()); Schema newSchema = Schema.newBuilder() .fromSchema(tableInfo.getSchema()) @@ -378,6 +393,62 @@ void testSchemaChanges() { .containsExactlyInAnyOrderElementsOf(allEvents)); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testUnchangedTableRegistration(boolean differentJsonFormatting) { + tableChangeWatcher.stop(); + TablePath tablePath = TablePath.of(DEFAULT_DB, "unchanged_table_registration"); + TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); + byte[] oldData = TableZNode.encode(table); + byte[] newData = + differentJsonFormatting + ? ("\n" + new String(oldData, StandardCharsets.UTF_8)) + .getBytes(StandardCharsets.UTF_8) + : oldData; + + // Deliver the callback synchronously so the absence of an event cannot pass before + // CuratorCache has observed the version bump. + CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); + listener.event( + CuratorCacheListener.Type.NODE_CHANGED, + tableNodeData(tablePath, oldData, 1), + tableNodeData(tablePath, newData, 2)); + + assertThat(eventManager.getEvents()).isEmpty(); + } + + @Test + void testChangedTableRegistrationContents() { + tableChangeWatcher.stop(); + TablePath tablePath = TablePath.of(DEFAULT_DB, "changed_table_registration"); + TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); + CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); + List expectedEvents = new ArrayList<>(); + for (TableRegistration updatedTable : + Arrays.asList( + table.newProperties( + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), "true"), + table.customProperties), + table.newProperties( + table.properties, + Collections.singletonMap("custom.key", "custom.value")), + table.newBucketCount(4))) { + listener.event( + CuratorCacheListener.Type.NODE_CHANGED, + tableNodeData(tablePath, TableZNode.encode(table), 1), + tableNodeData(tablePath, TableZNode.encode(updatedTable), 2)); + expectedEvents.add(new TableRegistrationChangeEvent(tablePath, updatedTable)); + assertThat(eventManager.getEvents()).containsExactlyElementsOf(expectedEvents); + } + } + + private static ChildData tableNodeData(TablePath tablePath, byte[] data, int version) { + Stat stat = new Stat(); + stat.setVersion(version); + return new ChildData(TableZNode.path(tablePath), stat, data); + } + @Test void testTableRegistrationChange() { // create a table diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index f5fb15b670b..6344eee5207 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -295,7 +295,12 @@ disabled, ordinary Fluss tables are not subject to Kafka-specific schema validat accepted on enabled tables. The `_kafka_metadata` contract is -`ROW`. -Enabling tables with this column is currently rejected until native append can assign and -persist each row's actual offset. Other optional system columns can be used independently. +`ROW`. +This optional physical column stores the input Kafka record timestamp, the batch timestamp type +(`-1` for no timestamp, `0` for CreateTime, `1` for LogAppendTime), and the requested partition. +Without a timestamp, the timestamp field is null; declared NOT NULL constraints still apply +when converting records. These values are available before Arrow encoding. Log offsets remain +native log metadata and are not part of this ROW, so enabling the column requires no append-time +offset backfill. CREATE and ALTER reject extra fields, including the former offset field, and +incorrect field names, ordering or types. These DDL contracts do not by themselves advertise Kafka Metadata or Produce API support. From b4e5a021235e5763b5ed2a07da640bdd736d0122 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 17:17:18 +0800 Subject: [PATCH 13/48] [kafka] Focus DDL mapping changes and streamline tests Create the configured Kafka default database when missing and preserve existing metadata. Fence schema and property ALTER with schema-parent version checks without rewriting table registrations, restoring the original table watcher. Remove redundant configuration, Flink conversion and mapping test coverage while retaining DDL validation, optional system columns and concurrent ALTER checks. Validation: targeted clean verify passed 173 tests with Checkstyle and Spotless. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 120/120 AI-Contributed/UT: 683/683 --- .../apache/fluss/config/ConfigOptions.java | 3 +- .../fluss/metadata/KafkaTableSchemaTest.java | 83 +----- .../flink/utils/FlinkConversionsTest.java | 45 ---- .../apache/fluss/kafka/KafkaConfigsTest.java | 59 ---- .../kafka/schema/KafkaTableMappingITCase.java | 255 ++++-------------- .../schema/KafkaTopicSchemaResolverTest.java | 28 -- .../server/coordinator/CoordinatorServer.java | 16 ++ .../server/coordinator/MetadataManager.java | 20 +- .../event/watcher/TableChangeWatcher.java | 21 +- .../fluss/server/zk/ZooKeeperClient.java | 54 +++- .../coordinator/KafkaDefaultDatabaseTest.java | 94 +++++++ .../KafkaTableSchemaAlterTest.java | 46 +++- .../event/watcher/TableChangeWatcherTest.java | 73 +---- website/docs/maintenance/configuration.md | 6 +- 14 files changed, 275 insertions(+), 528 deletions(-) delete mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 66f908d74d0..e05d981e995 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2943,7 +2943,8 @@ public class ConfigOptions { .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " - + "access other databases. Databases must already exist; this option does not create them."); + + "access other databases. When Kafka is enabled, the coordinator creates this " + + "database if it does not exist."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index 00aacbd51e5..458242feadd 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -18,9 +18,6 @@ package org.apache.fluss.metadata; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.FlussConfigUtils; -import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; import org.apache.fluss.metadata.KafkaTableSchema.ValueType; @@ -33,7 +30,6 @@ import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; -import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -42,22 +38,6 @@ /** Tests the shared DDL and Kafka conversion contract. */ class KafkaTableSchemaTest { - @Test - void testRegisteredAlterableOptionsAndReadTimeDefaults() { - for (String key : - Arrays.asList( - "table.kafka.enabled", - "table.kafka.convert.value.type", - "table.kafka.convert.value.layout", - "table.kafka.convert.key.type")) { - assertThat(FlussConfigUtils.TABLE_OPTIONS).containsKey(key); - assertThat(FlussConfigUtils.isAlterableTableOption(key)).isTrue(); - } - Configuration config = new Configuration(); - assertThat(new TableConfig(config).isKafkaEnabled()).isFalse(); - assertThat(config.toMap()).isEmpty(); - } - @Test void testExplicitOptInAndDisabledTables() { TableDescriptor disabled = @@ -106,35 +86,22 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { schema.column("_kafka_metadata", metadataType()); } TableDescriptor table = enabled(schema.build(), format).build(); - byte[] before = table.toJsonBytes(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueProjection().positions()).containsExactly(0); assertThat(resolved.valueProjection().dataTypeAt(0).isNullable()).isFalse(); - assertThat(resolved.valueLayout()).isNull(); assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); assertThat(resolved.metadataPosition() >= 0).isEqualTo((mask & 8) != 0); - assertThat(resolved.metadataPosition()) - .isEqualTo(table.getSchema().getRowType().getFieldIndex("_kafka_metadata")); - assertThat(table.toJsonBytes()).isEqualTo(before); assertThat(table.getProperties()) .doesNotContainKeys( "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); - assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) - .isEqualTo(resolved); } } @Test void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { - RowType nested = - DataTypes.ROW( - DataTypes.FIELD("_kafka_key", DataTypes.STRING()), - DataTypes.FIELD( - "items", - DataTypes.ARRAY( - DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())))); + RowType nested = DataTypes.ROW(DataTypes.FIELD("_kafka_key", DataTypes.STRING())); TableDescriptor table = enabled( Schema.newBuilder() @@ -159,14 +126,6 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { assertThat(resolved.rescuePosition()).isEqualTo(1); assertThat(resolved.metadataPosition()).isEqualTo(2); assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); - TableDescriptor noRescue = - enabled( - Schema.newBuilder() - .column("custom_rescue", DataTypes.STRING()) - .build(), - "parse_json") - .build(); - assertThat(KafkaTableSchema.resolve(noRescue).rescuePosition()).isEqualTo(-1); } @Test @@ -209,15 +168,6 @@ void testJsonRejectsNonStringMapKeysRecursively(String layout) { DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { TableDescriptor table = jsonTable(layout, type); assertInvalid(table, "only supports STRING map keys"); - assertThatCode( - () -> - KafkaTableSchema.validate( - TableDescriptor.builder(table) - .property( - ConfigOptions.TABLE_KAFKA_ENABLED, - false) - .build())) - .doesNotThrowAnyException(); } } } @@ -250,11 +200,7 @@ void testJsonAcceptsAllScalarTypesInNestedContainers(String layout) { DataTypes.ROW( DataTypes.FIELD("_kafka_key", scalar.copy(false))))); TableDescriptor table = jsonTable(layout, type); - byte[] before = table.toJsonBytes(); assertThatCode(() -> KafkaTableSchema.validate(table)).doesNotThrowAnyException(); - assertThat(table.toJsonBytes()).isEqualTo(before); - assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) - .isEqualTo(KafkaTableSchema.resolve(table)); } } @@ -276,13 +222,6 @@ void testJsonBusinessNestingDepthBoundary(String layout) { assertThatCode(() -> KafkaTableSchema.validate(valid)).doesNotThrowAnyException(); TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)); assertInvalid(tooDeep, "maximum nesting depth of 64"); - assertThatCode( - () -> - KafkaTableSchema.validate( - TableDescriptor.builder(tooDeep) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .build())) - .doesNotThrowAnyException(); } private static TableDescriptor jsonTable(String layout, DataType businessType) { @@ -325,7 +264,7 @@ void testReservedNamesAtValueMappingLevel(String name) { } @ParameterizedTest - @ValueSource(strings = {"raw", "json", "schema_id_prefix", "schema_latest", "avro", ""}) + @ValueSource(strings = {"raw", "avro", ""}) void testRejectsUnsupportedValueTypes(String type) { assertInvalid(enabled(valueSchema(), type).build(), "table.kafka.convert.value.type"); } @@ -446,8 +385,6 @@ void testMetadataPreservesDeclaredNullability() { TableDescriptor table = withSystem("_kafka_metadata", type); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())).isEqualTo(type); - assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) - .isEqualTo(resolved); } } @@ -522,7 +459,7 @@ void testUnknownAndPrototypeOptions() { } @Test - void testTableKindsAndCanonicalNameLimits() { + void testUnsupportedTableKinds() { Schema keyed = Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES().copy(false)) @@ -534,20 +471,6 @@ void testTableKindsAndCanonicalNameLimits() { "non-partitioned"); assertInvalid( enabled(valueSchema(), "binary").logFormat(LogFormat.INDEXED).build(), "Arrow"); - String database = String.join("", Collections.nCopies(124, "d")); - String tableName = String.join("", Collections.nCopies(124, "t")); - TableDescriptor table = enabled(valueSchema(), "binary").build(); - assertThatCode( - () -> - KafkaTableSchema.validateTablePath( - TablePath.of(database, tableName), table)) - .doesNotThrowAnyException(); - assertThatThrownBy( - () -> - KafkaTableSchema.validateTablePath( - TablePath.of(database, tableName + "t"), table)) - .isInstanceOf(InvalidTableException.class) - .hasMessageContaining("Kafka topic"); } private static RowType metadataType() { diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java index 56e990d8cd5..67f843a7276 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java @@ -19,7 +19,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.flink.catalog.TestSchemaResolver; -import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -76,50 +75,6 @@ /** Test for {@link FlinkConversions}. */ public class FlinkConversionsTest { - @Test - void testKafkaDdlUsesRegisteredPropertiesAndFixedColumns() { - ResolvedSchema schema = - new ResolvedSchema( - Arrays.asList( - Column.physical( - "_kafka_value", - org.apache.flink.table.api.DataTypes.BYTES()), - Column.physical( - "_kafka_key", org.apache.flink.table.api.DataTypes.BYTES()), - Column.physical( - "_kafka_headers", - org.apache.flink.table.api.DataTypes.MAP( - org.apache.flink.table.api.DataTypes.STRING(), - org.apache.flink.table.api.DataTypes.BYTES())), - Column.physical( - "_kafka_rescue", - org.apache.flink.table.api.DataTypes.STRING())), - Collections.emptyList(), - null); - Map options = new HashMap<>(); - options.put("table.kafka.enabled", "true"); - options.put("table.kafka.convert.value.type", "binary"); - CatalogTable table = - CatalogTable.of( - Schema.newBuilder().fromResolvedSchema(schema).build(), - "Kafka DDL", - Collections.emptyList(), - options); - TableDescriptor descriptor = - FlinkConversions.toFlussTable(new ResolvedCatalogTable(table, schema)); - assertThat(descriptor.getProperties()) - .containsAllEntriesOf(options) - .doesNotContainKeys( - "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); - assertThat(descriptor.getCustomProperties()) - .doesNotContainKeys("table.kafka.enabled", "table.kafka.convert.value.type"); - KafkaTableSchema mapping = KafkaTableSchema.resolve(descriptor); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - assertThat(mapping.keyPosition()).isEqualTo(1); - assertThat(mapping.headersPosition()).isEqualTo(2); - assertThat(mapping.rescuePosition()).isEqualTo(3); - } - @Test void testConvertBucketCountChange() { assertThat( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java deleted file mode 100644 index 17a8c6c0f9c..00000000000 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; - -/** Tests for Kafka configuration. */ -public class KafkaConfigsTest { - @Test - public void testFromMap() throws Exception { - Map map = new HashMap<>(); - map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); - map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "sales"); - Configuration configuration = Configuration.fromMap(map); - - assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); - assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) - .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("sales"); - } - - @Test - public void testFromDefault() throws Exception { - Configuration configuration = Configuration.fromMap(new HashMap<>()); - assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isFalse(); - assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) - .isEqualTo(Collections.singletonList("KAFKA")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("fluss"); - assertThat(configuration.toMap()).doesNotContainKey("kafka.default.database"); - } -} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 97e75987f89..a327df3d5a6 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -19,8 +19,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.exception.DatabaseNotExistException; -import org.apache.fluss.exception.InvalidConfigException; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.metadata.KafkaTableSchema; @@ -38,6 +36,7 @@ import org.apache.fluss.utils.json.DataTypeJsonSerde; import org.apache.fluss.utils.json.JsonSerdeUtils; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -67,7 +66,7 @@ class KafkaTableMappingITCase { static { CONFIG.set(ConfigOptions.KAFKA_ENABLED, true); - CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "kafka_default"); CONFIG.setString("kafka.database", "legacy_database"); } @@ -77,14 +76,24 @@ class KafkaTableMappingITCase { private CoordinatorGateway gateway; + @BeforeAll + static void verifyConfiguredDatabaseCreatedOnStartup() throws Exception { + assertThat( + CLUSTER.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("kafka_default")) + .get() + .isExists()) + .isTrue(); + } + @BeforeEach void setup() { gateway = CLUSTER.newCoordinatorClient(); } @Test - void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() throws Exception { - for (String database : Arrays.asList("missing_default", "kafka", "legacy_database")) { + void testConfiguredKafkaDatabaseAndQualifiedAccess() throws Exception { + for (String database : Arrays.asList("kafka", "legacy_database")) { assertThat(gateway.databaseExists(newDatabaseExistsRequest(database)).get().isExists()) .isFalse(); } @@ -99,22 +108,16 @@ void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() thro long salesId = createTable(CLUSTER, sales, enabled().build()); long warehouseId = createTable(CLUSTER, warehouse, enabled().build()); assertThat(salesId).isNotEqualTo(warehouseId); - assertThat(mapper.toTablePath("orders")) - .isEqualTo(TablePath.of("missing_default", "orders")); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - mapper.toTablePath("orders"), - enabled().build(), - false)) - .get()) - .hasCauseInstanceOf(DatabaseNotExistException.class); + assertThat(mapper.toTablePath("orders")).isEqualTo(TablePath.of("kafka_default", "orders")); + // The cluster fixture removes custom databases between tests; recreate this table's + // database through the usual DDL helper after checking startup creation in @BeforeAll. + createTable(CLUSTER, mapper.toTablePath("orders"), enabled().build()); assertThat( - gateway.databaseExists(newDatabaseExistsRequest("missing_default")) + gateway.databaseExists(newDatabaseExistsRequest("kafka_default")) .get() .isExists()) - .isFalse(); + .isTrue(); + assertThat(KafkaTableSchema.isEnabled(stored(mapper.toTablePath("orders")))).isTrue(); Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); KafkaTopicMapper aliasMapper = new KafkaTopicMapper(config); @@ -123,60 +126,36 @@ void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() thro assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); } - @Test - void testDefaultAliasUsesExistingFlussDatabase() throws Exception { - KafkaTopicMapper mapper = new KafkaTopicMapper(); - TablePath path = mapper.toTablePath("default_alias"); - assertThat(path).isEqualTo(TablePath.of(DEFAULT_DATABASE, "default_alias")); - createTable(CLUSTER, path, enabled().build()); - assertThat(info(mapper.toTablePath("fluss.default_alias")).getTableId()) - .isEqualTo(info(path).getTableId()); - assertThat(mapper.toTopicName(path)).isEqualTo("fluss.default_alias"); - } - @ParameterizedTest @ValueSource(strings = {"flat", "nested"}) void testJsonCreateAndEnableRejectUnsupportedBusinessSchema(String layout) throws Exception { gateway.createDatabase(newCreateDatabaseRequest("ddl_json", true)).get(); - DataType invalidMap = DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()); - int index = 0; - for (DataType type : - Arrays.asList( - invalidMap, - DataTypes.ARRAY(invalidMap), - DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), - DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { - TablePath path = TablePath.of("ddl_json", layout + "_invalid_" + index++); - TableDescriptor descriptor = jsonTable(layout, type).build(); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest(path, descriptor, false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("only supports STRING map keys"); - assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); - - // The identical schema is legal for an ordinary, explicitly disabled Fluss table. - createTable( - CLUSTER, - path, - TableDescriptor.builder(descriptor) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .build()); - GetTableInfoResponse before = info(path); - assertThatThrownBy( - () -> - set( - path, - Collections.singletonMap( - "table.kafka.enabled", "true"))) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("only supports STRING map keys"); - assertUnchanged(path, before); - assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); - } + TablePath path = TablePath.of("ddl_json", layout + "_invalid"); + DataType invalid = + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())))); + TableDescriptor descriptor = jsonTable(layout, invalid).build(); + assertThatThrownBy( + () -> + gateway.createTable(newCreateTableRequest(path, descriptor, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + + createTable( + CLUSTER, + path, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertUnchanged(path, before); } @ParameterizedTest @@ -238,41 +217,6 @@ void testJsonSchemaAlterRejectsUnsupportedNestedMapAtomically() throws Exception assertUnchanged(path, disabled); } - @ParameterizedTest - @ValueSource(strings = {"flat", "nested"}) - void testJsonDepthValidationThroughDdl(String layout) throws Exception { - DataType boundary = DataTypes.STRING(); - for (int i = 0; i < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; i++) { - boundary = DataTypes.ARRAY(boundary); - } - TablePath valid = TablePath.of("ddl_json", layout + "_depth_boundary"); - TableDescriptor descriptor = jsonTable(layout, boundary).build(); - createTable(CLUSTER, valid, descriptor); - assertThat(stored(valid).getSchema()).isEqualTo(descriptor.getSchema()); - TablePath invalid = TablePath.of("ddl_json", layout + "_too_deep"); - TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)).build(); - assertThatThrownBy( - () -> - gateway.createTable(newCreateTableRequest(invalid, tooDeep, false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("maximum nesting depth of 64"); - assertThat(gateway.tableExists(newTableExistsRequest(invalid)).get().isExists()).isFalse(); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(invalid)).isZero(); - createTable( - CLUSTER, - invalid, - TableDescriptor.builder(tooDeep) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .build()); - GetTableInfoResponse before = info(invalid); - assertThatThrownBy( - () -> set(invalid, Collections.singletonMap("table.kafka.enabled", "true"))) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("maximum nesting depth of 64"); - assertUnchanged(invalid, before); - } - private void assertUnchanged(TablePath path, GetTableInfoResponse before) throws Exception { GetTableInfoResponse after = info(path); assertThat(after.getTableJson()).isEqualTo(before.getTableJson()); @@ -374,10 +318,12 @@ void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exceptio addColumn(path, "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); assertThat(KafkaTableSchema.resolve(stored(path)).headersPosition()).isEqualTo(1); assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); - TableDescriptor withHeaders = stored(path); + addColumn(path, "_kafka_metadata", metadataType()); + assertThat(KafkaTableSchema.resolve(stored(path)).metadataPosition()).isEqualTo(2); + TableDescriptor beforeFailedRescue = stored(path); assertThatThrownBy(() -> addColumn(path, "_kafka_rescue", DataTypes.INT())) .hasCauseInstanceOf(InvalidTableException.class); - assertThat(stored(path)).isEqualTo(withHeaders); + assertThat(stored(path)).isEqualTo(beforeFailedRescue); set(path, Collections.singletonMap("table.kafka.enabled", "false")); addColumn(path, "unmapped_business_field", DataTypes.INT()); assertThat(stored(path).getSchema().getRowType().getFieldNames()) @@ -388,85 +334,18 @@ void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exceptio } @Test - void testCreateRejectsInvalidContractWithoutPublishingMetadata() throws Exception { - gateway.createDatabase(newCreateDatabaseRequest("ddl_invalid", true)).get(); - TablePath path = TablePath.of("ddl_invalid", "bad"); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - path, - enabled() - .property( - ConfigOptions - .TABLE_KAFKA_CONVERT_VALUE_TYPE, - "string") - .build(), - false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class); - assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); - // Non-null added columns are rejected by generic schema evolution first, so exercise - // Kafka-specific rescue nullability through CREATE, where such columns are allowed. - TableDescriptor nonNullableRescue = - enabled() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column("_kafka_rescue", DataTypes.STRING().copy(false)) - .build()) - .build(); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - path, nonNullableRescue, false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("nullable STRING"); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - path, - enabled() - .property( - "table.kafka.convert.rescue-column", - "rescue") - .build(), - false)) - .get()) - .hasCauseInstanceOf(InvalidConfigException.class); - assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); - createTable(CLUSTER, path, enabled().build()); - assertThat(KafkaTableSchema.isEnabled(stored(path))).isTrue(); - } - - @ParameterizedTest - @ValueSource(strings = {"binary", "string"}) - void testMetadataCanBeCreatedAndEnabledWithoutOffset(String format) throws Exception { + void testMetadataCanBeCreatedAndEnabledWithoutOffset() throws Exception { Schema schema = Schema.newBuilder() .column("_kafka_metadata", metadataType()) - .column( - "_kafka_value", - "binary".equals(format) ? DataTypes.BYTES() : DataTypes.STRING()) - .build(); - TableDescriptor descriptor = - enabled() - .schema(schema) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .column("_kafka_value", DataTypes.BYTES()) .build(); - TablePath created = TablePath.of("ddl", "metadata_create_" + format); + TableDescriptor descriptor = enabled().schema(schema).build(); + TablePath created = TablePath.of("ddl", "metadata_create"); createTable(CLUSTER, created, descriptor); - TableDescriptor persisted = stored(created); - assertThat(persisted.getSchema()).isEqualTo(schema); - assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); - assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); + assertThat(stored(created).getSchema()).isEqualTo(schema); - TablePath altered = TablePath.of("ddl", "metadata_enable_" + format); + TablePath altered = TablePath.of("ddl", "metadata_enable"); createTable( CLUSTER, altered, @@ -475,25 +354,9 @@ void testMetadataCanBeCreatedAndEnabledWithoutOffset(String format) throws Excep .build()); int schemaId = info(altered).getSchemaId(); set(altered, Collections.singletonMap("table.kafka.enabled", "true")); - persisted = stored(altered); assertThat(info(altered).getSchemaId()).isEqualTo(schemaId); - assertThat(persisted.getSchema()).isEqualTo(schema); - assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); - assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); - } - - @Test - void testMetadataCanBeAddedToEnabledTable() throws Exception { - TablePath path = TablePath.of("ddl", "metadata_add"); - createTable(CLUSTER, path, enabled().build()); - TableDescriptor before = stored(path); - addColumn(path, "_kafka_metadata", metadataType()); - TableDescriptor after = stored(path); - KafkaTableSchema resolved = KafkaTableSchema.resolve(after); - assertThat(resolved.metadataPosition()).isGreaterThanOrEqualTo(0); - assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())) - .isEqualTo(metadataType()); - assertThat(kafkaProperties(after)).isEqualTo(kafkaProperties(before)); + assertThat(stored(altered).getSchema()).isEqualTo(schema); + assertThat(KafkaTableSchema.resolve(stored(altered)).metadataPosition()).isZero(); } @Test diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 8327c6fb60e..119e8aff38f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -18,7 +18,6 @@ package org.apache.fluss.kafka.schema; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.types.DataTypes; @@ -32,33 +31,6 @@ class KafkaTopicSchemaResolverTest { private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); - @Test - void testEnabledContractUsesSharedValidation() { - TableDescriptor table = - table().schema( - Schema.newBuilder() - .column( - "_kafka_metadata", - DataTypes.ROW( - DataTypes.FIELD( - "timestamp", - DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD( - "timestamp_type", DataTypes.INT()), - DataTypes.FIELD( - "partition", DataTypes.INT()))) - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .build(); - assertThat(resolver.isEnabled(table)).isTrue(); - assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); - assertThat(resolver.resolve(table).metadataPosition()).isZero(); - assertThat(resolver.resolve(table).valueProjection().positions()).containsExactly(1); - assertThat(resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) - .isEqualTo(resolver.resolve(table)); - } - @Test void testCompatibleSchemaDoesNotEnableTable() { assertThat(resolver.isEnabled(table().build())).isFalse(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index aff8bb78761..ac161728107 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -548,11 +548,27 @@ private interface ThrowingRunnable { private void createDefaultDatabase() { MetadataManager metadataManager = new MetadataManager(zkClient, conf, lakeCatalogDynamicLoader); + createDefaultDatabases(metadataManager, conf); + } + + @VisibleForTesting + static void createDefaultDatabases(MetadataManager metadataManager, Configuration conf) { List databases = metadataManager.listDatabases(); if (databases.isEmpty()) { metadataManager.createDatabase(DEFAULT_DATABASE, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' because no database exists.", DEFAULT_DATABASE); } + if (conf.get(ConfigOptions.KAFKA_ENABLED)) { + String kafkaDatabase = conf.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); + // An empty cluster has already created the native default database above. + if (DEFAULT_DATABASE.equals(kafkaDatabase) && databases.isEmpty()) { + return; + } + if (!databases.contains(kafkaDatabase)) { + metadataManager.createDatabase(kafkaDatabase, DatabaseDescriptor.EMPTY, true); + LOG.info("Created default database '{}' for Kafka protocol.", kafkaDatabase); + } + } } /** diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index b1a5fae3d04..3354b869eb6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -474,7 +474,7 @@ public void alterTableSchema( ZkVersion.MATCH_ANY_VERSION.getVersion()); } - /** Alters schema with a table-version and coordinator-epoch fence against concurrent ALTER. */ + /** Alters schema with table, schema and coordinator version checks against concurrent ALTER. */ public void alterTableSchema( TablePath tablePath, List schemaChanges, @@ -485,6 +485,7 @@ public void alterTableSchema( try { ZooKeeperClient.VersionedData versionedTable = getTableRegistrationWithVersion(tablePath); + int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); TableInfo table = getTable(tablePath); TableDescriptor tableDescriptor = table.toTableDescriptor(); @@ -505,11 +506,12 @@ public void alterTableSchema( // Update Fluss schema (ZK) after Lake sync succeeds if (!newSchema.equals(table.getSchema())) { - zookeeperClient.registerSchemaAndBumpTableVersion( + zookeeperClient.registerSchemaWithVersionCheck( tablePath, newSchema, table.getSchemaId() + 1, - versionedTable, + versionedTable.zkVersion(), + schemaZkVersion, coordinatorZkVersion); } else { LOG.info( @@ -813,10 +815,10 @@ private boolean isTablePresent(TablePath tablePath) { } /** - * One attempt of the property ALTER read-modify-write. The ZK write is CAS-guarded by the table - * version read here plus the coordinator epoch version. Since {@link #preAlterTableProperties} - * may apply external lake side effects, a version conflict is surfaced instead of being retried - * automatically. + * One attempt of the property ALTER read-modify-write. The ZK write is guarded by the table and + * schema versions read here plus the coordinator epoch version. Since {@link + * #preAlterTableProperties} may apply external lake side effects, a version conflict is + * surfaced instead of being retried automatically. */ private void doAlterTablePropertiesOnce( TablePath tablePath, @@ -832,6 +834,7 @@ private void doAlterTablePropertiesOnce( getTableRegistrationWithVersion(tablePath); TableRegistration tableReg = versionedTableReg.data(); int tableZkVersion = versionedTableReg.zkVersion(); + int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); SchemaInfo schemaInfo = getLatestSchema(tablePath); // we can't use MetadataManager#getTable here, because it will add the default // lake options to the table properties, which may cause the validation failure @@ -901,7 +904,8 @@ private void doAlterTablePropertiesOnce( updatedTableRegistration, tableZkVersion, Collections.emptyMap(), - coordinatorZkVersion); + coordinatorZkVersion, + schemaZkVersion); } catch (KeeperException.BadVersionException e) { throw new FlussRuntimeException( String.format( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java index f38a7f0afe4..5e05877b4be 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java @@ -17,7 +17,6 @@ package org.apache.fluss.server.coordinator.event.watcher; -import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.metadata.PhysicalTablePath; @@ -88,8 +87,7 @@ public void stop() { } /** A listener to monitor the changes of table nodes in zookeeper. */ - @VisibleForTesting - final class TablePathChangeListener implements CuratorCacheListener { + private final class TablePathChangeListener implements CuratorCacheListener { @Override public void event(Type type, ChildData oldData, ChildData newData) { @@ -152,13 +150,13 @@ public void event(Type type, ChildData oldData, ChildData newData) { if (tablePath == null) { break; } - // Distinguish between table creation and registration updates. + // Distinguish between table creation and properties change. // If oldData exists and contains valid table registration data, - // compare its contents; otherwise, it's a table creation. + // it's a properties change; otherwise, it's a table creation. if (oldData != null && oldData.getData() != null && oldData.getData().length > 0) { - processTableRegistrationChange(tablePath, oldData, newData); + processTableRegistrationChange(tablePath, newData); } else { processCreateTable(tablePath, newData); } @@ -278,16 +276,9 @@ private void processCreatePartition( tablePath, tableId, partitionId, partitionName, partitionAssignment)); } - private void processTableRegistrationChange( - TablePath tablePath, ChildData oldData, ChildData newData) { - TableRegistration oldTable = TableZNode.decode(oldData.getData()); + private void processTableRegistrationChange(TablePath tablePath, ChildData newData) { TableRegistration newTable = TableZNode.decode(newData.getData()); - // Schema ALTER bumps the table znode version to fence concurrent registration updates, - // even when its contents are unchanged. Compare decoded registrations so differences - // in JSON serialization do not trigger a registration change either. - if (!oldTable.equals(newTable)) { - eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); - } + eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index a7a706c214e..99d2c6c01ae 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -1159,7 +1159,32 @@ public void updateTableWithPartitionBucketCountBackfill( Map> partitionBackfills, int expectedCoordinatorEpochZkVersion) throws Exception { + updateTableWithPartitionBucketCountBackfill( + tablePath, + tableRegistration, + expectedTableZkVersion, + partitionBackfills, + expectedCoordinatorEpochZkVersion, + ZkVersion.MATCH_ANY_VERSION.getVersion()); + } + + /** Updates table properties with an additional fence on the schema used for validation. */ + public void updateTableWithPartitionBucketCountBackfill( + TablePath tablePath, + TableRegistration tableRegistration, + int expectedTableZkVersion, + Map> partitionBackfills, + int expectedCoordinatorEpochZkVersion, + int expectedSchemaZkVersion) + throws Exception { List ops = new ArrayList<>(partitionBackfills.size() + 1); + if (expectedSchemaZkVersion != ZkVersion.MATCH_ANY_VERSION.getVersion()) { + ops.add( + zkClient.transactionOp() + .check() + .withVersion(expectedSchemaZkVersion) + .forPath(SchemasZNode.path(tablePath))); + } for (Map.Entry> entry : partitionBackfills.entrySet()) { String partitionPath = PartitionZNode.path(tablePath, entry.getKey()); @@ -1316,25 +1341,29 @@ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) thro } /** - * Publishes a schema and advances the table registration version in one fenced transaction. - * Property ALTER validates against a schema snapshot and compares the table version, so schema - * ALTER must participate in the same CAS to prevent enabling an incompatible Kafka contract. + * Publishes a schema and advances the schema version in one fenced transaction. The table + * registration is checked without being rewritten; property ALTER checks the schema version to + * prevent enabling a Kafka contract against a stale schema. */ - public void registerSchemaAndBumpTableVersion( + public void registerSchemaWithVersionCheck( TablePath tablePath, Schema schema, int schemaId, - VersionedData expectedTable, + int expectedTableZkVersion, + int expectedSchemaZkVersion, int expectedCoordinatorEpochZkVersion) throws Exception { List ops = new ArrayList<>(); + ops.add( + zkClient.transactionOp() + .check() + .withVersion(expectedTableZkVersion) + .forPath(TableZNode.path(tablePath))); ops.add( zkClient.transactionOp() .setData() - .withVersion(expectedTable.zkVersion()) - .forPath( - TableZNode.path(tablePath), - TableZNode.encode(expectedTable.data()))); + .withVersion(expectedSchemaZkVersion) + .forPath(SchemasZNode.path(tablePath), new byte[0])); ops.add( zkClient.transactionOp() .create() @@ -1345,6 +1374,13 @@ public void registerSchemaAndBumpTableVersion( .forOperations(wrapRequestsWithEpochCheck(ops, expectedCoordinatorEpochZkVersion)); } + /** Returns the schema parent znode version, read before loading a schema for DDL validation. */ + public int getSchemaZkVersion(TablePath tablePath) throws Exception { + Stat stat = new Stat(); + zkClient.getData().storingStatIn(stat).forPath(SchemasZNode.path(tablePath)); + return stat.getVersion(); + } + /** Get the specific schema by schema id in ZK metadata. */ public Optional getSchemaById(TablePath tablePath, int schemaId) throws Exception { Optional bytes = getOrEmpty(SchemaZNode.path(tablePath, schemaId)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java new file mode 100644 index 00000000000..d846c95f3e6 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.coordinator; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.ZkData.DatabasesZNode; +import org.apache.fluss.testutils.common.AllCallbackWrapper; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests startup database creation for native and Kafka default database configurations. */ +class KafkaDefaultDatabaseTest { + + @RegisterExtension + public static final AllCallbackWrapper ZK = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private MetadataManager manager; + + @BeforeEach + void setup() throws Exception { + ZooKeeperClient zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); + if (zk.getStat(DatabasesZNode.path()).isPresent()) { + zk.getCuratorClient() + .delete() + .deletingChildrenIfNeeded() + .forPath(DatabasesZNode.path()); + } + manager = + new MetadataManager( + zk, + new Configuration(), + new LakeCatalogDynamicLoader(new Configuration(), null, true)); + } + + @ParameterizedTest + @CsvSource({"fluss,false", "kafka_default,false", "fluss,true", "kafka_default,true"}) + void testCreateMissingDefaultAndPreserveExisting(String database, boolean exists) { + manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); + DatabaseDescriptor descriptor = + DatabaseDescriptor.builder().comment("existing default").build(); + if (exists) { + manager.createDatabase(database, descriptor, false); + } + Configuration config = kafkaConfiguration(database); + CoordinatorServer.createDefaultDatabases(manager, config); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()).containsExactlyInAnyOrder("existing", database); + assertThat(manager.getDatabase(database).getDatabaseDescriptor().toJsonBytes()) + .isEqualTo((exists ? descriptor : DatabaseDescriptor.EMPTY).toJsonBytes()); + } + + @Test + void testDisabledKafkaDoesNotCreateConfiguredDatabase() { + Configuration config = kafkaConfiguration("kafka_default"); + config.set(ConfigOptions.KAFKA_ENABLED, false); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()).containsExactly(DEFAULT_DATABASE); + } + + private static Configuration kafkaConfiguration(String database) { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, database); + return config; + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java index e76dbda662d..dc08ed10385 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java @@ -21,7 +21,6 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.metadata.DatabaseDescriptor; -import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -72,6 +71,9 @@ void setup() throws Exception { @Test void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { TablePath path = create("schema_wins"); + ZooKeeperClient.VersionedData before = + zk.getTableWithVersion(path).get(); + int schemaZkVersion = zk.getSchemaZkVersion(path); assertThatThrownBy( () -> enable( @@ -91,6 +93,10 @@ void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { epoch))) .isInstanceOf(FlussRuntimeException.class) .hasMessageContaining("Concurrent metadata change"); + ZooKeeperClient.VersionedData after = zk.getTableWithVersion(path).get(); + assertThat(after.zkVersion()).isEqualTo(before.zkVersion()); + assertThat(after.data()).isEqualTo(before.data()); + assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion + 1); assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isFalse(); assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) .contains("unmapped"); @@ -101,22 +107,23 @@ void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { TablePath path = create("enable_wins"); ZooKeeperClient.VersionedData snapshot = zk.getTableWithVersion(path).get(); + int schemaZkVersion = zk.getSchemaZkVersion(path); int schemaId = zk.getCurrentSchemaId(path); Schema incompatible = Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES()) .column("unmapped", DataTypes.STRING()) .build(); - // This candidate is valid for the disabled snapshot, but not after the competing enable. - KafkaTableSchema.validate( - TableDescriptor.builder(manager.getTable(path).toTableDescriptor()) - .schema(incompatible) - .build()); enable(path, (oldTable, updated) -> {}); assertThatThrownBy( () -> - zk.registerSchemaAndBumpTableVersion( - path, incompatible, schemaId + 1, snapshot, epoch)) + zk.registerSchemaWithVersionCheck( + path, + incompatible, + schemaId + 1, + snapshot.zkVersion(), + schemaZkVersion, + epoch)) .isInstanceOf(KeeperException.BadVersionException.class); assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isTrue(); @@ -125,26 +132,39 @@ void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { } @Test - void testSchemaFailureDoesNotAdvanceTableVersion() throws Exception { + void testSchemaFailureDoesNotAdvanceSchemaVersion() throws Exception { TablePath path = create("atomic_failure"); ZooKeeperClient.VersionedData snapshot = zk.getTableWithVersion(path).get(); + int schemaZkVersion = zk.getSchemaZkVersion(path); int schemaId = zk.getCurrentSchemaId(path); Schema schema = manager.getTable(path).getSchema(); assertThatThrownBy( () -> - zk.registerSchemaAndBumpTableVersion( - path, schema, schemaId, snapshot, epoch)) + zk.registerSchemaWithVersionCheck( + path, + schema, + schemaId, + snapshot.zkVersion(), + schemaZkVersion, + epoch)) .isInstanceOf(KeeperException.NodeExistsException.class); assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); zk.fenceBecomeCoordinatorLeader("next-coordinator"); assertThatThrownBy( () -> - zk.registerSchemaAndBumpTableVersion( - path, schema, schemaId + 1, snapshot, epoch)) + zk.registerSchemaWithVersionCheck( + path, + schema, + schemaId + 1, + snapshot.zkVersion(), + schemaZkVersion, + epoch)) .isInstanceOf(KeeperException.BadVersionException.class); assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java index 9a36280d891..28a4131ecd8 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java @@ -42,16 +42,11 @@ import org.apache.fluss.server.entity.TablePropertyChanges; import org.apache.fluss.server.zk.NOPErrorHandler; import org.apache.fluss.server.zk.ZooKeeperClient; -import org.apache.fluss.server.zk.ZooKeeperClient.VersionedData; import org.apache.fluss.server.zk.ZooKeeperExtension; import org.apache.fluss.server.zk.data.PartitionAssignment; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; -import org.apache.fluss.server.zk.data.ZkData.TableZNode; import org.apache.fluss.server.zk.data.ZkVersion; -import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCacheListener; -import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.data.Stat; import org.apache.fluss.testutils.common.AllCallbackWrapper; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; @@ -61,13 +56,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -311,7 +302,7 @@ void testPartitionedTable() throws Exception { } @Test - void testSchemaChanges() throws Exception { + void testSchemaChanges() { // create tables, collect create table events List expectedEvents = new ArrayList<>(); for (int i = 0; i < 10; i++) { @@ -358,8 +349,6 @@ void testSchemaChanges() throws Exception { } CreateTableEvent createTableEvent = (CreateTableEvent) coordinatorEvent; TableInfo tableInfo = createTableEvent.getTableInfo(); - VersionedData beforeAlter = - zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); metadataManager.alterTableSchema( tableInfo.getTablePath(), Collections.singletonList( @@ -370,10 +359,6 @@ void testSchemaChanges() throws Exception { TableChange.ColumnPosition.last())), false, null); - VersionedData afterAlter = - zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); - assertThat(afterAlter.zkVersion()).isEqualTo(beforeAlter.zkVersion() + 1); - assertThat(afterAlter.data()).isEqualTo(beforeAlter.data()); Schema newSchema = Schema.newBuilder() .fromSchema(tableInfo.getSchema()) @@ -393,62 +378,6 @@ void testSchemaChanges() throws Exception { .containsExactlyInAnyOrderElementsOf(allEvents)); } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - void testUnchangedTableRegistration(boolean differentJsonFormatting) { - tableChangeWatcher.stop(); - TablePath tablePath = TablePath.of(DEFAULT_DB, "unchanged_table_registration"); - TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); - byte[] oldData = TableZNode.encode(table); - byte[] newData = - differentJsonFormatting - ? ("\n" + new String(oldData, StandardCharsets.UTF_8)) - .getBytes(StandardCharsets.UTF_8) - : oldData; - - // Deliver the callback synchronously so the absence of an event cannot pass before - // CuratorCache has observed the version bump. - CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); - listener.event( - CuratorCacheListener.Type.NODE_CHANGED, - tableNodeData(tablePath, oldData, 1), - tableNodeData(tablePath, newData, 2)); - - assertThat(eventManager.getEvents()).isEmpty(); - } - - @Test - void testChangedTableRegistrationContents() { - tableChangeWatcher.stop(); - TablePath tablePath = TablePath.of(DEFAULT_DB, "changed_table_registration"); - TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); - CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); - List expectedEvents = new ArrayList<>(); - for (TableRegistration updatedTable : - Arrays.asList( - table.newProperties( - Collections.singletonMap( - ConfigOptions.TABLE_KAFKA_ENABLED.key(), "true"), - table.customProperties), - table.newProperties( - table.properties, - Collections.singletonMap("custom.key", "custom.value")), - table.newBucketCount(4))) { - listener.event( - CuratorCacheListener.Type.NODE_CHANGED, - tableNodeData(tablePath, TableZNode.encode(table), 1), - tableNodeData(tablePath, TableZNode.encode(updatedTable), 2)); - expectedEvents.add(new TableRegistrationChangeEvent(tablePath, updatedTable)); - assertThat(eventManager.getEvents()).containsExactlyElementsOf(expectedEvents); - } - } - - private static ChildData tableNodeData(TablePath tablePath, byte[] data, int version) { - Stat stat = new Stat(); - stat.setVersion(version); - return new ChildData(TableZNode.path(tablePath), stat, data); - } - @Test void testTableRegistrationChange() { // create a table diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 6344eee5207..5f4a5ef345a 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -254,14 +254,16 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, the coordinator creates this database if it is missing. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named `sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database` (by default, `fluss.orders`). Databases and tables must already exist. Reverse mapping always uses +`kafka.default.database` (by default, `fluss.orders`). The coordinator creates the configured default +database when Kafka is enabled and the database is missing. Existing databases are preserved; +other databases and all tables must be created explicitly. Reverse mapping always uses the fully qualified name. All TabletServers must use the same default database. Changing it redirects unqualified names, so migrate clients to fully qualified names before changing the configuration. The prototype `kafka.database` option is no longer used. From 7bcda8b4a17282b9e956137ea31ae281cbf5795b Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 20:11:19 +0800 Subject: [PATCH 14/48] [kafka] Simplify DDL validation and default database setup Reuse native initialization for the fluss database and create a custom Kafka default database only when it is missing. Remove PR03-only schema version and Coordinator epoch checks, preserving PR02 metadata writes and existing protections. Drop the corresponding concurrency tests while retaining Kafka DDL mapping validation. Validation: 121 focused tests passed; Maven verify checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 132/132 AI-Contributed/UT: 224/224 --- .../apache/fluss/config/ConfigOptions.java | 5 +- .../kafka/schema/KafkaTableMappingITCase.java | 2 +- .../server/coordinator/CoordinatorServer.java | 7 +- .../coordinator/CoordinatorService.java | 3 +- .../server/coordinator/MetadataManager.java | 41 +--- .../fluss/server/zk/ZooKeeperClient.java | 66 ------ .../coordinator/KafkaDefaultDatabaseTest.java | 20 +- .../KafkaTableSchemaAlterTest.java | 202 ------------------ website/docs/maintenance/configuration.md | 10 +- 9 files changed, 37 insertions(+), 319 deletions(-) delete mode 100644 fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index e05d981e995..6d6adb6a5ee 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2943,8 +2943,9 @@ public class ConfigOptions { .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " - + "access other databases. When Kafka is enabled, the coordinator creates this " - + "database if it does not exist."); + + "access other databases. When Kafka is enabled, a configured database other " + + "than 'fluss' is created if missing. The native 'fluss' database is created " + + "only when the cluster has no databases."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index a327df3d5a6..0c7ff17eda3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -271,7 +271,7 @@ void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { } @Test - void testEnableDisableAndFailedPropertyAlterAreAtomic() throws Exception { + void testEnableDisableAndRejectInvalidProperties() throws Exception { TablePath path = TablePath.of("ddl", "enable_disable"); TableDescriptor disabled = enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index ac161728107..3b93e2c8c50 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -560,11 +560,8 @@ static void createDefaultDatabases(MetadataManager metadataManager, Configuratio } if (conf.get(ConfigOptions.KAFKA_ENABLED)) { String kafkaDatabase = conf.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); - // An empty cluster has already created the native default database above. - if (DEFAULT_DATABASE.equals(kafkaDatabase) && databases.isEmpty()) { - return; - } - if (!databases.contains(kafkaDatabase)) { + // The native default database follows the cluster initialization logic above. + if (!DEFAULT_DATABASE.equals(kafkaDatabase) && !databases.contains(kafkaDatabase)) { metadataManager.createDatabase(kafkaDatabase, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' for Kafka protocol.", kafkaDatabase); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index e8e09dab26d..df8713338a8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -615,8 +615,7 @@ public CompletableFuture alterTable(AlterTableRequest reques tablePath, alterSchemaChanges, request.isIgnoreIfNotExists(), - currentSession().getPrincipal(), - coordinatorZkVersionSupplier.get()); + currentSession().getPrincipal()); } if (hasConfigChanges) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 3354b869eb6..41259eed034 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -59,7 +59,6 @@ import org.apache.fluss.server.zk.data.PartitionRegistration; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; -import org.apache.fluss.server.zk.data.ZkVersion; import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; import org.apache.fluss.utils.function.RunnableWithException; import org.apache.fluss.utils.function.ThrowingRunnable; @@ -466,26 +465,8 @@ public void alterTableSchema( boolean ignoreIfNotExists, FlussPrincipal flussPrincipal) throws TableNotExistException, TableNotPartitionedException { - alterTableSchema( - tablePath, - schemaChanges, - ignoreIfNotExists, - flussPrincipal, - ZkVersion.MATCH_ANY_VERSION.getVersion()); - } - - /** Alters schema with table, schema and coordinator version checks against concurrent ALTER. */ - public void alterTableSchema( - TablePath tablePath, - List schemaChanges, - boolean ignoreIfNotExists, - FlussPrincipal flussPrincipal, - int coordinatorZkVersion) - throws TableNotExistException, TableNotPartitionedException { try { - ZooKeeperClient.VersionedData versionedTable = - getTableRegistrationWithVersion(tablePath); - int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); + TableInfo table = getTable(tablePath); TableDescriptor tableDescriptor = table.toTableDescriptor(); @@ -506,13 +487,7 @@ public void alterTableSchema( // Update Fluss schema (ZK) after Lake sync succeeds if (!newSchema.equals(table.getSchema())) { - zookeeperClient.registerSchemaWithVersionCheck( - tablePath, - newSchema, - table.getSchemaId() + 1, - versionedTable.zkVersion(), - schemaZkVersion, - coordinatorZkVersion); + zookeeperClient.registerSchema(tablePath, newSchema, table.getSchemaId() + 1); } else { LOG.info( "Skipping schema evolution for table {} because the column(s) to add {} already exist.", @@ -815,10 +790,10 @@ private boolean isTablePresent(TablePath tablePath) { } /** - * One attempt of the property ALTER read-modify-write. The ZK write is guarded by the table and - * schema versions read here plus the coordinator epoch version. Since {@link - * #preAlterTableProperties} may apply external lake side effects, a version conflict is - * surfaced instead of being retried automatically. + * One attempt of the property ALTER read-modify-write. The ZK write is CAS-guarded by the table + * version read here plus the coordinator epoch version. Since {@link #preAlterTableProperties} + * may apply external lake side effects, a version conflict is surfaced instead of being retried + * automatically. */ private void doAlterTablePropertiesOnce( TablePath tablePath, @@ -834,7 +809,6 @@ private void doAlterTablePropertiesOnce( getTableRegistrationWithVersion(tablePath); TableRegistration tableReg = versionedTableReg.data(); int tableZkVersion = versionedTableReg.zkVersion(); - int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); SchemaInfo schemaInfo = getLatestSchema(tablePath); // we can't use MetadataManager#getTable here, because it will add the default // lake options to the table properties, which may cause the validation failure @@ -904,8 +878,7 @@ private void doAlterTablePropertiesOnce( updatedTableRegistration, tableZkVersion, Collections.emptyMap(), - coordinatorZkVersion, - schemaZkVersion); + coordinatorZkVersion); } catch (KeeperException.BadVersionException e) { throw new FlussRuntimeException( String.format( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index 99d2c6c01ae..01f897cff29 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -1159,32 +1159,7 @@ public void updateTableWithPartitionBucketCountBackfill( Map> partitionBackfills, int expectedCoordinatorEpochZkVersion) throws Exception { - updateTableWithPartitionBucketCountBackfill( - tablePath, - tableRegistration, - expectedTableZkVersion, - partitionBackfills, - expectedCoordinatorEpochZkVersion, - ZkVersion.MATCH_ANY_VERSION.getVersion()); - } - - /** Updates table properties with an additional fence on the schema used for validation. */ - public void updateTableWithPartitionBucketCountBackfill( - TablePath tablePath, - TableRegistration tableRegistration, - int expectedTableZkVersion, - Map> partitionBackfills, - int expectedCoordinatorEpochZkVersion, - int expectedSchemaZkVersion) - throws Exception { List ops = new ArrayList<>(partitionBackfills.size() + 1); - if (expectedSchemaZkVersion != ZkVersion.MATCH_ANY_VERSION.getVersion()) { - ops.add( - zkClient.transactionOp() - .check() - .withVersion(expectedSchemaZkVersion) - .forPath(SchemasZNode.path(tablePath))); - } for (Map.Entry> entry : partitionBackfills.entrySet()) { String partitionPath = PartitionZNode.path(tablePath, entry.getKey()); @@ -1340,47 +1315,6 @@ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) thro return schemaId; } - /** - * Publishes a schema and advances the schema version in one fenced transaction. The table - * registration is checked without being rewritten; property ALTER checks the schema version to - * prevent enabling a Kafka contract against a stale schema. - */ - public void registerSchemaWithVersionCheck( - TablePath tablePath, - Schema schema, - int schemaId, - int expectedTableZkVersion, - int expectedSchemaZkVersion, - int expectedCoordinatorEpochZkVersion) - throws Exception { - List ops = new ArrayList<>(); - ops.add( - zkClient.transactionOp() - .check() - .withVersion(expectedTableZkVersion) - .forPath(TableZNode.path(tablePath))); - ops.add( - zkClient.transactionOp() - .setData() - .withVersion(expectedSchemaZkVersion) - .forPath(SchemasZNode.path(tablePath), new byte[0])); - ops.add( - zkClient.transactionOp() - .create() - .withMode(CreateMode.PERSISTENT) - .forPath( - SchemaZNode.path(tablePath, schemaId), SchemaZNode.encode(schema))); - zkClient.transaction() - .forOperations(wrapRequestsWithEpochCheck(ops, expectedCoordinatorEpochZkVersion)); - } - - /** Returns the schema parent znode version, read before loading a schema for DDL validation. */ - public int getSchemaZkVersion(TablePath tablePath) throws Exception { - Stat stat = new Stat(); - zkClient.getData().storingStatIn(stat).forPath(SchemasZNode.path(tablePath)); - return stat.getVersion(); - } - /** Get the specific schema by schema id in ZK metadata. */ public Optional getSchemaById(TablePath tablePath, int schemaId) throws Exception { Optional bytes = getOrEmpty(SchemaZNode.path(tablePath, schemaId)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java index d846c95f3e6..5e4cc8bc3a5 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java @@ -30,7 +30,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; import static org.assertj.core.api.Assertions.assertThat; @@ -61,8 +61,22 @@ void setup() throws Exception { } @ParameterizedTest - @CsvSource({"fluss,false", "kafka_default,false", "fluss,true", "kafka_default,true"}) - void testCreateMissingDefaultAndPreserveExisting(String database, boolean exists) { + @ValueSource(booleans = {false, true}) + void testNativeDefaultFollowsClusterInitialization(boolean hasDatabase) { + if (hasDatabase) { + manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); + } + Configuration config = kafkaConfiguration(DEFAULT_DATABASE); + CoordinatorServer.createDefaultDatabases(manager, config); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()) + .containsExactly(hasDatabase ? "existing" : DEFAULT_DATABASE); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCreateMissingCustomDefaultAndPreserveExisting(boolean exists) { + String database = "kafka_default"; manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); DatabaseDescriptor descriptor = DatabaseDescriptor.builder().comment("existing default").build(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java deleted file mode 100644 index dc08ed10385..00000000000 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.server.coordinator; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.exception.FlussRuntimeException; -import org.apache.fluss.metadata.DatabaseDescriptor; -import org.apache.fluss.metadata.Schema; -import org.apache.fluss.metadata.TableChange; -import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.server.entity.TablePropertyChanges; -import org.apache.fluss.server.zk.NOPErrorHandler; -import org.apache.fluss.server.zk.ZooKeeperClient; -import org.apache.fluss.server.zk.ZooKeeperExtension; -import org.apache.fluss.server.zk.data.TableRegistration; -import org.apache.fluss.server.zk.data.ZkVersion; -import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; -import org.apache.fluss.testutils.common.AllCallbackWrapper; -import org.apache.fluss.types.DataTypes; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.util.Collections; -import java.util.function.BiConsumer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Deterministic interleavings of schema ALTER and Kafka enablement using real ZooKeeper. */ -class KafkaTableSchemaAlterTest { - - @RegisterExtension - public static final AllCallbackWrapper ZK = - new AllCallbackWrapper<>(new ZooKeeperExtension()); - - private ZooKeeperClient zk; - private MetadataManager manager; - private int epoch; - - @BeforeEach - void setup() throws Exception { - zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); - epoch = zk.fenceBecomeCoordinatorLeader("kafka-ddl").getCoordinatorEpochZkVersion(); - manager = - new MetadataManager( - zk, - new Configuration(), - new LakeCatalogDynamicLoader(new Configuration(), null, true)); - manager.createDatabase("ddl", DatabaseDescriptor.EMPTY, true); - } - - @Test - void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { - TablePath path = create("schema_wins"); - ZooKeeperClient.VersionedData before = - zk.getTableWithVersion(path).get(); - int schemaZkVersion = zk.getSchemaZkVersion(path); - assertThatThrownBy( - () -> - enable( - path, - (oldTable, updated) -> - manager.alterTableSchema( - path, - Collections.singletonList( - TableChange.addColumn( - "unmapped", - DataTypes.STRING(), - null, - TableChange.ColumnPosition - .last())), - false, - null, - epoch))) - .isInstanceOf(FlussRuntimeException.class) - .hasMessageContaining("Concurrent metadata change"); - ZooKeeperClient.VersionedData after = zk.getTableWithVersion(path).get(); - assertThat(after.zkVersion()).isEqualTo(before.zkVersion()); - assertThat(after.data()).isEqualTo(before.data()); - assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion + 1); - assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isFalse(); - assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) - .contains("unmapped"); - } - - @Test - void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { - TablePath path = create("enable_wins"); - ZooKeeperClient.VersionedData snapshot = - zk.getTableWithVersion(path).get(); - int schemaZkVersion = zk.getSchemaZkVersion(path); - int schemaId = zk.getCurrentSchemaId(path); - Schema incompatible = - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column("unmapped", DataTypes.STRING()) - .build(); - enable(path, (oldTable, updated) -> {}); - assertThatThrownBy( - () -> - zk.registerSchemaWithVersionCheck( - path, - incompatible, - schemaId + 1, - snapshot.zkVersion(), - schemaZkVersion, - epoch)) - .isInstanceOf(KeeperException.BadVersionException.class); - assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); - assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isTrue(); - assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) - .containsExactly("_kafka_value"); - } - - @Test - void testSchemaFailureDoesNotAdvanceSchemaVersion() throws Exception { - TablePath path = create("atomic_failure"); - ZooKeeperClient.VersionedData snapshot = - zk.getTableWithVersion(path).get(); - int schemaZkVersion = zk.getSchemaZkVersion(path); - int schemaId = zk.getCurrentSchemaId(path); - Schema schema = manager.getTable(path).getSchema(); - assertThatThrownBy( - () -> - zk.registerSchemaWithVersionCheck( - path, - schema, - schemaId, - snapshot.zkVersion(), - schemaZkVersion, - epoch)) - .isInstanceOf(KeeperException.NodeExistsException.class); - assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); - assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); - assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); - zk.fenceBecomeCoordinatorLeader("next-coordinator"); - assertThatThrownBy( - () -> - zk.registerSchemaWithVersionCheck( - path, - schema, - schemaId + 1, - snapshot.zkVersion(), - schemaZkVersion, - epoch)) - .isInstanceOf(KeeperException.BadVersionException.class); - assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); - assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); - assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); - } - - private TablePath create(String name) { - TablePath path = TablePath.of("ddl", name); - TableDescriptor table = - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .distributedBy(1) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") - .build() - .withReplicationFactor(1); - manager.createTable(path, zk.getDefaultRemoteDataDir(), table, null, false); - return path; - } - - private void enable( - TablePath path, - BiConsumer beforeUpdate) { - TablePropertyChanges.Builder changes = new TablePropertyChanges.Builder(); - changes.setTableProperty("table.kafka.enabled", "true"); - manager.alterTableProperties( - path, - Collections.singletonList(TableChange.set("table.kafka.enabled", "true")), - changes.build(), - false, - null, - beforeUpdate, - (oldTable, updated) -> {}, - ZkVersion.MATCH_ANY_VERSION.getVersion()); - } -} diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 5f4a5ef345a..b183783fd27 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -254,16 +254,18 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, the coordinator creates this database if it is missing. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, a configured database other than `fluss` is created if missing. The native `fluss` database is created only when the cluster has no databases. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named `sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database` (by default, `fluss.orders`). The coordinator creates the configured default -database when Kafka is enabled and the database is missing. Existing databases are preserved; -other databases and all tables must be created explicitly. Reverse mapping always uses +`kafka.default.database` (by default, `fluss.orders`). The native `fluss` database is created only +when the cluster has no databases; Kafka reuses this database when its default is `fluss`. +If Kafka is enabled and a different default database is configured, the coordinator creates it +if missing. Existing databases are preserved; other databases and all tables must be created +explicitly. Reverse mapping always uses the fully qualified name. All TabletServers must use the same default database. Changing it redirects unqualified names, so migrate clients to fully qualified names before changing the configuration. The prototype `kafka.database` option is no longer used. From 2b6fb3a9880d533d2fca9b0abdd8d9ee4c6aa210 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 15:33:27 +0800 Subject: [PATCH 15/48] [kafka] Use the DDL table contract in Metadata Integrate the DDL mapping prerequisite and use its resolver for Metadata discovery. Omit unsupported tables from all-topic queries and return per-topic mapping errors for named queries. Preserve metadata for compatible tables in mixed requests. Validation: Java 11, mvn -o -pl fluss-kafka clean verify (62 unit tests and 2 integration tests); Checkstyle, Spotless and RAT passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 742/742 AI-Contributed/UT: 571/571 --- .../fluss/kafka/KafkaProtocolPlugin.java | 3 +- .../fluss/kafka/KafkaRequestHandler.java | 22 +- .../kafka/api/metadata/MetadataHandler.java | 190 ++++++ .../metadata/GatewayKafkaMetadataBackend.java | 322 ++++++++++ .../metadata/KafkaClusterMetadata.java | 210 ++++++ .../metadata/KafkaMetadataBackend.java | 30 + .../backend/metadata/KafkaMetadataQuery.java | 102 +++ .../fluss/kafka/KafkaMetadataHandlerTest.java | 602 ++++++++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 8 +- 9 files changed, 1482 insertions(+), 7 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 9a9e2e9ff6e..fd185fdef12 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -66,6 +66,7 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " + service.getClass().getSimpleName()); } - return new KafkaRequestHandler(); + TabletServerGateway gateway = (TabletServerGateway) service; + return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index b3b555a8f9a..d6fb1a0430e 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,14 +17,20 @@ package org.apache.fluss.kafka; +import org.apache.fluss.kafka.api.metadata.MetadataHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import java.util.Collections; +import java.util.Arrays; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { @@ -32,11 +38,19 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaApiRegistry registry; private final KafkaRequestDispatcher dispatcher; - /** Creates a Kafka request handler with the implemented server capabilities. */ - public KafkaRequestHandler() { + /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ + public KafkaRequestHandler( + RpcGatewayService service, TabletServerGateway gateway, String kafkaDatabase) { + checkNotNull(service); + checkNotNull(gateway); + checkNotNull(kafkaDatabase); this.registry = new KafkaApiRegistry( - Collections.singletonList(new ApiVersionsHandler(this::apiRegistry))); + Arrays.asList( + new ApiVersionsHandler(this::apiRegistry), + new MetadataHandler( + new GatewayKafkaMetadataBackend( + service, gateway, kafkaDatabase)))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java new file mode 100644 index 00000000000..ee8bfe7a5a8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataBackend; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements Kafka Metadata versions 0 through 11 using a narrow Fluss metadata backend. */ +@Internal +public final class MetadataHandler implements KafkaApiHandler { + + private static final short MAX_SUPPORTED_VERSION = 11; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.METADATA, + ApiKeys.METADATA.oldestVersion(), + (short) Math.min(ApiKeys.METADATA.latestVersion(), MAX_SUPPORTED_VERSION), + true); + + private final KafkaMetadataBackend backend; + + /** Creates a Metadata handler. */ + public MetadataHandler(KafkaMetadataBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, MetadataRequest request) { + List validTopics = new ArrayList<>(); + List invalidTopics = new ArrayList<>(); + if (!request.isAllTopics()) { + for (MetadataRequestTopic topic : request.data().topics()) { + if (topic.name() == null) { + throw new InvalidRequestException( + "Topic name must be set because topic ID lookup is not supported by " + + "Metadata versions 10 and 11."); + } else if (!Topic.isValid(topic.name())) { + invalidTopics.add( + new KafkaClusterMetadata.Topic( + topic.name(), + topic.topicId(), + TopicError.INVALID_TOPIC, + Collections.emptyList())); + } else { + // Kafka added the topic ID fields in v10, but ID-based Metadata lookup was not + // implemented until v12. This handler intentionally stops at v11. + validTopics.add(new TopicReference(topic.name(), Uuid.ZERO_UUID)); + } + } + } + + KafkaMetadataQuery query = + new KafkaMetadataQuery( + request.isAllTopics(), + validTopics, + context.listenerName(), + clientAddress(context.remoteAddress())); + return backend.getMetadata(query) + .thenApply( + metadata -> { + List topics = + new ArrayList<>(metadata.topics()); + topics.addAll(invalidTopics); + return toResponse( + request.version(), + new KafkaClusterMetadata(metadata.brokers(), topics)); + }); + } + + private static MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { + MetadataResponseData data = + new MetadataResponseData() + .setThrottleTimeMs(0) + .setControllerId(MetadataResponse.NO_CONTROLLER_ID) + .setClusterAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Broker broker : metadata.brokers()) { + MetadataResponseData.MetadataResponseBroker responseBroker = + new MetadataResponseData.MetadataResponseBroker() + .setNodeId(broker.id()) + .setHost(broker.host()) + .setPort(broker.port()); + if (broker.rack() != null) { + responseBroker.setRack(broker.rack()); + } + data.brokers().add(responseBroker); + } + for (KafkaClusterMetadata.Topic topic : metadata.topics()) { + MetadataResponseData.MetadataResponseTopic responseTopic = + new MetadataResponseData.MetadataResponseTopic() + .setName(topic.name()) + .setTopicId(topic.topicId()) + .setErrorCode(toKafkaError(topic.error()).code()) + .setIsInternal(topic.name() != null && Topic.isInternal(topic.name())) + .setTopicAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Partition partition : topic.partitions()) { + responseTopic + .partitions() + .add( + new MetadataResponseData.MetadataResponsePartition() + .setErrorCode( + partition.leaderAvailable() + ? Errors.NONE.code() + : Errors.LEADER_NOT_AVAILABLE.code()) + .setPartitionIndex(partition.partitionId()) + .setLeaderId(partition.leaderId()) + .setLeaderEpoch(partition.leaderEpoch()) + .setReplicaNodes(partition.replicas()) + .setIsrNodes(partition.isr()) + .setOfflineReplicas(partition.offlineReplicas())); + } + data.topics().add(responseTopic); + } + return new MetadataResponse(data, version); + } + + private static Errors toKafkaError(TopicError error) { + switch (error) { + case NONE: + return Errors.NONE; + case UNKNOWN_TOPIC_OR_PARTITION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case UNKNOWN_TOPIC_ID: + return Errors.UNKNOWN_TOPIC_ID; + case INVALID_TOPIC: + return Errors.INVALID_TOPIC_EXCEPTION; + default: + throw new IllegalArgumentException("Unsupported metadata error " + error); + } + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java new file mode 100644 index 00000000000..5905624654f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.MetadataResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.Uuid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */ +@Internal +public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { + + private static final Logger LOG = LoggerFactory.getLogger(GatewayKafkaMetadataBackend.class); + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final String databaseName; + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + + /** Creates a metadata backend backed by the local TabletServer gateway. */ + public GatewayKafkaMetadataBackend( + RpcGatewayService service, TabletServerGateway gateway, String databaseName) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.databaseName = checkNotNull(databaseName); + this.topicMapper = new KafkaTopicMapper(databaseName); + } + + @Override + public CompletableFuture getMetadata(KafkaMetadataQuery query) { + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenCompose( + response -> + requestFlussMetadata( + query, + new LinkedHashSet<>(response.getTableNamesList()))); + } + + Set topicNames = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + topicNames.add(topic.topicName()); + } + } + return requestFlussMetadata(query, topicNames); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames) { + return requestFlussMetadata(query, topicNames, true); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { + MetadataRequest request = new MetadataRequest(); + for (String topicName : topicNames) { + TablePath tablePath = TablePath.of(databaseName, topicName); + if (!topicMapper.isMappedTable(tablePath)) { + continue; + } + tablePath = topicMapper.toTablePath(topicName); + request.addAllTablePaths( + Collections.singletonList( + new PbTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()))); + } + setCurrentSession(query); + try { + return gateway.metadata(request) + .handle( + (response, failure) -> + failure == null + ? CompletableFuture.completedFuture( + toKafkaMetadata(query, response)) + : recoverMetadataFailure( + query, failure, refreshAndRetry)) + .thenCompose(future -> future); + } catch (Throwable failure) { + return recoverMetadataFailure(query, failure, refreshAndRetry); + } + } + + private CompletableFuture recoverMetadataFailure( + KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) { + if (refreshAndRetry) { + return currentTopicNames(query) + .thenCompose(currentNames -> requestFlussMetadata(query, currentNames, false)); + } + LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure)); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(unwrap(failure)); + return failed; + } + + private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenApply( + response -> { + Set currentNames = + new LinkedHashSet<>(response.getTableNamesList()); + if (!query.allTopics() && !containsTopicId(query.topics())) { + Set requestedNames = new HashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + requestedNames.add(topic.topicName()); + } + } + currentNames.retainAll(requestedNames); + } + return currentNames; + }); + } + + private KafkaClusterMetadata toKafkaMetadata( + KafkaMetadataQuery query, MetadataResponse response) { + List brokers = new ArrayList<>(); + Set aliveBrokerIds = new HashSet<>(); + for (PbServerNode server : response.getTabletServersList()) { + brokers.add( + new Broker( + server.getNodeId(), + server.getHost(), + server.getPort(), + server.hasRack() ? server.getRack() : null)); + aliveBrokerIds.add(server.getNodeId()); + } + Collections.sort(brokers, Comparator.comparingInt(Broker::id)); + + Map topicsByName = new HashMap<>(); + Map topicsById = new HashMap<>(); + for (PbTableMetadata table : response.getTableMetadatasList()) { + TablePath tablePath = + TablePath.of( + table.getTablePath().getDatabaseName(), + table.getTablePath().getTableName()); + if (!topicMapper.isMappedTable(tablePath)) { + continue; + } + Topic topic = toKafkaTopic(table, aliveBrokerIds); + topicsByName.put(topic.name(), topic); + topicsById.put(topic.topicId(), topic); + } + + List topics = new ArrayList<>(); + if (query.allTopics()) { + for (Topic topic : topicsByName.values()) { + if (topic.error() == TopicError.NONE) { + topics.add(topic); + } + } + Collections.sort(topics, Comparator.comparing(Topic::name)); + } else { + for (TopicReference reference : query.topics()) { + Topic topic = + reference.hasTopicId() + ? topicsById.get(reference.topicId()) + : topicsByName.get(reference.topicName()); + if (topic != null && matches(reference, topic)) { + topics.add(topic); + } else { + topics.add(missingTopic(reference)); + } + } + } + return new KafkaClusterMetadata(brokers, topics); + } + + private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { + try { + schemaResolver.resolve(TableDescriptor.fromJsonBytes(table.getTableJson())); + } catch (IllegalArgumentException e) { + LOG.debug( + "Table {} does not define a supported Kafka mapping: {}", + table.getTablePath().getTableName(), + e.getMessage()); + return new Topic( + table.getTablePath().getTableName(), + topicMapper.toTopicId(table.getTableId()), + TopicError.INVALID_TOPIC, + Collections.emptyList()); + } + List partitions = new ArrayList<>(); + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + boolean leaderAvailable = + bucket.hasLeaderId() && aliveBrokerIds.contains(bucket.getLeaderId()); + List replicas = new ArrayList<>(); + List isr = new ArrayList<>(); + List offlineReplicas = new ArrayList<>(); + Set isrIds = new HashSet<>(); + if (bucket.hasBucketEpoch()) { + for (int isrId : bucket.getIsrs()) { + isrIds.add(isrId); + } + } else if (leaderAvailable) { + // Legacy metadata has no authoritative ISR. Report only the available leader + // as a conservative fallback; live followers may still be out of sync. + isrIds.add(bucket.getLeaderId()); + } + for (int replicaId : bucket.getReplicaIds()) { + replicas.add(replicaId); + if (isrIds.contains(replicaId)) { + isr.add(replicaId); + } + if (!aliveBrokerIds.contains(replicaId)) { + offlineReplicas.add(replicaId); + } + } + partitions.add( + new Partition( + bucket.getBucketId(), + leaderAvailable ? bucket.getLeaderId() : -1, + bucket.hasLeaderEpoch() ? bucket.getLeaderEpoch() : -1, + replicas, + isr, + offlineReplicas, + leaderAvailable)); + } + Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); + return new Topic( + table.getTablePath().getTableName(), + topicMapper.toTopicId(table.getTableId()), + TopicError.NONE, + partitions); + } + + private static Topic missingTopic(TopicReference reference) { + TopicError error = + reference.hasTopicId() + ? TopicError.UNKNOWN_TOPIC_ID + : TopicError.UNKNOWN_TOPIC_OR_PARTITION; + return new Topic( + reference.topicName(), reference.topicId(), error, Collections.emptyList()); + } + + private static boolean matches(TopicReference reference, Topic topic) { + return (reference.topicName() == null || reference.topicName().equals(topic.name())) + && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId())); + } + + private void setCurrentSession(KafkaMetadataQuery query) { + service.setCurrentSession( + new Session( + (short) 0, + query.listenerName(), + false, + query.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static boolean containsTopicId(List topics) { + for (TopicReference topic : topics) { + if (topic.hasTopicId()) { + return true; + } + } + return false; + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java new file mode 100644 index 00000000000..bbc634af6f0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Kafka-domain cluster metadata returned by a Fluss metadata backend. */ +@Internal +public final class KafkaClusterMetadata { + + private final List brokers; + private final List topics; + + /** Creates cluster metadata. */ + public KafkaClusterMetadata(List brokers, List topics) { + this.brokers = immutableCopy(brokers); + this.topics = immutableCopy(topics); + } + + /** Returns Kafka-reachable brokers. */ + public List brokers() { + return brokers; + } + + /** Returns topic metadata and topic-level errors. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Kafka-reachable broker information. */ + @Internal + public static final class Broker { + + private final int id; + private final String host; + private final int port; + private final @Nullable String rack; + + /** Creates broker information. */ + public Broker(int id, String host, int port, @Nullable String rack) { + this.id = id; + this.host = checkNotNull(host); + this.port = port; + this.rack = rack; + } + + /** Returns the Kafka broker ID. */ + public int id() { + return id; + } + + /** Returns the Kafka listener host. */ + public String host() { + return host; + } + + /** Returns the Kafka listener port. */ + public int port() { + return port; + } + + /** Returns the broker rack, if configured. */ + public @Nullable String rack() { + return rack; + } + } + + /** Topic-level error independent of a Kafka response schema version. */ + @Internal + public enum TopicError { + NONE, + UNKNOWN_TOPIC_OR_PARTITION, + UNKNOWN_TOPIC_ID, + INVALID_TOPIC + } + + /** Metadata for one Kafka topic. */ + @Internal + public static final class Topic { + + private final @Nullable String name; + private final Uuid topicId; + private final TopicError error; + private final List partitions; + + /** Creates topic metadata. */ + public Topic( + @Nullable String name, Uuid topicId, TopicError error, List partitions) { + this.name = name; + this.topicId = checkNotNull(topicId); + this.error = checkNotNull(error); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name, if known. */ + public @Nullable String name() { + return name; + } + + /** Returns the stable Kafka topic ID. */ + public Uuid topicId() { + return topicId; + } + + /** Returns the topic-level domain error. */ + public TopicError error() { + return error; + } + + /** Returns the topic partitions. */ + public List partitions() { + return partitions; + } + } + + /** Metadata for one Kafka partition backed by a Fluss bucket. */ + @Internal + public static final class Partition { + + private final int partitionId; + private final int leaderId; + private final int leaderEpoch; + private final List replicas; + private final List isr; + private final List offlineReplicas; + private final boolean leaderAvailable; + + /** Creates partition metadata. */ + public Partition( + int partitionId, + int leaderId, + int leaderEpoch, + List replicas, + List isr, + List offlineReplicas, + boolean leaderAvailable) { + this.partitionId = partitionId; + this.leaderId = leaderId; + this.leaderEpoch = leaderEpoch; + this.replicas = immutableCopy(replicas); + this.isr = immutableCopy(isr); + this.offlineReplicas = immutableCopy(offlineReplicas); + this.leaderAvailable = leaderAvailable; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the current leader ID, or {@code -1} when unavailable. */ + public int leaderId() { + return leaderId; + } + + /** Returns the leader epoch, or {@code -1} when unavailable. */ + public int leaderEpoch() { + return leaderEpoch; + } + + /** Returns assigned replica IDs. */ + public List replicas() { + return replicas; + } + + /** Returns replica IDs currently visible as in-sync. */ + public List isr() { + return isr; + } + + /** Returns assigned replicas whose TabletServers are unavailable. */ + public List offlineReplicas() { + return offlineReplicas; + } + + /** Returns whether the partition has a reachable leader. */ + public boolean leaderAvailable() { + return leaderAvailable; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java new file mode 100644 index 00000000000..abb02a920ed --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Metadata API. */ +@Internal +public interface KafkaMetadataBackend { + + /** Resolves Kafka-domain metadata asynchronously. */ + CompletableFuture getMetadata(KafkaMetadataQuery query); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java new file mode 100644 index 00000000000..01e21fa4440 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Domain query used by the Metadata API to access the Fluss adapter layer. */ +@Internal +public final class KafkaMetadataQuery { + + private final boolean allTopics; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a metadata query. */ + public KafkaMetadataQuery( + boolean allTopics, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.allTopics = allTopics; + this.topics = Collections.unmodifiableList(new ArrayList<>(checkNotNull(topics))); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns whether all Kafka topics should be returned. */ + public boolean allTopics() { + return allTopics; + } + + /** Returns the explicitly requested topic identities. */ + public List topics() { + return topics; + } + + /** Returns the Kafka listener used by the client connection. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client address when it is available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + /** Kafka topic name and ID supplied by a Metadata request. */ + @Internal + public static final class TopicReference { + + private final @Nullable String topicName; + private final Uuid topicId; + + /** Creates a topic reference. */ + public TopicReference(@Nullable String topicName, Uuid topicId) { + this.topicName = topicName; + this.topicId = checkNotNull(topicId); + } + + /** Returns the requested topic name, if present. */ + public @Nullable String topicName() { + return topicName; + } + + /** Returns the requested topic ID, or {@link Uuid#ZERO_UUID} when absent. */ + public Uuid topicId() { + return topicId; + } + + /** Returns whether this reference identifies a topic by ID. */ + public boolean hasTopicId() { + return !Uuid.ZERO_UUID.equals(topicId); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java new file mode 100644 index 00000000000..2614b2df3fd --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -0,0 +1,602 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol compatibility tests for the Kafka Metadata API. */ +public class KafkaMetadataHandlerTest { + + private static final Uuid TOPIC_ID = new Uuid(0x466c757373000000L, 123L); + + @Test + public void testNamedTopicForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Collections.singletonList("topic"))), + version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(100, new byte[] {1, 2, 3})); + request.data() + .topics() + .get(0) + .unknownTaggedFields() + .add(new RawTaggedField(101, new byte[] {4, 5, 6})); + } + MetadataResponse response = handle(service, request, version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.controller()).isNull(); + Node broker = response.brokers().iterator().next(); + assertThat(broker.host()).isEqualTo("broker-1"); + assertThat(broker.port()).isEqualTo(9092); + assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.partitionIndex()).isZero(); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).isEmpty(); + } + assertThat(service.lastListenerName).isEqualTo("KAFKA"); + } + + @Test + public void testAllTopicsForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = allTopicsRequest(version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(102, new byte[] {7, 8, 9})); + } + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("other", "topic"); + } + } + + @Test + public void testAllTopicsAndMissingAndInvalidTopic() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse allTopics = + handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); + assertThat(allTopics.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactlyInAnyOrder("other", "topic"); + + MetadataRequest requestedTopics = + new MetadataRequest.Builder(Arrays.asList("missing", "invalid topic"), false) + .build((short) 9); + MetadataResponse errors = handle(service, requestedTopics, (short) 9); + assertThat(errors.errors()) + .containsEntry("missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("invalid topic", Errors.INVALID_TOPIC_EXCEPTION); + } + + @Test + public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName("topic") + .setTopicId( + new Uuid( + 0x466c757373000000L, + 999L)))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(response.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + } + } + + @Test + public void testV10AndV11RejectTopicIdOnlyLookup() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Uuid requestedTopicId = new Uuid(0x466c757373000000L, 999L); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(null) + .setTopicId(requestedTopicId))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()).hasSize(1); + MetadataResponseTopic responseTopic = response.data().topics().iterator().next(); + assertThat(responseTopic.name()).isEmpty(); + assertThat(responseTopic.topicId()).isEqualTo(requestedTopicId); + assertThat(responseTopic.errorCode()).isEqualTo(Errors.INVALID_REQUEST.code()); + } + } + + @Test + public void testTopicIdentityAcrossDeleteAndRecreate() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse initial = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(initial.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + + service.removeTable("topic"); + MetadataResponse deleted = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(deleted.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + + service.putTable("topic", 223L); + Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); + MetadataResponse recreatedByName = + handle( + service, + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11), + (short) 11); + assertThat(recreatedByName.data().topics().find("topic").topicId()) + .isEqualTo(recreatedTopicId) + .isNotEqualTo(TOPIC_ID); + } + + @Test + public void testDeleteRaceBecomesUnknownTopicResult() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.removeTable("topic"); + service.failNextMetadataAsMissing = true; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + } + + @Test + public void testUnavailableLeaderUsesPartitionError() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicLeaderAvailable = false; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.leaderId()).isEqualTo(-1); + assertThat(partition.replicaNodes()).containsExactly(1, 2, 3); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).containsExactly(3); + } + + @Test + public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicIsr = new int[] {1}; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataUsesLeaderOnlyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + service.topicLeaderAvailable = false; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.isrNodes()).isEmpty(); + } + + @Test + public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { + for (int bucketEpoch : new int[] {7, -1}) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = bucketEpoch; + service.topicIsr = new int[0]; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.isrNodes()).isEmpty(); + } + } + + @Test + public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failMetadata = true; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 1)); + assertThat(response.brokers()).isEmpty(); + } + + @Test + public void testMetadataUsesDdlContractForEveryVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable( + "no_mapping", + 125L, + TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(2) + .build()); + service.putTable( + "bad_format", + 126L, + TableDescriptor.builder(defaultDescriptor()) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") + .build()); + service.putTable( + "primary_key", + 127L, + TableDescriptor.builder(defaultDescriptor()) + .schema( + Schema.newBuilder() + .column("body", DataTypes.BYTES().copy(false)) + .primaryKey("body") + .build()) + .build()); + service.putTable( + "partitioned", + 128L, + TableDescriptor.builder(defaultDescriptor()).partitionedBy("body").build()); + service.putTable( + "indexed", + 129L, + TableDescriptor.builder(defaultDescriptor()).logFormat(LogFormat.INDEXED).build()); + service.putTable("invalid topic", 130L); + List invalidMappings = + Arrays.asList("no_mapping", "bad_format", "primary_key", "partitioned", "indexed"); + for (short version = 0; version <= 11; version++) { + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("other", "topic"); + List requested = new ArrayList<>(invalidMappings); + requested.add("topic"); + requested.add("missing"); + MetadataResponse named = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + requested)), + version), + version); + for (String name : invalidMappings) { + MetadataResponseTopic topic = named.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(topic.partitions()).isEmpty(); + } + assertThat(named.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(named.data().topics().find("missing").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + } + + @Test + public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable( + "topic", + 123L, + TableDescriptor.builder(defaultDescriptor()) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .build()); + MetadataResponse invalid = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(invalid.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + service.putTable("topic", 123L); + MetadataResponse valid = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(valid.data().topics().find("topic").errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + } + + private static TableDescriptor defaultDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .build(); + } + + private static MetadataRequest namedTopicRequest(String topicName) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(topicName) + .setTopicId(Uuid.ZERO_UUID))), + (short) 11); + } + + private static MetadataRequest allTopicsRequest(short version) { + MetadataRequestData data = new MetadataRequestData(); + data.setTopics(version == 0 ? Collections.emptyList() : null); + return new MetadataRequest(data, version); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { + KafkaRequestHandler handler = new KafkaRequestHandler(service, service, "kafka"); + ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); + KafkaRequest request; + try { + request = + new KafkaRequest( + ApiKeys.METADATA, + version, + new RequestHeader(ApiKeys.METADATA, version, "client-id", 1), + requestBody, + "KAFKA", + requestBuffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + } finally { + // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest. + requestBuffer.release(); + } + handler.processRequest(request); + ByteBuf responseBuffer = request.responseBuffer(); + try { + return (MetadataResponse) + AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + assertThat(requestBuffer.refCnt()).isZero(); + } + } + + private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + + private final Map tables = new LinkedHashMap<>(); + private final Map descriptors = new LinkedHashMap<>(); + private String lastListenerName; + private boolean topicLeaderAvailable = true; + private int[] topicIsr = new int[] {1, 2}; + private Integer topicBucketEpoch = 7; + private boolean failMetadata; + private boolean failNextMetadataAsMissing; + + private TestingMetadataGatewayService() { + putTable("topic", 123L); + putTable("other", 124L); + } + + @Override + public CompletableFuture listTables(ListTablesRequest request) { + assertThat(request.getDatabaseName()).isEqualTo("kafka"); + return CompletableFuture.completedFuture( + new ListTablesResponse().addAllTableNames(new ArrayList<>(tables.keySet()))); + } + + @Override + public CompletableFuture metadata( + org.apache.fluss.rpc.messages.MetadataRequest request) { + lastListenerName = currentListenerName(); + if (failMetadata) { + CompletableFuture failure = + new CompletableFuture<>(); + failure.completeExceptionally(new IllegalStateException("metadata unavailable")); + return failure; + } + if (failNextMetadataAsMissing) { + failNextMetadataAsMissing = false; + throw new TableNotExistException("table was deleted"); + } + List topics = new ArrayList<>(); + for (PbTablePath tablePath : request.getTablePathsList()) { + Long tableId = tables.get(tablePath.getTableName()); + if (tableId != null) { + topics.add( + tableMetadata( + tablePath.getTableName(), + tableId, + !"topic".equals(tablePath.getTableName()) + || topicLeaderAvailable, + "topic".equals(tablePath.getTableName()) + ? topicIsr + : new int[] {1, 2}, + "topic".equals(tablePath.getTableName()) + ? topicBucketEpoch + : Integer.valueOf(7)) + .setTableJson( + descriptors + .get(tablePath.getTableName()) + .toJsonBytes())); + } + } + return CompletableFuture.completedFuture( + new org.apache.fluss.rpc.messages.MetadataResponse() + .addAllTabletServers( + Arrays.asList( + new PbServerNode() + .setNodeId(1) + .setHost("broker-1") + .setPort(9092) + .setRack("rack-a"), + new PbServerNode() + .setNodeId(2) + .setHost("broker-2") + .setPort(9093))) + .addAllTableMetadatas(topics)); + } + + private void putTable(String topic, long tableId) { + putTable(topic, tableId, defaultDescriptor()); + } + + private void putTable(String topic, long tableId, TableDescriptor descriptor) { + tables.put(topic, tableId); + descriptors.put(topic, descriptor); + } + + private void removeTable(String topic) { + tables.remove(topic); + descriptors.remove(topic); + } + + private static PbTableMetadata tableMetadata( + String topic, + long tableId, + boolean leaderAvailable, + int[] isr, + Integer bucketEpoch) { + PbTableMetadata table = + new PbTableMetadata() + .setTablePath( + new PbTablePath().setDatabaseName("kafka").setTableName(topic)) + .setTableId(tableId) + .addAllBucketMetadatas( + Arrays.asList( + new PbBucketMetadata() + .setBucketId(0) + .setLeaderId(leaderAvailable ? 1 : 3) + .setLeaderEpoch(5) + .setReplicaIds( + leaderAvailable + ? new int[] {1, 2} + : new int[] {1, 2, 3}) + .setIsrs( + bucketEpoch == null ? new int[0] : isr), + new PbBucketMetadata() + .setBucketId(1) + .setLeaderId(2) + .setLeaderEpoch(6) + .setReplicaIds(new int[] {1, 2}) + .setIsrs( + bucketEpoch == null + ? new int[0] + : new int[] {1, 2}))); + if (bucketEpoch != null) { + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + bucket.setBucketEpoch(bucketEpoch); + } + } + return table; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 40fdab04cc4..d00c436defe 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.kafka; +import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; @@ -145,7 +146,9 @@ private static void assertSuccessfulResponseDefaults(ApiVersionsResponse respons private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) - .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); + .containsExactly( + tuple(ApiKeys.METADATA.id, (short) 0, (short) 11), + tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } @ParameterizedTest @@ -292,6 +295,7 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + return new KafkaRequestHandler(service, service, "kafka"); } } From 947c1196cc794e23fa02ee6ea33a546c10f31251 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 14:39:43 +0800 Subject: [PATCH 16/48] [kafka] Discover qualified topics and isolate missing databases Route Metadata queries through the database.table mapping from PR03 and enumerate compatible topics across databases. Preserve brokers and valid topics when a requested database is absent or disappears during discovery; propagate other gateway failures. Cover qualified names, same-named tables across databases, synchronous and asynchronous lookup failures, and Metadata v0-v11 over a real listener. Validated on Java 11: 90 unit tests and 4 integration tests passed, along with Checkstyle, Spotless, RAT, and git diff --check. Five additional interleaved asynchronous review probes passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 129/129 AI-Contributed/UT: 514/514 --- .../fluss/kafka/KafkaProtocolPlugin.java | 2 +- .../fluss/kafka/KafkaRequestHandler.java | 7 +- .../kafka/api/metadata/MetadataHandler.java | 3 +- .../metadata/GatewayKafkaMetadataBackend.java | 116 +++++-- .../fluss/kafka/KafkaMetadataHandlerTest.java | 312 ++++++++++++++---- .../fluss/kafka/KafkaMetadataITCase.java | 200 +++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 2 +- 7 files changed, 531 insertions(+), 111 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index fd185fdef12..dddfff1f9eb 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -67,6 +67,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { + service.getClass().getSimpleName()); } TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE)); + return new KafkaRequestHandler(service, gateway); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index d6fb1a0430e..3278d8328a2 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -39,18 +39,15 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ - public KafkaRequestHandler( - RpcGatewayService service, TabletServerGateway gateway, String kafkaDatabase) { + public KafkaRequestHandler(RpcGatewayService service, TabletServerGateway gateway) { checkNotNull(service); checkNotNull(gateway); - checkNotNull(kafkaDatabase); this.registry = new KafkaApiRegistry( Arrays.asList( new ApiVersionsHandler(this::apiRegistry), new MetadataHandler( - new GatewayKafkaMetadataBackend( - service, gateway, kafkaDatabase)))); + new GatewayKafkaMetadataBackend(service, gateway)))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java index ee8bfe7a5a8..2ceca11fbe1 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -28,6 +28,7 @@ import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InvalidRequestException; @@ -85,7 +86,7 @@ public CompletableFuture handle( throw new InvalidRequestException( "Topic name must be set because topic ID lookup is not supported by " + "Metadata versions 10 and 11."); - } else if (!Topic.isValid(topic.name())) { + } else if (!KafkaTopicMapper.isValidTopic(topic.name())) { invalidTopics.add( new KafkaClusterMetadata.Topic( topic.name(), diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java index 5905624654f..202658b9f0f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.backend.metadata; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; @@ -29,7 +30,9 @@ import org.apache.fluss.metadata.TablePath; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; import org.apache.fluss.rpc.messages.MetadataRequest; import org.apache.fluss.rpc.messages.MetadataResponse; import org.apache.fluss.rpc.messages.PbBucketMetadata; @@ -38,6 +41,7 @@ import org.apache.fluss.rpc.messages.PbTablePath; import org.apache.fluss.rpc.netty.server.Session; import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.common.Uuid; import org.slf4j.Logger; @@ -65,29 +69,20 @@ public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { private final RpcGatewayService service; private final TabletServerGateway gateway; - private final String databaseName; - private final KafkaTopicMapper topicMapper; + private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); /** Creates a metadata backend backed by the local TabletServer gateway. */ - public GatewayKafkaMetadataBackend( - RpcGatewayService service, TabletServerGateway gateway, String databaseName) { + public GatewayKafkaMetadataBackend(RpcGatewayService service, TabletServerGateway gateway) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); - this.databaseName = checkNotNull(databaseName); - this.topicMapper = new KafkaTopicMapper(databaseName); } @Override public CompletableFuture getMetadata(KafkaMetadataQuery query) { if (query.allTopics() || containsTopicId(query.topics())) { - setCurrentSession(query); - return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) - .thenCompose( - response -> - requestFlussMetadata( - query, - new LinkedHashSet<>(response.getTableNamesList()))); + return currentTopicNames(query) + .thenCompose(topicNames -> requestFlussMetadata(query, topicNames)); } Set topicNames = new LinkedHashSet<>(); @@ -108,11 +103,7 @@ private CompletableFuture requestFlussMetadata( KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { MetadataRequest request = new MetadataRequest(); for (String topicName : topicNames) { - TablePath tablePath = TablePath.of(databaseName, topicName); - if (!topicMapper.isMappedTable(tablePath)) { - continue; - } - tablePath = topicMapper.toTablePath(topicName); + TablePath tablePath = topicMapper.toTablePath(topicName); request.addAllTablePaths( Collections.singletonList( new PbTablePath() @@ -148,12 +139,27 @@ private CompletableFuture recoverMetadataFailure( } private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { - setCurrentSession(query); - return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + CompletableFuture> databasesFuture; + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + databasesFuture = + gateway.listDatabases(new ListDatabasesRequest()) + .thenApply( + response -> + new LinkedHashSet<>(response.getDatabaseNamesList())); + } else { + Set databases = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + databases.add(topicMapper.toTablePath(topic.topicName()).getDatabaseName()); + } + } + databasesFuture = CompletableFuture.completedFuture(databases); + } + return databasesFuture + .thenCompose(databases -> listTopicNames(query, databases)) .thenApply( - response -> { - Set currentNames = - new LinkedHashSet<>(response.getTableNamesList()); + currentNames -> { if (!query.allTopics() && !containsTopicId(query.topics())) { Set requestedNames = new HashSet<>(); for (TopicReference topic : query.topics()) { @@ -167,6 +173,56 @@ private CompletableFuture> currentTopicNames(KafkaMetadataQuery quer }); } + private CompletableFuture> listTopicNames( + KafkaMetadataQuery query, Set databases) { + CompletableFuture> topicsFuture = + CompletableFuture.completedFuture(new LinkedHashSet<>()); + for (String database : databases) { + topicsFuture = + topicsFuture.thenCompose( + topicNames -> + listTables(query, database) + .thenApply( + response -> { + for (String tableName : + response.getTableNamesList()) { + TablePath tablePath = + TablePath.of( + database, tableName); + if (topicMapper.isMappedTable( + tablePath)) { + topicNames.add( + topicMapper.toTopicName( + tablePath)); + } + } + return topicNames; + })); + } + return topicsFuture; + } + + private CompletableFuture listTables( + KafkaMetadataQuery query, String database) { + setCurrentSession(query); + CompletableFuture tablesFuture; + try { + tablesFuture = gateway.listTables(new ListTablesRequest().setDatabaseName(database)); + } catch (Exception failure) { + tablesFuture = FutureUtils.completedExceptionally(failure); + } + return tablesFuture.exceptionally( + failure -> { + Throwable cause = unwrap(failure); + if (cause instanceof DatabaseNotExistException) { + // A database may be missing or deleted after listDatabases. Keep loading + // the remaining topics and brokers through the normal metadata request. + return new ListTablesResponse(); + } + throw new CompletionException(cause); + }); + } + private KafkaClusterMetadata toKafkaMetadata( KafkaMetadataQuery query, MetadataResponse response) { List brokers = new ArrayList<>(); @@ -192,7 +248,7 @@ private KafkaClusterMetadata toKafkaMetadata( if (!topicMapper.isMappedTable(tablePath)) { continue; } - Topic topic = toKafkaTopic(table, aliveBrokerIds); + Topic topic = toKafkaTopic(table, topicMapper.toTopicName(tablePath), aliveBrokerIds); topicsByName.put(topic.name(), topic); topicsById.put(topic.topicId(), topic); } @@ -221,16 +277,17 @@ private KafkaClusterMetadata toKafkaMetadata( return new KafkaClusterMetadata(brokers, topics); } - private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { + private Topic toKafkaTopic( + PbTableMetadata table, String topicName, Set aliveBrokerIds) { try { schemaResolver.resolve(TableDescriptor.fromJsonBytes(table.getTableJson())); } catch (IllegalArgumentException e) { LOG.debug( "Table {} does not define a supported Kafka mapping: {}", - table.getTablePath().getTableName(), + topicName, e.getMessage()); return new Topic( - table.getTablePath().getTableName(), + topicName, topicMapper.toTopicId(table.getTableId()), TopicError.INVALID_TOPIC, Collections.emptyList()); @@ -273,10 +330,7 @@ private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { } Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); return new Topic( - table.getTablePath().getTableName(), - topicMapper.toTopicId(table.getTableId()), - TopicError.NONE, - partitions); + topicName, topicMapper.toTopicId(table.getTableId()), TopicError.NONE, partitions); } private static Topic missingTopic(TopicReference reference) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 2614b2df3fd..4c33a51635a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -17,12 +17,15 @@ package org.apache.fluss.kafka; +import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; +import org.apache.fluss.rpc.messages.ListDatabasesResponse; import org.apache.fluss.rpc.messages.ListTablesRequest; import org.apache.fluss.rpc.messages.ListTablesResponse; import org.apache.fluss.rpc.messages.PbBucketMetadata; @@ -32,6 +35,7 @@ import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.common.Node; import org.apache.kafka.common.Uuid; @@ -47,14 +51,20 @@ import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; @@ -72,7 +82,7 @@ public void testNamedTopicForEverySupportedVersion() { new MetadataRequestData() .setTopics( MetadataRequest.convertToMetadataRequestTopic( - Collections.singletonList("topic"))), + Collections.singletonList("kafka.topic"))), version); if (version >= 9) { request.data() @@ -92,7 +102,7 @@ public void testNamedTopicForEverySupportedVersion() { assertThat(broker.host()).isEqualTo("broker-1"); assertThat(broker.port()).isEqualTo(9092); assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); - MetadataResponseTopic topic = response.data().topics().find("topic"); + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); assertThat(topic.partitions()).hasSize(2); assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); @@ -121,7 +131,7 @@ public void testAllTopicsForEverySupportedVersion() { assertThat(response.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactly("other", "topic"); + .containsExactly("kafka.other", "kafka.topic"); } } @@ -133,15 +143,16 @@ public void testAllTopicsAndMissingAndInvalidTopic() { handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); assertThat(allTopics.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactlyInAnyOrder("other", "topic"); + .containsExactlyInAnyOrder("kafka.other", "kafka.topic"); MetadataRequest requestedTopics = - new MetadataRequest.Builder(Arrays.asList("missing", "invalid topic"), false) + new MetadataRequest.Builder( + Arrays.asList("kafka.missing", "kafka.invalid topic"), false) .build((short) 9); MetadataResponse errors = handle(service, requestedTopics, (short) 9); assertThat(errors.errors()) - .containsEntry("missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) - .containsEntry("invalid topic", Errors.INVALID_TOPIC_EXCEPTION); + .containsEntry("kafka.missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("kafka.invalid topic", Errors.INVALID_TOPIC_EXCEPTION); } @Test @@ -154,7 +165,7 @@ public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { .setTopics( Collections.singletonList( new MetadataRequestTopic() - .setName("topic") + .setName("kafka.topic") .setTopicId( new Uuid( 0x466c757373000000L, @@ -164,7 +175,7 @@ public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { MetadataResponse response = handle(service, request, version); assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); - assertThat(response.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + assertThat(response.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); } } @@ -197,24 +208,24 @@ public void testV10AndV11RejectTopicIdOnlyLookup() { public void testTopicIdentityAcrossDeleteAndRecreate() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - MetadataResponse initial = handle(service, namedTopicRequest("topic"), (short) 11); - assertThat(initial.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + MetadataResponse initial = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + assertThat(initial.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); - service.removeTable("topic"); - MetadataResponse deleted = handle(service, namedTopicRequest("topic"), (short) 11); + service.removeTable("kafka.topic"); + MetadataResponse deleted = handle(service, namedTopicRequest("kafka.topic"), (short) 11); assertThat(deleted.errorCounts()) .containsExactlyEntriesOf( Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); - service.putTable("topic", 223L); + service.putTable("kafka.topic", 223L); Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); MetadataResponse recreatedByName = handle( service, - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11), (short) 11); - assertThat(recreatedByName.data().topics().find("topic").topicId()) + assertThat(recreatedByName.data().topics().find("kafka.topic").topicId()) .isEqualTo(recreatedTopicId) .isNotEqualTo(TOPIC_ID); } @@ -222,10 +233,10 @@ public void testTopicIdentityAcrossDeleteAndRecreate() { @Test public void testDeleteRaceBecomesUnknownTopicResult() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.removeTable("topic"); + service.removeTable("kafka.topic"); service.failNextMetadataAsMissing = true; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); assertThat(response.errorCounts()) .containsExactlyEntriesOf( @@ -237,12 +248,12 @@ public void testUnavailableLeaderUsesPartitionError() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicLeaderAvailable = false; MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11); MetadataResponse response = handle(service, request, (short) 11); - MetadataResponseTopic topic = response.data().topics().find("topic"); + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); MetadataResponsePartition partition = topic.partitions().get(0); assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); @@ -257,13 +268,13 @@ public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicIsr = new int[] {1}; MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11); MetadataResponse response = handle(service, request, (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.replicaNodes()).containsExactly(1, 2); assertThat(partition.isrNodes()).containsExactly(1); assertThat(partition.offlineReplicas()).isEmpty(); @@ -274,10 +285,10 @@ public void testLegacyMetadataUsesLeaderOnlyIsr() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicBucketEpoch = null; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.leaderId()).isEqualTo(1); assertThat(partition.replicaNodes()).containsExactly(1, 2); assertThat(partition.isrNodes()).containsExactly(1); @@ -290,10 +301,10 @@ public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() { service.topicBucketEpoch = null; service.topicLeaderAvailable = false; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); assertThat(partition.isrNodes()).isEmpty(); } @@ -305,10 +316,11 @@ public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { service.topicBucketEpoch = bucketEpoch; service.topicIsr = new int[0]; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = + handle(service, namedTopicRequest("kafka.topic"), (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.isrNodes()).isEmpty(); } } @@ -318,7 +330,7 @@ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.failMetadata = true; MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11); MetadataResponse response = handle(service, request, (short) 11); @@ -332,20 +344,20 @@ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { public void testMetadataUsesDdlContractForEveryVersion() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.putTable( - "no_mapping", + "kafka.no_mapping", 125L, TableDescriptor.builder() .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) .distributedBy(2) .build()); service.putTable( - "bad_format", + "kafka.bad_format", 126L, TableDescriptor.builder(defaultDescriptor()) .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") .build()); service.putTable( - "primary_key", + "kafka.primary_key", 127L, TableDescriptor.builder(defaultDescriptor()) .schema( @@ -355,24 +367,29 @@ public void testMetadataUsesDdlContractForEveryVersion() { .build()) .build()); service.putTable( - "partitioned", + "kafka.partitioned", 128L, TableDescriptor.builder(defaultDescriptor()).partitionedBy("body").build()); service.putTable( - "indexed", + "kafka.indexed", 129L, TableDescriptor.builder(defaultDescriptor()).logFormat(LogFormat.INDEXED).build()); - service.putTable("invalid topic", 130L); + service.putTable("kafka.invalid topic", 130L); List invalidMappings = - Arrays.asList("no_mapping", "bad_format", "primary_key", "partitioned", "indexed"); + Arrays.asList( + "kafka.no_mapping", + "kafka.bad_format", + "kafka.primary_key", + "kafka.partitioned", + "kafka.indexed"); for (short version = 0; version <= 11; version++) { MetadataResponse all = handle(service, allTopicsRequest(version), version); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactly("other", "topic"); + .containsExactly("kafka.other", "kafka.topic"); List requested = new ArrayList<>(invalidMappings); - requested.add("topic"); - requested.add("missing"); + requested.add("kafka.topic"); + requested.add("kafka.missing"); MetadataResponse named = handle( service, @@ -388,9 +405,9 @@ public void testMetadataUsesDdlContractForEveryVersion() { assertThat(topic.errorCode()).isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); assertThat(topic.partitions()).isEmpty(); } - assertThat(named.data().topics().find("topic").errorCode()) + assertThat(named.data().topics().find("kafka.topic").errorCode()) .isEqualTo(Errors.NONE.code()); - assertThat(named.data().topics().find("missing").errorCode()) + assertThat(named.data().topics().find("kafka.missing").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); } } @@ -399,18 +416,135 @@ public void testMetadataUsesDdlContractForEveryVersion() { public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.putTable( - "topic", + "kafka.topic", 123L, TableDescriptor.builder(defaultDescriptor()) .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") .build()); - MetadataResponse invalid = handle(service, namedTopicRequest("topic"), (short) 11); - assertThat(invalid.data().topics().find("topic").errorCode()) + MetadataResponse invalid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + assertThat(invalid.data().topics().find("kafka.topic").errorCode()) .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); - service.putTable("topic", 123L); - MetadataResponse valid = handle(service, namedTopicRequest("topic"), (short) 11); - assertThat(valid.data().topics().find("topic").errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(valid.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + service.putTable("kafka.topic", 123L); + MetadataResponse valid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + assertThat(valid.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(valid.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); + } + + @Test + public void testSameTableNameInDifferentDatabasesForEveryVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("sales.topic", 223L); + for (short version = 0; version <= 11; version++) { + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic", "sales.topic"); + MetadataResponse named = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList( + "kafka.topic", "sales.topic"))), + version), + version); + assertThat(named.data().topics()).hasSize(2); + assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); + if (version >= 10) { + assertThat(named.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); + assertThat(named.data().topics().find("sales.topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 223L)); + } + } + } + + @Test + public void testRejectUnqualifiedAndMalformedNamesWithoutHidingValidTopics() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + List invalidNames = Arrays.asList("topic", ".topic", "kafka.", "kafka.topic.extra"); + List names = new ArrayList<>(invalidNames); + names.add("kafka.topic"); + MetadataResponse response = + handle( + service, + new MetadataRequest.Builder(names, false).build((short) 11), + (short) 11); + for (String name : invalidNames) { + assertThat(response.data().topics().find(name).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + } + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMissingDatabaseRetainsBrokersAndValidTopics(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failListTablesAsFuture = failedFuture; + for (short version = 0; version <= 11; version++) { + for (List names : + Arrays.asList( + Collections.singletonList("missing_db.topic"), + Arrays.asList("missing_db.topic", "kafka.topic"))) { + MetadataResponse response = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest + .convertToMetadataRequestTopic( + names)), + version), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(names.size()); + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + if (names.contains("kafka.topic")) { + MetadataResponseTopic valid = response.data().topics().find("kafka.topic"); + assertThat(valid.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.partitions()).hasSize(2); + } + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testDatabaseDeletedDuringDiscoveryRetainsOtherTopics(boolean failedFuture) { + for (short version = 0; version <= 11; version++) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("deleted_db.topic", 223L); + service.deleteDatabaseBeforeListing = "deleted_db"; + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, allTopicsRequest(version), version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic"); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testUnexpectedListTablesFailureIsNotTreatedAsMissingDatabase(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failNextMetadataAsMissing = true; + service.listTablesFailure = new IllegalStateException("metadata unavailable"); + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_SERVER_ERROR.code()); } private static TableDescriptor defaultDescriptor() { @@ -441,7 +575,7 @@ private static MetadataRequest allTopicsRequest(short version) { private static MetadataResponse handle( TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { - KafkaRequestHandler handler = new KafkaRequestHandler(service, service, "kafka"); + KafkaRequestHandler handler = new KafkaRequestHandler(service, service); ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); KafkaRequest request; try { @@ -472,6 +606,7 @@ private static MetadataResponse handle( private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + private final Set databases = new LinkedHashSet<>(); private final Map tables = new LinkedHashMap<>(); private final Map descriptors = new LinkedHashMap<>(); private String lastListenerName; @@ -480,17 +615,49 @@ private static final class TestingMetadataGatewayService extends TestingTabletGa private Integer topicBucketEpoch = 7; private boolean failMetadata; private boolean failNextMetadataAsMissing; + private boolean failListTablesAsFuture; + private String deleteDatabaseBeforeListing; + private RuntimeException listTablesFailure; private TestingMetadataGatewayService() { - putTable("topic", 123L); - putTable("other", 124L); + putTable("kafka.topic", 123L); + putTable("kafka.other", 124L); + } + + @Override + public CompletableFuture listDatabases( + ListDatabasesRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + return CompletableFuture.completedFuture( + new ListDatabasesResponse().addAllDatabaseNames(databases)); } @Override public CompletableFuture listTables(ListTablesRequest request) { - assertThat(request.getDatabaseName()).isEqualTo("kafka"); + assertThat(currentListenerName()).isEqualTo("KAFKA"); + String database = request.getDatabaseName(); + if (database.equals(deleteDatabaseBeforeListing)) { + databases.remove(database); + tables.keySet().removeIf(name -> name.startsWith(database + ".")); + } + RuntimeException failure = listTablesFailure; + if (failure == null && !databases.contains(database)) { + failure = new DatabaseNotExistException("Database does not exist."); + } + if (failure != null) { + if (failListTablesAsFuture) { + return FutureUtils.completedExceptionally(new CompletionException(failure)); + } + throw failure; + } + String prefix = request.getDatabaseName() + "."; + List names = + tables.keySet().stream() + .filter(name -> name.startsWith(prefix)) + .map(name -> name.substring(prefix.length())) + .collect(Collectors.toList()); return CompletableFuture.completedFuture( - new ListTablesResponse().addAllTableNames(new ArrayList<>(tables.keySet()))); + new ListTablesResponse().addAllTableNames(names)); } @Override @@ -509,25 +676,23 @@ public CompletableFuture metadat } List topics = new ArrayList<>(); for (PbTablePath tablePath : request.getTablePathsList()) { - Long tableId = tables.get(tablePath.getTableName()); - if (tableId != null) { - topics.add( - tableMetadata( - tablePath.getTableName(), - tableId, - !"topic".equals(tablePath.getTableName()) - || topicLeaderAvailable, - "topic".equals(tablePath.getTableName()) - ? topicIsr - : new int[] {1, 2}, - "topic".equals(tablePath.getTableName()) - ? topicBucketEpoch - : Integer.valueOf(7)) - .setTableJson( - descriptors - .get(tablePath.getTableName()) - .toJsonBytes())); + String topicName = tablePath.getDatabaseName() + "." + tablePath.getTableName(); + Long tableId = tables.get(topicName); + if (tableId == null) { + throw new TableNotExistException("Table does not exist: " + topicName); } + topics.add( + tableMetadata( + topicName, + tableId, + !"kafka.topic".equals(topicName) || topicLeaderAvailable, + "kafka.topic".equals(topicName) + ? topicIsr + : new int[] {1, 2}, + "kafka.topic".equals(topicName) + ? topicBucketEpoch + : Integer.valueOf(7)) + .setTableJson(descriptors.get(topicName).toJsonBytes())); } return CompletableFuture.completedFuture( new org.apache.fluss.rpc.messages.MetadataResponse() @@ -550,6 +715,7 @@ private void putTable(String topic, long tableId) { } private void putTable(String topic, long tableId, TableDescriptor descriptor) { + databases.add(topic.substring(0, topic.indexOf('.'))); tables.put(topic, tableId); descriptors.put(topic, descriptor); } @@ -568,7 +734,9 @@ private static PbTableMetadata tableMetadata( PbTableMetadata table = new PbTableMetadata() .setTablePath( - new PbTablePath().setDatabaseName("kafka").setTableName(topic)) + new PbTablePath() + .setDatabaseName(topic.substring(0, topic.indexOf('.'))) + .setTableName(topic.substring(topic.indexOf('.') + 1))) .setTableId(tableId) .addAllBucketMetadatas( Arrays.asList( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java new file mode 100644 index 00000000000..2f18ffe495d --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.dropDatabase; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies qualified Metadata discovery over a real Kafka listener and TabletServer. */ +public class KafkaMetadataITCase { + + @RegisterExtension + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfiguration()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + public void testQualifiedMetadataAcrossDatabases() throws Exception { + TableDescriptor descriptor = topicDescriptor(); + long first = createTable(CLUSTER, TablePath.of("first_db", "events"), descriptor); + long second = createTable(CLUSTER, TablePath.of("second_db", "events"), descriptor); + CLUSTER.waitUntilTableReady(first); + CLUSTER.waitUntilTableReady(second); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + query( + node, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList( + "first_db.events", + "second_db.events", + "first_db.missing"))), + version)); + assertThat(named.data().topics().find("first_db.events").partitions()).hasSize(2); + assertThat(named.data().topics().find("second_db.events").partitions()).hasSize(2); + assertThat(named.data().topics().find("first_db.missing").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(named.data().topics().find("first_db.events").topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373000000L, first) : Uuid.ZERO_UUID); + assertThat(named.data().topics().find("second_db.events").topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373000000L, second) : Uuid.ZERO_UUID); + assertThat(named.brokers()).hasSize(1); + assertThat(named.brokers().iterator().next().port()).isEqualTo(node.port()); + + MetadataRequestData allTopics = new MetadataRequestData(); + allTopics.setTopics(version == 0 ? Collections.emptyList() : null); + MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("first_db.events", "second_db.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @Test + public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exception { + TableDescriptor descriptor = topicDescriptor(); + long existing = createTable(CLUSTER, TablePath.of("existing_db", "events"), descriptor); + long deleted = createTable(CLUSTER, TablePath.of("deleted_db", "events"), descriptor); + CLUSTER.waitUntilTableReady(existing); + CLUSTER.waitUntilTableReady(deleted); + dropDatabase(CLUSTER, "deleted_db"); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + + for (short version = 0; version <= 11; version++) { + MetadataResponse missing = + query( + node, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Collections.singletonList( + "deleted_db.events"))), + version)); + assertThat(missing.brokers()).hasSize(1); + assertThat(missing.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + + MetadataResponse mixed = + query( + node, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList( + "deleted_db.events", + "existing_db.events"))), + version)); + assertThat(mixed.brokers()).hasSize(1); + assertThat(mixed.brokers().iterator().next().port()).isEqualTo(node.port()); + assertThat(mixed.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(mixed.data().topics().find("existing_db.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(mixed.data().topics().find("existing_db.events").partitions()).hasSize(2); + + MetadataRequestData allTopics = new MetadataRequestData(); + allTopics.setTopics(version == 0 ? Collections.emptyList() : null); + MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); + assertThat(all.brokers()).hasSize(1); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("existing_db.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + private static TableDescriptor topicDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty("kafka.value.format", "raw") + .build(); + } + + private static Configuration clusterConfiguration() { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.KAFKA_ENABLED, true); + conf.set(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS, 3); + return conf; + } + + private static MetadataResponse query(ServerNode node, MetadataRequest request) + throws Exception { + RequestHeader header = + new RequestHeader(ApiKeys.METADATA, request.version(), "metadata-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] requestBytes = new byte[encoded.remaining()]; + encoded.get(requestBytes); + try (Socket socket = new Socket(node.host(), node.port())) { + socket.setSoTimeout(10000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + byte[] responseBytes = new byte[input.readInt()]; + input.readFully(responseBytes); + return (MetadataResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(responseBytes), header); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index d00c436defe..89b855f7538 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -296,6 +296,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { private static KafkaRequestHandler createKafkaRequestHandler() { TestingTabletGatewayService service = new TestingTabletGatewayService(); - return new KafkaRequestHandler(service, service, "kafka"); + return new KafkaRequestHandler(service, service); } } From 14b73e955af479eeef8ebf40a5a90e37e05d8e98 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 18:14:14 +0800 Subject: [PATCH 17/48] [kafka] Preserve Metadata regressions on the updated PR03 base Keep the ApiVersions wire fallback assertion limited to its retry range, while successful negotiation verifies both registered API ranges. Cover Metadata v0-v11 handling of native gateway visibility filtering in named queries, all-topic discovery, and missing-database retries. Verify Metadata v12 is rejected before invoking the gateway. Retain the existing qualified-topic discovery contract on prerequisite c508de6a8df974727c30539080d5e58f56de930a. New Core-dependent aliases and persisted table enablement remain deferred. Validation: Java 11 reactor verify passed 105 unit tests and 5 integration tests, with Spotless, Checkstyle, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 71/71 --- .../fluss/kafka/KafkaMetadataHandlerTest.java | 66 +++++++++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 5 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 4c33a51635a..53dcbeb9e48 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -480,6 +480,62 @@ public void testRejectUnqualifiedAndMalformedNamesWithoutHidingValidTopics() { .isEqualTo(Errors.NONE.code()); } + @Test + public void testUnauthorizedTopicsRemainHiddenForEveryVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.hiddenTopics.add("kafka.topic"); + for (short version = 0; version <= 11; version++) { + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other"); + for (List names : + Arrays.asList( + Arrays.asList("kafka.topic", "kafka.other"), + Arrays.asList("kafka.topic", "kafka.other", "missing_db.topic"))) { + MetadataResponse named = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest + .convertToMetadataRequestTopic( + names)), + version), + version); + assertThat(named.brokers()).hasSize(2); + MetadataResponseTopic hidden = named.data().topics().find("kafka.topic"); + assertThat(hidden.errorCode()).isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(hidden.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(hidden.partitions()).isEmpty(); + MetadataResponseTopic visible = named.data().topics().find("kafka.other"); + assertThat(visible.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(visible.partitions()).hasSize(2); + assertThat(named.data().topics()).hasSize(names.size()); + if (names.contains("missing_db.topic")) { + assertThat(named.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + } + } + } + + @Test + public void testMetadataV12IsRejectedBeforeCallingGateway() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) + .build((short) 12); + + MetadataResponse response = handle(service, request, (short) 12); + + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + assertThat(service.gatewayCalls).isZero(); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) public void testMissingDatabaseRetainsBrokersAndValidTopics(boolean failedFuture) { @@ -607,8 +663,10 @@ private static MetadataResponse handle( private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { private final Set databases = new LinkedHashSet<>(); + private final Set hiddenTopics = new LinkedHashSet<>(); private final Map tables = new LinkedHashMap<>(); private final Map descriptors = new LinkedHashMap<>(); + private int gatewayCalls; private String lastListenerName; private boolean topicLeaderAvailable = true; private int[] topicIsr = new int[] {1, 2}; @@ -627,6 +685,7 @@ private TestingMetadataGatewayService() { @Override public CompletableFuture listDatabases( ListDatabasesRequest request) { + gatewayCalls++; assertThat(currentListenerName()).isEqualTo("KAFKA"); return CompletableFuture.completedFuture( new ListDatabasesResponse().addAllDatabaseNames(databases)); @@ -634,6 +693,7 @@ public CompletableFuture listDatabases( @Override public CompletableFuture listTables(ListTablesRequest request) { + gatewayCalls++; assertThat(currentListenerName()).isEqualTo("KAFKA"); String database = request.getDatabaseName(); if (database.equals(deleteDatabaseBeforeListing)) { @@ -653,6 +713,7 @@ public CompletableFuture listTables(ListTablesRequest reques String prefix = request.getDatabaseName() + "."; List names = tables.keySet().stream() + .filter(name -> !hiddenTopics.contains(name)) .filter(name -> name.startsWith(prefix)) .map(name -> name.substring(prefix.length())) .collect(Collectors.toList()); @@ -663,6 +724,7 @@ public CompletableFuture listTables(ListTablesRequest reques @Override public CompletableFuture metadata( org.apache.fluss.rpc.messages.MetadataRequest request) { + gatewayCalls++; lastListenerName = currentListenerName(); if (failMetadata) { CompletableFuture failure = @@ -677,6 +739,10 @@ public CompletableFuture metadat List topics = new ArrayList<>(); for (PbTablePath tablePath : request.getTablePathsList()) { String topicName = tablePath.getDatabaseName() + "." + tablePath.getTableName(); + // Native Metadata and listTables omit tables without DESCRIBE permission. + if (hiddenTopics.contains(topicName)) { + continue; + } Long tableId = tables.get(topicName); if (tableId == null) { throw new TableNotExistException("Table does not exist: " + topicName); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 89b855f7538..0d06962e301 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -77,7 +77,10 @@ public void testUnsupportedWireVersionFallsBackAndAllowsRenegotiation() { ApiVersionsResponse unsupportedResponse = parseWireResponse(channel, (short) 0, 41); assertThat(unsupportedResponse.data().errorCode()) .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); - assertBrokerCapabilities(unsupportedResponse); + // The version error only advertises the range needed to retry ApiVersions. + assertThat(unsupportedResponse.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); ApiVersionsRequestData data = new ApiVersionsRequestData() From 02d5074506658b85cd1efda9cc65c65a692dec1e Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 11:14:01 +0800 Subject: [PATCH 18/48] [test] Wait for coordinator metadata before snapshot lease retry Wait for all server gateways to advertise the restarted coordinator before acquire, release and drop lease checks. Assert that the client still holds its cached address so each operation exercises metadata refresh and retry. Rebased PR04 onto PR03 at 86d2735ba6c4f61eddf9d86ad9b4ef3732714972. Validation: restart case passed in the full Admin suite and focused rerun; Kafka 110 tests, RPC retry 6 tests, Spotless and Checkstyle passed. The full Admin suite passed 46/47; the unchanged rolling-upgrade health test first observed RED instead of GREEN and passed the subsequent focused rerun. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 11/11 --- .../apache/fluss/client/admin/FlussAdminITCase.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index b39739c6b01..8938993fa5a 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -19,8 +19,10 @@ import org.apache.fluss.client.Connection; import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.FlussConnection; import org.apache.fluss.client.metadata.KvSnapshotMetadata; import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.MetadataUpdater; import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.writer.AppendWriter; @@ -1211,19 +1213,24 @@ void testKvSnapshotLeaseAfterCoordinatorServerRestart() throws Exception { assertThat(zkClient.getKvSnapshotLeaseMetadata(lease.leaseId())).isPresent(); restartCoordinatorServer(zkClient); - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); - lease.dropLease().get(); assertThat(zkClient.getKvSnapshotLeaseMetadata(lease.leaseId())).isNotPresent(); } private void restartCoordinatorServer(ZooKeeperClient zkClient) throws Exception { + MetadataUpdater metadataUpdater = ((FlussConnection) conn).getMetadataUpdater(); + ServerNode cachedCoordinator = metadataUpdater.getCoordinatorServer(); FLUSS_CLUSTER_EXTENSION.stopCoordinatorServer(); waitUntil( () -> !zkClient.getCoordinatorLeaderAddress().isPresent(), Duration.ofMinutes(1), "Coordinator server node still exists in ZooKeeper"); FLUSS_CLUSTER_EXTENSION.startCoordinatorServer(); + // Election completes before the new coordinator address reaches every tablet server. + // Wait for server metadata to converge so that the client's retry can discover it. + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + // Keep the client cache stale to exercise the lease's metadata refresh and retry. + assertThat(metadataUpdater.getCoordinatorServer()).isEqualTo(cachedCoordinator); } @Test From 543932966b23977640c6018587ffa2dca44624e0 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 17:40:33 +0800 Subject: [PATCH 19/48] [kafka] Apply persisted topic enablement to Metadata Use the actual server default database and the PR03 shared mapper/schema contract. Resolve aliases by TablePath, preserve requested names, and retain valid aliases when a missing database triggers metadata discovery retry. Expose only tables with persisted table.kafka.enabled. Return unknown-topic for disabled or missing tables and invalid-topic for damaged enabled contracts in named and all-topic requests. Discovery emits canonical names without duplicate aliases and retains native authorization and routing. Verify configuration injection, no fallback lookup or database creation, enablement/reset changes, invalid historical contracts and v0-v11 behavior through real Kafka listeners. Preserve the coordinator snapshot lease fix. Validation: Java 11 clean reactor verify passed 160 distinct cases: 118 Kafka (including 6 Metadata integration cases), 31 shared schema, 3 atomic DDL, 6 RPC retry and 2 snapshot lease cases. Spotless, Checkstyle and RAT passed. The four existing PR04 commits were rebased unchanged onto PR03 4a9aa6b007053ba7af5e8c0a0222a106c477a9f2 and checked with range-diff. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 155/155 AI-Contributed/UT: 571/571 --- .../fluss/kafka/KafkaProtocolPlugin.java | 2 +- .../fluss/kafka/KafkaRequestHandler.java | 10 +- .../kafka/api/metadata/MetadataHandler.java | 18 +- .../metadata/GatewayKafkaMetadataBackend.java | 124 +++---- .../fluss/kafka/KafkaMetadataHandlerTest.java | 265 ++++++++++++++- .../fluss/kafka/KafkaMetadataITCase.java | 303 +++++++++++++++++- .../fluss/kafka/KafkaRequestHandlerTest.java | 3 +- 7 files changed, 642 insertions(+), 83 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index dddfff1f9eb..d0363e3f6f4 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -67,6 +67,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { + service.getClass().getSimpleName()); } TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(service, gateway); + return new KafkaRequestHandler(conf, service, gateway); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 3278d8328a2..46dc00355bf 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,12 +17,14 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; import org.apache.fluss.kafka.api.metadata.MetadataHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; @@ -39,15 +41,19 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ - public KafkaRequestHandler(RpcGatewayService service, TabletServerGateway gateway) { + public KafkaRequestHandler( + Configuration conf, RpcGatewayService service, TabletServerGateway gateway) { checkNotNull(service); checkNotNull(gateway); + KafkaTopicMapper topicMapper = new KafkaTopicMapper(checkNotNull(conf)); this.registry = new KafkaApiRegistry( Arrays.asList( new ApiVersionsHandler(this::apiRegistry), new MetadataHandler( - new GatewayKafkaMetadataBackend(service, gateway)))); + new GatewayKafkaMetadataBackend( + service, gateway, topicMapper), + topicMapper))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java index 2ceca11fbe1..2787cf83124 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -32,6 +32,7 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.internals.Topic; import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; import org.apache.kafka.common.message.MetadataResponseData; @@ -64,10 +65,12 @@ public final class MetadataHandler implements KafkaApiHandler { true); private final KafkaMetadataBackend backend; + private final KafkaTopicMapper topicMapper; /** Creates a Metadata handler. */ - public MetadataHandler(KafkaMetadataBackend backend) { + public MetadataHandler(KafkaMetadataBackend backend, KafkaTopicMapper topicMapper) { this.backend = checkNotNull(backend); + this.topicMapper = checkNotNull(topicMapper); } @Override @@ -86,7 +89,7 @@ public CompletableFuture handle( throw new InvalidRequestException( "Topic name must be set because topic ID lookup is not supported by " + "Metadata versions 10 and 11."); - } else if (!KafkaTopicMapper.isValidTopic(topic.name())) { + } else if (!isValidTopic(topic.name())) { invalidTopics.add( new KafkaClusterMetadata.Topic( topic.name(), @@ -119,6 +122,17 @@ public CompletableFuture handle( }); } + private boolean isValidTopic(String name) { + try { + // Validate the resolved path too: a bare name's canonical form includes the configured + // default database and must fit the same Kafka name limit as an explicit name. + topicMapper.toTablePath(name); + return true; + } catch (InvalidTopicException e) { + return false; + } + } + private static MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { MetadataResponseData data = new MetadataResponseData() diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java index 202658b9f0f..0d4ed5a78c3 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -44,9 +44,12 @@ import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -69,41 +72,42 @@ public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { private final RpcGatewayService service; private final TabletServerGateway gateway; - private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); + private final KafkaTopicMapper topicMapper; private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); /** Creates a metadata backend backed by the local TabletServer gateway. */ - public GatewayKafkaMetadataBackend(RpcGatewayService service, TabletServerGateway gateway) { + public GatewayKafkaMetadataBackend( + RpcGatewayService service, TabletServerGateway gateway, KafkaTopicMapper topicMapper) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); + this.topicMapper = checkNotNull(topicMapper); } @Override public CompletableFuture getMetadata(KafkaMetadataQuery query) { if (query.allTopics() || containsTopicId(query.topics())) { - return currentTopicNames(query) - .thenCompose(topicNames -> requestFlussMetadata(query, topicNames)); + return currentTablePaths(query) + .thenCompose(tablePaths -> requestFlussMetadata(query, tablePaths)); } - Set topicNames = new LinkedHashSet<>(); + Set tablePaths = new LinkedHashSet<>(); for (TopicReference topic : query.topics()) { if (topic.topicName() != null) { - topicNames.add(topic.topicName()); + tablePaths.add(topicMapper.toTablePath(topic.topicName())); } } - return requestFlussMetadata(query, topicNames); + return requestFlussMetadata(query, tablePaths); } private CompletableFuture requestFlussMetadata( - KafkaMetadataQuery query, Set topicNames) { - return requestFlussMetadata(query, topicNames, true); + KafkaMetadataQuery query, Set tablePaths) { + return requestFlussMetadata(query, tablePaths, true); } private CompletableFuture requestFlussMetadata( - KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { + KafkaMetadataQuery query, Set tablePaths, boolean refreshAndRetry) { MetadataRequest request = new MetadataRequest(); - for (String topicName : topicNames) { - TablePath tablePath = topicMapper.toTablePath(topicName); + for (TablePath tablePath : tablePaths) { request.addAllTablePaths( Collections.singletonList( new PbTablePath() @@ -129,8 +133,8 @@ private CompletableFuture requestFlussMetadata( private CompletableFuture recoverMetadataFailure( KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) { if (refreshAndRetry) { - return currentTopicNames(query) - .thenCompose(currentNames -> requestFlussMetadata(query, currentNames, false)); + return currentTablePaths(query) + .thenCompose(currentPaths -> requestFlussMetadata(query, currentPaths, false)); } LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure)); CompletableFuture failed = new CompletableFuture<>(); @@ -138,7 +142,7 @@ private CompletableFuture recoverMetadataFailure( return failed; } - private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { + private CompletableFuture> currentTablePaths(KafkaMetadataQuery query) { CompletableFuture> databasesFuture; if (query.allTopics() || containsTopicId(query.topics())) { setCurrentSession(query); @@ -157,49 +161,46 @@ private CompletableFuture> currentTopicNames(KafkaMetadataQuery quer databasesFuture = CompletableFuture.completedFuture(databases); } return databasesFuture - .thenCompose(databases -> listTopicNames(query, databases)) + .thenCompose(databases -> listTablePaths(query, databases)) .thenApply( - currentNames -> { + currentPaths -> { if (!query.allTopics() && !containsTopicId(query.topics())) { - Set requestedNames = new HashSet<>(); + Set requestedTables = new HashSet<>(); for (TopicReference topic : query.topics()) { if (topic.topicName() != null) { - requestedNames.add(topic.topicName()); + requestedTables.add( + topicMapper.toTablePath(topic.topicName())); } } - currentNames.retainAll(requestedNames); + // Canonical names and aliases share one table identity, including + // when a missing database triggers this discovery retry. + currentPaths.retainAll(requestedTables); } - return currentNames; + return currentPaths; }); } - private CompletableFuture> listTopicNames( + private CompletableFuture> listTablePaths( KafkaMetadataQuery query, Set databases) { - CompletableFuture> topicsFuture = + CompletableFuture> tablesFuture = CompletableFuture.completedFuture(new LinkedHashSet<>()); for (String database : databases) { - topicsFuture = - topicsFuture.thenCompose( - topicNames -> + tablesFuture = + tablesFuture.thenCompose( + tablePaths -> listTables(query, database) .thenApply( response -> { for (String tableName : response.getTableNamesList()) { - TablePath tablePath = + tablePaths.add( TablePath.of( - database, tableName); - if (topicMapper.isMappedTable( - tablePath)) { - topicNames.add( - topicMapper.toTopicName( - tablePath)); - } + database, tableName)); } - return topicNames; + return tablePaths; })); } - return topicsFuture; + return tablesFuture; } private CompletableFuture listTables( @@ -238,37 +239,40 @@ private KafkaClusterMetadata toKafkaMetadata( } Collections.sort(brokers, Comparator.comparingInt(Broker::id)); - Map topicsByName = new HashMap<>(); + Map topicsByTable = new HashMap<>(); Map topicsById = new HashMap<>(); for (PbTableMetadata table : response.getTableMetadatasList()) { TablePath tablePath = TablePath.of( table.getTablePath().getDatabaseName(), table.getTablePath().getTableName()); - if (!topicMapper.isMappedTable(tablePath)) { - continue; + Topic topic = toKafkaTopic(table, tablePath, aliveBrokerIds); + if (topic != null) { + topicsByTable.put(tablePath, topic); + topicsById.put(topic.topicId(), topic); } - Topic topic = toKafkaTopic(table, topicMapper.toTopicName(tablePath), aliveBrokerIds); - topicsByName.put(topic.name(), topic); - topicsById.put(topic.topicId(), topic); } List topics = new ArrayList<>(); if (query.allTopics()) { - for (Topic topic : topicsByName.values()) { - if (topic.error() == TopicError.NONE) { - topics.add(topic); - } - } + // Invalid enabled tables remain visible as errors. Only disabled tables are omitted. + topics.addAll(topicsByTable.values()); Collections.sort(topics, Comparator.comparing(Topic::name)); } else { for (TopicReference reference : query.topics()) { Topic topic = reference.hasTopicId() ? topicsById.get(reference.topicId()) - : topicsByName.get(reference.topicName()); + : topicsByTable.get(topicMapper.toTablePath(reference.topicName())); if (topic != null && matches(reference, topic)) { - topics.add(topic); + topics.add( + new Topic( + reference.topicName() == null + ? topic.name() + : reference.topicName(), + topic.topicId(), + topic.error(), + topic.partitions())); } else { topics.add(missingTopic(reference)); } @@ -277,13 +281,20 @@ private KafkaClusterMetadata toKafkaMetadata( return new KafkaClusterMetadata(brokers, topics); } + @Nullable private Topic toKafkaTopic( - PbTableMetadata table, String topicName, Set aliveBrokerIds) { + PbTableMetadata table, TablePath tablePath, Set aliveBrokerIds) { + String topicName = tablePath.toString(); try { - schemaResolver.resolve(TableDescriptor.fromJsonBytes(table.getTableJson())); - } catch (IllegalArgumentException e) { + TableDescriptor descriptor = TableDescriptor.fromJsonBytes(table.getTableJson()); + if (!schemaResolver.isEnabled(descriptor)) { + return null; + } + topicMapper.toTopicName(tablePath); + schemaResolver.resolve(descriptor); + } catch (IllegalArgumentException | InvalidTopicException e) { LOG.debug( - "Table {} does not define a supported Kafka mapping: {}", + "Enabled table {} has an invalid Kafka contract: {}", topicName, e.getMessage()); return new Topic( @@ -342,8 +353,11 @@ private static Topic missingTopic(TopicReference reference) { reference.topicName(), reference.topicId(), error, Collections.emptyList()); } - private static boolean matches(TopicReference reference, Topic topic) { - return (reference.topicName() == null || reference.topicName().equals(topic.name())) + private boolean matches(TopicReference reference, Topic topic) { + return (reference.topicName() == null + || topicMapper + .toTablePath(reference.topicName()) + .equals(topicMapper.toTablePath(topic.name()))) && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId())); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 53dcbeb9e48..bd06a8cb78e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -17,9 +17,10 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.exception.TableNotExistException; -import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; @@ -347,14 +348,17 @@ public void testMetadataUsesDdlContractForEveryVersion() { "kafka.no_mapping", 125L, TableDescriptor.builder() - .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) .distributedBy(2) .build()); service.putTable( "kafka.bad_format", 126L, TableDescriptor.builder(defaultDescriptor()) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "unimplemented") .build()); service.putTable( "kafka.primary_key", @@ -362,14 +366,14 @@ public void testMetadataUsesDdlContractForEveryVersion() { TableDescriptor.builder(defaultDescriptor()) .schema( Schema.newBuilder() - .column("body", DataTypes.BYTES().copy(false)) - .primaryKey("body") + .column("_kafka_value", DataTypes.BYTES().copy(false)) + .primaryKey("_kafka_value") .build()) .build()); service.putTable( "kafka.partitioned", 128L, - TableDescriptor.builder(defaultDescriptor()).partitionedBy("body").build()); + TableDescriptor.builder(defaultDescriptor()).partitionedBy("_kafka_value").build()); service.putTable( "kafka.indexed", 129L, @@ -377,17 +381,24 @@ public void testMetadataUsesDdlContractForEveryVersion() { service.putTable("kafka.invalid topic", 130L); List invalidMappings = Arrays.asList( - "kafka.no_mapping", "kafka.bad_format", "kafka.primary_key", "kafka.partitioned", - "kafka.indexed"); + "kafka.indexed", + "kafka.invalid topic"); for (short version = 0; version <= 11; version++) { MetadataResponse all = handle(service, allTopicsRequest(version), version); + List exposed = new ArrayList<>(invalidMappings); + exposed.addAll(Arrays.asList("kafka.other", "kafka.topic")); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactly("kafka.other", "kafka.topic"); + .containsExactlyInAnyOrderElementsOf(exposed); + for (String name : invalidMappings) { + assertThat(all.data().topics().find(name).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + } List requested = new ArrayList<>(invalidMappings); + requested.add("kafka.no_mapping"); requested.add("kafka.topic"); requested.add("kafka.missing"); MetadataResponse named = @@ -409,6 +420,8 @@ public void testMetadataUsesDdlContractForEveryVersion() { .isEqualTo(Errors.NONE.code()); assertThat(named.data().topics().find("kafka.missing").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(named.data().topics().find("kafka.no_mapping").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); } } @@ -419,7 +432,7 @@ public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { "kafka.topic", 123L, TableDescriptor.builder(defaultDescriptor()) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "string") .build()); MetadataResponse invalid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); assertThat(invalid.data().topics().find("kafka.topic").errorCode()) @@ -462,9 +475,9 @@ public void testSameTableNameInDifferentDatabasesForEveryVersion() { } @Test - public void testRejectUnqualifiedAndMalformedNamesWithoutHidingValidTopics() { + public void testRejectMalformedNamesWithoutHidingValidTopics() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - List invalidNames = Arrays.asList("topic", ".topic", "kafka.", "kafka.topic.extra"); + List invalidNames = Arrays.asList(".topic", "kafka.", "kafka.topic.extra"); List names = new ArrayList<>(invalidNames); names.add("kafka.topic"); MetadataResponse response = @@ -603,12 +616,220 @@ public void testUnexpectedListTablesFailureIsNotTreatedAsMissingDatabase(boolean .isEqualTo(Errors.UNKNOWN_SERVER_ERROR.code()); } + @ParameterizedTest + @ValueSource(strings = {"fluss", "custom_db"}) + public void testAliasesPreserveRequestedNamesAndShareIdentity(String defaultDatabase) { + Configuration config = new Configuration(); + if (!"fluss".equals(defaultDatabase)) { + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, defaultDatabase); + } + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + String canonical = defaultDatabase + ".topic"; + service.putTable(canonical, 777L); + service.putTable("sales.topic", 778L); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle( + service, + topicsRequest(version, "topic", canonical, "sales.topic"), + version, + config); + assertThat(named.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("topic", canonical, "sales.topic"); + assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = named.data().topics().find("topic"); + MetadataResponseTopic qualified = named.data().topics().find(canonical); + assertThat(alias.partitions()).isEqualTo(qualified.partitions()); + assertThat(alias.topicId()).isEqualTo(qualified.topicId()); + if (version >= 10) { + assertThat(alias.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)) + .isNotEqualTo(named.data().topics().find("sales.topic").topicId()); + } + MetadataResponse all = handle(service, allTopicsRequest(version), version, config); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .contains(canonical, "sales.topic") + .doesNotContain("topic") + .doesNotHaveDuplicates(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testAliasSurvivesMissingDatabaseRetry(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + service.failListTablesAsFuture = failedFuture; + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + handle( + service, + topicsRequest( + version, + "topic", + "missing_db.topic", + "fluss.topic", + "kafka.topic"), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(4); + for (String name : Arrays.asList("topic", "fluss.topic", "kafka.topic")) { + assertThat(response.data().topics().find(name).errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(response.data().topics().find(name).partitions()).hasSize(2); + } + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("topic").topicId()) + .isEqualTo(response.data().topics().find("fluss.topic").topicId()); + } + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.listedDatabases) + .containsExactlyInAnyOrder("fluss", "kafka", "missing_db"); + } + + @Test + public void testMissingDefaultDatabaseNeverSearchesOtherDatabases() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + MetadataResponse response = + handle( + service, + topicsRequest((short) 11, "topic", "kafka.topic"), + (short) 11, + config); + assertThat(response.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.requestedTables) + .containsExactlyInAnyOrder("missing_default.topic", "kafka.topic"); + assertThat(service.listedDatabases).containsExactlyInAnyOrder("missing_default", "kafka"); + } + + @Test + public void testAliasAuthorizationDoesNotFallbackToSameNamedVisibleTable() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + service.hiddenTopics.add("fluss.topic"); + service.putTable("sales.topic", 778L); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + handle( + service, + topicsRequest( + version, + "topic", + "fluss.topic", + "sales.topic", + "missing_db.topic"), + version); + for (String hidden : Arrays.asList("topic", "fluss.topic")) { + MetadataResponseTopic topic = response.data().topics().find(hidden); + assertThat(topic.errorCode()).isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + } + assertThat(response.data().topics().find("sales.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .contains("sales.topic") + .doesNotContain("topic", "fluss.topic"); + } + assertThat(service.requestedTables).doesNotContain("sales.other"); + } + + @Test + public void testPersistedEnablementControlsBothAliasesAndDiscovery() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + TableDescriptor missingFlag = + TableDescriptor.builder() + .schema(defaultDescriptor().getSchema()) + .distributedBy(2) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + TableDescriptor disabledInvalid = + TableDescriptor.builder(defaultDescriptor()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "unimplemented") + .build(); + for (TableDescriptor disabled : Arrays.asList(missingFlag, disabledInvalid)) { + service.putTable("fluss.topic", 777L, disabled); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + assertThat(named.errors()) + .containsEntry("topic", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("fluss.topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(named.data().topics()) + .allSatisfy( + topic -> { + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + }); + assertThat(handle(service, allTopicsRequest(version), version).data().topics()) + .extracting(MetadataResponseTopic::name) + .doesNotContain("topic", "fluss.topic"); + } + } + service.putTable("fluss.topic", 777L); + MetadataResponse enabled = + handle(service, topicsRequest((short) 11, "topic", "fluss.topic"), (short) 11); + assertThat(enabled.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(enabled.data().topics().find("topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)); + service.putTable("fluss.topic", 777L, disabledInvalid); + assertThat(handle(service, namedTopicRequest("topic"), (short) 11).errors()) + .containsEntry("topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + + @Test + public void testResolvedAliasNameLimitDoesNotHideValidTopic() { + String database = String.join("", Collections.nCopies(125, "d")); + String table = String.join("", Collections.nCopies(125, "t")); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, database); + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + MetadataResponse response = + handle( + service, + topicsRequest((short) 11, table, "kafka.topic"), + (short) 11, + config); + assertThat(response.errors()).containsEntry(table, Errors.INVALID_TOPIC_EXCEPTION); + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(service.requestedTables).containsExactly("kafka.topic"); + + // A historical enabled table with an unrepresentable canonical name must be visible as + // an error during discovery rather than silently classified as disabled. + service.putTable(database + "." + table, 777L); + assertThat(handle(service, allTopicsRequest((short) 11), (short) 11, config).errors()) + .containsEntry(database + "." + table, Errors.INVALID_TOPIC_EXCEPTION); + } + + private static MetadataRequest topicsRequest(short version, String... names) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList(names))), + version); + } + private static TableDescriptor defaultDescriptor() { return TableDescriptor.builder() - .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) .distributedBy(2) .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") .build(); } @@ -631,7 +852,15 @@ private static MetadataRequest allTopicsRequest(short version) { private static MetadataResponse handle( TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { - KafkaRequestHandler handler = new KafkaRequestHandler(service, service); + return handle(service, requestBody, version, new Configuration()); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, + MetadataRequest requestBody, + short version, + Configuration config) { + KafkaRequestHandler handler = new KafkaRequestHandler(config, service, service); ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); KafkaRequest request; try { @@ -667,6 +896,9 @@ private static final class TestingMetadataGatewayService extends TestingTabletGa private final Map tables = new LinkedHashMap<>(); private final Map descriptors = new LinkedHashMap<>(); private int gatewayCalls; + private int listDatabasesCalls; + private final Set requestedTables = new LinkedHashSet<>(); + private final Set listedDatabases = new LinkedHashSet<>(); private String lastListenerName; private boolean topicLeaderAvailable = true; private int[] topicIsr = new int[] {1, 2}; @@ -686,6 +918,7 @@ private TestingMetadataGatewayService() { public CompletableFuture listDatabases( ListDatabasesRequest request) { gatewayCalls++; + listDatabasesCalls++; assertThat(currentListenerName()).isEqualTo("KAFKA"); return CompletableFuture.completedFuture( new ListDatabasesResponse().addAllDatabaseNames(databases)); @@ -696,6 +929,7 @@ public CompletableFuture listTables(ListTablesRequest reques gatewayCalls++; assertThat(currentListenerName()).isEqualTo("KAFKA"); String database = request.getDatabaseName(); + listedDatabases.add(database); if (database.equals(deleteDatabaseBeforeListing)) { databases.remove(database); tables.keySet().removeIf(name -> name.startsWith(database + ".")); @@ -739,6 +973,7 @@ public CompletableFuture metadat List topics = new ArrayList<>(); for (PbTablePath tablePath : request.getTablePathsList()) { String topicName = tablePath.getDatabaseName() + "." + tablePath.getTableName(); + requestedTables.add(topicName); // Native Metadata and listTables omit tables without DESCRIBE permission. if (hiddenTopics.contains(topicName)) { continue; diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java index 2f18ffe495d..d162ed837a4 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -25,6 +25,7 @@ import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.data.TableRegistration; import org.apache.fluss.types.DataTypes; import org.apache.kafka.common.Uuid; @@ -43,11 +44,17 @@ import java.io.DataOutputStream; import java.net.Socket; import java.nio.ByteBuffer; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.dropDatabase; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newDatabaseExistsRequest; +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.assertj.core.api.Assertions.assertThat; /** Verifies qualified Metadata discovery over a real Kafka listener and TabletServer. */ @@ -56,7 +63,7 @@ public class KafkaMetadataITCase { @RegisterExtension public static final FlussClusterExtension CLUSTER = FlussClusterExtension.builder() - .setNumOfTabletServers(1) + .setNumOfTabletServers(2) .setClusterConf(clusterConfiguration()) .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") .build(); @@ -93,7 +100,7 @@ public void testQualifiedMetadataAcrossDatabases() throws Exception { assertThat(named.data().topics().find("second_db.events").topicId()) .isEqualTo( version >= 10 ? new Uuid(0x466c757373000000L, second) : Uuid.ZERO_UUID); - assertThat(named.brokers()).hasSize(1); + assertThat(named.brokers()).hasSize(2); assertThat(named.brokers().iterator().next().port()).isEqualTo(node.port()); MetadataRequestData allTopics = new MetadataRequestData(); @@ -127,7 +134,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio Collections.singletonList( "deleted_db.events"))), version)); - assertThat(missing.brokers()).hasSize(1); + assertThat(missing.brokers()).hasSize(2); assertThat(missing.data().topics().find("deleted_db.events").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); @@ -142,7 +149,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio "deleted_db.events", "existing_db.events"))), version)); - assertThat(mixed.brokers()).hasSize(1); + assertThat(mixed.brokers()).hasSize(2); assertThat(mixed.brokers().iterator().next().port()).isEqualTo(node.port()); assertThat(mixed.data().topics().find("deleted_db.events").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); @@ -153,7 +160,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio MetadataRequestData allTopics = new MetadataRequestData(); allTopics.setTopics(version == 0 ? Collections.emptyList() : null); MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); - assertThat(all.brokers()).hasSize(1); + assertThat(all.brokers()).hasSize(2); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) .containsExactly("existing_db.events"); @@ -161,18 +168,300 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio } } + @Test + public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception { + long other = createTable(CLUSTER, TablePath.of("sales", "events"), topicDescriptor()); + CLUSTER.waitUntilTableReady(other); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + for (short version = 0; version <= 11; version++) { + MetadataResponse before = query(node, named(version, "events", "sales.events")); + assertThat(before.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(before.data().topics().find("sales.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + long tableId = createTable(CLUSTER, TablePath.of("fluss", "events"), topicDescriptor()); + CLUSTER.waitUntilTableReady(tableId); + for (short version = 0; version <= 11; version++) { + for (ServerNode broker : CLUSTER.getTabletServerNodes("KAFKA")) { + // The missing database forces the same discovery retry that used to discard + // aliases. + MetadataResponse response = + query( + broker, + named( + version, + "events", + "fluss.events", + "sales.events", + "missing_db.events")); + MetadataResponseTopic alias = response.data().topics().find("events"); + MetadataResponseTopic canonical = response.data().topics().find("fluss.events"); + assertThat(alias.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(alias.partitions()).isEqualTo(canonical.partitions()).hasSize(2); + assertThat(alias.partitions()) + .allSatisfy( + partition -> { + assertThat(partition.leaderId()).isIn(0, 1); + assertThat(partition.replicaNodes()) + .containsExactlyInAnyOrder(0, 1); + assertThat(partition.isrNodes()) + .containsExactlyInAnyOrder(0, 1); + }); + assertThat(alias.topicId()).isEqualTo(canonical.topicId()); + if (version >= 10) { + assertThat(alias.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, tableId)) + .isNotEqualTo(response.data().topics().find("sales.events").topicId()); + } + assertThat(response.data().topics().find("missing_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(query(broker, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events", "sales.events"); + } + } + } + + @Test + public void testPersistedEnablementChangesAreVisibleThroughBothNames() throws Exception { + TablePath path = TablePath.of("fluss", "events"); + TableDescriptor disabled = + TableDescriptor.builder(topicDescriptor()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build(); + TableDescriptor unset = + TableDescriptor.builder() + .schema(topicDescriptor().getSchema()) + .distributedBy(2) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + long tableId = createTable(CLUSTER, path, disabled); + long unsetId = createTable(CLUSTER, TablePath.of("sales", "unset"), unset); + CLUSTER.waitUntilTableReady(tableId); + CLUSTER.waitUntilTableReady(unsetId); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + assertExposure(node, false, tableId); + setEnabled(CLUSTER, path, true); + assertExposure(node, true, tableId); + setEnabled(CLUSTER, path, false); + assertExposure(node, false, tableId); + setEnabled(CLUSTER, path, true); + assertExposure(node, true, tableId); + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList(ConfigOptions.TABLE_KAFKA_ENABLED.key()), + Collections.emptyList(), + false)) + .get(); + assertExposure(node, false, tableId); + } + + @Test + public void testInvalidEnabledContractRemainsVisibleInNamedAndAllTopics() throws Exception { + TablePath path = TablePath.of("fluss", "events"); + long tableId = createTable(CLUSTER, path, topicDescriptor()); + long validId = createTable(CLUSTER, TablePath.of("sales", "valid"), topicDescriptor()); + CLUSTER.waitUntilTableReady(tableId); + CLUSTER.waitUntilTableReady(validId); + TableRegistration registration = CLUSTER.getZooKeeperClient().getTable(path).get(); + Map invalidProperties = new HashMap<>(registration.properties); + invalidProperties.put(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "string"); + // Bypass current DDL validation to model historical persisted metadata with a damaged + // contract. Metadata must expose the enabled table's error, not hide it as disabled. + CLUSTER.getZooKeeperClient() + .updateTable( + path, + registration.newProperties( + invalidProperties, registration.customProperties)); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + query(node, named((short) 11, "events")) + .data() + .topics() + .find("events") + .errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code())); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + query(node, named(version, "events", "fluss.events", "sales.valid")); + for (String topic : Arrays.asList("events", "fluss.events")) { + assertThat(named.data().topics().find(topic).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(named.data().topics().find(topic).partitions()).isEmpty(); + } + MetadataResponse all = query(node, allTopics(version)); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events", "sales.valid"); + assertThat(all.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(all.data().topics().find("sales.valid").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + } + + @Test + public void testCustomDefaultDatabaseIsInjectedAndNeverCreatedByMetadata() throws Exception { + Configuration config = clusterConfiguration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "custom_default"); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + FlussClusterExtension cluster = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(config) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + cluster.start(); + try { + long nativeDefaultId = + createTable(cluster, TablePath.of("fluss", "events"), topicDescriptor()); + cluster.waitUntilTableReady(nativeDefaultId); + ServerNode node = cluster.getTabletServerInfos().get(0).node("KAFKA"); + assertThat( + cluster.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("custom_default")) + .get() + .isExists()) + .isFalse(); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = query(node, named(version, "events", "fluss.events")); + assertThat(response.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + long customId = + createTable( + cluster, TablePath.of("custom_default", "events"), topicDescriptor()); + cluster.waitUntilTableReady(customId); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + query( + node, + named(version, "events", "custom_default.events", "fluss.events")); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = response.data().topics().find("events"); + assertThat(alias.partitions()) + .isEqualTo( + response.data() + .topics() + .find("custom_default.events") + .partitions()); + if (version >= 10) { + assertThat(alias.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, customId)) + .isNotEqualTo(response.data().topics().find("fluss.events").topicId()); + } + assertThat(query(node, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("custom_default.events", "fluss.events"); + } + dropDatabase(cluster, "custom_default"); + MetadataResponse deleted = query(node, named((short) 11, "events", "fluss.events")); + assertThat(deleted.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(deleted.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat( + cluster.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("custom_default")) + .get() + .isExists()) + .isFalse(); + } finally { + cluster.close(); + } + } + + private static void assertExposure(ServerNode node, boolean enabled, long tableId) + throws Exception { + short expected = enabled ? Errors.NONE.code() : Errors.UNKNOWN_TOPIC_OR_PARTITION.code(); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + query(node, named((short) 11, "events")) + .data() + .topics() + .find("events") + .errorCode()) + .isEqualTo(expected)); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + query(node, named(version, "events", "fluss.events", "sales.unset")); + for (String name : Arrays.asList("events", "fluss.events")) { + MetadataResponseTopic topic = response.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(expected); + assertThat(topic.topicId()) + .isEqualTo( + enabled && version >= 10 + ? new Uuid(0x466c757373000000L, tableId) + : Uuid.ZERO_UUID); + } + assertThat(response.data().topics().find("sales.unset").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + if (enabled) { + assertThat(query(node, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events"); + } else { + assertThat(query(node, allTopics(version)).data().topics()).isEmpty(); + } + } + } + + private static void setEnabled(FlussClusterExtension cluster, TablePath path, boolean enabled) + throws Exception { + cluster.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled)), + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private static MetadataRequest named(short version, String... names) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList(names))), + version); + } + + private static MetadataRequest allTopics(short version) { + return new MetadataRequest( + new MetadataRequestData().setTopics(version == 0 ? Collections.emptyList() : null), + version); + } + private static TableDescriptor topicDescriptor() { return TableDescriptor.builder() - .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) .distributedBy(2) .logFormat(LogFormat.ARROW) - .customProperty("kafka.value.format", "raw") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") .build(); } private static Configuration clusterConfiguration() { Configuration conf = new Configuration(); conf.set(ConfigOptions.KAFKA_ENABLED, true); + conf.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 2); conf.set(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS, 3); return conf; } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 0d06962e301..f55b1832c0b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; @@ -299,6 +300,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { private static KafkaRequestHandler createKafkaRequestHandler() { TestingTabletGatewayService service = new TestingTabletGatewayService(); - return new KafkaRequestHandler(service, service); + return new KafkaRequestHandler(new Configuration(), service, service); } } From 2cf97cd9aae567fbb41d253c07bc79952367b4d3 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 18:44:54 +0800 Subject: [PATCH 20/48] [kafka] Give topic aliases distinct stable wire IDs Keep canonical topic IDs unchanged and encode alias wire IDs in a separate namespace using the same Fluss table ID. Kafka clients can then cache both names and detect table recreation even when the new leader epoch is lower. Cover Kafka Metadata v10/v11 full and partial updates, recreation from epoch 5 to 0 without a missing-topic response, and native Metadata responses. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 31/31 AI-Contributed/UT: 135/135 --- .../kafka/api/metadata/MetadataHandler.java | 12 ++- .../fluss/kafka/mapping/KafkaTopicMapper.java | 19 +++- .../fluss/kafka/KafkaMetadataHandlerTest.java | 96 ++++++++++++++++++- .../fluss/kafka/KafkaMetadataITCase.java | 23 ++++- .../kafka/mapping/KafkaTopicMapperTest.java | 16 ++++ 5 files changed, 155 insertions(+), 11 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java index 2787cf83124..bffc28d6de2 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -133,7 +133,7 @@ private boolean isValidTopic(String name) { } } - private static MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { + private MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { MetadataResponseData data = new MetadataResponseData() .setThrottleTimeMs(0) @@ -155,7 +155,15 @@ private static MetadataResponse toResponse(short version, KafkaClusterMetadata m MetadataResponseData.MetadataResponseTopic responseTopic = new MetadataResponseData.MetadataResponseTopic() .setName(topic.name()) - .setTopicId(topic.topicId()) + // The backend retains the shared table identity. Distinguish names + // only on the wire so clients can cache both and detect recreation. + .setTopicId( + topic.name() != null + && topic.name().indexOf('.') < 0 + && topicMapper.isMappedTopicId(topic.topicId()) + ? topicMapper.toAliasTopicId( + topicMapper.toTableId(topic.topicId())) + : topic.topicId()) .setErrorCode(toKafkaError(topic.error()).code()) .setIsInternal(topic.name() != null && Topic.isInternal(topic.name())) .setTopicAuthorizedOperations( diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index 704d322f2e0..fbd54ef77e4 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -35,6 +35,8 @@ public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + // ASCII "FlussA": Kafka clients require distinct IDs for distinct wire topic names. + private static final long ALIAS_TOPIC_ID_NAMESPACE = 0x466c757373410000L; private final String defaultDatabase; @@ -101,16 +103,29 @@ public static boolean isValidTopic(String topicName) { topicName.substring(0, separator), topicName.substring(separator + 1))); } - /** Maps a Fluss table ID to a stable Kafka topic ID. */ + /** Maps a Fluss table ID to the canonical name's stable Kafka topic ID. */ public Uuid toTopicId(long tableId) { checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); return new Uuid(TOPIC_ID_NAMESPACE, tableId); } + /** + * Maps a Fluss table ID to the default-database alias's stable Kafka topic ID. + * + *

The wire ID must differ from the canonical name's ID because Kafka clients maintain a + * one-to-one ID-to-name map. It must also change on recreation so a client can accept a new + * table's lower leader epoch without first observing a missing-topic response. + */ + public Uuid toAliasTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(ALIAS_TOPIC_ID_NAMESPACE, tableId); + } + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ public boolean isMappedTopicId(Uuid topicId) { return topicId != null - && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && (topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + || topicId.getMostSignificantBits() == ALIAS_TOPIC_ID_NAMESPACE) && topicId.getLeastSignificantBits() >= 0L; } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index bd06a8cb78e..73801d678c5 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -38,8 +38,11 @@ import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.concurrent.FutureUtils; +import org.apache.kafka.clients.Metadata; import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.message.MetadataRequestData; import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; @@ -51,6 +54,7 @@ import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.utils.LogContext; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -641,10 +645,13 @@ public void testAliasesPreserveRequestedNamesAndShareIdentity(String defaultData MetadataResponseTopic alias = named.data().topics().find("topic"); MetadataResponseTopic qualified = named.data().topics().find(canonical); assertThat(alias.partitions()).isEqualTo(qualified.partitions()); - assertThat(alias.topicId()).isEqualTo(qualified.topicId()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373410000L, 777L) : Uuid.ZERO_UUID); if (version >= 10) { - assertThat(alias.topicId()) + assertThat(qualified.topicId()) .isEqualTo(new Uuid(0x466c757373000000L, 777L)) + .isNotEqualTo(alias.topicId()) .isNotEqualTo(named.data().topics().find("sales.topic").topicId()); } MetadataResponse all = handle(service, allTopicsRequest(version), version, config); @@ -683,7 +690,12 @@ public void testAliasSurvivesMissingDatabaseRetry(boolean failedFuture) { assertThat(response.data().topics().find("missing_db.topic").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); assertThat(response.data().topics().find("topic").topicId()) - .isEqualTo(response.data().topics().find("fluss.topic").topicId()); + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373410000L, 777L) : Uuid.ZERO_UUID); + if (version >= 10) { + assertThat(response.data().topics().find("fluss.topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)); + } } assertThat(service.listDatabasesCalls).isZero(); assertThat(service.listedDatabases) @@ -783,6 +795,8 @@ public void testPersistedEnablementControlsBothAliasesAndDiscovery() { handle(service, topicsRequest((short) 11, "topic", "fluss.topic"), (short) 11); assertThat(enabled.errorCounts()).containsOnlyKeys(Errors.NONE); assertThat(enabled.data().topics().find("topic").topicId()) + .isEqualTo(new Uuid(0x466c757373410000L, 777L)); + assertThat(enabled.data().topics().find("fluss.topic").topicId()) .isEqualTo(new Uuid(0x466c757373000000L, 777L)); service.putTable("fluss.topic", 777L, disabledInvalid); assertThat(handle(service, namedTopicRequest("topic"), (short) 11).errors()) @@ -814,6 +828,82 @@ public void testResolvedAliasNameLimitDoesNotHideValidTopic() { .containsEntry(database + "." + table, Errors.INVALID_TOPIC_EXCEPTION); } + @ParameterizedTest + @ValueSource(shorts = {10, 11}) + public void testKafkaClientCanCacheCanonicalNameAndAliasTogether(short version) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + Metadata client = + new Metadata(0L, 0L, 60000L, new LogContext(), new ClusterResourceListeners()); + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "fluss.topic"), version), false, 1L); + // A producer discovers the alias after already caching the canonical name. Nonzero + // duplicate IDs would make Kafka's MetadataSnapshot throw while building topicNames. + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "topic"), version), true, 2L); + assertThat(client.fetch().partitionsForTopic("topic")).hasSize(2); + assertThat(client.fetch().partitionsForTopic("fluss.topic")).hasSize(2); + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "topic", "fluss.topic"), version), + false, + 3L); + assertThat(client.fetch().partitionsForTopic("topic")).hasSize(2); + assertThat(client.fetch().partitionsForTopic("fluss.topic")).hasSize(2); + assertThat(client.topicIds()).containsOnlyKeys("topic", "fluss.topic"); + assertThat(client.topicNames()) + .containsEntry(new Uuid(0x466c757373000000L, 777L), "fluss.topic") + .containsEntry(new Uuid(0x466c757373410000L, 777L), "topic"); + client.close(); + } + + @ParameterizedTest + @ValueSource(shorts = {10, 11}) + public void testKafkaClientRefreshesRecreatedAliasWithLowerLeaderEpoch(short version) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + Metadata client = + new Metadata(0L, 0L, 60000L, new LogContext(), new ClusterResourceListeners()); + MetadataResponse original = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + original.data() + .topics() + .forEach( + topic -> + topic.partitions() + .forEach(p -> p.setLeaderId(1).setLeaderEpoch(5))); + client.updateWithCurrentRequestVersion(original, false, 1L); + for (String name : Arrays.asList("topic", "fluss.topic")) { + assertThat(client.fetch().leaderFor(new TopicPartition(name, 0)).id()).isEqualTo(1); + assertThat(client.lastSeenLeaderEpoch(new TopicPartition(name, 0))).contains(5); + } + + // The client never observes the missing table between deletion and recreation. + service.removeTable("fluss.topic"); + service.putTable("fluss.topic", 778L); + MetadataResponse recreated = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + recreated + .data() + .topics() + .forEach( + topic -> + topic.partitions() + .forEach(p -> p.setLeaderId(2).setLeaderEpoch(0))); + client.updateWithCurrentRequestVersion(recreated, false, 2L); + for (String name : Arrays.asList("fluss.topic", "topic")) { + assertThat(client.fetch().leaderFor(new TopicPartition(name, 0)).id()) + .as("Leader after recreating %s", name) + .isEqualTo(2); + assertThat(client.lastSeenLeaderEpoch(new TopicPartition(name, 0))).contains(0); + } + assertThat(client.topicNames()) + .containsEntry(new Uuid(0x466c757373000000L, 778L), "fluss.topic") + .containsEntry(new Uuid(0x466c757373410000L, 778L), "topic") + .doesNotContainKeys( + new Uuid(0x466c757373000000L, 777L), new Uuid(0x466c757373410000L, 777L)); + client.close(); + } + private static MetadataRequest topicsRequest(short version, String... names) { return new MetadataRequest( new MetadataRequestData() diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java index d162ed837a4..94b5c35f14f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -208,10 +208,15 @@ public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception assertThat(partition.isrNodes()) .containsExactlyInAnyOrder(0, 1); }); - assertThat(alias.topicId()).isEqualTo(canonical.topicId()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, tableId) + : Uuid.ZERO_UUID); if (version >= 10) { - assertThat(alias.topicId()) + assertThat(canonical.topicId()) .isEqualTo(new Uuid(0x466c757373000000L, tableId)) + .isNotEqualTo(alias.topicId()) .isNotEqualTo(response.data().topics().find("sales.events").topicId()); } assertThat(response.data().topics().find("missing_db.events").errorCode()) @@ -355,9 +360,15 @@ public void testCustomDefaultDatabaseIsInjectedAndNeverCreatedByMetadata() throw .topics() .find("custom_default.events") .partitions()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, customId) + : Uuid.ZERO_UUID); if (version >= 10) { - assertThat(alias.topicId()) + assertThat(response.data().topics().find("custom_default.events").topicId()) .isEqualTo(new Uuid(0x466c757373000000L, customId)) + .isNotEqualTo(alias.topicId()) .isNotEqualTo(response.data().topics().find("fluss.events").topicId()); } assertThat(query(node, allTopics(version)).data().topics()) @@ -403,7 +414,11 @@ private static void assertExposure(ServerNode node, boolean enabled, long tableI assertThat(topic.topicId()) .isEqualTo( enabled && version >= 10 - ? new Uuid(0x466c757373000000L, tableId) + ? new Uuid( + name.indexOf('.') >= 0 + ? 0x466c757373000000L + : 0x466c757373410000L, + tableId) : Uuid.ZERO_UUID); } assertThat(response.data().topics().find("sales.unset").errorCode()) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index f70c7ffca22..42c45627319 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -162,6 +162,22 @@ public void testTopicIdMapping() { assertThat(mapper.toTableId(firstTableTopicId)).isZero(); } + @Test + public void testAliasTopicIdsKeepTableIdentityAndDetectRecreation() { + for (long tableId : new long[] {0L, 123L, Long.MAX_VALUE}) { + Uuid canonical = mapper.toTopicId(tableId); + Uuid alias = mapper.toAliasTopicId(tableId); + assertThat(alias) + .isNotIn(canonical, Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID) + .isEqualTo(new KafkaTopicMapper().toAliasTopicId(tableId)); + assertThat(mapper.isMappedTopicId(alias)).isTrue(); + assertThat(mapper.toTableId(alias)).isEqualTo(mapper.toTableId(canonical)); + } + assertThat(mapper.toAliasTopicId(124L)).isNotEqualTo(mapper.toAliasTopicId(123L)); + assertThatThrownBy(() -> mapper.toAliasTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + } + @Test public void testRejectsInvalidTopicIds() { assertThatThrownBy(() -> mapper.toTopicId(-1L)) From 408a85ba01accb03d76ebd2600f09c1d56577539 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 14:30:55 +0800 Subject: [PATCH 21/48] [kafka] Validate Metadata against the updated PR03 contract Adapt the decoder regression to the configured request handler. Cover optional three-field metadata columns and legacy offset rejection across Metadata v0-v11, including native DDL and both real TabletServer listeners. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 109/109 --- .../fluss/kafka/KafkaCommandDecoderTest.java | 5 +- .../fluss/kafka/KafkaMetadataHandlerTest.java | 59 +++++++++++++++++++ .../fluss/kafka/KafkaMetadataITCase.java | 45 ++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index dda2e52bde5..147b39dd834 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -17,6 +17,8 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; @@ -70,7 +72,8 @@ public void testUnsupportedApiVersionsRequestReturnsVersionZeroErrorResponse() { assertThat(parsedRequest.hasUnsupportedRequestVersion()).isTrue(); assertThat(parsedRequest.version()).isZero(); - new KafkaRequestHandler().processRequest(request); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + new KafkaRequestHandler(new Configuration(), service, service).processRequest(request); channel.runPendingTasks(); responseBuffer = channel.readOutbound(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 73801d678c5..528932410a1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -36,6 +36,7 @@ import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.clients.Metadata; @@ -429,6 +430,64 @@ public void testMetadataUsesDdlContractForEveryVersion() { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMetadataColumnContractForEveryVersion(boolean legacyOffset) { + RowType metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + if (legacyOffset) { + metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT())); + } + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + // Also model an invalid historical descriptor that native DDL would now reject. + service.putTable( + "fluss.metadata_events", + 777L, + TableDescriptor.builder(defaultDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_metadata", metadataType) + .build()) + .build()); + short expected = legacyOffset ? Errors.INVALID_TOPIC_EXCEPTION.code() : Errors.NONE.code(); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle( + service, + topicsRequest( + version, + "metadata_events", + "fluss.metadata_events", + "kafka.topic"), + version); + for (String name : Arrays.asList("metadata_events", "fluss.metadata_events")) { + MetadataResponseTopic topic = named.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(expected); + assertThat(topic.partitions()).hasSize(legacyOffset ? 0 : 2); + } + assertThat(named.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics().find("fluss.metadata_events").errorCode()) + .isEqualTo(expected); + assertThat(all.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .doesNotContain("metadata_events"); + } + } + @Test public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java index 94b5c35f14f..0ac5279c401 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -113,6 +113,51 @@ public void testQualifiedMetadataAcrossDatabases() throws Exception { } } + @Test + public void testOptionalMetadataColumnOverRealListeners() throws Exception { + TableDescriptor descriptor = + TableDescriptor.builder(topicDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()))) + .build()) + .build(); + long tableId = createTable(CLUSTER, TablePath.of("fluss", "metadata_events"), descriptor); + CLUSTER.waitUntilTableReady(tableId); + for (int serverId = 0; serverId < 2; serverId++) { + ServerNode node = CLUSTER.getTabletServerInfos().get(serverId).node("KAFKA"); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + query(node, named(version, "metadata_events", "fluss.metadata_events")); + assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = named.data().topics().find("metadata_events"); + MetadataResponseTopic canonical = + named.data().topics().find("fluss.metadata_events"); + assertThat(alias.partitions()).hasSize(2).isEqualTo(canonical.partitions()); + if (version >= 10) { + assertThat(canonical.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, tableId)); + assertThat(alias.topicId()).isEqualTo(new Uuid(0x466c757373410000L, tableId)); + } + MetadataResponse all = query(node, allTopics(version)); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.metadata_events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + } + @Test public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exception { TableDescriptor descriptor = topicDescriptor(); From ecbb368832efd3d5f542880c190e419f3ad5604b Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 20:24:12 +0800 Subject: [PATCH 22/48] [kafka] Simplify Metadata wiring and regression tests Pass one RPC service through the request handler and validate its TabletServerGateway capability inside the Metadata backend. Consolidate overlapping Metadata tests while retaining protocol-version, alias identity, client cache, authorization and discovery regressions. Align the custom default database test with PR03 startup creation and verify that Metadata does not recreate a deleted default database. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 35/35 AI-Contributed/UT: 546/546 --- .../fluss/kafka/KafkaProtocolPlugin.java | 9 +- .../fluss/kafka/KafkaRequestHandler.java | 9 +- .../metadata/GatewayKafkaMetadataBackend.java | 17 +- .../fluss/kafka/KafkaCommandDecoderTest.java | 2 +- .../fluss/kafka/KafkaMetadataHandlerTest.java | 375 +++--------------- .../fluss/kafka/KafkaMetadataITCase.java | 167 +++----- .../fluss/kafka/KafkaRequestHandlerTest.java | 2 +- 7 files changed, 123 insertions(+), 458 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index d0363e3f6f4..2d0fa9220e3 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -20,7 +20,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.rpc.RpcGatewayService; -import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; @@ -61,12 +60,6 @@ public ChannelHandler createChannelHandler( @Override public RequestHandler createRequestHandler(RpcGatewayService service) { - if (!(service instanceof TabletServerGateway)) { - throw new IllegalArgumentException( - "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " - + service.getClass().getSimpleName()); - } - TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(conf, service, gateway); + return new KafkaRequestHandler(conf, service); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 46dc00355bf..31da9143823 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -26,7 +26,6 @@ import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.rpc.RpcGatewayService; -import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; @@ -41,18 +40,14 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ - public KafkaRequestHandler( - Configuration conf, RpcGatewayService service, TabletServerGateway gateway) { - checkNotNull(service); - checkNotNull(gateway); + public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { KafkaTopicMapper topicMapper = new KafkaTopicMapper(checkNotNull(conf)); this.registry = new KafkaApiRegistry( Arrays.asList( new ApiVersionsHandler(this::apiRegistry), new MetadataHandler( - new GatewayKafkaMetadataBackend( - service, gateway, topicMapper), + new GatewayKafkaMetadataBackend(service, topicMapper), topicMapper))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java index 0d4ed5a78c3..2c2ff42e117 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -62,6 +62,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */ @@ -75,11 +76,19 @@ public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { private final KafkaTopicMapper topicMapper; private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); - /** Creates a metadata backend backed by the local TabletServer gateway. */ - public GatewayKafkaMetadataBackend( - RpcGatewayService service, TabletServerGateway gateway, KafkaTopicMapper topicMapper) { + /** + * Creates a metadata backend backed by the local TabletServer gateway. + * + * @throws IllegalArgumentException if the service does not implement {@link + * TabletServerGateway} + */ + public GatewayKafkaMetadataBackend(RpcGatewayService service, KafkaTopicMapper topicMapper) { this.service = checkNotNull(service); - this.gateway = checkNotNull(gateway); + checkArgument( + service instanceof TabletServerGateway, + "Kafka protocol endpoints can only be enabled on TabletServers, but the service is %s", + service.getClass().getSimpleName()); + this.gateway = (TabletServerGateway) service; this.topicMapper = checkNotNull(topicMapper); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index 147b39dd834..f745fe7fe8b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -73,7 +73,7 @@ public void testUnsupportedApiVersionsRequestReturnsVersionZeroErrorResponse() { assertThat(parsedRequest.version()).isZero(); TestingTabletGatewayService service = new TestingTabletGatewayService(); - new KafkaRequestHandler(new Configuration(), service, service).processRequest(request); + new KafkaRequestHandler(new Configuration(), service).processRequest(request); channel.runPendingTasks(); responseBuffer = channel.readOutbound(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 528932410a1..b1359f1c705 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -83,13 +83,7 @@ public class KafkaMetadataHandlerTest { public void testNamedTopicForEverySupportedVersion() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { - MetadataRequest request = - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Collections.singletonList("kafka.topic"))), - version); + MetadataRequest request = topicsRequest(version, "kafka.topic"); if (version >= 9) { request.data() .unknownTaggedFields() @@ -141,26 +135,6 @@ public void testAllTopicsForEverySupportedVersion() { } } - @Test - public void testAllTopicsAndMissingAndInvalidTopic() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - - MetadataResponse allTopics = - handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); - assertThat(allTopics.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactlyInAnyOrder("kafka.other", "kafka.topic"); - - MetadataRequest requestedTopics = - new MetadataRequest.Builder( - Arrays.asList("kafka.missing", "kafka.invalid topic"), false) - .build((short) 9); - MetadataResponse errors = handle(service, requestedTopics, (short) 9); - assertThat(errors.errors()) - .containsEntry("kafka.missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) - .containsEntry("kafka.invalid topic", Errors.INVALID_TOPIC_EXCEPTION); - } - @Test public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -210,32 +184,6 @@ public void testV10AndV11RejectTopicIdOnlyLookup() { } } - @Test - public void testTopicIdentityAcrossDeleteAndRecreate() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - - MetadataResponse initial = handle(service, namedTopicRequest("kafka.topic"), (short) 11); - assertThat(initial.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); - - service.removeTable("kafka.topic"); - MetadataResponse deleted = handle(service, namedTopicRequest("kafka.topic"), (short) 11); - assertThat(deleted.errorCounts()) - .containsExactlyEntriesOf( - Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); - - service.putTable("kafka.topic", 223L); - Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); - MetadataResponse recreatedByName = - handle( - service, - new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) - .build((short) 11), - (short) 11); - assertThat(recreatedByName.data().topics().find("kafka.topic").topicId()) - .isEqualTo(recreatedTopicId) - .isNotEqualTo(TOPIC_ID); - } - @Test public void testDeleteRaceBecomesUnknownTopicResult() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -253,9 +201,7 @@ public void testDeleteRaceBecomesUnknownTopicResult() { public void testUnavailableLeaderUsesPartitionError() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicLeaderAvailable = false; - MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) - .build((short) 11); + MetadataRequest request = namedTopicRequest("kafka.topic"); MetadataResponse response = handle(service, request, (short) 11); @@ -273,9 +219,7 @@ public void testUnavailableLeaderUsesPartitionError() { public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicIsr = new int[] {1}; - MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) - .build((short) 11); + MetadataRequest request = namedTopicRequest("kafka.topic"); MetadataResponse response = handle(service, request, (short) 11); @@ -335,9 +279,7 @@ public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.failMetadata = true; - MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) - .build((short) 11); + MetadataRequest request = namedTopicRequest("kafka.topic"); MetadataResponse response = handle(service, request, (short) 11); @@ -346,90 +288,6 @@ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { assertThat(response.brokers()).isEmpty(); } - @Test - public void testMetadataUsesDdlContractForEveryVersion() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.putTable( - "kafka.no_mapping", - 125L, - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .distributedBy(2) - .build()); - service.putTable( - "kafka.bad_format", - 126L, - TableDescriptor.builder(defaultDescriptor()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "unimplemented") - .build()); - service.putTable( - "kafka.primary_key", - 127L, - TableDescriptor.builder(defaultDescriptor()) - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES().copy(false)) - .primaryKey("_kafka_value") - .build()) - .build()); - service.putTable( - "kafka.partitioned", - 128L, - TableDescriptor.builder(defaultDescriptor()).partitionedBy("_kafka_value").build()); - service.putTable( - "kafka.indexed", - 129L, - TableDescriptor.builder(defaultDescriptor()).logFormat(LogFormat.INDEXED).build()); - service.putTable("kafka.invalid topic", 130L); - List invalidMappings = - Arrays.asList( - "kafka.bad_format", - "kafka.primary_key", - "kafka.partitioned", - "kafka.indexed", - "kafka.invalid topic"); - for (short version = 0; version <= 11; version++) { - MetadataResponse all = handle(service, allTopicsRequest(version), version); - List exposed = new ArrayList<>(invalidMappings); - exposed.addAll(Arrays.asList("kafka.other", "kafka.topic")); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactlyInAnyOrderElementsOf(exposed); - for (String name : invalidMappings) { - assertThat(all.data().topics().find(name).errorCode()) - .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); - } - List requested = new ArrayList<>(invalidMappings); - requested.add("kafka.no_mapping"); - requested.add("kafka.topic"); - requested.add("kafka.missing"); - MetadataResponse named = - handle( - service, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - requested)), - version), - version); - for (String name : invalidMappings) { - MetadataResponseTopic topic = named.data().topics().find(name); - assertThat(topic.errorCode()).isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); - assertThat(topic.partitions()).isEmpty(); - } - assertThat(named.data().topics().find("kafka.topic").errorCode()) - .isEqualTo(Errors.NONE.code()); - assertThat(named.data().topics().find("kafka.missing").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(named.data().topics().find("kafka.no_mapping").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - } - } - @ParameterizedTest @ValueSource(booleans = {false, true}) public void testMetadataColumnContractForEveryVersion(boolean legacyOffset) { @@ -488,55 +346,6 @@ public void testMetadataColumnContractForEveryVersion(boolean legacyOffset) { } } - @Test - public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.putTable( - "kafka.topic", - 123L, - TableDescriptor.builder(defaultDescriptor()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "string") - .build()); - MetadataResponse invalid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); - assertThat(invalid.data().topics().find("kafka.topic").errorCode()) - .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); - service.putTable("kafka.topic", 123L); - MetadataResponse valid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); - assertThat(valid.data().topics().find("kafka.topic").errorCode()) - .isEqualTo(Errors.NONE.code()); - assertThat(valid.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); - } - - @Test - public void testSameTableNameInDifferentDatabasesForEveryVersion() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.putTable("sales.topic", 223L); - for (short version = 0; version <= 11; version++) { - MetadataResponse all = handle(service, allTopicsRequest(version), version); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactly("kafka.other", "kafka.topic", "sales.topic"); - MetadataResponse named = - handle( - service, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Arrays.asList( - "kafka.topic", "sales.topic"))), - version), - version); - assertThat(named.data().topics()).hasSize(2); - assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); - if (version >= 10) { - assertThat(named.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); - assertThat(named.data().topics().find("sales.topic").topicId()) - .isEqualTo(new Uuid(0x466c757373000000L, 223L)); - } - } - } - @Test public void testRejectMalformedNamesWithoutHidingValidTopics() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -556,48 +365,6 @@ public void testRejectMalformedNamesWithoutHidingValidTopics() { .isEqualTo(Errors.NONE.code()); } - @Test - public void testUnauthorizedTopicsRemainHiddenForEveryVersion() { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.hiddenTopics.add("kafka.topic"); - for (short version = 0; version <= 11; version++) { - MetadataResponse all = handle(service, allTopicsRequest(version), version); - assertThat(all.brokers()).hasSize(2); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactly("kafka.other"); - for (List names : - Arrays.asList( - Arrays.asList("kafka.topic", "kafka.other"), - Arrays.asList("kafka.topic", "kafka.other", "missing_db.topic"))) { - MetadataResponse named = - handle( - service, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest - .convertToMetadataRequestTopic( - names)), - version), - version); - assertThat(named.brokers()).hasSize(2); - MetadataResponseTopic hidden = named.data().topics().find("kafka.topic"); - assertThat(hidden.errorCode()).isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(hidden.topicId()).isEqualTo(Uuid.ZERO_UUID); - assertThat(hidden.partitions()).isEmpty(); - MetadataResponseTopic visible = named.data().topics().find("kafka.other"); - assertThat(visible.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(visible.partitions()).hasSize(2); - assertThat(named.data().topics()).hasSize(names.size()); - if (names.contains("missing_db.topic")) { - assertThat(named.data().topics().find("missing_db.topic").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - } - } - } - } - @Test public void testMetadataV12IsRejectedBeforeCallingGateway() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -614,36 +381,47 @@ public void testMetadataV12IsRejectedBeforeCallingGateway() { @ParameterizedTest @ValueSource(booleans = {false, true}) - public void testMissingDatabaseRetainsBrokersAndValidTopics(boolean failedFuture) { + public void testMissingDatabaseRetainsBrokersAndAliases(boolean failedFuture) { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); service.failListTablesAsFuture = failedFuture; for (short version = 0; version <= 11; version++) { for (List names : Arrays.asList( Collections.singletonList("missing_db.topic"), - Arrays.asList("missing_db.topic", "kafka.topic"))) { + Arrays.asList( + "missing_db.topic", "topic", "fluss.topic", "kafka.topic"))) { MetadataResponse response = handle( service, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest - .convertToMetadataRequestTopic( - names)), - version), + topicsRequest(version, names.toArray(new String[0])), version); assertThat(response.brokers()).hasSize(2); assertThat(response.data().topics()).hasSize(names.size()); assertThat(response.data().topics().find("missing_db.topic").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - if (names.contains("kafka.topic")) { - MetadataResponseTopic valid = response.data().topics().find("kafka.topic"); - assertThat(valid.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(valid.partitions()).hasSize(2); + if (names.contains("topic")) { + for (String name : Arrays.asList("topic", "fluss.topic", "kafka.topic")) { + MetadataResponseTopic valid = response.data().topics().find(name); + assertThat(valid.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.partitions()).hasSize(2); + } + assertThat(response.data().topics().find("topic").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, 777L) + : Uuid.ZERO_UUID); + assertThat(response.data().topics().find("fluss.topic").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373000000L, 777L) + : Uuid.ZERO_UUID); } } } + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.listedDatabases) + .containsExactlyInAnyOrder("fluss", "kafka", "missing_db"); } @ParameterizedTest @@ -722,45 +500,6 @@ public void testAliasesPreserveRequestedNamesAndShareIdentity(String defaultData } } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - public void testAliasSurvivesMissingDatabaseRetry(boolean failedFuture) { - TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.putTable("fluss.topic", 777L); - service.failListTablesAsFuture = failedFuture; - for (short version = 0; version <= 11; version++) { - MetadataResponse response = - handle( - service, - topicsRequest( - version, - "topic", - "missing_db.topic", - "fluss.topic", - "kafka.topic"), - version); - assertThat(response.brokers()).hasSize(2); - assertThat(response.data().topics()).hasSize(4); - for (String name : Arrays.asList("topic", "fluss.topic", "kafka.topic")) { - assertThat(response.data().topics().find(name).errorCode()) - .isEqualTo(Errors.NONE.code()); - assertThat(response.data().topics().find(name).partitions()).hasSize(2); - } - assertThat(response.data().topics().find("missing_db.topic").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(response.data().topics().find("topic").topicId()) - .isEqualTo( - version >= 10 ? new Uuid(0x466c757373410000L, 777L) : Uuid.ZERO_UUID); - if (version >= 10) { - assertThat(response.data().topics().find("fluss.topic").topicId()) - .isEqualTo(new Uuid(0x466c757373000000L, 777L)); - } - } - assertThat(service.listDatabasesCalls).isZero(); - assertThat(service.listedDatabases) - .containsExactlyInAnyOrder("fluss", "kafka", "missing_db"); - } - @Test public void testMissingDefaultDatabaseNeverSearchesOtherDatabases() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -783,37 +522,46 @@ public void testMissingDefaultDatabaseNeverSearchesOtherDatabases() { } @Test - public void testAliasAuthorizationDoesNotFallbackToSameNamedVisibleTable() { + public void testUnauthorizedAliasesRemainHiddenWithAndWithoutRetry() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.putTable("fluss.topic", 777L); service.hiddenTopics.add("fluss.topic"); service.putTable("sales.topic", 778L); for (short version = 0; version <= 11; version++) { - MetadataResponse response = - handle( - service, - topicsRequest( - version, - "topic", - "fluss.topic", - "sales.topic", - "missing_db.topic"), - version); - for (String hidden : Arrays.asList("topic", "fluss.topic")) { - MetadataResponseTopic topic = response.data().topics().find(hidden); - assertThat(topic.errorCode()).isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); - assertThat(topic.partitions()).isEmpty(); + for (List names : + Arrays.asList( + Arrays.asList("topic", "fluss.topic", "sales.topic"), + Arrays.asList( + "topic", "fluss.topic", "sales.topic", "missing_db.topic"))) { + MetadataResponse response = + handle( + service, + topicsRequest(version, names.toArray(new String[0])), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(names.size()); + for (String hidden : Arrays.asList("topic", "fluss.topic")) { + MetadataResponseTopic topic = response.data().topics().find(hidden); + assertThat(topic.errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + } + MetadataResponseTopic visible = response.data().topics().find("sales.topic"); + assertThat(visible.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(visible.partitions()).hasSize(2); + if (names.contains("missing_db.topic")) { + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } } - assertThat(response.data().topics().find("sales.topic").errorCode()) - .isEqualTo(Errors.NONE.code()); MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.brokers()).hasSize(2); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) .contains("sales.topic") .doesNotContain("topic", "fluss.topic"); } - assertThat(service.requestedTables).doesNotContain("sales.other"); } @Test @@ -983,14 +731,7 @@ private static TableDescriptor defaultDescriptor() { } private static MetadataRequest namedTopicRequest(String topicName) { - return new MetadataRequest( - new MetadataRequestData() - .setTopics( - Collections.singletonList( - new MetadataRequestTopic() - .setName(topicName) - .setTopicId(Uuid.ZERO_UUID))), - (short) 11); + return topicsRequest((short) 11, topicName); } private static MetadataRequest allTopicsRequest(short version) { @@ -1009,7 +750,7 @@ private static MetadataResponse handle( MetadataRequest requestBody, short version, Configuration config) { - KafkaRequestHandler handler = new KafkaRequestHandler(config, service, service); + KafkaRequestHandler handler = new KafkaRequestHandler(config, service); ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); KafkaRequest request; try { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java index 0ac5279c401..3a5e582ee71 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -28,6 +28,7 @@ import org.apache.fluss.server.zk.data.TableRegistration; import org.apache.fluss.types.DataTypes; +import org.apache.kafka.common.Node; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.message.MetadataRequestData; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; @@ -68,96 +69,6 @@ public class KafkaMetadataITCase { .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") .build(); - @Test - public void testQualifiedMetadataAcrossDatabases() throws Exception { - TableDescriptor descriptor = topicDescriptor(); - long first = createTable(CLUSTER, TablePath.of("first_db", "events"), descriptor); - long second = createTable(CLUSTER, TablePath.of("second_db", "events"), descriptor); - CLUSTER.waitUntilTableReady(first); - CLUSTER.waitUntilTableReady(second); - ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); - - for (short version = 0; version <= 11; version++) { - MetadataResponse named = - query( - node, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Arrays.asList( - "first_db.events", - "second_db.events", - "first_db.missing"))), - version)); - assertThat(named.data().topics().find("first_db.events").partitions()).hasSize(2); - assertThat(named.data().topics().find("second_db.events").partitions()).hasSize(2); - assertThat(named.data().topics().find("first_db.missing").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(named.data().topics().find("first_db.events").topicId()) - .isEqualTo( - version >= 10 ? new Uuid(0x466c757373000000L, first) : Uuid.ZERO_UUID); - assertThat(named.data().topics().find("second_db.events").topicId()) - .isEqualTo( - version >= 10 ? new Uuid(0x466c757373000000L, second) : Uuid.ZERO_UUID); - assertThat(named.brokers()).hasSize(2); - assertThat(named.brokers().iterator().next().port()).isEqualTo(node.port()); - - MetadataRequestData allTopics = new MetadataRequestData(); - allTopics.setTopics(version == 0 ? Collections.emptyList() : null); - MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactly("first_db.events", "second_db.events"); - assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); - } - } - - @Test - public void testOptionalMetadataColumnOverRealListeners() throws Exception { - TableDescriptor descriptor = - TableDescriptor.builder(topicDescriptor()) - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column( - "_kafka_metadata", - DataTypes.ROW( - DataTypes.FIELD( - "timestamp", - DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD( - "timestamp_type", DataTypes.INT()), - DataTypes.FIELD( - "partition", DataTypes.INT()))) - .build()) - .build(); - long tableId = createTable(CLUSTER, TablePath.of("fluss", "metadata_events"), descriptor); - CLUSTER.waitUntilTableReady(tableId); - for (int serverId = 0; serverId < 2; serverId++) { - ServerNode node = CLUSTER.getTabletServerInfos().get(serverId).node("KAFKA"); - for (short version = 0; version <= 11; version++) { - MetadataResponse named = - query(node, named(version, "metadata_events", "fluss.metadata_events")); - assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); - MetadataResponseTopic alias = named.data().topics().find("metadata_events"); - MetadataResponseTopic canonical = - named.data().topics().find("fluss.metadata_events"); - assertThat(alias.partitions()).hasSize(2).isEqualTo(canonical.partitions()); - if (version >= 10) { - assertThat(canonical.topicId()) - .isEqualTo(new Uuid(0x466c757373000000L, tableId)); - assertThat(alias.topicId()).isEqualTo(new Uuid(0x466c757373410000L, tableId)); - } - MetadataResponse all = query(node, allTopics(version)); - assertThat(all.data().topics()) - .extracting(MetadataResponseTopic::name) - .containsExactly("fluss.metadata_events"); - assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); - } - } - } - @Test public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exception { TableDescriptor descriptor = topicDescriptor(); @@ -169,31 +80,13 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); for (short version = 0; version <= 11; version++) { - MetadataResponse missing = - query( - node, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Collections.singletonList( - "deleted_db.events"))), - version)); + MetadataResponse missing = query(node, named(version, "deleted_db.events")); assertThat(missing.brokers()).hasSize(2); assertThat(missing.data().topics().find("deleted_db.events").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); MetadataResponse mixed = - query( - node, - new MetadataRequest( - new MetadataRequestData() - .setTopics( - MetadataRequest.convertToMetadataRequestTopic( - Arrays.asList( - "deleted_db.events", - "existing_db.events"))), - version)); + query(node, named(version, "deleted_db.events", "existing_db.events")); assertThat(mixed.brokers()).hasSize(2); assertThat(mixed.brokers().iterator().next().port()).isEqualTo(node.port()); assertThat(mixed.data().topics().find("deleted_db.events").errorCode()) @@ -202,9 +95,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio .isEqualTo(Errors.NONE.code()); assertThat(mixed.data().topics().find("existing_db.events").partitions()).hasSize(2); - MetadataRequestData allTopics = new MetadataRequestData(); - allTopics.setTopics(version == 0 ? Collections.emptyList() : null); - MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); + MetadataResponse all = query(node, allTopics(version)); assertThat(all.brokers()).hasSize(2); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) @@ -214,7 +105,7 @@ public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exceptio } @Test - public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception { + public void testAliasesAndOptionalMetadataAcrossDatabases() throws Exception { long other = createTable(CLUSTER, TablePath.of("sales", "events"), topicDescriptor()); CLUSTER.waitUntilTableReady(other); ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); @@ -225,7 +116,25 @@ public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception assertThat(before.data().topics().find("sales.events").errorCode()) .isEqualTo(Errors.NONE.code()); } - long tableId = createTable(CLUSTER, TablePath.of("fluss", "events"), topicDescriptor()); + // Exercise PR03's optional metadata column through the same alias and discovery paths. + TableDescriptor withMetadata = + TableDescriptor.builder(topicDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()))) + .build()) + .build(); + long tableId = createTable(CLUSTER, TablePath.of("fluss", "events"), withMetadata); CLUSTER.waitUntilTableReady(tableId); for (short version = 0; version <= 11; version++) { for (ServerNode broker : CLUSTER.getTabletServerNodes("KAFKA")) { @@ -239,7 +148,20 @@ public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception "events", "fluss.events", "sales.events", + "fluss.missing", "missing_db.events")); + assertThat(response.brokers()).hasSize(2); + assertThat(response.brokers()).extracting(Node::port).contains(broker.port()); + for (String name : Arrays.asList("fluss.events", "sales.events")) { + MetadataResponseTopic topic = response.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + } + assertThat(response.data().topics().find("sales.events").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373000000L, other) + : Uuid.ZERO_UUID); MetadataResponseTopic alias = response.data().topics().find("events"); MetadataResponseTopic canonical = response.data().topics().find("fluss.events"); assertThat(alias.errorCode()).isEqualTo(Errors.NONE.code()); @@ -264,11 +186,15 @@ public void testDefaultAliasesAndNoFallbackOverRealListeners() throws Exception .isNotEqualTo(alias.topicId()) .isNotEqualTo(response.data().topics().find("sales.events").topicId()); } - assertThat(response.data().topics().find("missing_db.events").errorCode()) - .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); - assertThat(query(broker, allTopics(version)).data().topics()) + for (String missing : Arrays.asList("fluss.missing", "missing_db.events")) { + assertThat(response.data().topics().find(missing).errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + MetadataResponse all = query(broker, allTopics(version)); + assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) .containsExactly("fluss.events", "sales.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); } } } @@ -359,7 +285,7 @@ public void testInvalidEnabledContractRemainsVisibleInNamedAndAllTopics() throws } @Test - public void testCustomDefaultDatabaseIsInjectedAndNeverCreatedByMetadata() throws Exception { + public void testCustomDefaultDatabaseAliasAndNoRecreationByMetadata() throws Exception { Configuration config = clusterConfiguration(); config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "custom_default"); config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); @@ -375,12 +301,13 @@ public void testCustomDefaultDatabaseIsInjectedAndNeverCreatedByMetadata() throw createTable(cluster, TablePath.of("fluss", "events"), topicDescriptor()); cluster.waitUntilTableReady(nativeDefaultId); ServerNode node = cluster.getTabletServerInfos().get(0).node("KAFKA"); + // PR03 creates the configured database at startup, before any Metadata request. assertThat( cluster.newCoordinatorClient() .databaseExists(newDatabaseExistsRequest("custom_default")) .get() .isExists()) - .isFalse(); + .isTrue(); for (short version = 0; version <= 11; version++) { MetadataResponse response = query(node, named(version, "events", "fluss.events")); assertThat(response.data().topics().find("events").errorCode()) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index f55b1832c0b..3c1bfe6128b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -300,6 +300,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { private static KafkaRequestHandler createKafkaRequestHandler() { TestingTabletGatewayService service = new TestingTabletGatewayService(); - return new KafkaRequestHandler(new Configuration(), service, service); + return new KafkaRequestHandler(new Configuration(), service); } } From 39504352518bed5996feb00decf87605e76f9ac5 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:18:03 +0800 Subject: [PATCH 23/48] [kafka] Add independently testable Produce protocol handling Validate non-idempotent Produce v3-v11 and isolate invalid partitions before invoking a protocol-independent backend. Preserve partition order, acknowledgements and owned record data. Leave production registration to the append integration PR. Validation: Java 11, mvn -o -pl fluss-kafka spotless:apply clean verify; 69 unit tests and 2 integration tests passed, including Checkstyle, Spotless and RAT. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 621/621 AI-Contributed/UT: 366/366 --- .../kafka/api/produce/ProduceHandler.java | 284 ++++++++++++++ .../backend/produce/KafkaProduceBackend.java | 29 ++ .../backend/produce/KafkaProduceCommand.java | 197 ++++++++++ .../backend/produce/KafkaProduceResult.java | 111 ++++++ .../fluss/kafka/KafkaProduceProtocolTest.java | 366 ++++++++++++++++++ 5 files changed, 987 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java new file mode 100644 index 00000000000..0eb14b26ff2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidRequiredAcksException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.BaseRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements non-idempotent Kafka Produce versions 3 through 11. */ +@Internal +public final class ProduceHandler implements KafkaApiHandler { + + private static final short MIN_SUPPORTED_VERSION = 3; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec(ApiKeys.PRODUCE, MIN_SUPPORTED_VERSION, (short) 11, true); + + private final KafkaProduceBackend backend; + + /** Creates a non-idempotent Produce handler. */ + public ProduceHandler(KafkaProduceBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ProduceRequest request) { + validateRequest(request); + List topics = new ArrayList<>(); + Map failures = new HashMap<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + try { + if (!Topic.isValid(topic.name())) { + throw new InvalidTopicException("Invalid Kafka topic name " + topic.name()); + } + if (partition.index() < 0) { + throw new InvalidRequestException("Negative Kafka partition ID."); + } + partitions.add( + new PartitionWrite( + partition.index(), + copyRecords(request.version(), partition.records()))); + } catch (RuntimeException failure) { + failures.put( + new TopicPartition(topic.name(), partition.index()), + failedPartition(partition.index(), failure)); + } + } + if (!partitions.isEmpty()) { + topics.add(new TopicWrite(topic.name(), partitions)); + } + } + KafkaProduceCommand command = + new KafkaProduceCommand( + request.acks(), + request.timeout(), + topics, + context.listenerName(), + clientAddress(context.remoteAddress())); + CompletableFuture result; + try { + result = + topics.isEmpty() + ? CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList())) + : checkNotNull(backend.write(command)); + } catch (RuntimeException failure) { + result = new CompletableFuture<>(); + result.completeExceptionally(failure); + } + return result.handle( + (written, failure) -> { + Map results = new HashMap<>(failures); + if (failure == null && written != null) { + for (TopicResult topic : written.topics()) { + for (PartitionResult partition : topic.partitions()) { + results.putIfAbsent( + new TopicPartition( + topic.topicName(), partition.partitionId()), + partition); + } + } + } + List ordered = new ArrayList<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + TopicPartition key = + new TopicPartition(topic.name(), partition.index()); + PartitionResult value = results.get(key); + if (value == null) { + value = + failedPartition( + partition.index(), + failure == null + ? new IllegalStateException( + "Produce backend omitted this partition.") + : failure); + } + partitions.add(value); + } + ordered.add(new TopicResult(topic.name(), partitions)); + } + return toResponse(new KafkaProduceResult(ordered)); + }); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + while (failure instanceof CompletionException && failure.getCause() != null) { + failure = failure.getCause(); + } + return new PartitionResult( + partitionId, Errors.forException(failure), -1L, failure.getMessage()); + } + + private static void validateRequest(ProduceRequest request) { + Set names = new HashSet<>(); + for (TopicProduceData topic : request.data().topicData()) { + if (!names.add(topic.name())) { + throw new InvalidRequestException("Duplicate Kafka topic in Produce request."); + } + Set partitions = new HashSet<>(); + for (PartitionProduceData partition : topic.partitionData()) { + if (!partitions.add(partition.index())) { + throw new InvalidRequestException( + "Duplicate Kafka partition in Produce request."); + } + } + } + if (request.transactionalId() != null) { + throw new InvalidRequestException( + "Transactional Produce is not supported by the Fluss Kafka compatibility layer."); + } + if (request.acks() != -1 && request.acks() != 0 && request.acks() != 1) { + throw new InvalidRequiredAcksException("Invalid required acks " + request.acks()); + } + } + + private static List copyRecords( + short version, BaseRecords baseRecords) { + if (!(baseRecords instanceof Records)) { + throw new InvalidRequestException("Unsupported Kafka records representation."); + } + ProduceRequest.validateRecords(version, baseRecords); + Records records = (Records) baseRecords; + if (records.sizeInBytes() == 0) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + List copied = new ArrayList<>(); + int validBytes = 0; + for (RecordBatch batch : records.batches()) { + validBytes += batch.sizeInBytes(); + batch.ensureValid(); + if (batch.hasProducerId() || batch.isTransactional() || batch.isControlBatch()) { + throw new InvalidRequestException( + "Idempotent, transactional, and control record batches are not supported."); + } + for (org.apache.kafka.common.record.Record record : batch) { + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } + } + if (copied.isEmpty() || validBytes != records.sizeInBytes()) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + return copied; + } + + private static ProduceResponse toResponse(KafkaProduceResult result) { + ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0); + for (TopicResult topic : result.topics()) { + ProduceResponseData.TopicProduceResponse topicResponse = + new ProduceResponseData.TopicProduceResponse().setName(topic.topicName()); + for (PartitionResult partition : topic.partitions()) { + topicResponse + .partitionResponses() + .add( + new ProduceResponseData.PartitionProduceResponse() + .setIndex(partition.partitionId()) + .setErrorCode(partition.error().code()) + .setBaseOffset(partition.baseOffset()) + .setLogAppendTimeMs(-1L) + .setLogStartOffset(-1L) + .setErrorMessage(partition.errorMessage())); + } + data.responses().add(topicResponse); + } + return new ProduceResponse(data); + } + + private static List copyHeaders(Header[] headers) { + List copied = new ArrayList<>(headers.length); + for (Header header : headers) { + copied.add(new RecordHeader(header.key(), header.value())); + } + return copied; + } + + private static byte[] copyBuffer(ByteBuffer buffer) { + if (buffer == null) { + return null; + } + ByteBuffer duplicate = buffer.duplicate(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return bytes; + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java new file mode 100644 index 00000000000..3b8428c5670 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Produce API. */ +@Internal +public interface KafkaProduceBackend { + /** Writes copied Kafka records through the native Fluss write path. */ + CompletableFuture write(KafkaProduceCommand command); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java new file mode 100644 index 00000000000..722f3d3d245 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Protocol-independent write command used by the Kafka Produce backend. */ +@Internal +public final class KafkaProduceCommand { + + private final short acks; + private final int timeoutMs; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a Kafka write command. */ + public KafkaProduceCommand( + short acks, + int timeoutMs, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.acks = acks; + this.timeoutMs = timeoutMs; + this.topics = immutableCopy(topics); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns Kafka required acknowledgements. */ + public short acks() { + return acks; + } + + /** Returns the Produce timeout in milliseconds. */ + public int timeoutMs() { + return timeoutMs; + } + + /** Returns the topic writes in request order. */ + public List topics() { + return topics; + } + + /** Returns the listener that received the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client network address when available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Records addressed to one Kafka topic. */ + @Internal + public static final class TopicWrite { + private final String topicName; + private final List partitions; + + /** Creates the writes for one topic. */ + public TopicWrite(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns partition writes in request order. */ + public List partitions() { + return partitions; + } + } + + /** Records addressed to one Kafka partition. */ + @Internal + public static final class PartitionWrite { + private final int partitionId; + private final List records; + + /** Creates the writes for one partition. */ + public PartitionWrite(int partitionId, List records) { + this.partitionId = partitionId; + this.records = immutableCopy(records); + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns copied records in append order. */ + public List records() { + return records; + } + } + + /** A copied Kafka record whose lifetime is independent of the network request buffer. */ + @Internal + public static final class Record { + private final long timestamp; + private final @Nullable byte[] key; + private final @Nullable byte[] value; + private final List headers; + + /** Creates a copied Kafka record. */ + public Record( + long timestamp, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { + this.timestamp = timestamp; + this.key = copyNullable(key); + this.value = copyNullable(value); + this.headers = immutableCopy(headers); + } + + /** Returns the Kafka record timestamp. */ + public long timestamp() { + return timestamp; + } + + /** Returns a copy of the nullable Kafka record key. */ + public @Nullable byte[] key() { + return copyNullable(key); + } + + /** Returns a copy of the nullable Kafka record value. */ + public @Nullable byte[] value() { + return copyNullable(value); + } + + /** Returns the copied Kafka headers in record order. */ + public List headers() { + return headers; + } + + private static @Nullable byte[] copyNullable(@Nullable byte[] value) { + return value == null ? null : value.clone(); + } + } + + /** A copied Kafka record header. */ + @Internal + public static final class RecordHeader { + private final String name; + private final @Nullable byte[] value; + + /** Creates a copied Kafka record header. */ + public RecordHeader(String name, @Nullable byte[] value) { + this.name = checkNotNull(name); + this.value = value == null ? null : value.clone(); + } + + /** Returns the header name. */ + public String name() { + return name; + } + + /** Returns a copy of the nullable header value. */ + public @Nullable byte[] value() { + return value == null ? null : value.clone(); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java new file mode 100644 index 00000000000..218c566bde4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Result of a Kafka Produce backend invocation. */ +@Internal +public final class KafkaProduceResult { + private final List topics; + + /** Creates a Produce result. */ + public KafkaProduceResult(List topics) { + this.topics = immutableCopy(topics); + } + + /** Returns topic results in request order. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Results for one topic. */ + @Internal + public static final class TopicResult { + private final String topicName; + private final List partitions; + + /** Creates the result for one topic. */ + public TopicResult(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns the partition results in request order. */ + public List partitions() { + return partitions; + } + } + + /** Result for one partition. */ + @Internal + public static final class PartitionResult { + private final int partitionId; + private final Errors error; + private final long baseOffset; + private final @Nullable String errorMessage; + + /** Creates the result for one partition. */ + public PartitionResult( + int partitionId, Errors error, long baseOffset, @Nullable String errorMessage) { + this.partitionId = partitionId; + this.error = checkNotNull(error); + this.baseOffset = baseOffset; + this.errorMessage = errorMessage; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the Kafka protocol error. */ + public Errors error() { + return error; + } + + /** Returns the first appended offset, or {@code -1} on failure. */ + public long baseOffset() { + return baseOffset; + } + + /** Returns an optional diagnostic error message. */ + public @Nullable String errorMessage() { + return errorMessage; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java new file mode 100644 index 00000000000..bf043823689 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.kafka.api.produce.ProduceHandler; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; + +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol tests independent of the Fluss append and record conversion backends. */ +class KafkaProduceProtocolTest { + + @Test + void testSupportedVersionsCopyRecordsAndForwardContext() { + for (short version = 3; version <= 11; version++) { + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request(version, (short) -1, partition(0, records())), + command -> { + copied.set(command); + return successful(command); + }); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(42L); + KafkaProduceCommand command = copied.get(); + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(4321); + assertThat(command.listenerName()).isEqualTo("KAFKA"); + KafkaProduceCommand.Record record = + command.topics().get(0).partitions().get(0).records().get(0); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()).hasSize(1); + assertThat(record.headers().get(0).value()).isNull(); + byte[] key = record.key(); + key[0] = 9; + assertThat(record.key()).containsExactly((byte) 1); + } + } + + @Test + void testCorruptPartitionDoesNotSuppressValidPartition() { + MemoryRecords corrupt = records(); + corrupt.buffer().put(corrupt.sizeInBytes() - 1, (byte) 99); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, corrupt), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + assertThat(command.topics().get(0).partitions().get(0).partitionId()) + .isEqualTo(1); + return successful(command); + }); + assertErrors(response, Errors.CORRUPT_MESSAGE, Errors.NONE); + } + + @Test + void testInvalidTopicIsIsolated() { + ProduceRequest request = request((short) 11, (short) 1, partition(0, records())); + request.data() + .topicData() + .add( + new TopicProduceData() + .setName("bad/name") + .setPartitionData( + Collections.singletonList(partition(0, records())))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()).hasSize(1); + return successful(command); + }); + assertThat(response.errorCounts()) + .containsEntry(Errors.NONE, 1) + .containsEntry(Errors.INVALID_TOPIC_EXCEPTION, 1); + } + + @Test + void testEmptyNegativeAndIdempotentPartitionsAreRejectedLocally() { + MemoryRecords idempotent = + MemoryRecords.withIdempotentRecords( + Compression.NONE, 10L, (short) 0, 0, new SimpleRecord(new byte[] {1})); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(-1, records()), + partition(2, idempotent), + partition(3, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + return successful(command); + }); + assertErrors( + response, + Errors.INVALID_RECORD, + Errors.INVALID_REQUEST, + Errors.INVALID_REQUEST, + Errors.NONE); + } + + @Test + void testRequestValidationNeverCallsBackend() { + KafkaProduceBackend unused = + command -> { + throw new AssertionError("Backend must not be called"); + }; + assertErrors( + dispatch(request((short) 11, (short) 2, partition(0, records())), unused), + Errors.INVALID_REQUIRED_ACKS); + ProduceRequest transactional = request((short) 11, (short) 1, partition(0, records())); + transactional.data().setTransactionalId("transaction"); + assertErrors( + dispatch(new ProduceRequest(transactional.data(), transactional.version()), unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch( + request( + (short) 11, + (short) 1, + partition(0, records()), + partition(0, records())), + unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch(request((short) 2, (short) 1, partition(0, records())), unused), + Errors.UNSUPPORTED_VERSION); + } + + @Test + void testBackendFailuresAndMissingResultsPreserveValidationErrors() { + for (boolean synchronous : Arrays.asList(true, false)) { + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(1, records())), + command -> { + if (synchronous) { + throw new TimeoutException("append timed out"); + } + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally( + new TimeoutException("append timed out")); + return failed; + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.REQUEST_TIMED_OUT); + } + assertErrors( + dispatch( + request((short) 11, (short) 1, partition(0, records())), + command -> + CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList()))), + Errors.UNKNOWN_SERVER_ERROR); + } + + @Test + void testAcksZeroSuccessAndFailureReleaseBufferWithoutSendingResponse() { + for (boolean failure : Arrays.asList(false, true)) { + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + ProduceRequest body = request((short) 11, (short) 0, partition(0, records())); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, (short) 11, "producer", 1); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + CompletableFuture pending = new CompletableFuture<>(); + try { + channel.writeInbound(buffer); + KafkaRequest parsed = (KafkaRequest) requests.pollRequest(1000); + dispatcher(command -> pending) + .dispatch(parsed) + .whenComplete( + (response, error) -> { + if (error == null) { + parsed.complete(response); + } else { + parsed.fail(error); + } + }); + assertThat(parsed.future()).isNotDone(); + if (failure) { + pending.completeExceptionally(new TimeoutException("failure")); + } else { + pending.complete( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, Errors.NONE, 42L, null)))))); + } + channel.runPendingTasks(); + assertThat(parsed.future()).isDone(); + assertThat((Object) channel.readOutbound()).isNull(); + assertThat(buffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + } + + private static ProduceResponse dispatch(ProduceRequest body, KafkaProduceBackend backend) { + ByteBuf buffer = Unpooled.buffer(1); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, body.version(), "producer", 1); + KafkaRequest request = + new KafkaRequest( + ApiKeys.PRODUCE, + body.version(), + header, + body, + "KAFKA", + buffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + buffer.release(); + request.complete(dispatcher(backend).dispatch(request).join()); + ByteBuf response = request.responseBuffer(); + try { + ByteBuffer bytes = response.nioBuffer(); + ResponseHeader.parse(bytes, header.toResponseHeader().headerVersion()); + return (ProduceResponse) + AbstractResponse.parseResponse(ApiKeys.PRODUCE, bytes, body.version()); + } finally { + response.release(); + } + } + + private static KafkaRequestDispatcher dispatcher(KafkaProduceBackend backend) { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ProduceHandler(backend)); + registry.freeze(); + return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); + } + + private static CompletableFuture successful(KafkaProduceCommand command) { + List topics = new ArrayList<>(); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + List partitions = new ArrayList<>(); + for (KafkaProduceCommand.PartitionWrite partition : topic.partitions()) { + partitions.add( + new PartitionResult(partition.partitionId(), Errors.NONE, 42L, null)); + } + topics.add(new TopicResult(topic.topicName(), partitions)); + } + return CompletableFuture.completedFuture(new KafkaProduceResult(topics)); + } + + private static ProduceRequest request( + short version, short acks, PartitionProduceData... partitions) { + TopicProduceData topic = + new TopicProduceData().setName("topic").setPartitionData(Arrays.asList(partitions)); + return new ProduceRequest( + new ProduceRequestData() + .setAcks(acks) + .setTimeoutMs(4321) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topic).iterator())), + version); + } + + private static PartitionProduceData partition(int index, MemoryRecords records) { + return new PartitionProduceData().setIndex(index).setRecords(records); + } + + private static MemoryRecords records() { + return MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.NONE, + new SimpleRecord( + 123L, + new byte[] {1}, + new byte[] {2}, + new Header[] {new RecordHeader("header", null)})); + } + + private static void assertErrors(ProduceResponse response, Errors... errors) { + assertThat(response.data().responses().find("topic").partitionResponses()) + .extracting(PartitionProduceResponse::errorCode) + .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); + } +} From 42c427f162625251e7af90ce7a920d6567d38396 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 17:04:41 +0800 Subject: [PATCH 24/48] [kafka] Validate Produce batch offsets and stream compressed records Read compressed records through a closeable streaming iterator to avoid preallocating a record list from the declared batch count. Reject incoming Kafka batches whose offset range disagrees with their record count before copying records or invoking the backend. Keep Fluss-assigned offsets independent of this input validation and preserve partition error isolation. Cover all five codecs, owned record copies, malformed counts and offset ranges, and backend-assigned offsets. Java 11 validation passed: 117 unit tests and 4 integration tests, plus Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 42/42 AI-Contributed/UT: 168/168 --- .../kafka/api/produce/ProduceHandler.java | 42 ++++- .../fluss/kafka/KafkaProduceProtocolTest.java | 168 ++++++++++++++++++ 2 files changed, 202 insertions(+), 8 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java index 0eb14b26ff2..a598c0d6cfa 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -30,6 +30,7 @@ import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.kafka.common.InvalidRecordException; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.InvalidRequiredAcksException; @@ -47,6 +48,8 @@ import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ProduceRequest; import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.CloseableIterator; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -219,14 +222,37 @@ private static List copyRecords( throw new InvalidRequestException( "Idempotent, transactional, and control record batches are not supported."); } - for (org.apache.kafka.common.record.Record record : batch) { - record.ensureValid(); - copied.add( - new KafkaProduceCommand.Record( - record.timestamp(), - copyBuffer(record.hasKey() ? record.key() : null), - copyBuffer(record.hasValue() ? record.value() : null), - copyHeaders(record.headers()))); + // Validate the incoming Kafka batch before copying away its offset metadata. Fluss + // assigns its own storage offsets later, independently of this input validation. + long countFromOffsets = batch.lastOffset() - batch.baseOffset() + 1; + Integer recordCount = batch.countOrNull(); + if (countFromOffsets <= 0 + || recordCount == null + || recordCount <= 0 + || countFromOffsets != recordCount) { + throw new InvalidRecordException( + "Invalid Kafka record batch offset range [" + + batch.baseOffset() + + ", " + + batch.lastOffset() + + "] for record count " + + recordCount + + "."); + } + // The ordinary compressed iterator preallocates a list using the untrusted record + // count. Stream records instead and close the decompressor on success and failure. + try (CloseableIterator iterator = + batch.streamingIterator(BufferSupplier.NO_CACHING)) { + while (iterator.hasNext()) { + org.apache.kafka.common.record.Record record = iterator.next(); + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } } } if (copied.isEmpty() || validBytes != records.sizeInBytes()) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java index bf043823689..c2160805510 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -41,6 +41,8 @@ import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.DefaultRecordBatch; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.SimpleRecord; @@ -50,7 +52,10 @@ import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.requests.ResponseHeader; +import org.apache.kafka.common.utils.Crc32C; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -102,6 +107,161 @@ void testSupportedVersionsCopyRecordsAndForwardContext() { } } + @ParameterizedTest + @EnumSource(CompressionType.class) + void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType) { + byte[] value = new byte[32 * 1024]; + Arrays.fill(value, (byte) 7); + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord( + 123L, + new byte[] {1}, + value, + new Header[] { + new RecordHeader("header", new byte[] {2, 3}), + new RecordHeader("header", null) + }), + new SimpleRecord(456L, null, new byte[0])); + AtomicReference copied = new AtomicReference<>(); + + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + copied.set(command); + return successful(command); + }); + assertErrors(response, Errors.NONE); + ByteBuffer input = records.buffer().duplicate(); + while (input.hasRemaining()) { + input.put((byte) 0); + } + + List result = + copied.get().topics().get(0).partitions().get(0).records(); + assertThat(result).hasSize(2); + assertThat(result.get(0).timestamp()).isEqualTo(123L); + assertThat(result.get(0).key()).containsExactly((byte) 1); + assertThat(result.get(0).value()).containsExactly(value); + assertThat(result.get(0).headers()) + .extracting(KafkaProduceCommand.RecordHeader::name) + .containsExactly("header", "header"); + assertThat(result.get(0).headers().get(0).value()).containsExactly((byte) 2, (byte) 3); + assertThat(result.get(0).headers().get(1).value()).isNull(); + assertThat(result.get(1).timestamp()).isEqualTo(456L); + assertThat(result.get(1).key()).isNull(); + assertThat(result.get(1).value()).isEmpty(); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testIncorrectRecordCountDoesNotSuppressValidPartition(CompressionType compressionType) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + ByteBuffer bytes = malformed.buffer(); + bytes.putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, 32); + bytes.putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, 31); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testInvalidBatchOffsetRangeDoesNotSuppressValidPartition(CompressionType compressionType) { + for (int[] invalidHeader : + new int[][] {{1, -1}, {1, 9}, {1, Integer.MAX_VALUE}, {0, 0}, {-1, 0}}) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + int recordCount = invalidHeader[0]; + int lastOffsetDelta = invalidHeader[1]; + malformed.buffer().putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, recordCount); + malformed.buffer().putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, lastOffsetDelta); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testValidBatchUsesOffsetAssignedByBackend(CompressionType compressionType) { + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1}), + new SimpleRecord(new byte[] {2}), + new SimpleRecord(new byte[] {3})); + long backendBaseOffset = 1L << 40; + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + assertThat(command.topics().get(0).partitions().get(0).records()) + .hasSize(3); + return CompletableFuture.completedFuture( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, + Errors.NONE, + backendBaseOffset, + null)))))); + }); + assertErrors(response, Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(backendBaseOffset); + } + @Test void testCorruptPartitionDoesNotSuppressValidPartition() { MemoryRecords corrupt = records(); @@ -363,4 +523,12 @@ private static void assertErrors(ProduceResponse response, Errors... errors) { .extracting(PartitionProduceResponse::errorCode) .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); } + + private static void updateBatchChecksum(MemoryRecords records) { + ByteBuffer bytes = records.buffer(); + int checksumStart = DefaultRecordBatch.CRC_OFFSET + Integer.BYTES; + bytes.putInt( + DefaultRecordBatch.CRC_OFFSET, + (int) Crc32C.compute(bytes, checksumStart, bytes.limit() - checksumStart)); + } } From 65804e71b12ff548c74598540c8468b42c872575 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 18:25:10 +0800 Subject: [PATCH 25/48] [kafka] Preserve batch timestamp types in Produce commands Carry timestamp provenance through a protocol-independent enum on each copied record. Read the type from its batch for compressed and uncompressed records; keep timestamp presence separate. Retain the existing constructor with an explicit UNKNOWN type instead of guessing provenance from the numeric value. Cover mixed batch types across partitions, absent timestamps, owned data and legacy constructors. Keep partition validation and acks=0 behavior, and check that ApiVersions advertises the registered Produce range. Metadata row storage and production append registration remain outside this protocol-only change. Validated with Java 11 and this worktree's clean reactor: 152 unit tests and 5 integration tests passed, including ApiVersions and Metadata regressions. Spotless, Checkstyle, RAT and git diff --check passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 64/64 AI-Contributed/UT: 326/326 --- .../kafka/api/produce/ProduceHandler.java | 16 ++ .../backend/produce/KafkaProduceCommand.java | 48 ++++- .../fluss/kafka/KafkaProduceProtocolTest.java | 159 ++++++++++++++++- .../api/versions/ApiVersionsHandlerTest.java | 7 + .../produce/KafkaProduceCommandTest.java | 164 ++++++++++++++++++ 5 files changed, 388 insertions(+), 6 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java index a598c0d6cfa..a679636a2c9 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -23,6 +23,7 @@ import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; @@ -239,6 +240,7 @@ private static List copyRecords( + recordCount + "."); } + TimestampType timestampType = timestampType(batch); // The ordinary compressed iterator preallocates a list using the untrusted record // count. Stream records instead and close the decompressor on success and failure. try (CloseableIterator iterator = @@ -249,6 +251,7 @@ private static List copyRecords( copied.add( new KafkaProduceCommand.Record( record.timestamp(), + timestampType, copyBuffer(record.hasKey() ? record.key() : null), copyBuffer(record.hasValue() ? record.value() : null), copyHeaders(record.headers()))); @@ -261,6 +264,19 @@ private static List copyRecords( return copied; } + private static TimestampType timestampType(RecordBatch batch) { + switch (batch.timestampType()) { + case CREATE_TIME: + return TimestampType.CREATE_TIME; + case LOG_APPEND_TIME: + return TimestampType.LOG_APPEND_TIME; + case NO_TIMESTAMP_TYPE: + return TimestampType.UNKNOWN; + default: + throw new InvalidRecordException("Unknown batch timestamp type."); + } + } + private static ProduceResponse toResponse(KafkaProduceResult result) { ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0); for (TopicResult topic : result.topics()) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index 722f3d3d245..9a4f035ad19 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -127,31 +127,75 @@ public List records() { } } + /** + * Timestamp provenance supplied by a batch, independent of whether a record has a timestamp. + */ + @Internal + public enum TimestampType { + /** No batch timestamp type was supplied, including by the legacy record constructor. */ + UNKNOWN, + /** The timestamp was assigned when the record was created. */ + CREATE_TIME, + /** The timestamp was assigned when the batch was appended to a log. */ + LOG_APPEND_TIME + } + /** A copied Kafka record whose lifetime is independent of the network request buffer. */ @Internal public static final class Record { + /** Sentinel used when the input record has no timestamp. */ + public static final long NO_TIMESTAMP = -1L; + private final long timestamp; + private final TimestampType timestampType; private final @Nullable byte[] key; private final @Nullable byte[] value; private final List headers; - /** Creates a copied Kafka record. */ + /** + * Creates a copied record without batch timestamp provenance. + * + *

The type is {@link TimestampType#UNKNOWN} even when a timestamp is present. Callers + * that know the batch type should use the explicit constructor instead of inferring it from + * the timestamp value. + */ public Record( long timestamp, @Nullable byte[] key, @Nullable byte[] value, List headers) { + this(timestamp, TimestampType.UNKNOWN, key, value, headers); + } + + /** Creates a copied record with the timestamp type supplied by its source batch. */ + public Record( + long timestamp, + TimestampType timestampType, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { this.timestamp = timestamp; + this.timestampType = checkNotNull(timestampType); this.key = copyNullable(key); this.value = copyNullable(value); this.headers = immutableCopy(headers); } - /** Returns the Kafka record timestamp. */ + /** Returns the record timestamp in milliseconds, or {@link #NO_TIMESTAMP} when absent. */ public long timestamp() { return timestamp; } + /** Returns whether the input record has a timestamp, independently of its batch type. */ + public boolean hasTimestamp() { + return timestamp != NO_TIMESTAMP; + } + + /** Returns the source batch's timestamp type, never inferred from the timestamp value. */ + public TimestampType timestampType() { + return timestampType; + } + /** Returns a copy of the nullable Kafka record key. */ public @Nullable byte[] key() { return copyNullable(key); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java index c2160805510..63d9f8b7c6d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -20,6 +20,7 @@ import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; @@ -44,6 +45,7 @@ import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.DefaultRecordBatch; import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.MemoryRecordsBuilder; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.SimpleRecord; import org.apache.kafka.common.requests.AbstractResponse; @@ -97,6 +99,8 @@ void testSupportedVersionsCopyRecordsAndForwardContext() { KafkaProduceCommand.Record record = command.topics().get(0).partitions().get(0).records().get(0); assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(record.hasTimestamp()).isTrue(); assertThat(record.key()).containsExactly((byte) 1); assertThat(record.value()).containsExactly((byte) 2); assertThat(record.headers()).hasSize(1); @@ -107,6 +111,120 @@ void testSupportedVersionsCopyRecordsAndForwardContext() { } } + @ParameterizedTest + @EnumSource(CompressionType.class) + void testTimestampTypesRemainLocalToTheirBatches(CompressionType compressionType) { + MemoryRecords createTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + 123L, + 456L); + MemoryRecords appendTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + 123L, + 789L, + 987L); + MemoryRecords missingCreateTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP); + MemoryRecords missingAppendTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP); + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, createTime), + partition(1, appendTime), + partition(2, missingCreateTime), + partition(3, missingAppendTime)), + command -> { + copied.set(command); + return successful(command); + }); + assertErrors(response, Errors.NONE, Errors.NONE, Errors.NONE, Errors.NONE); + for (MemoryRecords input : + Arrays.asList(createTime, appendTime, missingCreateTime, missingAppendTime)) { + ByteBuffer buffer = input.buffer().duplicate(); + while (buffer.hasRemaining()) { + buffer.put((byte) 0); + } + } + + List partitions = + copied.get().topics().get(0).partitions(); + assertThat(partitions).hasSize(4); + assertThat(partitions.get(0).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(123L, 456L); + // The same numeric timestamp can have a different batch type. + assertThat(partitions.get(1).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(123L, 123L); + for (int i = 0; i < partitions.size(); i++) { + TimestampType expectedType = + i % 2 == 0 ? TimestampType.CREATE_TIME : TimestampType.LOG_APPEND_TIME; + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::timestampType) + .containsExactly(expectedType, expectedType); + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::hasTimestamp) + .containsExactly(i < 2, i < 2); + if (i >= 2) { + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(RecordBatch.NO_TIMESTAMP, RecordBatch.NO_TIMESTAMP); + } + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testMultipleBatchesPerPartitionRemainInvalid(CompressionType compressionType) { + MemoryRecords first = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + 123L); + MemoryRecords second = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + 456L, + 123L); + ByteBuffer combined = ByteBuffer.allocate(first.sizeInBytes() + second.sizeInBytes()); + combined.put(first.buffer().duplicate()).put(second.buffer().duplicate()).flip(); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.readableRecords(combined)), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + @ParameterizedTest @EnumSource(CompressionType.class) void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType) { @@ -145,6 +263,7 @@ void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType copied.get().topics().get(0).partitions().get(0).records(); assertThat(result).hasSize(2); assertThat(result.get(0).timestamp()).isEqualTo(123L); + assertThat(result.get(0).timestampType()).isEqualTo(TimestampType.CREATE_TIME); assertThat(result.get(0).key()).containsExactly((byte) 1); assertThat(result.get(0).value()).containsExactly(value); assertThat(result.get(0).headers()) @@ -153,6 +272,7 @@ void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType assertThat(result.get(0).headers().get(0).value()).containsExactly((byte) 2, (byte) 3); assertThat(result.get(0).headers().get(1).value()).isNull(); assertThat(result.get(1).timestamp()).isEqualTo(456L); + assertThat(result.get(1).timestampType()).isEqualTo(TimestampType.CREATE_TIME); assertThat(result.get(1).key()).isNull(); assertThat(result.get(1).value()).isEmpty(); } @@ -409,7 +529,19 @@ void testAcksZeroSuccessAndFailureReleaseBufferWithoutSendingResponse() { try { channel.writeInbound(buffer); KafkaRequest parsed = (KafkaRequest) requests.pollRequest(1000); - dispatcher(command -> pending) + dispatcher( + command -> { + assertThat( + command.topics() + .get(0) + .partitions() + .get(0) + .records() + .get(0) + .timestampType()) + .isEqualTo(TimestampType.CREATE_TIME); + return pending; + }) .dispatch(parsed) .whenComplete( (response, error) -> { @@ -469,9 +601,8 @@ private static ProduceResponse dispatch(ProduceRequest body, KafkaProduceBackend } private static KafkaRequestDispatcher dispatcher(KafkaProduceBackend backend) { - KafkaApiRegistry registry = new KafkaApiRegistry(); - registry.register(new ProduceHandler(backend)); - registry.freeze(); + KafkaApiRegistry registry = + new KafkaApiRegistry(Collections.singletonList(new ProduceHandler(backend))); return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @@ -518,6 +649,26 @@ private static MemoryRecords records() { new Header[] {new RecordHeader("header", null)})); } + private static MemoryRecords timestampedRecords( + CompressionType compressionType, + org.apache.kafka.common.record.TimestampType timestampType, + long logAppendTime, + long... timestamps) { + try (MemoryRecordsBuilder builder = + MemoryRecords.builder( + ByteBuffer.allocate(1024), + RecordBatch.MAGIC_VALUE_V2, + Compression.of(compressionType).build(), + timestampType, + 0L, + logAppendTime)) { + for (long timestamp : timestamps) { + builder.append(new SimpleRecord(timestamp, new byte[] {1}, new byte[] {2})); + } + return builder.build(); + } + } + private static void assertErrors(ProduceResponse response, Errors... errors) { assertThat(response.data().responses().find("topic").partitionResponses()) .extracting(PartitionProduceResponse::errorCode) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java index 0492ee3572f..9d7912b5b42 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.api.versions; import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; @@ -59,6 +60,11 @@ void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { new KafkaApiRegistry( Arrays.asList( handler, + new ProduceHandler( + command -> { + throw new AssertionError( + "Capability advertisement must not invoke the Produce backend."); + }), metadataHandler, new TestingApiHandler( new KafkaApiSpec( @@ -78,6 +84,7 @@ void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, (short) 11), tuple(ApiKeys.METADATA.id, (short) 2, (short) 7), tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java new file mode 100644 index 00000000000..01fa6825b28 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests timestamp provenance and ownership of protocol-independent Produce commands. */ +class KafkaProduceCommandTest { + + @ParameterizedTest + @ValueSource(longs = {-1L, 0L, 123L}) + void testLegacyConstructorDoesNotInferBatchType(long timestamp) { + Record record = new Record(timestamp, null, null, Collections.emptyList()); + + assertThat(record.timestamp()).isEqualTo(timestamp); + assertThat(record.hasTimestamp()).isEqualTo(timestamp != Record.NO_TIMESTAMP); + assertThat(record.timestampType()).isEqualTo(TimestampType.UNKNOWN); + assertThat(record.key()).isNull(); + assertThat(record.value()).isNull(); + } + + @ParameterizedTest + @EnumSource(TimestampType.class) + void testTimestampPresenceDoesNotChangeExplicitBatchType(TimestampType timestampType) { + for (long timestamp : new long[] {Record.NO_TIMESTAMP, 0L, 123L}) { + Record record = + new Record(timestamp, timestampType, null, null, Collections.emptyList()); + + assertThat(record.timestampType()).isEqualTo(timestampType); + assertThat(record.timestamp()).isEqualTo(timestamp); + assertThat(record.hasTimestamp()).isEqualTo(timestamp != Record.NO_TIMESTAMP); + } + } + + @Test + void testExplicitBatchTypeMustNotBeNull() { + assertThatThrownBy(() -> new Record(123L, null, null, null, Collections.emptyList())) + .isInstanceOf(NullPointerException.class); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testBothConstructorsOwnTheirRecordData(boolean explicitTimestampType) { + byte[] key = {1}; + byte[] value = {2}; + byte[] headerValue = {3}; + List headers = + new ArrayList<>( + Arrays.asList( + new RecordHeader("header", headerValue), + new RecordHeader("header", null))); + Record record = + explicitTimestampType + ? new Record(123L, TimestampType.LOG_APPEND_TIME, key, value, headers) + : new Record(123L, key, value, headers); + key[0] = 4; + value[0] = 5; + headerValue[0] = 6; + headers.clear(); + record.key()[0] = 7; + record.value()[0] = 8; + record.headers().get(0).value()[0] = 9; + + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()) + .extracting(RecordHeader::name) + .containsExactly("header", "header"); + assertThat(record.headers().get(0).value()).containsExactly((byte) 3); + assertThat(record.headers().get(1).value()).isNull(); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.timestampType()) + .isEqualTo( + explicitTimestampType + ? TimestampType.LOG_APPEND_TIME + : TimestampType.UNKNOWN); + assertThatThrownBy(() -> record.headers().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { + List records = + new ArrayList<>( + Arrays.asList( + new Record( + 123L, + TimestampType.CREATE_TIME, + null, + null, + Collections.emptyList()), + new Record( + 123L, + TimestampType.LOG_APPEND_TIME, + null, + null, + Collections.emptyList()), + new Record( + Record.NO_TIMESTAMP, null, null, Collections.emptyList()))); + List partitions = + new ArrayList<>(Collections.singletonList(new PartitionWrite(2, records))); + List topics = + new ArrayList<>(Collections.singletonList(new TopicWrite("db.topic", partitions))); + KafkaProduceCommand command = + new KafkaProduceCommand((short) -1, 1000, topics, "KAFKA", null); + records.clear(); + partitions.clear(); + topics.clear(); + + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(1000); + assertThat(command.topics()).extracting(TopicWrite::topicName).containsExactly("db.topic"); + assertThat(command.topics().get(0).partitions()) + .extracting(PartitionWrite::partitionId) + .containsExactly(2); + List copied = command.topics().get(0).partitions().get(0).records(); + assertThat(copied) + .extracting(Record::timestampType) + .containsExactly( + TimestampType.CREATE_TIME, + TimestampType.LOG_APPEND_TIME, + TimestampType.UNKNOWN); + assertThat(copied) + .extracting(Record::timestamp) + .containsExactly(123L, 123L, Record.NO_TIMESTAMP); + assertThatThrownBy(() -> command.topics().clear()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> command.topics().get(0).partitions().clear()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(copied::clear).isInstanceOf(UnsupportedOperationException.class); + } +} From 0dde872227c68deb96e5959192af7f801fed9861 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 17:50:34 +0800 Subject: [PATCH 26/48] [kafka] Verify Produce topic aliases preserve request names Document that the protocol backend receives original topic names and returns results keyed by those names, leaving database resolution to the adapter. Cover bare and qualified names together across Produce v3-v11, reordered backend results, local validation failures, backend errors and invalid acks. Keep null and empty record values distinct in the owned protocol command. Validated on Java 11 with this worktree's clean reactor: 162 Kafka unit tests, 23 integration tests, and 34 Common/Server schema and DDL tests passed. Spotless, Checkstyle, RAT and git diff --check passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 11/11 AI-Contributed/UT: 150/150 --- .../kafka/api/produce/ProduceHandler.java | 7 +- .../backend/produce/KafkaProduceCommand.java | 2 +- .../backend/produce/KafkaProduceResult.java | 2 +- .../fluss/kafka/KafkaProduceProtocolTest.java | 150 +++++++++++++++++- 4 files changed, 155 insertions(+), 6 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java index a679636a2c9..c485e618152 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -68,7 +68,12 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Implements non-idempotent Kafka Produce versions 3 through 11. */ +/** + * Implements non-idempotent Kafka Produce versions 3 through 11. + * + *

Topic names cross the backend boundary unchanged. The backend resolves bare and qualified + * names and returns results keyed by the original request names. + */ @Internal public final class ProduceHandler implements KafkaApiHandler { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index 9a4f035ad19..aadaa36f737 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -93,7 +93,7 @@ public TopicWrite(String topicName, List partitions) { this.partitions = immutableCopy(partitions); } - /** Returns the Kafka topic name. */ + /** Returns the original request topic name, including any default-database alias. */ public String topicName() { return topicName; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java index 218c566bde4..8f41864f85b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java @@ -60,7 +60,7 @@ public TopicResult(String topicName, List partitions) { this.partitions = immutableCopy(partitions); } - /** Returns the Kafka topic name. */ + /** Returns the original request topic name, rather than a resolved table path. */ public String topicName() { return topicName; } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java index 63d9f8b7c6d..1a801cd55ed 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -40,6 +40,7 @@ import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.message.ProduceResponseData.TopicProduceResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.CompressionType; @@ -58,6 +59,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -68,6 +70,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; /** Protocol tests independent of the Fluss append and record conversion backends. */ class KafkaProduceProtocolTest { @@ -111,6 +114,134 @@ void testSupportedVersionsCopyRecordsAndForwardContext() { } } + @ParameterizedTest + @ValueSource(shorts = {3, 4, 5, 6, 7, 8, 9, 10, 11}) + void testBareAndQualifiedTopicNamesRemainUnchanged(short version) { + MemoryRecords nullableValues = + MemoryRecords.withRecords( + Compression.NONE, + new SimpleRecord(123L, new byte[] {1}, null), + new SimpleRecord(456L, null, new byte[0])); + ProduceRequest request = + requestForTopics( + version, + (short) 1, + topic("orders", partition(0, nullableValues)), + topic("fluss.orders", partition(0, records())), + topic("sales.orders", partition(0, records()))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()) + .extracting(KafkaProduceCommand.TopicWrite::topicName) + .containsExactly("orders", "fluss.orders", "sales.orders"); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + assertThat(topic.partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(0); + } + List copied = + command.topics().get(0).partitions().get(0).records(); + assertThat(copied).hasSize(2); + // The protocol command preserves null and empty values for the adapter. + assertThat(copied.get(0).value()).isNull(); + assertThat(copied.get(1).value()).isEmpty(); + List results = + new ArrayList<>(successful(command).join().topics()); + Collections.reverse(results); + return CompletableFuture.completedFuture( + new KafkaProduceResult(results)); + }); + + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactly("orders", "fluss.orders", "sales.orders"); + for (TopicProduceResponse topic : response.data().responses()) { + assertThat(topic.partitionResponses()) + .extracting( + PartitionProduceResponse::index, + PartitionProduceResponse::errorCode, + PartitionProduceResponse::baseOffset) + .containsExactly(tuple(0, Errors.NONE.code(), 42L)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"orders", "fluss.orders"}) + void testLocalAndBackendErrorsAreIsolatedByRequestTopicName(String invalidTopic) { + String validTopic = invalidTopic.equals("orders") ? "fluss.orders" : "orders"; + ProduceResponse response = + dispatch( + requestForTopics( + (short) 11, + (short) 1, + topic(invalidTopic, partition(0, MemoryRecords.EMPTY)), + topic(validTopic, partition(0, records())), + topic("sales.orders", partition(0, records()))), + command -> { + assertThat(command.topics()) + .extracting(KafkaProduceCommand.TopicWrite::topicName) + .containsExactly(validTopic, "sales.orders"); + return CompletableFuture.completedFuture( + new KafkaProduceResult( + Arrays.asList( + new TopicResult( + "sales.orders", + Collections.singletonList( + new PartitionResult( + 0, + Errors + .TOPIC_AUTHORIZATION_FAILED, + -1L, + null))), + new TopicResult( + validTopic, + Collections.singletonList( + new PartitionResult( + 0, + Errors.NONE, + 42L, + null)))))); + }); + + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactly(invalidTopic, validTopic, "sales.orders"); + assertTopicError(response, invalidTopic, Errors.INVALID_RECORD); + assertTopicError(response, validTopic, Errors.NONE); + assertTopicError(response, "sales.orders", Errors.TOPIC_AUTHORIZATION_FAILED); + } + + @ParameterizedTest + @ValueSource(shorts = {3, 4, 5, 6, 7, 8, 9, 10, 11}) + void testRequestAndBackendFailuresPreserveTopicNames(short version) { + for (boolean invalidAcks : Arrays.asList(false, true)) { + ProduceResponse response = + dispatch( + requestForTopics( + version, + invalidAcks ? (short) 2 : (short) 1, + topic("orders", partition(0, records())), + topic("fluss.orders", partition(0, records())), + topic("sales.orders", partition(0, records()))), + command -> { + assertThat(invalidAcks).isFalse(); + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally(new TimeoutException("append failed")); + return failed; + }); + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactlyInAnyOrder("orders", "fluss.orders", "sales.orders"); + Errors error = invalidAcks ? Errors.INVALID_REQUIRED_ACKS : Errors.REQUEST_TIMED_OUT; + for (String name : Arrays.asList("orders", "fluss.orders", "sales.orders")) { + assertTopicError(response, name, error); + } + } + } + @ParameterizedTest @EnumSource(CompressionType.class) void testTimestampTypesRemainLocalToTheirBatches(CompressionType compressionType) { @@ -621,18 +752,25 @@ private static CompletableFuture successful(KafkaProduceComm private static ProduceRequest request( short version, short acks, PartitionProduceData... partitions) { - TopicProduceData topic = - new TopicProduceData().setName("topic").setPartitionData(Arrays.asList(partitions)); + return requestForTopics(version, acks, topic("topic", partitions)); + } + + private static ProduceRequest requestForTopics( + short version, short acks, TopicProduceData... topics) { return new ProduceRequest( new ProduceRequestData() .setAcks(acks) .setTimeoutMs(4321) .setTopicData( new ProduceRequestData.TopicProduceDataCollection( - Collections.singletonList(topic).iterator())), + Arrays.asList(topics).iterator())), version); } + private static TopicProduceData topic(String name, PartitionProduceData... partitions) { + return new TopicProduceData().setName(name).setPartitionData(Arrays.asList(partitions)); + } + private static PartitionProduceData partition(int index, MemoryRecords records) { return new PartitionProduceData().setIndex(index).setRecords(records); } @@ -675,6 +813,12 @@ private static void assertErrors(ProduceResponse response, Errors... errors) { .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); } + private static void assertTopicError(ProduceResponse response, String topic, Errors error) { + assertThat(response.data().responses().find(topic).partitionResponses()) + .extracting(PartitionProduceResponse::index, PartitionProduceResponse::errorCode) + .containsExactly(tuple(0, error.code())); + } + private static void updateBatchChecksum(MemoryRecords records) { ByteBuffer bytes = records.buffer(); int checksumStart = DefaultRecordBatch.CRC_OFFSET + Integer.BYTES; From e40fc47b7050f5aa289e205e257a6497c9640755 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:21:35 +0800 Subject: [PATCH 27/48] [kafka] Transcode mapped raw and string records to Arrow Reuse the DDL contract to assemble key, value, timestamp and ordered headers into physical rows. Decode strings with strict UTF-8 validation and preserve nullable values. Return owned heap log bytes and recycle Arrow writers on success and failure. Validation: Java 11, mvn -o -pl fluss-kafka spotless:apply clean verify; 75 unit tests and 2 integration tests passed, including Checkstyle, Spotless and RAT. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 314/314 AI-Contributed/UT: 326/326 --- .../transcode/ArrowKafkaRecordTranscoder.java | 87 +++++ .../transcode/FlussArrowRecordEncoder.java | 71 ++++ .../KafkaRecordEncodingException.java | 35 ++ .../transcode/KafkaRecordTranscoder.java | 32 ++ .../kafka/transcode/KafkaRowAssembler.java | 89 +++++ .../ArrowKafkaRecordTranscoderTest.java | 326 ++++++++++++++++++ 6 files changed, 640 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java new file mode 100644 index 00000000000..717f5cd33ff --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; + +import javax.annotation.Nullable; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Converts raw/string Kafka records using the DDL mapping into owned Fluss Arrow log bytes. */ +@Internal +public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); + + @Override + public BytesView transcode(List records, TableInfo tableInfo) throws Exception { + checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); + KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaRowAssembler assembler = new KafkaRowAssembler(schema); + List rows = new ArrayList<>(records.size()); + for (Record record : records) { + Object[] key = + schema.keyFormat() == null + ? new Object[0] + : new Object[] {decode(schema.keyFormat(), record.key())}; + rows.add( + assembler.assemble( + key, + new Object[] {decode(schema.valueFormat(), record.value())}, + record.timestamp(), + record.headers())); + } + return arrowRecordEncoder.encode(rows, tableInfo); + } + + private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) { + if (bytes == null || format == KafkaDataFormat.RAW) { + return bytes; + } + if (format != KafkaDataFormat.STRING) { + throw new KafkaRecordEncodingException("Unsupported Kafka data format: " + format); + } + try { + return BinaryString.fromString( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString()); + } catch (CharacterCodingException e) { + throw new KafkaRecordEncodingException("Kafka string field is not valid UTF-8.", e); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java new file mode 100644 index 00000000000..f74680ccaa2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.memory.UnmanagedPagedOutputView; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.MemoryLogRecordsArrowBuilder; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.arrow.ArrowWriter; +import org.apache.fluss.row.arrow.ArrowWriterPool; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator; + +import java.util.List; + +/** Encodes assembled physical Fluss rows into one native Arrow log batch. */ +@Internal +public final class FlussArrowRecordEncoder { + + private static final int INITIAL_PAGE_SIZE = 4096; + + /** Encodes all rows using the table's current schema ID and Arrow compression settings. */ + public BytesView encode(List rows, TableInfo tableInfo) throws Exception { + try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + ArrowWriterPool provider = new ArrowWriterPool(allocator)) { + ArrowWriter writer = + provider.getOrCreateWriter( + tableInfo.getTableId(), + tableInfo.getSchemaId(), + Integer.MAX_VALUE, + tableInfo.getRowType(), + tableInfo.getTableConfig().getArrowCompressionInfo()); + long epoch = writer.getEpoch(); + try { + MemoryLogRecordsArrowBuilder builder = + MemoryLogRecordsArrowBuilder.builder( + tableInfo.getSchemaId(), + writer, + new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), + true, + null); + for (GenericRow row : rows) { + builder.append(ChangeType.APPEND_ONLY, row); + } + // The output view owns heap pages. The result remains valid after the Arrow + // writer and allocator close, including across asynchronous append completion. + return builder.build(); + } finally { + writer.recycle(epoch); + } + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java new file mode 100644 index 00000000000..b48dd9d894f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that Kafka record bytes cannot be decoded using the configured data format. */ +@Internal +public final class KafkaRecordEncodingException extends IllegalArgumentException { + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message) { + super(message); + } + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java new file mode 100644 index 00000000000..eeff2f3d14f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; + +import java.util.List; + +/** Converts copied Kafka records into the native Fluss log representation. */ +@Internal +public interface KafkaRecordTranscoder { + /** Transcodes records according to the target Fluss table schema and log format. */ + BytesView transcode(List records, TableInfo tableInfo) throws Exception; +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java new file mode 100644 index 00000000000..4ebd506dc3c --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.schema.KafkaFieldProjection; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; + +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Assembles decoded Kafka key, value, and metadata into a physical Fluss row. */ +@Internal +public final class KafkaRowAssembler { + + private final KafkaTopicSchema topicSchema; + + /** Creates an assembler for the resolved topic schema. */ + public KafkaRowAssembler(KafkaTopicSchema topicSchema) { + this.topicSchema = checkNotNull(topicSchema); + } + + /** Assembles one Fluss row. */ + public GenericRow assemble( + Object[] keyValues, Object[] valueValues, long timestamp, List headers) { + GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); + setProjectedFields(row, topicSchema.keyProjection(), keyValues); + setProjectedFields(row, topicSchema.valueProjection(), valueValues); + if (topicSchema.timestampPosition() >= 0) { + row.setField(topicSchema.timestampPosition(), TimestampLtz.fromEpochMillis(timestamp)); + } + if (topicSchema.headersPosition() >= 0) { + row.setField(topicSchema.headersPosition(), toHeaders(headers)); + } + return row; + } + + private static void setProjectedFields( + GenericRow row, KafkaFieldProjection projection, Object[] values) { + if (values.length != projection.size()) { + throw new IllegalArgumentException( + "Kafka decoder returned " + + values.length + + " fields for a projection of " + + projection.size() + + "."); + } + for (int i = 0; i < values.length; i++) { + Object value = values[i]; + if (value == null && !projection.dataTypeAt(i).isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka record cannot populate NOT NULL field '" + + projection.nameAt(i) + + "' with null."); + } + row.setField(projection.positionAt(i), value); + } + } + + private static GenericArray toHeaders(List headers) { + Object[] rows = new Object[headers.size()]; + for (int i = 0; i < headers.size(); i++) { + RecordHeader header = headers.get(i); + rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.value()); + } + return new GenericArray(rows); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java new file mode 100644 index 00000000000..6936b5c8d61 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.CloseableIterator; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; + +/** Verifies mapped Arrow records after transcoding resources have closed. */ +class ArrowKafkaRecordTranscoderTest { + private final ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + + @Test + void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { + for (boolean string : new boolean[] {false, true}) { + TableInfo table = envelope(string); + Record record = + new Record( + 123L, + bytes("键"), + bytes("message"), + Arrays.asList( + new RecordHeader("duplicate", bytes("first")), + new RecordHeader("duplicate", null))); + BytesView encoded = transcoder.transcode(Arrays.asList(record, record), table); + read( + encoded, + table, + 2, + row -> { + if (string) { + assertThat(row.getString(0).toString()).isEqualTo("message"); + assertThat(row.getString(3).toString()).isEqualTo("键"); + } else { + assertThat(row.getBytes(0)).isEqualTo(bytes("message")); + assertThat(row.getBytes(3)).isEqualTo(bytes("键")); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + InternalArray headers = row.getArray(2); + assertThat(headers.size()).isEqualTo(2); + assertThat(headers.getRow(0, 2).getString(0).toString()) + .isEqualTo("duplicate"); + assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("first")); + assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); + }); + } + } + + @Test + void testNullsAndEmptyBytesRemainDistinct() throws Exception { + TableInfo table = envelope(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, null, null, Collections.emptyList())), + table), + table, + 1, + row -> { + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.isNullAt(3)).isTrue(); + assertThat(row.getArray(2).size()).isZero(); + }); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, new byte[0], new byte[0], Collections.emptyList())), + table), + table, + 1, + row -> { + assertThat(row.getBytes(0)).isEmpty(); + assertThat(row.getBytes(3)).isEmpty(); + }); + } + + @Test + void testUnmappedKeyIsIgnoredAndMixedFormatsWork() throws Exception { + TableInfo valueOnly = valueTable(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + new byte[] {(byte) 0xff}, + bytes("value"), + Collections.emptyList())), + valueOnly), + valueOnly, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("value")); + Schema schema = + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("id", DataTypes.STRING()) + .build(); + TableInfo mixed = + table( + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .build()); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + bytes("key"), + new byte[] {(byte) 0xff}, + Collections.emptyList())), + mixed), + mixed, + 1, + row -> { + assertThat(row.getString(1).toString()).isEqualTo("key"); + assertThat(row.getBytes(0)).containsExactly((byte) 0xff); + }); + } + + @Test + void testRejectsMalformedUtf8AndNotNullViolationsThenRemainsUsable() throws Exception { + TableInfo table = valueTable(false); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + new byte[] {(byte) 0xc3, 0x28}, + Collections.emptyList())), + table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("UTF-8"); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, null, null, Collections.emptyList())), + valueTable(true))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL"); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, null, bytes("ok"), Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("ok")); + } + + @Test + void testRejectsInvalidMappingAndEmptyPartition() { + TableInfo invalid = + table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING()) + .build()) + .distributedBy(1) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + bytes("value"), + Collections.emptyList())), + invalid)) + .isInstanceOf(KafkaTopicSchemaException.class); + assertThatThrownBy(() -> transcoder.transcode(Collections.emptyList(), valueTable(false))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("empty"); + } + + @Test + void testArrowWriterFailureClosesAllocatorWithoutLeaking() { + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encode( + Collections.singletonList(GenericRow.of(123)), + valueTable(false))); + assertThat(failure).isInstanceOf(ClassCastException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + + private static TableInfo valueTable(boolean notNull) { + return table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING().copy(!notNull)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .build()); + } + + private static TableInfo envelope(boolean string) { + Schema schema = + Schema.newBuilder() + .column("body", string ? DataTypes.STRING() : DataTypes.BYTES()) + .column("time", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "attrs", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES())))) + .column("id", string ? DataTypes.STRING() : DataTypes.BYTES()) + .build(); + return table( + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty( + KafkaDataFormat.VALUE_FORMAT_CONFIG, string ? "string" : "raw") + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, string ? "string" : "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "time") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attrs") + .build()); + } + + private static TableInfo table(TableDescriptor descriptor) { + return TableInfo.of(TablePath.of("kafka", "topic"), 42L, 3, descriptor, null, 1L, 1L); + } + + private static void read( + BytesView encoded, TableInfo table, int count, Consumer verify) + throws Exception { + ByteBuf buffer = encoded.getByteBuf(); + try { + LogRecordBatch batch = + MemoryLogRecords.pointToByteBuffer(buffer.nioBuffer()) + .batches() + .iterator() + .next(); + batch.ensureValid(); + assertThat(batch.schemaId()).isEqualTo((short) table.getSchemaId()); + assertThat(batch.getRecordCount()).isEqualTo(count); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + table.getRowType(), + table.getSchemaId(), + new TestingSchemaGetter( + new SchemaInfo( + table.getSchema(), table.getSchemaId()))); + CloseableIterator records = batch.records(context)) { + for (int i = 0; i < count; i++) { + assertThat(records.hasNext()).isTrue(); + verify.accept(records.next().getRow()); + } + assertThat(records.hasNext()).isFalse(); + } + } finally { + buffer.release(); + } + } + + private static byte[] bytes(String text) { + return text.getBytes(StandardCharsets.UTF_8); + } +} From 0d7a442f701a35550149514e2cc9bbbe049d9206 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 17:19:52 +0800 Subject: [PATCH 28/48] [kafka] Stream record transcoding and reuse owned payloads Decode and append one record at a time instead of retaining an entire row list. Borrow command-owned key, value and header bytes during synchronous conversion while preserving defensive-copy accessors. Cover row and payload reuse, batches beyond initial Arrow capacity, and resource cleanup after partial conversion failures. Validation on Java 11: reactor clean verify passed with 130 unit tests and 4 integration tests, plus Checkstyle, Spotless, RAT and git diff --check. An additional 25 boundary scenarios passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 101/101 AI-Contributed/UT: 224/224 --- .../backend/produce/KafkaProduceCommand.java | 30 ++++ .../transcode/ArrowKafkaRecordTranscoder.java | 36 +++-- .../transcode/FlussArrowRecordEncoder.java | 26 +++- .../kafka/transcode/KafkaRowAssembler.java | 9 +- .../produce/KafkaProduceCommandTest.java | 47 ++++++ .../ArrowKafkaRecordTranscoderTest.java | 144 ++++++++++++++++++ 6 files changed, 271 insertions(+), 21 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index aadaa36f737..db13f63b314 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -201,11 +201,31 @@ public TimestampType timestampType() { return copyNullable(key); } + /** + * Borrows the nullable key owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #key()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedKey() { + return key; + } + /** Returns a copy of the nullable Kafka record value. */ public @Nullable byte[] value() { return copyNullable(value); } + /** + * Borrows the nullable value owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } + /** Returns the copied Kafka headers in record order. */ public List headers() { return headers; @@ -237,5 +257,15 @@ public String name() { public @Nullable byte[] value() { return value == null ? null : value.clone(); } + + /** + * Borrows the nullable header value for synchronous conversion. + * + *

The returned array is owned by this header and must not be modified or retained after + * conversion. Use {@link #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 717f5cd33ff..9fcaaaabf31 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -25,7 +25,6 @@ import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.BinaryString; -import org.apache.fluss.row.GenericRow; import javax.annotation.Nullable; @@ -33,7 +32,6 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -49,20 +47,26 @@ public BytesView transcode(List records, TableInfo tableInfo) throws Exc checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); KafkaRowAssembler assembler = new KafkaRowAssembler(schema); - List rows = new ArrayList<>(records.size()); - for (Record record : records) { - Object[] key = - schema.keyFormat() == null - ? new Object[0] - : new Object[] {decode(schema.keyFormat(), record.key())}; - rows.add( - assembler.assemble( - key, - new Object[] {decode(schema.valueFormat(), record.value())}, - record.timestamp(), - record.headers())); - } - return arrowRecordEncoder.encode(rows, tableInfo); + return arrowRecordEncoder.encodeStreaming( + consumer -> { + for (Record record : records) { + Object[] key = + schema.keyFormat() == null + ? new Object[0] + : new Object[] { + decode(schema.keyFormat(), record.borrowedKey()) + }; + consumer.append( + assembler.assemble( + key, + new Object[] { + decode(schema.valueFormat(), record.borrowedValue()) + }, + record.timestamp(), + record.headers())); + } + }, + tableInfo); } private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java index f74680ccaa2..c84dd20d690 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java @@ -39,6 +39,16 @@ public final class FlussArrowRecordEncoder { /** Encodes all rows using the table's current schema ID and Arrow compression settings. */ public BytesView encode(List rows, TableInfo tableInfo) throws Exception { + return encodeStreaming( + consumer -> { + for (GenericRow row : rows) { + consumer.append(row); + } + }, + tableInfo); + } + + BytesView encodeStreaming(RowSource rowSource, TableInfo tableInfo) throws Exception { try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); ArrowWriterPool provider = new ArrowWriterPool(allocator)) { ArrowWriter writer = @@ -57,9 +67,7 @@ public BytesView encode(List rows, TableInfo tableInfo) throws Excep new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), true, null); - for (GenericRow row : rows) { - builder.append(ChangeType.APPEND_ONLY, row); - } + rowSource.produce(row -> builder.append(ChangeType.APPEND_ONLY, row)); // The output view owns heap pages. The result remains valid after the Arrow // writer and allocator close, including across asynchronous append completion. return builder.build(); @@ -68,4 +76,16 @@ public BytesView encode(List rows, TableInfo tableInfo) throws Excep } } } + + @FunctionalInterface + interface RowSource { + /** Produces rows synchronously and never retains the supplied consumer. */ + void produce(RowConsumer consumer) throws Exception; + } + + @FunctionalInterface + interface RowConsumer { + /** Serializes one row before returning and never retains the row. */ + void append(GenericRow row) throws Exception; + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index 4ebd506dc3c..cac8ac3e841 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -41,7 +41,12 @@ public KafkaRowAssembler(KafkaTopicSchema topicSchema) { this.topicSchema = checkNotNull(topicSchema); } - /** Assembles one Fluss row. */ + /** + * Assembles one Fluss row for immediate synchronous encoding. + * + *

The row borrows decoded values and header payloads; it must not modify them or retain them + * after conversion. + */ public GenericRow assemble( Object[] keyValues, Object[] valueValues, long timestamp, List headers) { GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); @@ -82,7 +87,7 @@ private static GenericArray toHeaders(List headers) { Object[] rows = new Object[headers.size()]; for (int i = 0; i < headers.size(); i++) { RecordHeader header = headers.get(i); - rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.value()); + rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.borrowedValue()); } return new GenericArray(rows); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java index 01fa6825b28..3b9c8f15ec0 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -161,4 +161,51 @@ void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { .isInstanceOf(UnsupportedOperationException.class); assertThatThrownBy(copied::clear).isInstanceOf(UnsupportedOperationException.class); } + + @Test + void testBorrowedPayloadsReuseOwnedArraysWhileCopyAccessorsRemainDefensive() { + byte[] key = {1}; + byte[] value = {2}; + byte[] headerValue = {3}; + RecordHeader header = new RecordHeader("header", headerValue); + List headers = new ArrayList<>(); + headers.add(header); + Record record = new Record(1L, key, value, headers); + + assertThat(record.borrowedKey()).isNotSameAs(key); + assertThat(record.borrowedValue()).isNotSameAs(value); + assertThat(header.borrowedValue()).isNotSameAs(headerValue); + key[0] = 10; + value[0] = 20; + headerValue[0] = 30; + headers.clear(); + + assertThat(record.borrowedKey()).containsExactly((byte) 1); + assertThat(record.borrowedValue()).containsExactly((byte) 2); + assertThat(header.borrowedValue()).containsExactly((byte) 3); + assertThat(record.headers()).containsExactly(header); + assertThat(record.borrowedKey()).isSameAs(record.borrowedKey()); + assertThat(record.borrowedValue()).isSameAs(record.borrowedValue()); + assertThat(header.borrowedValue()).isSameAs(header.borrowedValue()); + + record.key()[0] = 40; + record.value()[0] = 50; + header.value()[0] = 60; + assertThat(record.key()).containsExactly((byte) 1).isNotSameAs(record.borrowedKey()); + assertThat(record.value()).containsExactly((byte) 2).isNotSameAs(record.borrowedValue()); + assertThat(header.value()).containsExactly((byte) 3).isNotSameAs(header.borrowedValue()); + } + + @Test + void testBorrowedPayloadsPreserveNullAndEmptyBytes() { + Record nullRecord = new Record(1L, null, null, Collections.emptyList()); + assertThat(nullRecord.borrowedKey()).isNull(); + assertThat(nullRecord.borrowedValue()).isNull(); + assertThat(new RecordHeader("header", null).borrowedValue()).isNull(); + + Record emptyRecord = new Record(1L, new byte[0], new byte[0], Collections.emptyList()); + assertThat(emptyRecord.borrowedKey()).isEmpty(); + assertThat(emptyRecord.borrowedValue()).isEmpty(); + assertThat(new RecordHeader("header", new byte[0]).borrowedValue()).isEmpty(); + } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 6936b5c8d61..f1df39504d2 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -33,18 +33,27 @@ import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.record.TestingSchemaGetter; import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.InternalArray; import org.apache.fluss.row.InternalRow; +import org.apache.fluss.row.TimestampLtz; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.CloseableIterator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; @@ -242,6 +251,141 @@ void testArrowWriterFailureClosesAllocatorWithoutLeaking() { assertThat(failure.getSuppressed()).isEmpty(); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean string) + throws Exception { + TableInfo table = envelope(string); + List records = new ArrayList<>(); + for (int i = 0; i < 2050; i++) { + records.add( + new Record( + i, + bytes("key-" + i), + bytes("消息-" + i), + Arrays.asList( + new RecordHeader("duplicate", bytes("header-" + i)), + new RecordHeader("duplicate", null)))); + } + BytesView encoded = transcoder.transcode(records, table); + AtomicInteger nextRecord = new AtomicInteger(); + read( + encoded, + table, + records.size(), + row -> { + int i = nextRecord.getAndIncrement(); + if (string) { + assertThat(row.getString(0).toString()).isEqualTo("消息-" + i); + assertThat(row.getString(3).toString()).isEqualTo("key-" + i); + } else { + assertThat(row.getBytes(0)).isEqualTo(bytes("消息-" + i)); + assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + InternalArray headers = row.getArray(2); + assertThat(headers.size()).isEqualTo(2); + assertThat(headers.getRow(0, 2).getString(0).toString()).isEqualTo("duplicate"); + assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("header-" + i)); + assertThat(headers.getRow(1, 2).getString(0).toString()).isEqualTo("duplicate"); + assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); + }); + } + + @Test + void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exception { + TableInfo table = envelope(false); + byte[] key = new byte[1]; + byte[] value = new byte[1]; + byte[] headerValue = new byte[1]; + GenericRow row = + GenericRow.of( + value, + TimestampLtz.fromEpochMillis(1L), + new GenericArray( + new Object[] { + GenericRow.of(BinaryString.fromString("header"), headerValue) + }), + key); + BytesView encoded = + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + for (int i = 0; i < 3; i++) { + key[0] = (byte) i; + value[0] = (byte) (i + 10); + headerValue[0] = (byte) (i + 20); + row.setField(1, TimestampLtz.fromEpochMillis(i)); + consumer.append(row); + } + Arrays.fill(key, (byte) -1); + Arrays.fill(value, (byte) -1); + Arrays.fill(headerValue, (byte) -1); + row.setField(0, null); + }, + table); + AtomicInteger nextRecord = new AtomicInteger(); + read( + encoded, + table, + 3, + actual -> { + int i = nextRecord.getAndIncrement(); + assertThat(actual.getBytes(3)).containsExactly((byte) i); + assertThat(actual.getBytes(0)).containsExactly((byte) (i + 10)); + assertThat(actual.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + assertThat(actual.getArray(2).getRow(0, 2).getBytes(1)) + .containsExactly((byte) (i + 20)); + }); + } + + @Test + void testDecodeFailureAfterValidRecordClosesAllocatorAndRemainsUsable() throws Exception { + TableInfo table = valueTable(true); + Record valid = new Record(1L, null, bytes("valid"), Collections.emptyList()); + for (byte[] invalidValue : new byte[][] {null, {(byte) 0xc3, 0x28}}) { + Throwable failure = + catchThrowable( + () -> + transcoder.transcode( + Arrays.asList( + valid, + new Record( + 2L, + null, + invalidValue, + Collections.emptyList())), + table)); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + read( + transcoder.transcode(Collections.singletonList(valid), table), + table, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); + } + + @Test + void testStreamingSourceFailureClosesAllocator() { + IOException expected = new IOException("source failed after the first row"); + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + consumer.append( + GenericRow.of( + BinaryString.fromString( + "valid"))); + throw expected; + }, + valueTable(false))); + assertThat(failure).isSameAs(expected); + assertThat(failure.getSuppressed()).isEmpty(); + } + private static TableInfo valueTable(boolean notNull) { return table( TableDescriptor.builder() From 3cde643e6045f0e1101dff3e5131ed065494477e Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:31:14 +0800 Subject: [PATCH 29/48] [kafka] Encode timestamp metadata as UTC TIMESTAMP values Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 13/13 AI-Contributed/UT: 75/75 --- .../kafka/transcode/KafkaRowAssembler.java | 13 +++- .../ArrowKafkaRecordTranscoderTest.java | 75 +++++++++++++++++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index cac8ac3e841..01886849d2b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -24,7 +24,7 @@ import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.GenericArray; import org.apache.fluss.row.GenericRow; -import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; import java.util.List; @@ -53,7 +53,16 @@ public GenericRow assemble( setProjectedFields(row, topicSchema.keyProjection(), keyValues); setProjectedFields(row, topicSchema.valueProjection(), valueValues); if (topicSchema.timestampPosition() >= 0) { - row.setField(topicSchema.timestampPosition(), TimestampLtz.fromEpochMillis(timestamp)); + int position = topicSchema.timestampPosition(); + if (timestamp == -1L) { + if (!topicSchema.rowType().getTypeAt(position).isNullable()) { + throw new KafkaRecordEncodingException( + "Missing Kafka timestamp cannot populate a NOT NULL timestamp column."); + } + } else { + // Epoch milliseconds define UTC calendar fields, independently of the JVM zone. + row.setField(position, TimestampNtz.fromMillis(timestamp)); + } } if (topicSchema.headersPosition() >= 0) { row.setField(topicSchema.headersPosition(), toHeaders(headers)); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index f1df39504d2..1b91bcdbc53 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -38,7 +38,7 @@ import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.InternalArray; import org.apache.fluss.row.InternalRow; -import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.CloseableIterator; @@ -49,10 +49,14 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.TimeZone; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -89,7 +93,7 @@ void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { assertThat(row.getBytes(0)).isEqualTo(bytes("message")); assertThat(row.getBytes(3)).isEqualTo(bytes("键")); } - assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(123L); InternalArray headers = row.getArray(2); assertThat(headers.size()).isEqualTo(2); assertThat(headers.getRow(0, 2).getString(0).toString()) @@ -100,6 +104,59 @@ void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { } } + @Test + void testTimestampCalendarValuesAreIndependentOfDefaultTimeZone() throws Exception { + TimeZone original = TimeZone.getDefault(); + long millis = 1700000000123L; + try { + for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/Los_Angeles"}) { + TimeZone.setDefault(TimeZone.getTimeZone(zone)); + TableInfo table = envelope(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + millis, + null, + bytes("value"), + Collections.emptyList())), + table), + table, + 1, + row -> + assertThat(row.getTimestampNtz(1, 3).toLocalDateTime()) + .isEqualTo( + LocalDateTime.ofInstant( + Instant.ofEpochMilli(millis), + ZoneOffset.UTC))); + } + } finally { + TimeZone.setDefault(original); + } + } + + @Test + void testMissingTimestampRespectsColumnNullability() throws Exception { + List records = + Collections.singletonList( + new Record(-1L, null, bytes("value"), Collections.emptyList())); + TableInfo nullable = envelope(false, true); + read( + transcoder.transcode(records, nullable), + nullable, + 1, + row -> assertThat(row.isNullAt(1)).isTrue()); + assertThatThrownBy(() -> transcoder.transcode(records, envelope(false))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("Missing Kafka timestamp"); + TableInfo unmapped = valueTable(false); + read( + transcoder.transcode(records, unmapped), + unmapped, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("value")); + } + @Test void testNullsAndEmptyBytesRemainDistinct() throws Exception { TableInfo table = envelope(false); @@ -282,7 +339,7 @@ void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean str assertThat(row.getBytes(0)).isEqualTo(bytes("消息-" + i)); assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); } - assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(i); InternalArray headers = row.getArray(2); assertThat(headers.size()).isEqualTo(2); assertThat(headers.getRow(0, 2).getString(0).toString()).isEqualTo("duplicate"); @@ -301,7 +358,7 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce GenericRow row = GenericRow.of( value, - TimestampLtz.fromEpochMillis(1L), + TimestampNtz.fromMillis(1L), new GenericArray( new Object[] { GenericRow.of(BinaryString.fromString("header"), headerValue) @@ -315,7 +372,7 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce key[0] = (byte) i; value[0] = (byte) (i + 10); headerValue[0] = (byte) (i + 20); - row.setField(1, TimestampLtz.fromEpochMillis(i)); + row.setField(1, TimestampNtz.fromMillis(i)); consumer.append(row); } Arrays.fill(key, (byte) -1); @@ -333,7 +390,7 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce int i = nextRecord.getAndIncrement(); assertThat(actual.getBytes(3)).containsExactly((byte) i); assertThat(actual.getBytes(0)).containsExactly((byte) (i + 10)); - assertThat(actual.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + assertThat(actual.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(i); assertThat(actual.getArray(2).getRow(0, 2).getBytes(1)) .containsExactly((byte) (i + 20)); }); @@ -400,10 +457,14 @@ private static TableInfo valueTable(boolean notNull) { } private static TableInfo envelope(boolean string) { + return envelope(string, false); + } + + private static TableInfo envelope(boolean string, boolean nullableTimestamp) { Schema schema = Schema.newBuilder() .column("body", string ? DataTypes.STRING() : DataTypes.BYTES()) - .column("time", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("time", DataTypes.TIMESTAMP(3).copy(nullableTimestamp)) .column( "attrs", DataTypes.ARRAY( From b59b5273d610efa01a95e752ac4b528b566af48e Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 18:05:19 +0800 Subject: [PATCH 30/48] [kafka] Transcode binary and string into fixed system columns Use the shared KafkaTableSchema contract for the fixed value column, raw optional keys, and last-value-wins header maps. Validate final map and key nullability, keep optional rescue values null, and remove prototype field projections and timestamp-column mapping. Reject metadata until native per-row offsets are available; JSON decoding remains a later stage. Preserve synchronous streaming, borrowed payloads, owned Arrow output and failure-path cleanup. Cover all optional system-column combinations, raw bytes, strict UTF-8, header deduplication, multi-page output and partial conversion failures without changing null-value filtering. Validation: Java 11 current-worktree clean reactor, 277 unit tests and 27 integration tests passed, including cumulative Kafka, shared schema, DDL and Arrow coverage. The final 23 transcoder cases passed again after strengthening the multi-page test. Spotless, Checkstyle, RAT and diff checks passed. All SNAPSHOT test dependencies came from this worktree. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 95/95 AI-Contributed/UT: 741/741 --- .../transcode/ArrowKafkaRecordTranscoder.java | 30 +- .../kafka/transcode/KafkaRowAssembler.java | 65 +- .../ArrowKafkaRecordTranscoderTest.java | 741 +++++++++++------- 3 files changed, 530 insertions(+), 306 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 9fcaaaabf31..88711f42323 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -19,9 +19,9 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.kafka.schema.KafkaTopicSchema; import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.BinaryString; @@ -36,7 +36,7 @@ import static org.apache.fluss.utils.Preconditions.checkArgument; -/** Converts raw/string Kafka records using the DDL mapping into owned Fluss Arrow log bytes. */ +/** Converts binary/string Kafka values and fixed system columns into owned Arrow log bytes. */ @Internal public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); @@ -45,37 +45,31 @@ public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { @Override public BytesView transcode(List records, TableInfo tableInfo) throws Exception { checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); - KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaTableSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + if (schema.valueType() != ValueType.BINARY && schema.valueType() != ValueType.STRING) { + throw new KafkaRecordEncodingException( + "Unsupported Kafka value conversion type: " + schema.valueType()); + } KafkaRowAssembler assembler = new KafkaRowAssembler(schema); return arrowRecordEncoder.encodeStreaming( consumer -> { for (Record record : records) { - Object[] key = - schema.keyFormat() == null - ? new Object[0] - : new Object[] { - decode(schema.keyFormat(), record.borrowedKey()) - }; consumer.append( assembler.assemble( - key, new Object[] { - decode(schema.valueFormat(), record.borrowedValue()) + decode(schema.valueType(), record.borrowedValue()) }, - record.timestamp(), + record.borrowedKey(), record.headers())); } }, tableInfo); } - private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) { - if (bytes == null || format == KafkaDataFormat.RAW) { + private static @Nullable Object decode(ValueType format, @Nullable byte[] bytes) { + if (bytes == null || format == ValueType.BINARY) { return bytes; } - if (format != KafkaDataFormat.STRING) { - throw new KafkaRecordEncodingException("Unsupported Kafka data format: " + format); - } try { return BinaryString.fromString( StandardCharsets.UTF_8 diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index 01886849d2b..00747192122 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -19,25 +19,29 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; -import org.apache.fluss.kafka.schema.KafkaFieldProjection; -import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; -import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; -import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.types.MapType; +import javax.annotation.Nullable; + +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Assembles decoded Kafka key, value, and metadata into a physical Fluss row. */ +/** Assembles decoded values, raw keys and headers using the shared fixed-column contract. */ @Internal public final class KafkaRowAssembler { - private final KafkaTopicSchema topicSchema; + private final KafkaTableSchema topicSchema; /** Creates an assembler for the resolved topic schema. */ - public KafkaRowAssembler(KafkaTopicSchema topicSchema) { + public KafkaRowAssembler(KafkaTableSchema topicSchema) { this.topicSchema = checkNotNull(topicSchema); } @@ -48,25 +52,24 @@ public KafkaRowAssembler(KafkaTopicSchema topicSchema) { * after conversion. */ public GenericRow assemble( - Object[] keyValues, Object[] valueValues, long timestamp, List headers) { + Object[] valueValues, @Nullable byte[] key, List headers) { GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); - setProjectedFields(row, topicSchema.keyProjection(), keyValues); setProjectedFields(row, topicSchema.valueProjection(), valueValues); - if (topicSchema.timestampPosition() >= 0) { - int position = topicSchema.timestampPosition(); - if (timestamp == -1L) { - if (!topicSchema.rowType().getTypeAt(position).isNullable()) { - throw new KafkaRecordEncodingException( - "Missing Kafka timestamp cannot populate a NOT NULL timestamp column."); - } - } else { - // Epoch milliseconds define UTC calendar fields, independently of the JVM zone. - row.setField(position, TimestampNtz.fromMillis(timestamp)); + if (topicSchema.keyPosition() >= 0) { + int position = topicSchema.keyPosition(); + if (key == null && !topicSchema.rowType().getTypeAt(position).isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka key cannot populate NOT NULL _kafka_key."); } + row.setField(position, key); } if (topicSchema.headersPosition() >= 0) { - row.setField(topicSchema.headersPosition(), toHeaders(headers)); + int position = topicSchema.headersPosition(); + row.setField( + position, + toHeaders(headers, (MapType) topicSchema.rowType().getTypeAt(position))); } + // A declared _kafka_rescue stays null for binary/string; omitted columns do not exist. return row; } @@ -92,12 +95,22 @@ private static void setProjectedFields( } } - private static GenericArray toHeaders(List headers) { - Object[] rows = new Object[headers.size()]; - for (int i = 0; i < headers.size(); i++) { - RecordHeader header = headers.get(i); - rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.borrowedValue()); + private static GenericMap toHeaders(List headers, MapType type) { + Map values = new LinkedHashMap<>(); + for (RecordHeader header : headers) { + values.put(BinaryString.fromString(header.name()), header.borrowedValue()); + } + // Validate the final map: an overwritten null must not reject a non-null last value. + for (Map.Entry entry : values.entrySet()) { + if (entry.getKey() == null && !type.getKeyType().isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka header name cannot populate NOT NULL _kafka_headers map keys."); + } + if (entry.getValue() == null && !type.getValueType().isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka header value cannot populate NOT NULL _kafka_headers map values."); + } } - return new GenericArray(rows); + return new GenericMap(values); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 1b91bcdbc53..ee01c5e4e60 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -17,10 +17,12 @@ package org.apache.fluss.kafka.transcode; +import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; -import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.SchemaInfo; @@ -34,12 +36,12 @@ import org.apache.fluss.record.TestingSchemaGetter; import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.BinaryString; -import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; -import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalMap; import org.apache.fluss.row.InternalRow; -import org.apache.fluss.row.TimestampNtz; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.CloseableIterator; @@ -49,270 +51,405 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; -import java.util.TimeZone; +import java.util.Map; +import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; -/** Verifies mapped Arrow records after transcoding resources have closed. */ +/** Verifies fixed-column Arrow records after transcoding resources have closed. */ class ArrowKafkaRecordTranscoderTest { private final ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); - @Test - void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { - for (boolean string : new boolean[] {false, true}) { - TableInfo table = envelope(string); - Record record = - new Record( - 123L, - bytes("键"), - bytes("message"), - Arrays.asList( - new RecordHeader("duplicate", bytes("first")), - new RecordHeader("duplicate", null))); - BytesView encoded = transcoder.transcode(Arrays.asList(record, record), table); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) throws Exception { + // The value looks like JSON but binary/string must not interpret or rescue its members. + byte[] value = bytes("{\"_kafka_rescue\":\"消息\",\"_kafka_key\":123}"); + byte[] key = {(byte) 0xff, 0, (byte) 0xc3}; + Record record = + new Record( + 123L, + TimestampType.LOG_APPEND_TIME, + key, + value, + Arrays.asList( + new RecordHeader("duplicate", bytes("first")), + new RecordHeader("nullable", bytes("discarded")), + new RecordHeader("duplicate", bytes("last")), + new RecordHeader("nullable", null))); + for (int mask = 0; mask < 8; mask++) { + TableInfo table = envelope(string, mask); + TableDescriptor before = table.toTableDescriptor(); + KafkaTableSchema contract = KafkaTableSchema.resolve(before); read( - encoded, + transcoder.transcode(Collections.singletonList(record), table), table, - 2, + 1, row -> { - if (string) { - assertThat(row.getString(0).toString()).isEqualTo("message"); - assertThat(row.getString(3).toString()).isEqualTo("键"); - } else { - assertThat(row.getBytes(0)).isEqualTo(bytes("message")); - assertThat(row.getBytes(3)).isEqualTo(bytes("键")); + assertThat(row.getFieldCount()) + .isEqualTo(table.getRowType().getFieldCount()); + assertValue(row, contract.valueProjection().positionAt(0), string, value); + if (contract.keyPosition() >= 0) { + assertThat(row.getBytes(contract.keyPosition())).isEqualTo(key); + } + if (contract.headersPosition() >= 0) { + Map headers = + headers(row.getMap(contract.headersPosition())); + assertThat(headers).hasSize(2).containsKey("nullable"); + assertThat(headers.get("duplicate")).isEqualTo(bytes("last")); + assertThat(headers.get("nullable")).isNull(); + } + if (contract.rescuePosition() >= 0) { + assertThat(row.isNullAt(contract.rescuePosition())).isTrue(); } - assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(123L); - InternalArray headers = row.getArray(2); - assertThat(headers.size()).isEqualTo(2); - assertThat(headers.getRow(0, 2).getString(0).toString()) - .isEqualTo("duplicate"); - assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("first")); - assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); }); + assertThat(table.toTableDescriptor()).isEqualTo(before); + assertThat(before.getProperties()).doesNotContainKey("table.kafka.convert.key.type"); + assertThat(before.getCustomProperties()).isEmpty(); } + assertThat(record.timestampType()).isEqualTo(TimestampType.LOG_APPEND_TIME); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.borrowedKey()).isEqualTo(key); + assertThat(record.borrowedValue()).isEqualTo(value); } - @Test - void testTimestampCalendarValuesAreIndependentOfDefaultTimeZone() throws Exception { - TimeZone original = TimeZone.getDefault(); - long millis = 1700000000123L; - try { - for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/Los_Angeles"}) { - TimeZone.setDefault(TimeZone.getTimeZone(zone)); - TableInfo table = envelope(false); - read( - transcoder.transcode( - Collections.singletonList( - new Record( - millis, - null, - bytes("value"), - Collections.emptyList())), - table), - table, - 1, - row -> - assertThat(row.getTimestampNtz(1, 3).toLocalDateTime()) - .isEqualTo( - LocalDateTime.ofInstant( - Instant.ofEpochMilli(millis), - ZoneOffset.UTC))); - } - } finally { - TimeZone.setDefault(original); - } - } - - @Test - void testMissingTimestampRespectsColumnNullability() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testNullAndEmptyPayloadsRemainDistinct(boolean string) throws Exception { + TableInfo table = envelope(string, 7); + // This converter retains its existing nullability checks; ingress filtering is separate. List records = - Collections.singletonList( - new Record(-1L, null, bytes("value"), Collections.emptyList())); - TableInfo nullable = envelope(false, true); + Arrays.asList( + new Record(Record.NO_TIMESTAMP, null, null, Collections.emptyList()), + new Record(0L, new byte[0], new byte[0], Collections.emptyList())); + AtomicInteger next = new AtomicInteger(); read( - transcoder.transcode(records, nullable), - nullable, - 1, - row -> assertThat(row.isNullAt(1)).isTrue()); - assertThatThrownBy(() -> transcoder.transcode(records, envelope(false))) + transcoder.transcode(records, table), + table, + 2, + row -> { + int value = table.getRowType().getFieldIndex("_kafka_value"); + int key = table.getRowType().getFieldIndex("_kafka_key"); + if (next.getAndIncrement() == 0) { + assertThat(row.isNullAt(value)).isTrue(); + assertThat(row.isNullAt(key)).isTrue(); + } else { + assertValue(row, value, string, new byte[0]); + assertThat(row.getBytes(key)).isEmpty(); + } + assertThat( + row.getMap(table.getRowType().getFieldIndex("_kafka_headers")) + .size()) + .isZero(); + assertThat(row.isNullAt(table.getRowType().getFieldIndex("_kafka_rescue"))) + .isTrue(); + }); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES().copy(false)) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + bytes("valid"), + Collections.emptyList())), + table)) .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("Missing Kafka timestamp"); - TableInfo unmapped = valueTable(false); + .hasMessageContaining("NOT NULL _kafka_key"); + byte[] invalidUtf8Key = {(byte) 0xff}; read( - transcoder.transcode(records, unmapped), - unmapped, + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + invalidUtf8Key, + bytes("valid"), + Collections.emptyList())), + table), + table, 1, - row -> assertThat(row.getString(0).toString()).isEqualTo("value")); + row -> assertThat(row.getBytes(0)).isEqualTo(invalidUtf8Key)); } - @Test - void testNullsAndEmptyBytesRemainDistinct() throws Exception { - TableInfo table = envelope(false); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING() + .copy(false), + DataTypes.BYTES() + .copy( + nullableValue)) + .copy(false)) + .column("_kafka_value", DataTypes.STRING()) + .build(), + true) + .build()); + List overwrittenNull = + Arrays.asList( + new RecordHeader("same", null), + new RecordHeader("", new byte[0]), + new RecordHeader("same", bytes("last"))); read( transcoder.transcode( Collections.singletonList( - new Record(1L, null, null, Collections.emptyList())), + new Record(1L, null, bytes("v"), overwrittenNull)), table), table, 1, row -> { - assertThat(row.isNullAt(0)).isTrue(); - assertThat(row.isNullAt(3)).isTrue(); - assertThat(row.getArray(2).size()).isZero(); + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(2); + assertThat(map.get("same")).isEqualTo(bytes("last")); + assertThat(map.get("")).isEmpty(); }); + Record finalNull = + new Record( + 1L, + null, + bytes("v"), + Arrays.asList( + new RecordHeader("same", bytes("first")), + new RecordHeader("same", null))); + if (nullableValue) { + read( + transcoder.transcode(Collections.singletonList(finalNull), table), + table, + 1, + row -> + assertThat(headers(row.getMap(0))) + .containsOnlyKeys("same") + .containsEntry("same", null)); + } else { + assertThatThrownBy( + () -> transcoder.transcode(Collections.singletonList(finalNull), table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL _kafka_headers map values"); + } read( transcoder.transcode( Collections.singletonList( - new Record(1L, new byte[0], new byte[0], Collections.emptyList())), + new Record(1L, null, bytes("v"), Collections.emptyList())), table), table, 1, + row -> assertThat(row.getMap(0).size()).isZero()); + } + + @Test + void testOmittedSystemColumnsIgnoreTimestampKeyAndHeaders() throws Exception { + TableInfo table = valueTable(false); + List records = new ArrayList<>(); + for (TimestampType type : TimestampType.values()) { + for (long timestamp : new long[] {Record.NO_TIMESTAMP, 0L, Long.MAX_VALUE}) { + records.add( + new Record( + timestamp, + type, + new byte[] {(byte) 0xff}, + bytes("v"), + Collections.singletonList(new RecordHeader("ignored", null)))); + } + } + read( + transcoder.transcode(records, table), + table, + records.size(), row -> { - assertThat(row.getBytes(0)).isEmpty(); - assertThat(row.getBytes(3)).isEmpty(); + assertThat(row.getFieldCount()).isEqualTo(1); + assertThat(row.getString(0).toString()).isEqualTo("v"); }); } @Test - void testUnmappedKeyIsIgnoredAndMixedFormatsWork() throws Exception { - TableInfo valueOnly = valueTable(false); - read( + void testBinaryValuePreservesArbitraryBytesAcrossOutputPages() throws Exception { + byte[] value = new byte[32768]; + new Random(42L).nextBytes(value); + value[0] = (byte) 0xff; + TableInfo table = envelope(false, 4); + BytesView encoded = transcoder.transcode( Collections.singletonList( - new Record( - 1L, - new byte[] {(byte) 0xff}, - bytes("value"), - Collections.emptyList())), - valueOnly), - valueOnly, - 1, - row -> assertThat(row.getString(0).toString()).isEqualTo("value")); - Schema schema = - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("id", DataTypes.STRING()) - .build(); - TableInfo mixed = - table( - TableDescriptor.builder() - .schema(schema) - .distributedBy(1) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty( - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .build()); + new Record(1L, null, value, Collections.emptyList())), + table); + assertThat(encoded.getBytesLength()).isGreaterThan(4096); read( - transcoder.transcode( - Collections.singletonList( - new Record( - 1L, - bytes("key"), - new byte[] {(byte) 0xff}, - Collections.emptyList())), - mixed), - mixed, + encoded, + table, 1, row -> { - assertThat(row.getString(1).toString()).isEqualTo("key"); - assertThat(row.getBytes(0)).containsExactly((byte) 0xff); + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.getBytes(1)).isEqualTo(value); }); } - @Test - void testRejectsMalformedUtf8AndNotNullViolationsThenRemainsUsable() throws Exception { - TableInfo table = valueTable(false); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testValueNotNullConstraint(boolean string) { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_value", + valueType(string).copy(false)) + .build(), + string) + .build()); assertThatThrownBy( () -> transcoder.transcode( Collections.singletonList( new Record( - 1L, - null, - new byte[] {(byte) 0xc3, 0x28}, - Collections.emptyList())), + 1L, null, null, Collections.emptyList())), table)) .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("UTF-8"); + .hasMessageContaining("NOT NULL field '_kafka_value'"); + } + + @Test + void testRejectsMetadataUntilActualOffsetsCanBePersisted() { + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))) + .build(); assertThatThrownBy( () -> transcoder.transcode( Collections.singletonList( new Record( - 1L, null, null, Collections.emptyList())), - valueTable(true))) - .isInstanceOf(KafkaRecordEncodingException.class) - .hasMessageContaining("NOT NULL"); - read( - transcoder.transcode( - Collections.singletonList( - new Record(1L, null, bytes("ok"), Collections.emptyList())), - table), - table, - 1, - row -> assertThat(row.getString(0).toString()).isEqualTo("ok")); + 123L, + TimestampType.CREATE_TIME, + null, + bytes("v"), + Collections.emptyList())), + table(descriptor(schema, true).build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("actual per-row offsets"); } @Test - void testRejectsInvalidMappingAndEmptyPartition() { - TableInfo invalid = - table( - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("body", DataTypes.STRING()) - .build()) - .distributedBy(1) - .build()); + void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { + List records = + Collections.singletonList( + new Record(1L, null, bytes("v"), Collections.emptyList())); + Schema fixed = Schema.newBuilder().column("_kafka_value", DataTypes.STRING()).build(); assertThatThrownBy( () -> transcoder.transcode( - Collections.singletonList( - new Record( - 1L, - null, - bytes("value"), - Collections.emptyList())), - invalid)) - .isInstanceOf(KafkaTopicSchemaException.class); + records, + table( + descriptor(fixed, true) + .property( + ConfigOptions.TABLE_KAFKA_ENABLED, + false) + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); + assertThatThrownBy( + () -> + transcoder.transcode( + records, + table( + descriptor( + Schema.newBuilder() + .column( + "body", + DataTypes.STRING()) + .build(), + true) + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value"); + assertThatThrownBy( + () -> + transcoder.transcode( + records, + table( + descriptor(fixed, true) + .customProperty( + "kafka.value.format", "string") + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("custom property"); assertThatThrownBy(() -> transcoder.transcode(Collections.emptyList(), valueTable(false))) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("empty"); } @Test - void testArrowWriterFailureClosesAllocatorWithoutLeaking() { - Throwable failure = - catchThrowable( - () -> - new FlussArrowRecordEncoder() - .encode( - Collections.singletonList(GenericRow.of(123)), - valueTable(false))); - assertThat(failure).isInstanceOf(ClassCastException.class); - assertThat(failure.getSuppressed()).isEmpty(); + void testJsonIsNotActivatedEvenForNullValues() { + TableInfo table = + table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build()); + for (byte[] value : new byte[][] {null, bytes("{}")}) { + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + value, + Collections.emptyList())), + table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("PARSE_JSON"); + } } @ParameterizedTest @ValueSource(booleans = {false, true}) void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean string) throws Exception { - TableInfo table = envelope(string); + TableInfo table = envelope(string, 7); List records = new ArrayList<>(); for (int i = 0; i < 2050; i++) { records.add( @@ -321,48 +458,57 @@ void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean str bytes("key-" + i), bytes("消息-" + i), Arrays.asList( - new RecordHeader("duplicate", bytes("header-" + i)), - new RecordHeader("duplicate", null)))); + new RecordHeader("duplicate", null), + new RecordHeader("duplicate", bytes("header-" + i))))); } - BytesView encoded = transcoder.transcode(records, table); - AtomicInteger nextRecord = new AtomicInteger(); + AtomicInteger next = new AtomicInteger(); read( - encoded, + transcoder.transcode(records, table), table, records.size(), row -> { - int i = nextRecord.getAndIncrement(); - if (string) { - assertThat(row.getString(0).toString()).isEqualTo("消息-" + i); - assertThat(row.getString(3).toString()).isEqualTo("key-" + i); - } else { - assertThat(row.getBytes(0)).isEqualTo(bytes("消息-" + i)); - assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); + int i = next.getAndIncrement(); + assertValue(row, 2, string, bytes("消息-" + i)); + assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(1); + assertThat(map.get("duplicate")).isEqualTo(bytes("header-" + i)); + assertThat(row.isNullAt(1)).isTrue(); + }); + } + + @Test + void testOneRecordWithManyDistinctHeadersGrowsArrowMapVectors() throws Exception { + TableInfo table = envelope(false, 2); + List input = new ArrayList<>(); + for (int i = 0; i < 2050; i++) { + input.add(new RecordHeader("header-" + i, bytes("value-" + i))); + } + read( + transcoder.transcode( + Collections.singletonList(new Record(1L, null, bytes("v"), input)), table), + table, + 1, + row -> { + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(2050); + for (int i = 0; i < 2050; i++) { + assertThat(map.get("header-" + i)).isEqualTo(bytes("value-" + i)); } - assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(i); - InternalArray headers = row.getArray(2); - assertThat(headers.size()).isEqualTo(2); - assertThat(headers.getRow(0, 2).getString(0).toString()).isEqualTo("duplicate"); - assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("header-" + i)); - assertThat(headers.getRow(1, 2).getString(0).toString()).isEqualTo("duplicate"); - assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); }); } @Test void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exception { - TableInfo table = envelope(false); + TableInfo table = envelope(false, 7); byte[] key = new byte[1]; byte[] value = new byte[1]; byte[] headerValue = new byte[1]; GenericRow row = GenericRow.of( + GenericMap.of(BinaryString.fromString("header"), headerValue), + null, value, - TimestampNtz.fromMillis(1L), - new GenericArray( - new Object[] { - GenericRow.of(BinaryString.fromString("header"), headerValue) - }), key); BytesView encoded = new FlussArrowRecordEncoder() @@ -372,49 +518,50 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce key[0] = (byte) i; value[0] = (byte) (i + 10); headerValue[0] = (byte) (i + 20); - row.setField(1, TimestampNtz.fromMillis(i)); consumer.append(row); } Arrays.fill(key, (byte) -1); Arrays.fill(value, (byte) -1); Arrays.fill(headerValue, (byte) -1); - row.setField(0, null); + row.setField(2, null); }, table); - AtomicInteger nextRecord = new AtomicInteger(); + AtomicInteger next = new AtomicInteger(); read( encoded, table, 3, actual -> { - int i = nextRecord.getAndIncrement(); + int i = next.getAndIncrement(); assertThat(actual.getBytes(3)).containsExactly((byte) i); - assertThat(actual.getBytes(0)).containsExactly((byte) (i + 10)); - assertThat(actual.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(i); - assertThat(actual.getArray(2).getRow(0, 2).getBytes(1)) + assertThat(actual.getBytes(2)).containsExactly((byte) (i + 10)); + assertThat(headers(actual.getMap(0)).get("header")) .containsExactly((byte) (i + 20)); }); } @Test - void testDecodeFailureAfterValidRecordClosesAllocatorAndRemainsUsable() throws Exception { + void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { TableInfo table = valueTable(true); Record valid = new Record(1L, null, bytes("valid"), Collections.emptyList()); - for (byte[] invalidValue : new byte[][] {null, {(byte) 0xc3, 0x28}}) { + for (byte[] invalid : new byte[][] {null, {(byte) 0xc3, 0x28}, {(byte) 0xff}}) { + AtomicReference output = new AtomicReference<>(); Throwable failure = catchThrowable( () -> - transcoder.transcode( - Arrays.asList( - valid, - new Record( - 2L, - null, - invalidValue, - Collections.emptyList())), - table)); + output.set( + transcoder.transcode( + Arrays.asList( + valid, + new Record( + 2L, + null, + invalid, + Collections.emptyList())), + table))); assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); } read( transcoder.transcode(Collections.singletonList(valid), table), @@ -423,6 +570,63 @@ void testDecodeFailureAfterValidRecordClosesAllocatorAndRemainsUsable() throws E row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); } + @Test + void testHeaderFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.BYTES().copy(false))) + .build(), + true) + .build()); + Record valid = + new Record( + 1L, + null, + bytes("v"), + Collections.singletonList(new RecordHeader("h", bytes("v")))); + Record invalid = + new Record( + 1L, + null, + bytes("v"), + Collections.singletonList(new RecordHeader("h", null))); + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + Arrays.asList(valid, invalid), table))); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + read( + transcoder.transcode(Collections.singletonList(valid), table), + table, + 1, + row -> assertThat(headers(row.getMap(1)).get("h")).isEqualTo(bytes("v"))); + } + + @Test + void testArrowWriterFailureClosesAllocatorWithoutLeaking() { + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encode( + Collections.singletonList(GenericRow.of(123)), + valueTable(false))); + assertThat(failure).isInstanceOf(ClassCastException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + @Test void testStreamingSourceFailureClosesAllocator() { IOException expected = new IOException("source failed after the first row"); @@ -445,52 +649,65 @@ void testStreamingSourceFailureClosesAllocator() { private static TableInfo valueTable(boolean notNull) { return table( - TableDescriptor.builder() - .schema( + descriptor( Schema.newBuilder() - .column("body", DataTypes.STRING().copy(!notNull)) - .build()) - .distributedBy(1) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .column("_kafka_value", DataTypes.STRING().copy(!notNull)) + .build(), + true) .build()); } - private static TableInfo envelope(boolean string) { - return envelope(string, false); + private static TableInfo envelope(boolean string, int mask) { + Schema.Builder schema = Schema.newBuilder(); + if ((mask & 2) != 0) { + schema.column("_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + } + if ((mask & 4) != 0) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + schema.column("_kafka_value", valueType(string)); + if ((mask & 1) != 0) { + schema.column("_kafka_key", DataTypes.BYTES()); + } + return table(descriptor(schema.build(), string).build()); + } + + private static DataType valueType(boolean string) { + return string ? DataTypes.STRING() : DataTypes.BYTES(); } - private static TableInfo envelope(boolean string, boolean nullableTimestamp) { - Schema schema = - Schema.newBuilder() - .column("body", string ? DataTypes.STRING() : DataTypes.BYTES()) - .column("time", DataTypes.TIMESTAMP(3).copy(nullableTimestamp)) - .column( - "attrs", - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES())))) - .column("id", string ? DataTypes.STRING() : DataTypes.BYTES()) - .build(); - return table( - TableDescriptor.builder() - .schema(schema) - .distributedBy(1) - .logFormat(LogFormat.ARROW) - .customProperty( - KafkaDataFormat.VALUE_FORMAT_CONFIG, string ? "string" : "raw") - .customProperty( - KafkaDataFormat.KEY_FORMAT_CONFIG, string ? "string" : "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "time") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attrs") - .build()); + private static TableDescriptor.Builder descriptor(Schema schema, boolean string) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, string ? "string" : "binary"); } private static TableInfo table(TableDescriptor descriptor) { - return TableInfo.of(TablePath.of("kafka", "topic"), 42L, 3, descriptor, null, 1L, 1L); + return TableInfo.of(TablePath.of("fluss", "topic"), 42L, 3, descriptor, null, 1L, 1L); + } + + private static Map headers(InternalMap map) { + Map result = new LinkedHashMap<>(); + for (int i = 0; i < map.size(); i++) { + assertThat(map.keyArray().isNullAt(i)).isFalse(); + String key = map.keyArray().getString(i).toString(); + assertThat(result).doesNotContainKey(key); + result.put(key, map.valueArray().isNullAt(i) ? null : map.valueArray().getBytes(i)); + } + return result; + } + + private static void assertValue(InternalRow row, int position, boolean string, byte[] value) { + if (string) { + assertThat(row.getString(position).toString()) + .isEqualTo(new String(value, StandardCharsets.UTF_8)); + } else { + assertThat(row.getBytes(position)).isEqualTo(value); + } } private static void read( From 675bd55561586af5c6ea8459f81929ae9c8bd2dc Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 17:02:56 +0800 Subject: [PATCH 31/48] [kafka] Preserve metadata rejection on the updated PR05 base The shared schema now accepts three-field Kafka metadata without offsets. Reject it explicitly during row assembler construction so binary/string conversion cannot silently emit null metadata. Keep the existing streaming and borrowed-payload paths unchanged. Cover nullable and non-null metadata in both formats, and update the legacy offset-field regression to the current schema validation message. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 5/5 AI-Contributed/UT: 41/41 --- .../kafka/transcode/KafkaRowAssembler.java | 5 +++ .../ArrowKafkaRecordTranscoderTest.java | 41 ++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index 00747192122..b3ab60dbf82 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -19,6 +19,7 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; import org.apache.fluss.metadata.KafkaFieldProjection; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; @@ -43,6 +44,10 @@ public final class KafkaRowAssembler { /** Creates an assembler for the resolved topic schema. */ public KafkaRowAssembler(KafkaTableSchema topicSchema) { this.topicSchema = checkNotNull(topicSchema); + if (topicSchema.metadataPosition() >= 0) { + throw new KafkaTopicSchemaException( + "_kafka_metadata conversion is not supported by this transcoder yet."); + } } /** diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index ee01c5e4e60..ab343b831a1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -337,8 +337,43 @@ void testValueNotNullConstraint(boolean string) { .hasMessageContaining("NOT NULL field '_kafka_value'"); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRejectsMetadataUntilTranscodingIsSupported(boolean string) { + DataType metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + for (boolean nullable : new boolean[] {false, true}) { + TableDescriptor descriptor = + descriptor( + Schema.newBuilder() + .column("_kafka_value", valueType(string)) + .column("_kafka_metadata", metadataType.copy(nullable)) + .build(), + string) + .build(); + // DDL accepts the new contract, but encoding must not silently drop its metadata. + assertThat(KafkaTableSchema.resolve(descriptor).metadataPosition()).isEqualTo(1); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 123L, + TimestampType.CREATE_TIME, + null, + bytes("v"), + Collections.emptyList())), + table(descriptor))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_metadata conversion is not supported"); + } + } + @Test - void testRejectsMetadataUntilActualOffsetsCanBePersisted() { + void testRejectsLegacyMetadataOffsetField() { Schema schema = Schema.newBuilder() .column("_kafka_value", DataTypes.STRING()) @@ -362,7 +397,9 @@ void testRejectsMetadataUntilActualOffsetsCanBePersisted() { Collections.emptyList())), table(descriptor(schema, true).build()))) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("actual per-row offsets"); + .hasMessageContaining( + "_kafka_metadata must be ROW"); } @Test From ef271929f85adaa6f4ac0bf8ae1f54e513c8ea66 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 23:09:59 +0800 Subject: [PATCH 32/48] [kafka] Transcode three-field metadata from Produce records Populate TIMESTAMP_LTZ(3), Kafka batch timestamp type and the requested partition before Arrow encoding. Preserve timestamp provenance independently of timestamp presence, and enforce nested timestamp NOT NULL constraints. Keep synchronous streaming and borrowed key/value/header payloads. Cover all 16 optional-column combinations, timestamp boundaries, partition isolation, Arrow vector growth and failure atomicity for binary and string values. Validation: Java 11 offline clean verify; 348 tests passed (317 unit and 31 integration), with Spotless, Checkstyle and RAT checks passing. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 74/74 AI-Contributed/UT: 233/233 --- .../transcode/ArrowKafkaRecordTranscoder.java | 8 +- .../transcode/KafkaRecordTranscoder.java | 8 +- .../kafka/transcode/KafkaRowAssembler.java | 58 +++-- .../ArrowKafkaRecordTranscoderTest.java | 233 ++++++++++++++---- 4 files changed, 244 insertions(+), 63 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 88711f42323..dcbc0143c89 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -43,7 +43,9 @@ public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); @Override - public BytesView transcode(List records, TableInfo tableInfo) throws Exception { + public BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception { + checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTableSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); if (schema.valueType() != ValueType.BINARY && schema.valueType() != ValueType.STRING) { @@ -59,8 +61,8 @@ public BytesView transcode(List records, TableInfo tableInfo) throws Exc new Object[] { decode(schema.valueType(), record.borrowedValue()) }, - record.borrowedKey(), - record.headers())); + record, + partitionId)); } }, tableInfo); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java index eeff2f3d14f..9bcc465b2a2 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -27,6 +27,10 @@ /** Converts copied Kafka records into the native Fluss log representation. */ @Internal public interface KafkaRecordTranscoder { - /** Transcodes records according to the target Fluss table schema and log format. */ - BytesView transcode(List records, TableInfo tableInfo) throws Exception; + /** + * Transcodes records from the requested Kafka partition according to the target Fluss table + * schema and log format. + */ + BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index b3ab60dbf82..b5cb170b483 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -18,16 +18,17 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; -import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; import org.apache.fluss.metadata.KafkaFieldProjection; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.GenericMap; import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; import org.apache.fluss.types.MapType; - -import javax.annotation.Nullable; +import org.apache.fluss.types.RowType; import java.util.LinkedHashMap; import java.util.List; @@ -35,7 +36,7 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Assembles decoded values, raw keys and headers using the shared fixed-column contract. */ +/** Assembles decoded values and system columns using the shared fixed-column contract. */ @Internal public final class KafkaRowAssembler { @@ -44,24 +45,21 @@ public final class KafkaRowAssembler { /** Creates an assembler for the resolved topic schema. */ public KafkaRowAssembler(KafkaTableSchema topicSchema) { this.topicSchema = checkNotNull(topicSchema); - if (topicSchema.metadataPosition() >= 0) { - throw new KafkaTopicSchemaException( - "_kafka_metadata conversion is not supported by this transcoder yet."); - } } /** - * Assembles one Fluss row for immediate synchronous encoding. + * Assembles one Fluss row from a record and its requested partition for immediate synchronous + * encoding. * - *

The row borrows decoded values and header payloads; it must not modify them or retain them - * after conversion. + *

The row borrows decoded values, keys and header payloads; it must not modify them or + * retain them after conversion. */ - public GenericRow assemble( - Object[] valueValues, @Nullable byte[] key, List headers) { + public GenericRow assemble(Object[] valueValues, Record record, int partitionId) { GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); setProjectedFields(row, topicSchema.valueProjection(), valueValues); if (topicSchema.keyPosition() >= 0) { int position = topicSchema.keyPosition(); + byte[] key = record.borrowedKey(); if (key == null && !topicSchema.rowType().getTypeAt(position).isNullable()) { throw new KafkaRecordEncodingException( "Null Kafka key cannot populate NOT NULL _kafka_key."); @@ -72,12 +70,44 @@ public GenericRow assemble( int position = topicSchema.headersPosition(); row.setField( position, - toHeaders(headers, (MapType) topicSchema.rowType().getTypeAt(position))); + toHeaders( + record.headers(), (MapType) topicSchema.rowType().getTypeAt(position))); + } + if (topicSchema.metadataPosition() >= 0) { + int position = topicSchema.metadataPosition(); + RowType type = (RowType) topicSchema.rowType().getTypeAt(position); + if (!record.hasTimestamp() && !type.getTypeAt(0).isNullable()) { + throw new KafkaRecordEncodingException( + "Missing Kafka timestamp cannot populate NOT NULL _kafka_metadata.timestamp."); + } + row.setField( + position, + GenericRow.of( + record.hasTimestamp() + ? TimestampLtz.fromEpochMillis(record.timestamp()) + : null, + timestampTypeId(record.timestampType()), + partitionId)); } // A declared _kafka_rescue stays null for binary/string; omitted columns do not exist. return row; } + private static int timestampTypeId(TimestampType type) { + // Preserve batch provenance even when a record's timestamp is absent. The legacy record + // constructor supplies UNKNOWN, so a present timestamp alone cannot imply CreateTime. + switch (type) { + case UNKNOWN: + return -1; + case CREATE_TIME: + return 0; + case LOG_APPEND_TIME: + return 1; + default: + throw new KafkaRecordEncodingException("Unsupported Kafka timestamp type: " + type); + } + } + private static void setProjectedFields( GenericRow row, KafkaFieldProjection projection, Object[] values) { if (values.length != projection.size()) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index ab343b831a1..2ffa89093c3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -87,12 +87,12 @@ void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) thro new RecordHeader("nullable", bytes("discarded")), new RecordHeader("duplicate", bytes("last")), new RecordHeader("nullable", null))); - for (int mask = 0; mask < 8; mask++) { + for (int mask = 0; mask < 16; mask++) { TableInfo table = envelope(string, mask); TableDescriptor before = table.toTableDescriptor(); KafkaTableSchema contract = KafkaTableSchema.resolve(before); read( - transcoder.transcode(Collections.singletonList(record), table), + transcoder.transcode(7, Collections.singletonList(record), table), table, 1, row -> { @@ -109,6 +109,9 @@ void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) thro assertThat(headers.get("duplicate")).isEqualTo(bytes("last")); assertThat(headers.get("nullable")).isNull(); } + if (contract.metadataPosition() >= 0) { + assertMetadata(row, contract.metadataPosition(), 123L, 1, 7); + } if (contract.rescuePosition() >= 0) { assertThat(row.isNullAt(contract.rescuePosition())).isTrue(); } @@ -134,7 +137,7 @@ void testNullAndEmptyPayloadsRemainDistinct(boolean string) throws Exception { new Record(0L, new byte[0], new byte[0], Collections.emptyList())); AtomicInteger next = new AtomicInteger(); read( - transcoder.transcode(records, table), + transcoder.transcode(0, records, table), table, 2, row -> { @@ -171,6 +174,7 @@ void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { assertThatThrownBy( () -> transcoder.transcode( + 0, Collections.singletonList( new Record( 1L, @@ -183,6 +187,7 @@ void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { byte[] invalidUtf8Key = {(byte) 0xff}; read( transcoder.transcode( + 0, Collections.singletonList( new Record( 1L, @@ -222,6 +227,7 @@ void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) thr new RecordHeader("same", bytes("last"))); read( transcoder.transcode( + 0, Collections.singletonList( new Record(1L, null, bytes("v"), overwrittenNull)), table), @@ -243,7 +249,7 @@ void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) thr new RecordHeader("same", null))); if (nullableValue) { read( - transcoder.transcode(Collections.singletonList(finalNull), table), + transcoder.transcode(0, Collections.singletonList(finalNull), table), table, 1, row -> @@ -252,12 +258,15 @@ void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) thr .containsEntry("same", null)); } else { assertThatThrownBy( - () -> transcoder.transcode(Collections.singletonList(finalNull), table)) + () -> + transcoder.transcode( + 0, Collections.singletonList(finalNull), table)) .isInstanceOf(KafkaRecordEncodingException.class) .hasMessageContaining("NOT NULL _kafka_headers map values"); } read( transcoder.transcode( + 0, Collections.singletonList( new Record(1L, null, bytes("v"), Collections.emptyList())), table), @@ -282,7 +291,7 @@ void testOmittedSystemColumnsIgnoreTimestampKeyAndHeaders() throws Exception { } } read( - transcoder.transcode(records, table), + transcoder.transcode(0, records, table), table, records.size(), row -> { @@ -299,6 +308,7 @@ void testBinaryValuePreservesArbitraryBytesAcrossOutputPages() throws Exception TableInfo table = envelope(false, 4); BytesView encoded = transcoder.transcode( + 0, Collections.singletonList( new Record(1L, null, value, Collections.emptyList())), table); @@ -329,6 +339,7 @@ void testValueNotNullConstraint(boolean string) { assertThatThrownBy( () -> transcoder.transcode( + 0, Collections.singletonList( new Record( 1L, null, null, Collections.emptyList())), @@ -339,39 +350,117 @@ void testValueNotNullConstraint(boolean string) { @ParameterizedTest @ValueSource(booleans = {false, true}) - void testRejectsMetadataUntilTranscodingIsSupported(boolean string) { - DataType metadataType = - DataTypes.ROW( - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), - DataTypes.FIELD("partition", DataTypes.INT())); + void testMetadataPreservesInputTimestampBatchTypeAndRequestedPartition(boolean string) + throws Exception { + long[] timestamps = {Record.NO_TIMESTAMP, 0L, 123L, 1728912345678L, Long.MAX_VALUE}; + TimestampType[] types = { + TimestampType.UNKNOWN, TimestampType.CREATE_TIME, TimestampType.LOG_APPEND_TIME + }; + int[] typeIds = {-1, 0, 1}; + List records = new ArrayList<>(); + for (TimestampType type : types) { + for (long timestamp : timestamps) { + records.add(new Record(timestamp, type, null, bytes("v"), Collections.emptyList())); + } + } for (boolean nullable : new boolean[] {false, true}) { - TableDescriptor descriptor = - descriptor( - Schema.newBuilder() - .column("_kafka_value", valueType(string)) - .column("_kafka_metadata", metadataType.copy(nullable)) - .build(), - string) - .build(); - // DDL accepts the new contract, but encoding must not silently drop its metadata. - assertThat(KafkaTableSchema.resolve(descriptor).metadataPosition()).isEqualTo(1); - assertThatThrownBy( - () -> - transcoder.transcode( - Collections.singletonList( - new Record( - 123L, - TimestampType.CREATE_TIME, - null, - bytes("v"), - Collections.emptyList())), - table(descriptor))) - .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining("_kafka_metadata conversion is not supported"); + TableInfo table = metadataTable(string, nullable, true); + TableDescriptor before = table.toTableDescriptor(); + // Reuse the same converter and records across requests without retaining partition + // state. + for (int partition : new int[] {7, 0, 3}) { + AtomicInteger next = new AtomicInteger(); + read( + transcoder.transcode(partition, records, table), + table, + records.size(), + row -> { + int i = next.getAndIncrement(); + assertMetadata( + row, + 0, + timestamps[i % timestamps.length], + typeIds[i / timestamps.length], + partition); + assertValue(row, 1, string, bytes("v")); + }); + } + assertThat(table.toTableDescriptor()).isEqualTo(before); } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMetadataTimestampNotNullFailureHasNoPartialOutputAndRemainsUsable(boolean string) + throws Exception { + Record valid = + new Record( + 0L, TimestampType.CREATE_TIME, null, bytes("v"), Collections.emptyList()); + for (boolean nullable : new boolean[] {false, true}) { + TableInfo table = metadataTable(string, nullable, false); + for (TimestampType type : TimestampType.values()) { + Record invalid = + new Record( + Record.NO_TIMESTAMP, + type, + null, + bytes("v"), + Collections.emptyList()); + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + 7, Arrays.asList(valid, invalid), table))); + assertThat(failure) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL _kafka_metadata.timestamp"); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + read( + transcoder.transcode(3, Collections.singletonList(valid), table), + table, + 1, + row -> { + assertMetadata(row, 0, 0L, 0, 3); + assertValue(row, 1, string, bytes("v")); + }); + } + } + } + + @Test + void testLegacyRecordDoesNotInferBatchTimestampType() throws Exception { + TableInfo table = metadataTable(true, false, false); + read( + transcoder.transcode( + 7, + Collections.singletonList( + new Record(123L, null, bytes("v"), Collections.emptyList())), + table), + table, + 1, + row -> assertMetadata(row, 0, 123L, -1, 7)); + } + + @Test + void testRejectsNegativePartitionId() { + assertThatThrownBy( + () -> + transcoder.transcode( + -1, + Collections.singletonList( + new Record( + Record.NO_TIMESTAMP, + null, + bytes("v"), + Collections.emptyList())), + envelope(false, 8))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Kafka partition ID must be non-negative"); + } + @Test void testRejectsLegacyMetadataOffsetField() { Schema schema = @@ -388,6 +477,7 @@ void testRejectsLegacyMetadataOffsetField() { assertThatThrownBy( () -> transcoder.transcode( + 0, Collections.singletonList( new Record( 123L, @@ -411,6 +501,7 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { assertThatThrownBy( () -> transcoder.transcode( + 0, records, table( descriptor(fixed, true) @@ -423,6 +514,7 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { assertThatThrownBy( () -> transcoder.transcode( + 0, records, table( descriptor( @@ -438,6 +530,7 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { assertThatThrownBy( () -> transcoder.transcode( + 0, records, table( descriptor(fixed, true) @@ -446,7 +539,8 @@ void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() { .build()))) .isInstanceOf(KafkaTopicSchemaException.class) .hasMessageContaining("custom property"); - assertThatThrownBy(() -> transcoder.transcode(Collections.emptyList(), valueTable(false))) + assertThatThrownBy( + () -> transcoder.transcode(0, Collections.emptyList(), valueTable(false))) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("empty"); } @@ -470,6 +564,7 @@ void testJsonIsNotActivatedEvenForNullValues() { assertThatThrownBy( () -> transcoder.transcode( + 0, Collections.singletonList( new Record( 1L, @@ -486,12 +581,17 @@ void testJsonIsNotActivatedEvenForNullValues() { @ValueSource(booleans = {false, true}) void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean string) throws Exception { - TableInfo table = envelope(string, 7); + TableInfo table = envelope(string, 15); List records = new ArrayList<>(); for (int i = 0; i < 2050; i++) { records.add( new Record( - i, + i % 3 == 0 ? Record.NO_TIMESTAMP : i, + i % 3 == 0 + ? TimestampType.UNKNOWN + : (i % 3 == 1 + ? TimestampType.CREATE_TIME + : TimestampType.LOG_APPEND_TIME), bytes("key-" + i), bytes("消息-" + i), Arrays.asList( @@ -500,7 +600,7 @@ void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean str } AtomicInteger next = new AtomicInteger(); read( - transcoder.transcode(records, table), + transcoder.transcode(7, records, table), table, records.size(), row -> { @@ -511,6 +611,7 @@ void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean str assertThat(map).hasSize(1); assertThat(map.get("duplicate")).isEqualTo(bytes("header-" + i)); assertThat(row.isNullAt(1)).isTrue(); + assertMetadata(row, 4, i % 3 == 0 ? Record.NO_TIMESTAMP : i, i % 3 - 1, 7); }); } @@ -523,7 +624,9 @@ void testOneRecordWithManyDistinctHeadersGrowsArrowMapVectors() throws Exception } read( transcoder.transcode( - Collections.singletonList(new Record(1L, null, bytes("v"), input)), table), + 0, + Collections.singletonList(new Record(1L, null, bytes("v"), input)), + table), table, 1, row -> { @@ -588,6 +691,7 @@ void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw () -> output.set( transcoder.transcode( + 0, Arrays.asList( valid, new Record( @@ -601,7 +705,7 @@ void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw assertThat(output).hasNullValue(); } read( - transcoder.transcode(Collections.singletonList(valid), table), + transcoder.transcode(0, Collections.singletonList(valid), table), table, 1, row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); @@ -640,12 +744,12 @@ void testHeaderFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw () -> output.set( transcoder.transcode( - Arrays.asList(valid, invalid), table))); + 0, Arrays.asList(valid, invalid), table))); assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); assertThat(failure.getSuppressed()).isEmpty(); assertThat(output).hasNullValue(); read( - transcoder.transcode(Collections.singletonList(valid), table), + transcoder.transcode(0, Collections.singletonList(valid), table), table, 1, row -> assertThat(headers(row.getMap(1)).get("h")).isEqualTo(bytes("v"))); @@ -706,9 +810,33 @@ private static TableInfo envelope(boolean string, int mask) { if ((mask & 1) != 0) { schema.column("_kafka_key", DataTypes.BYTES()); } + if ((mask & 8) != 0) { + schema.column("_kafka_metadata", metadataType(true)); + } return table(descriptor(schema.build(), string).build()); } + private static DataType metadataType(boolean nullableTimestamp) { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3).copy(nullableTimestamp)), + DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))); + } + + private static TableInfo metadataTable( + boolean string, boolean nullableRow, boolean nullableTimestamp) { + return table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_metadata", + metadataType(nullableTimestamp).copy(nullableRow)) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + } + private static DataType valueType(boolean string) { return string ? DataTypes.STRING() : DataTypes.BYTES(); } @@ -716,7 +844,7 @@ private static DataType valueType(boolean string) { private static TableDescriptor.Builder descriptor(Schema schema, boolean string) { return TableDescriptor.builder() .schema(schema) - .distributedBy(1) + .distributedBy(8) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) .property( @@ -747,6 +875,23 @@ private static void assertValue(InternalRow row, int position, boolean string, b } } + private static void assertMetadata( + InternalRow row, int position, long timestamp, int timestampType, int partition) { + assertThat(row.isNullAt(position)).isFalse(); + InternalRow metadata = row.getRow(position, 3); + assertThat(metadata.getFieldCount()).isEqualTo(3); + if (timestamp == Record.NO_TIMESTAMP) { + assertThat(metadata.isNullAt(0)).isTrue(); + } else { + assertThat(metadata.isNullAt(0)).isFalse(); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(timestamp); + } + assertThat(metadata.isNullAt(1)).isFalse(); + assertThat(metadata.getInt(1)).isEqualTo(timestampType); + assertThat(metadata.isNullAt(2)).isFalse(); + assertThat(metadata.getInt(2)).isEqualTo(partition); + } + private static void read( BytesView encoded, TableInfo table, int count, Consumer verify) throws Exception { From af46cfc1ffaeb283ca7cc1cb9877d7c960e58b87 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:31:15 +0800 Subject: [PATCH 33/48] [kafka] Connect Produce to native Fluss append Register Produce v3-v11 and adapt the local TabletServer gateway using the shared qualified-topic DDL contract and raw/string transcoder. Preserve the resolved TablePath across metadata lookup and conversion, isolate table/partition failures, and map invalid topic names correctly. Drain deferred actions immediately after the synchronous append invocation so acks=all does not wait for its own delayed replica fetches. Base on PR06 04a9093b3 and independent core PR #3412 at 969f985ab. Validation: Java 11 reactor verify; 139 Kafka unit tests, 7 Kafka integration tests, and 10 targeted RPC/server tests passed. Three-replica tests cover raw/string, gzip, acks=0/1/all, offsets, timestamp/headers, native readback, and independent writes to same-named tables in different databases. Checkstyle, Spotless, RAT, and git diff --check passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 319/319 AI-Contributed/UT: 606/606 --- fluss-kafka/pom.xml | 6 + .../fluss/kafka/KafkaRequestHandler.java | 11 +- .../produce/GatewayKafkaProduceBackend.java | 306 +++++++++++++++ .../fluss/kafka/KafkaProduceAppendITCase.java | 240 ++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 1 + .../GatewayKafkaProduceBackendTest.java | 365 ++++++++++++++++++ 6 files changed, 928 insertions(+), 1 deletion(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index b77695848ca..e38487db564 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,6 +64,12 @@ + + ${project.groupId} + fluss-client + ${project.version} + test + org.apache.curator curator-test diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 31da9143823..9e985dbea7f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -19,13 +19,17 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.kafka.api.metadata.MetadataHandler; +import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; +import org.apache.fluss.kafka.backend.produce.GatewayKafkaProduceBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; @@ -48,7 +52,12 @@ public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { new ApiVersionsHandler(this::apiRegistry), new MetadataHandler( new GatewayKafkaMetadataBackend(service, topicMapper), - topicMapper))); + topicMapper), + new ProduceHandler( + new GatewayKafkaProduceBackend( + service, + (TabletServerGateway) service, + new ArrowKafkaRecordTranscoder())))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java new file mode 100644 index 00000000000..dd07df63733 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.protocol.Errors; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the local TabletServer write gateway to the Kafka Produce backend contract. */ +@Internal +public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final KafkaRecordTranscoder transcoder; + + /** Creates a Produce backend backed by the local TabletServer gateway. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + KafkaRecordTranscoder transcoder) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.transcoder = checkNotNull(transcoder); + } + + @Override + public CompletableFuture write(KafkaProduceCommand command) { + List> futures = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + futures.add(writeTopic(command, topic)); + } + CompletableFuture all = + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + return all.thenApply( + ignored -> { + List results = new ArrayList<>(); + for (CompletableFuture future : futures) { + results.add(future.join()); + } + return new KafkaProduceResult(results); + }); + } + + private CompletableFuture writeTopic( + KafkaProduceCommand command, TopicWrite topic) { + try { + TablePath path = topicMapper.toTablePath(topic.topicName()); + setCurrentSession(command); + GetTableInfoRequest request = new GetTableInfoRequest(); + request.setTablePath() + .setDatabaseName(path.getDatabaseName()) + .setTableName(path.getTableName()); + return gateway.getTableInfo(request) + .thenCompose( + response -> produceTopic(command, topic, toTableInfo(path, response))) + .exceptionally(failure -> failedTopic(topic, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedTopic(topic, failure)); + } + } + + private CompletableFuture produceTopic( + KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { + // Admission must match Metadata, including when the schema changes between requests. + schemaResolver.resolve(tableInfo.toTableDescriptor()); + ProduceLogRequest request = + new ProduceLogRequest() + .setTableId(tableInfo.getTableId()) + .setAcks(command.acks()) + .setTimeoutMs(command.timeoutMs()); + Map failures = new HashMap<>(); + for (PartitionWrite partition : topic.partitions()) { + if (partition.partitionId() < 0 + || partition.partitionId() >= tableInfo.getNumBuckets()) { + failures.put( + partition.partitionId(), + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_TOPIC_OR_PARTITION, + -1L, + "Partition is outside the table bucket range.")); + continue; + } + try { + BytesView records = transcoder.transcode(partition.records(), tableInfo); + request.addBucketsReq() + .setBucketId(partition.partitionId()) + .setRecordsBytesView(records); + } catch (Exception failure) { + failures.put( + partition.partitionId(), failedPartition(partition.partitionId(), failure)); + } + } + if (request.getBucketsReqsCount() == 0) { + return CompletableFuture.completedFuture( + toTopicResult(topic, new ProduceLogResponse(), failures)); + } + try { + setCurrentSession(command); + CompletableFuture appended; + try { + appended = gateway.produceLog(request); + } finally { + // produceLog appends synchronously, but acks=-1 can finish later. Wake delayed + // fetches now so replicas can fetch the append needed to complete that response. + service.tryCompleteActions(); + } + return appended.handle( + (response, failure) -> + failure == null + ? toTopicResult(topic, response, failures) + : failedAppend(topic, failures, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedAppend(topic, failures, failure)); + } + } + + private static TopicResult failedAppend( + TopicWrite topic, Map failures, Throwable failure) { + List results = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult local = failures.get(partition.partitionId()); + results.add(local == null ? failedPartition(partition.partitionId(), failure) : local); + } + return new TopicResult(topic.topicName(), results); + } + + private static TableInfo toTableInfo(TablePath path, GetTableInfoResponse response) { + return TableInfo.of( + path, + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null, + response.getCreatedTime(), + response.getModifiedTime()); + } + + private static TopicResult toTopicResult( + TopicWrite topic, ProduceLogResponse response, Map failures) { + Map responses = new HashMap<>(); + for (PbProduceLogRespForBucket bucket : response.getBucketsRespsList()) { + responses.put(bucket.getBucketId(), bucket); + } + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult localFailure = failures.get(partition.partitionId()); + if (localFailure != null) { + partitions.add(localFailure); + continue; + } + PbProduceLogRespForBucket bucket = responses.get(partition.partitionId()); + if (bucket == null) { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_SERVER_ERROR, + -1L, + "Fluss Produce response omitted this bucket.")); + } else if (bucket.hasErrorCode() && bucket.getErrorCode() != 0) { + partitions.add( + new PartitionResult( + partition.partitionId(), + toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forCode( + bucket.getErrorCode())), + -1L, + bucket.hasErrorMessage() ? bucket.getErrorMessage() : null)); + } else { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.NONE, + bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, + null)); + } + } + return new TopicResult(topic.topicName(), partitions); + } + + private static TopicResult failedTopic(TopicWrite topic, Throwable failure) { + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + partitions.add(failedPartition(partition.partitionId(), failure)); + } + return new TopicResult(topic.topicName(), partitions); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + Throwable cause = unwrap(failure); + Errors kafkaError = + cause instanceof KafkaTopicSchemaException || cause instanceof InvalidTopicException + ? Errors.INVALID_TOPIC_EXCEPTION + : cause instanceof KafkaRecordEncodingException + ? Errors.CORRUPT_MESSAGE + : cause instanceof IllegalArgumentException + ? Errors.INVALID_REQUEST + : toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forException( + cause)); + return new PartitionResult(partitionId, kafkaError, -1L, cause.getMessage()); + } + + private static Errors toKafkaError(org.apache.fluss.rpc.protocol.Errors error) { + switch (error) { + case NONE: + return Errors.NONE; + case TABLE_NOT_EXIST: + case UNKNOWN_TABLE_OR_BUCKET_EXCEPTION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case NOT_LEADER_OR_FOLLOWER: + return Errors.NOT_LEADER_OR_FOLLOWER; + case LEADER_NOT_AVAILABLE_EXCEPTION: + return Errors.LEADER_NOT_AVAILABLE; + case RECORD_TOO_LARGE_EXCEPTION: + return Errors.MESSAGE_TOO_LARGE; + case CORRUPT_MESSAGE: + case CORRUPT_RECORD_EXCEPTION: + return Errors.CORRUPT_MESSAGE; + case INVALID_REQUIRED_ACKS: + return Errors.INVALID_REQUIRED_ACKS; + case REQUEST_TIME_OUT: + return Errors.REQUEST_TIMED_OUT; + case NOT_ENOUGH_REPLICAS_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS; + case NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND; + case AUTHORIZATION_EXCEPTION: + return Errors.TOPIC_AUTHORIZATION_FAILED; + case LOG_STORAGE_EXCEPTION: + case STORAGE_EXCEPTION: + case DISK_WRITE_LOCKED: + return Errors.KAFKA_STORAGE_ERROR; + default: + return Errors.UNKNOWN_SERVER_ERROR; + } + } + + private void setCurrentSession(KafkaProduceCommand command) { + service.setCurrentSession( + new Session( + (short) 0, + command.listenerName(), + false, + command.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java new file mode 100644 index 00000000000..8f6fab3a2e5 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Exercises native DDL, Kafka discovery and Produce, replication, and native Fluss readback. */ +class KafkaProduceAppendITCase { + private static final String DATABASE = "kafka"; + private static final byte[] KEY = "key".getBytes(StandardCharsets.UTF_8); + private static final byte[] VALUE = "value".getBytes(StandardCharsets.UTF_8); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @ParameterizedTest + @ValueSource(strings = {"raw", "string"}) + void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception { + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, "produce_" + format + "_" + acks); + String topic = path.toString(); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer(acks)) { + assertThat(producer.partitionsFor(topic)).hasSize(1); + ProducerRecord record = + new ProducerRecord<>( + topic, + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + RecordMetadata first = producer.send(record).get(30, TimeUnit.SECONDS); + RecordMetadata second = producer.send(record).get(30, TimeUnit.SECONDS); + if (!acks.equals("0")) { + assertThat(first.offset()).isZero(); + assertThat(second.offset()).isEqualTo(1L); + } + producer.flush(); + assertReadback(connection, path, format, VALUE); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testSameNamedTablesAcrossDatabases() throws Exception { + Map tables = new LinkedHashMap<>(); + tables.put(TablePath.of("produce_first", "events"), VALUE); + tables.put( + TablePath.of("produce_second", "events"), + "second database".getBytes(StandardCharsets.UTF_8)); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + try { + for (TablePath path : tables.keySet()) { + admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true) + .get(); + admin.createTable(path, descriptor("raw"), false).get(); + } + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all")) { + for (Map.Entry entry : tables.entrySet()) { + String topic = entry.getKey().toString(); + assertThat(producer.partitionsFor(topic)).hasSize(1); + ProducerRecord record = + new ProducerRecord<>( + topic, + 0, + 123L, + KEY, + entry.getValue(), + Arrays.asList( + new RecordHeader("source", entry.getValue()), + new RecordHeader("source", null))); + assertThat(producer.send(record).get(30, TimeUnit.SECONDS).offset()) + .isZero(); + assertThat(producer.send(record).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(1L); + } + for (Map.Entry entry : tables.entrySet()) { + assertReadback(connection, entry.getKey(), "raw", entry.getValue()); + } + } + } finally { + for (TablePath path : tables.keySet()) { + admin.dropTable(path, true).get(); + } + } + } + } + + private static void assertReadback( + Connection connection, TablePath path, String format, byte[] expectedValue) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < 2 && System.nanoTime() < deadline) { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + InternalRow row = record.getRow(); + if (format.equals("raw")) { + assertThat(row.getBytes(0)).isEqualTo(expectedValue); + assertThat(row.getBytes(3)).isEqualTo(KEY); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo(new String(expectedValue, StandardCharsets.UTF_8)); + assertThat(row.getString(3).toString()).isEqualTo("key"); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(row.getArray(2).size()).isEqualTo(2); + assertThat(row.getArray(2).getRow(0, 2).getString(0).toString()) + .isEqualTo("source"); + assertThat(row.getArray(2).getRow(1, 2).isNullAt(1)).isTrue(); + count++; + } + } + assertThat(count).isEqualTo(2); + } + } + + private static TableDescriptor descriptor(String format) { + boolean raw = format.equals("raw"); + return TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", raw ? DataTypes.BYTES() : DataTypes.STRING()) + .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "attributes", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD( + "value", DataTypes.BYTES())))) + .column("message_key", raw ? DataTypes.BYTES() : DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format) + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format) + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .build(); + } + + private static KafkaProducer producer(String acks) { + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 3c1bfe6128b..9642ed9bfb1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -151,6 +151,7 @@ private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, (short) 11), tuple(ApiKeys.METADATA.id, (short) 0, (short) 11), tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java new file mode 100644 index 00000000000..d556d910db7 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.protocol.Errors; +import org.junit.jupiter.api.Test; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests append admission, error isolation, session propagation and delayed-fetch completion. */ +class GatewayKafkaProduceBackendTest { + + @Test + void testDrainsImmediatelyWhileAcksAllResponseIsPending() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.append.getAcks()).isEqualTo(-1); + assertThat(service.append.getTimeoutMs()).isEqualTo(4321); + assertThat(service.append.getTableId()).isEqualTo(42L); + assertThat(service.drains).isEqualTo(1); + assertThat(result).isNotDone(); + assertThat(service.pendingAppend).isNotDone(); + service.pendingAppend.complete(success(0)); + assertThat(result.join().topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testDrainsAfterSynchronousAppendFailure() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.throwAppend = true; + assertErrors( + backend(service).write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.UNKNOWN_SERVER_ERROR); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception { + TestingProduceService service = new TestingProduceService(); + PartitionWrite malformed = + new PartitionWrite( + 0, + Collections.singletonList( + new Record( + 1L, + null, + new byte[] {(byte) 0xff}, + Collections.emptyList()))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("kafka.topic", malformed, good(1), good(8)))) + .join(); + assertErrors( + result, Errors.CORRUPT_MESSAGE, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testInvalidAndMissingTablesDoNotSuppressOtherTopics() throws Exception { + for (boolean async : new boolean[] {false, true}) { + TestingProduceService service = new TestingProduceService(); + service.asyncMissing = async; + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("kafka.invalid", good(0)), + topic("kafka.missing", good(0)), + topic("kafka.topic", good(0)))) + .join(); + assertThat(result.topics()).hasSize(3); + assertThat(result.topics().get(0).partitions().get(0).error()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION); + assertThat(result.topics().get(1).partitions().get(0).error()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(result.topics().get(2).partitions().get(0).error()).isEqualTo(Errors.NONE); + assertThat(service.drains).isEqualTo(1); + } + } + + @Test + void testNoAppendOrDrainWhenAllPartitionsFailConversion() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service).write(command((short) 1, topic("kafka.topic", good(8)))).join(); + assertErrors(result, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append).isNull(); + assertThat(service.drains).isZero(); + } + + @Test + void testMapsReplicationFailuresAndMissingResponsesPerPartition() throws Exception { + org.apache.fluss.rpc.protocol.Errors[] flussErrors = { + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.REQUEST_TIME_OUT, + org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER + }; + Errors[] kafkaErrors = { + Errors.NOT_ENOUGH_REPLICAS, + Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, + Errors.REQUEST_TIMED_OUT, + Errors.NOT_LEADER_OR_FOLLOWER + }; + for (int i = 0; i < flussErrors.length; i++) { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = + CompletableFuture.completedFuture( + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setErrorCode(flussErrors[i].code()), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setErrorCode(0) + .setBaseOffset(17L)))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) -1, + topic("kafka.topic", good(0), good(1), good(2)))) + .join(); + assertErrors(result, kafkaErrors[i], Errors.NONE, Errors.UNKNOWN_SERVER_ERROR); + assertThat(result.topics().get(0).partitions().get(1).baseOffset()).isEqualTo(17L); + } + } + + @Test + void testAsyncMetadataCompletionRestoresSessionAndAcksZero() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) 0, topic("kafka.topic", good(0)))); + CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))).join(); + assertErrors(result.join(), Errors.NONE); + assertThat(service.append.getAcks()).isZero(); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testQualifiedTopicsPreserveDatabaseAndTranscoderTablePath() throws Exception { + TestingProduceService service = new TestingProduceService(); + List transcodedPaths = new ArrayList<>(); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + (records, tableInfo) -> { + transcodedPaths.add(tableInfo.getTablePath()); + return transcoder.transcode(records, tableInfo); + }); + + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("kafka.topic", good(0)), + topic("other.topic", good(0)))) + .join(); + + assertThat(service.metadataPaths) + .containsExactly(TablePath.of("kafka", "topic"), TablePath.of("other", "topic")); + assertThat(transcodedPaths).containsExactlyElementsOf(service.metadataPaths); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 43L); + assertThat(result.topics()) + .extracting(KafkaProduceResult.TopicResult::topicName) + .containsExactly("kafka.topic", "other.topic"); + assertThat(result.topics()) + .allSatisfy( + topic -> + assertThat(topic.partitions().get(0).error()) + .isEqualTo(Errors.NONE)); + } + + @Test + void testInvalidQualifiedNamesDoNotReachGatewayOrSuppressValidTopic() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("topic", good(0)), + topic("kafka.topic.extra", good(0)), + topic("kafka.__internal", good(0)), + topic("kafka.topic", good(0)))) + .join(); + + assertThat(result.topics()) + .extracting(topic -> topic.partitions().get(0).error()) + .containsExactly( + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION, + Errors.NONE); + assertThat(service.metadataPaths).containsExactly(TablePath.of("kafka", "topic")); + assertThat(service.appends).hasSize(1); + } + + private static GatewayKafkaProduceBackend backend(TestingProduceService service) { + return new GatewayKafkaProduceBackend(service, service, new ArrowKafkaRecordTranscoder()); + } + + private static KafkaProduceCommand command(short acks, TopicWrite... topics) throws Exception { + return new KafkaProduceCommand( + acks, 4321, Arrays.asList(topics), "KAFKA", InetAddress.getLoopbackAddress()); + } + + private static TopicWrite topic(String name, PartitionWrite... partitions) { + return new TopicWrite(name, Arrays.asList(partitions)); + } + + private static PartitionWrite good(int id) { + return new PartitionWrite( + id, + Collections.singletonList( + new Record(123L, null, new byte[] {65}, Collections.emptyList()))); + } + + private static GetTableInfoResponse metadata(boolean invalid) { + TableDescriptor.Builder descriptor = + TableDescriptor.builder() + .schema(Schema.newBuilder().column("value", DataTypes.STRING()).build()) + .distributedBy(3) + .logFormat(LogFormat.ARROW); + if (!invalid) { + descriptor.customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"); + } + return new GetTableInfoResponse() + .setTableId(42L) + .setSchemaId(1) + .setTableJson(descriptor.build().toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L); + } + + private static ProduceLogResponse success(int id) { + return new ProduceLogResponse() + .addAllBucketsResps( + Collections.singletonList( + new PbProduceLogRespForBucket() + .setBucketId(id) + .setBaseOffset(17L))); + } + + private static void assertErrors(KafkaProduceResult result, Errors... errors) { + List partitions = result.topics().get(0).partitions(); + assertThat(partitions).extracting(PartitionResult::error).containsExactly(errors); + for (PartitionResult partition : partitions) { + if (partition.error() != Errors.NONE) { + assertThat(partition.baseOffset()).isEqualTo(-1L); + } + } + } + + private static class TestingProduceService extends TestingTabletGatewayService { + private final List metadataPaths = new ArrayList<>(); + private final List appends = new ArrayList<>(); + private ProduceLogRequest append; + private CompletableFuture pendingAppend; + private CompletableFuture pendingMetadata; + private boolean throwAppend; + private boolean asyncMissing; + private int drains; + + @Override + public CompletableFuture getTableInfo(GetTableInfoRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + String database = request.getTablePath().getDatabaseName(); + String name = request.getTablePath().getTableName(); + metadataPaths.add(TablePath.of(database, name)); + if (name.equals("missing")) { + if (!asyncMissing) { + throw new TableNotExistException("missing"); + } + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(new TableNotExistException("missing")); + return failed; + } + return pendingMetadata == null + ? CompletableFuture.completedFuture( + GatewayKafkaProduceBackendTest.metadata(name.equals("invalid")) + .setTableId(database.equals("other") ? 43L : 42L)) + : pendingMetadata; + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(currentSession().getInetAddress()) + .isEqualTo(InetAddress.getLoopbackAddress()); + append = request; + appends.add(request); + if (throwAppend) { + throw new IllegalStateException("append failed"); + } + return pendingAppend == null + ? CompletableFuture.completedFuture( + success(request.getBucketsReqsList().get(0).getBucketId())) + : pendingAppend; + } + + @Override + public void tryCompleteActions() { + assertThat(append).isNotNull(); + drains++; + } + } +} From b7c0f29f083891998af9edc96f251789d2b517c8 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 19:33:20 +0800 Subject: [PATCH 34/48] [kafka] Cache Produce plans and bound conversion execution Reuse parsed table metadata and immutable raw/string write plans while refreshing native metadata and authorization for every topic write. Offload metadata lookup, conversion and native submission to bounded FIFO workers, preserving connection order without waiting for acknowledgements. Close protocol-owned workers after transport and RPC processors stop. Validation: Java 11 reactor verify passed 160 Kafka unit tests, 8 Kafka integration tests and 5 NettyServer tests. Checkstyle, Spotless, RAT and git diff --check passed. Tests cover cache invalidation, authorization refresh, saturation, shutdown, pending acknowledgements and pipelined three-replica append order. Source content is unchanged since validation. ZooKeeper query reduction and byte-based admission remain follow-up work. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 812/812 AI-Contributed/UT: 1047/1047 --- .../fluss/kafka/KafkaProtocolPlugin.java | 40 ++- .../fluss/kafka/KafkaRequestHandler.java | 14 +- .../produce/GatewayKafkaProduceBackend.java | 100 +++++-- .../KafkaProduceConversionExecutor.java | 163 ++++++++++ .../backend/produce/KafkaTableInfoCache.java | 179 +++++++++++ .../transcode/ArrowKafkaRecordTranscoder.java | 24 +- .../KafkaCompiledWritePlanCache.java | 161 ++++++++++ .../transcode/KafkaRecordTranscoder.java | 20 +- .../kafka/transcode/KafkaTopicWritePlan.java | 54 ++++ .../fluss/kafka/KafkaProduceAppendITCase.java | 53 ++++ .../GatewayKafkaProduceBackendTest.java | 141 +++++++++ .../KafkaProduceConversionExecutorTest.java | 184 ++++++++++++ .../produce/KafkaTableInfoCacheTest.java | 276 +++++++++++++++++ .../KafkaCompiledWritePlanCacheTest.java | 278 ++++++++++++++++++ .../fluss/rpc/netty/server/NettyServer.java | 60 ++-- .../rpc/protocol/NetworkProtocolPlugin.java | 6 + .../rpc/netty/server/NettyServerTest.java | 115 ++++++++ 17 files changed, 1811 insertions(+), 57 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 2d0fa9220e3..07bdbbfe86b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.backend.produce.KafkaProduceConversionExecutor; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.rpc.netty.server.RequestHandler; @@ -26,11 +27,16 @@ import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkState; /** The Kafka protocol plugin. */ public class KafkaProtocolPlugin implements NetworkProtocolPlugin { private Configuration conf; + private KafkaProduceConversionExecutor conversionExecutor; + private CompletableFuture closeFuture; @Override public String name() { @@ -39,7 +45,7 @@ public String name() { @Override public void setup(Configuration conf) { - this.conf = conf; + this.conf = new Configuration(conf); } @Override @@ -59,7 +65,35 @@ public ChannelHandler createChannelHandler( } @Override - public RequestHandler createRequestHandler(RpcGatewayService service) { - return new KafkaRequestHandler(conf, service); + public synchronized RequestHandler createRequestHandler(RpcGatewayService service) { + checkState(closeFuture == null, "Kafka protocol plugin has already been closed."); + if (conversionExecutor == null) { + // Mirror the RPC worker concurrency, with a separate queue capped at 1024 requests. + int queueCapacity = + Math.max( + 1, + Math.min( + 1024, + conf.get(ConfigOptions.NETTY_SERVER_MAX_QUEUED_REQUESTS))); + int threads = + Math.max( + 1, + Math.min( + queueCapacity, + conf.get(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS))); + conversionExecutor = new KafkaProduceConversionExecutor(threads, queueCapacity); + } + return new KafkaRequestHandler(conf, service, conversionExecutor); + } + + @Override + public synchronized CompletableFuture closeAsync() { + if (closeFuture == null) { + closeFuture = + conversionExecutor == null + ? CompletableFuture.completedFuture(null) + : conversionExecutor.closeAsync(); + } + return closeFuture; } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 9e985dbea7f..2b48e91bbed 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -23,6 +23,7 @@ import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; import org.apache.fluss.kafka.backend.produce.GatewayKafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceConversionExecutor; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; @@ -33,6 +34,8 @@ import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; +import javax.annotation.Nullable; + import java.util.Arrays; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -45,6 +48,14 @@ public class KafkaRequestHandler implements RequestHandler { /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { + this(conf, service, null); + } + + /** Creates a handler with a plugin-owned executor for blocking Produce work. */ + public KafkaRequestHandler( + Configuration conf, + RpcGatewayService service, + @Nullable KafkaProduceConversionExecutor conversionExecutor) { KafkaTopicMapper topicMapper = new KafkaTopicMapper(checkNotNull(conf)); this.registry = new KafkaApiRegistry( @@ -57,7 +68,8 @@ public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { new GatewayKafkaProduceBackend( service, (TabletServerGateway) service, - new ArrowKafkaRecordTranscoder())))); + new ArrowKafkaRecordTranscoder(), + conversionExecutor)))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index dd07df63733..2b3755408b6 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -24,9 +24,9 @@ import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; -import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; @@ -44,12 +44,19 @@ import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.protocol.Errors; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -60,21 +67,48 @@ public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { private final RpcGatewayService service; private final TabletServerGateway gateway; private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); - private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); private final KafkaRecordTranscoder transcoder; + private final KafkaTableInfoCache tableInfoCache = new KafkaTableInfoCache(); + private final @Nullable KafkaProduceConversionExecutor conversionExecutor; /** Creates a Produce backend backed by the local TabletServer gateway. */ public GatewayKafkaProduceBackend( RpcGatewayService service, TabletServerGateway gateway, KafkaRecordTranscoder transcoder) { + this(service, gateway, transcoder, null); + } + + /** Creates a backend that submits work to a caller-owned bounded conversion executor. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + KafkaRecordTranscoder transcoder, + @Nullable KafkaProduceConversionExecutor conversionExecutor) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); this.transcoder = checkNotNull(transcoder); + this.conversionExecutor = conversionExecutor; } @Override public CompletableFuture write(KafkaProduceCommand command) { + if (conversionExecutor != null) { + return conversionExecutor + .submit(Thread.currentThread().getId(), () -> writeTopics(command)) + .exceptionally( + failure -> { + List results = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + results.add(failedTopic(topic, failure)); + } + return new KafkaProduceResult(results); + }); + } + return writeTopics(command); + } + + private CompletableFuture writeTopics(KafkaProduceCommand command) { List> futures = new ArrayList<>(); for (TopicWrite topic : command.topics()) { futures.add(writeTopic(command, topic)); @@ -94,16 +128,27 @@ public CompletableFuture write(KafkaProduceCommand command) private CompletableFuture writeTopic( KafkaProduceCommand command, TopicWrite topic) { try { + checkInterrupted(); TablePath path = topicMapper.toTablePath(topic.topicName()); setCurrentSession(command); GetTableInfoRequest request = new GetTableInfoRequest(); request.setTablePath() .setDatabaseName(path.getDatabaseName()) .setTableName(path.getTableName()); - return gateway.getTableInfo(request) - .thenCompose( + CompletableFuture metadata = gateway.getTableInfo(request); + if (conversionExecutor != null) { + // The local gateway is synchronous. Waiting here also preserves FIFO append + // order if an alternative gateway returns asynchronously; only this worker waits. + GetTableInfoResponse response = + metadata.get(Math.max(0, command.timeoutMs()), TimeUnit.MILLISECONDS); + return produceTopic(command, topic, toTableInfo(path, response)); + } + return metadata.thenCompose( response -> produceTopic(command, topic, toTableInfo(path, response))) .exceptionally(failure -> failedTopic(topic, failure)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + return CompletableFuture.completedFuture(failedTopic(topic, failure)); } catch (Exception failure) { return CompletableFuture.completedFuture(failedTopic(topic, failure)); } @@ -112,7 +157,7 @@ private CompletableFuture writeTopic( private CompletableFuture produceTopic( KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { // Admission must match Metadata, including when the schema changes between requests. - schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaTopicWritePlan writePlan = transcoder.prepare(tableInfo); ProduceLogRequest request = new ProduceLogRequest() .setTableId(tableInfo.getTableId()) @@ -132,7 +177,7 @@ private CompletableFuture produceTopic( continue; } try { - BytesView records = transcoder.transcode(partition.records(), tableInfo); + BytesView records = transcoder.transcode(partition.records(), writePlan); request.addBucketsReq() .setBucketId(partition.partitionId()) .setRecordsBytesView(records); @@ -146,6 +191,7 @@ private CompletableFuture produceTopic( toTopicResult(topic, new ProduceLogResponse(), failures)); } try { + checkInterrupted(); setCurrentSession(command); CompletableFuture appended; try { @@ -175,7 +221,11 @@ private static TopicResult failedAppend( return new TopicResult(topic.topicName(), results); } - private static TableInfo toTableInfo(TablePath path, GetTableInfoResponse response) { + private TableInfo toTableInfo(TablePath path, GetTableInfoResponse response) { + return tableInfoCache.getOrLoad(path, response, () -> parseTableInfo(path, response)); + } + + private static TableInfo parseTableInfo(TablePath path, GetTableInfoResponse response) { return TableInfo.of( path, response.getTableId(), @@ -183,7 +233,8 @@ private static TableInfo toTableInfo(TablePath path, GetTableInfoResponse respon TableDescriptor.fromJsonBytes(response.getTableJson()), response.hasRemoteDataDir() ? response.getRemoteDataDir() : null, response.getCreatedTime(), - response.getModifiedTime()); + response.getModifiedTime(), + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L); } private static TopicResult toTopicResult( @@ -239,15 +290,21 @@ private static TopicResult failedTopic(TopicWrite topic, Throwable failure) { private static PartitionResult failedPartition(int partitionId, Throwable failure) { Throwable cause = unwrap(failure); Errors kafkaError = - cause instanceof KafkaTopicSchemaException || cause instanceof InvalidTopicException - ? Errors.INVALID_TOPIC_EXCEPTION - : cause instanceof KafkaRecordEncodingException - ? Errors.CORRUPT_MESSAGE - : cause instanceof IllegalArgumentException - ? Errors.INVALID_REQUEST - : toKafkaError( - org.apache.fluss.rpc.protocol.Errors.forException( - cause)); + cause instanceof RejectedExecutionException + || cause instanceof TimeoutException + || cause instanceof InterruptedException + || cause instanceof CancellationException + ? Errors.REQUEST_TIMED_OUT + : cause instanceof KafkaTopicSchemaException + || cause instanceof InvalidTopicException + ? Errors.INVALID_TOPIC_EXCEPTION + : cause instanceof KafkaRecordEncodingException + ? Errors.CORRUPT_MESSAGE + : cause instanceof IllegalArgumentException + ? Errors.INVALID_REQUEST + : toKafkaError( + org.apache.fluss.rpc.protocol.Errors + .forException(cause)); return new PartitionResult(partitionId, kafkaError, -1L, cause.getMessage()); } @@ -298,9 +355,16 @@ private void setCurrentSession(KafkaProduceCommand command) { private static Throwable unwrap(Throwable failure) { Throwable current = failure; - while (current instanceof CompletionException && current.getCause() != null) { + while ((current instanceof CompletionException || current instanceof ExecutionException) + && current.getCause() != null) { current = current.getCause(); } return current; } + + private void checkInterrupted() { + if (conversionExecutor != null && Thread.currentThread().isInterrupted()) { + throw new CancellationException("Kafka conversion worker was interrupted."); + } + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java new file mode 100644 index 00000000000..e3953f001e0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; + +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Dedicated, bounded FIFO lanes for Kafka metadata lookup, conversion, and native submission. + * + *

The caller uses its RPC processor identity as the ordering key. The server routes each + * connection to one processor, so its requests remain ordered when moved off that processor. An + * operation must invoke native append before returning, but need not wait for acknowledgements. + */ +@Internal +public final class KafkaProduceConversionExecutor { + private final ThreadPoolExecutor[] workers; + private volatile boolean closed; + private CompletableFuture closeFuture; + + /** Creates fixed single-worker lanes whose combined queue capacity is bounded. */ + public KafkaProduceConversionExecutor(int threads, int queueCapacity) { + checkArgument(threads > 0, "Conversion thread count must be positive."); + checkArgument( + queueCapacity >= threads, "Queue capacity must be at least the thread count."); + workers = new ThreadPoolExecutor[threads]; + for (int i = 0; i < threads; i++) { + int capacity = queueCapacity / threads + (i < queueCapacity % threads ? 1 : 0); + workers[i] = + new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(capacity), + new ExecutorThreadFactory("kafka-produce-conversion-" + i), + new ThreadPoolExecutor.AbortPolicy()); + } + } + + /** Submits ordered work without ever running rejected work on the caller's thread. */ + public CompletableFuture submit( + long orderingKey, Supplier> operation) { + ConversionTask task = new ConversionTask<>(checkNotNull(operation)); + try { + if (closed) { + throw new RejectedExecutionException("Kafka conversion executor is closed."); + } + workers[(int) Math.floorMod(orderingKey, (long) workers.length)].execute(task); + } catch (RejectedExecutionException failure) { + task.cancel(failure); + } + return task.result; + } + + /** + * Stops new work, rejects queued requests, interrupts workers, and awaits their termination. + */ + public synchronized CompletableFuture closeAsync() { + if (closeFuture != null) { + return closeFuture; + } + closed = true; + RejectedExecutionException failure = + new RejectedExecutionException("Kafka conversion executor is shutting down."); + for (ThreadPoolExecutor worker : workers) { + List queued = worker.shutdownNow(); + for (Runnable task : queued) { + ((ConversionTask) task).cancel(failure); + } + } + closeFuture = + CompletableFuture.runAsync( + () -> { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + try { + for (ThreadPoolExecutor worker : workers) { + if (!worker.awaitTermination( + Math.max(0L, deadline - System.nanoTime()), + TimeUnit.NANOSECONDS)) { + throw new IllegalStateException( + "Kafka conversion worker did not terminate."); + } + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted closing Kafka conversion workers.", + interrupted); + } + }); + return closeFuture; + } + + private static final class ConversionTask implements Runnable { + private final AtomicReference>> operation; + private final CompletableFuture result = new CompletableFuture<>(); + private final AtomicBoolean claimed = new AtomicBoolean(); + + private ConversionTask(Supplier> operation) { + this.operation = new AtomicReference<>(operation); + } + + @Override + public void run() { + if (!claimed.compareAndSet(false, true)) { + return; + } + Supplier> action = operation.getAndSet(null); + if (result.isDone()) { + return; + } + try { + checkNotNull(action.get(), "conversion result") + .whenComplete( + (value, failure) -> { + if (failure == null) { + result.complete(value); + } else { + result.completeExceptionally(failure); + } + }); + } catch (Throwable failure) { + result.completeExceptionally(failure); + } + } + + private void cancel(Throwable failure) { + if (claimed.compareAndSet(false, true)) { + operation.set(null); + result.completeExceptionally(failure); + } + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java new file mode 100644 index 00000000000..1d53a5eed8a --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded cache for parsed Kafka topic {@link TableInfo} snapshots. + * + *

The cache never replaces the authoritative {@code GetTableInfo} call. Every Produce request + * still obtains a fresh response (and therefore retains its authorization and freshness semantics). + * This cache only avoids parsing an unchanged descriptor JSON response again. + * + *

An entry is reused only when all response identity fields and the complete descriptor bytes + * match. Comparing the complete descriptor is intentional: a {@code kafka.*} property change may + * alter decoding semantics without changing the schema ID. Cache slots serialize parsing for one + * table, while independent topics remain concurrent. + */ +@Internal +@ThreadSafe +final class KafkaTableInfoCache { + + static final long DEFAULT_MAXIMUM_SIZE = 1_000L; + + private final long maximumSize; + private final ConcurrentMap slots; + private final Object slotCreationLock = new Object(); + + KafkaTableInfoCache() { + this(DEFAULT_MAXIMUM_SIZE); + } + + KafkaTableInfoCache(long maximumSize) { + checkArgument(maximumSize > 0, "Kafka TableInfo cache size must be greater than zero."); + this.maximumSize = maximumSize; + this.slots = new ConcurrentHashMap<>(); + } + + TableInfo getOrLoad( + TablePath tablePath, + GetTableInfoResponse response, + Supplier tableInfoLoader) { + checkNotNull(tablePath); + checkNotNull(response); + checkNotNull(tableInfoLoader); + + while (true) { + CacheSlot slot = getOrCreateSlot(tablePath); + synchronized (slot) { + if (slots.get(tablePath) != slot) { + continue; + } + if (slot.entry != null && slot.entry.matches(response)) { + return slot.entry.tableInfo; + } + TableInfo tableInfo = checkNotNull(tableInfoLoader.get()); + slot.entry = new CacheEntry(response, tableInfo); + return tableInfo; + } + } + } + + void invalidate(TablePath tablePath) { + CacheSlot slot = slots.get(checkNotNull(tablePath)); + if (slot == null) { + return; + } + synchronized (slot) { + slot.entry = null; + slots.remove(tablePath, slot); + } + } + + long estimatedSize() { + return slots.size(); + } + + private CacheSlot getOrCreateSlot(TablePath tablePath) { + CacheSlot slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + synchronized (slotCreationLock) { + slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + if (slots.size() >= maximumSize) { + evictOneSlot(); + } + slot = new CacheSlot(); + slots.put(tablePath, slot); + return slot; + } + } + + private void evictOneSlot() { + for (Map.Entry entry : slots.entrySet()) { + CacheSlot slot = entry.getValue(); + synchronized (slot) { + if (slots.remove(entry.getKey(), slot)) { + return; + } + } + } + } + + private static final class CacheSlot { + private CacheEntry entry; + } + + private static final class CacheEntry { + private final long tableId; + private final int schemaId; + private final byte[] tableJson; + private final long createdTime; + private final long modifiedTime; + private final long bucketCountEpoch; + private final String remoteDataDir; + private final TableInfo tableInfo; + + private CacheEntry(GetTableInfoResponse response, TableInfo tableInfo) { + this.tableId = response.getTableId(); + this.schemaId = response.getSchemaId(); + this.tableJson = response.getTableJson().clone(); + this.createdTime = response.getCreatedTime(); + this.modifiedTime = response.getModifiedTime(); + this.bucketCountEpoch = + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L; + this.remoteDataDir = response.hasRemoteDataDir() ? response.getRemoteDataDir() : null; + this.tableInfo = tableInfo; + } + + private boolean matches(GetTableInfoResponse response) { + return tableId == response.getTableId() + && schemaId == response.getSchemaId() + && createdTime == response.getCreatedTime() + && modifiedTime == response.getModifiedTime() + && bucketCountEpoch + == (response.hasBucketCountEpoch() + ? response.getBucketCountEpoch() + : 0L) + && Objects.equals( + remoteDataDir, + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null) + && Arrays.equals(tableJson, response.getTableJson()); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index dcbc0143c89..7852657b3ba 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -41,18 +41,32 @@ public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); + private final KafkaCompiledWritePlanCache writePlanCache = new KafkaCompiledWritePlanCache(); @Override - public BytesView transcode(int partitionId, List records, TableInfo tableInfo) - throws Exception { + public KafkaTopicWritePlan prepare(TableInfo tableInfo) { + return writePlanCache.getOrCompile( + tableInfo, + () -> + new KafkaTopicWritePlan( + tableInfo, schemaResolver.resolve(tableInfo.toTableDescriptor()))); + } + + @Override + public BytesView transcode(int partitionId, List records, TableInfo tableInfo) throws Exception { + return transcode(partitionId, records, prepare(tableInfo)); + } + + @Override + public BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); - KafkaTableSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaTableSchema schema = plan.topicSchema(); if (schema.valueType() != ValueType.BINARY && schema.valueType() != ValueType.STRING) { throw new KafkaRecordEncodingException( "Unsupported Kafka value conversion type: " + schema.valueType()); } - KafkaRowAssembler assembler = new KafkaRowAssembler(schema); + KafkaRowAssembler assembler = plan.rowAssembler(); return arrowRecordEncoder.encodeStreaming( consumer -> { for (Record record : records) { @@ -65,7 +79,7 @@ public BytesView transcode(int partitionId, List records, TableInfo tabl partitionId)); } }, - tableInfo); + plan.tableInfo()); } private static @Nullable Object decode(ValueType format, @Nullable byte[] bytes) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java new file mode 100644 index 00000000000..258e8c557b8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded, single-flight cache for immutable Kafka topic write plans. + * + *

The bounded map is keyed by {@link TablePath}, while each slot fences its plan with the exact + * {@link TableInfo} instance from the authoritative metadata cache. This avoids the recursive and + * allocation-heavy {@link TableInfo#hashCode()} on every Produce request. An equivalent but + * separately constructed {@code TableInfo} is deliberately treated as a cache miss. In the + * production path, {@code KafkaTableInfoCache} returns the same instance only when the complete + * {@code GetTableInfo} response is unchanged. + * + *

Consequently a delete/recreate, schema change, or property-only Kafka contract change receives + * a different instance and cannot reuse an older plan. Slots are synchronized individually so a + * concurrent cold start compiles once per table version. + */ +@Internal +@ThreadSafe +final class KafkaCompiledWritePlanCache { + + static final long DEFAULT_MAXIMUM_SIZE = 1_000L; + + private final long maximumSize; + private final ConcurrentMap slots; + private final Object slotCreationLock = new Object(); + + KafkaCompiledWritePlanCache() { + this(DEFAULT_MAXIMUM_SIZE); + } + + KafkaCompiledWritePlanCache(long maximumSize) { + checkArgument( + maximumSize > 0, "Kafka compiled write plan cache size must be greater than zero."); + this.maximumSize = maximumSize; + this.slots = new ConcurrentHashMap<>(); + } + + KafkaTopicWritePlan getIfPresent(TableInfo tableInfo) { + checkNotNull(tableInfo); + CacheSlot slot = slots.get(tableInfo.getTablePath()); + if (slot == null) { + return null; + } + synchronized (slot) { + if (slots.get(tableInfo.getTablePath()) != slot) { + return null; + } + return slot.tableInfo == tableInfo ? slot.plan : null; + } + } + + KafkaTopicWritePlan getOrCompile( + TableInfo tableInfo, Supplier planCompiler) { + checkNotNull(tableInfo); + checkNotNull(planCompiler); + + TablePath tablePath = tableInfo.getTablePath(); + while (true) { + CacheSlot slot = getOrCreateSlot(tablePath); + synchronized (slot) { + if (slots.get(tablePath) != slot) { + continue; + } + if (slot.tableInfo == tableInfo && slot.plan != null) { + return slot.plan; + } + KafkaTopicWritePlan plan = checkNotNull(planCompiler.get()); + slot.tableInfo = tableInfo; + slot.plan = plan; + return plan; + } + } + } + + void invalidate(TableInfo tableInfo) { + checkNotNull(tableInfo); + TablePath tablePath = tableInfo.getTablePath(); + CacheSlot slot = slots.get(tablePath); + if (slot == null) { + return; + } + synchronized (slot) { + if (slot.tableInfo != tableInfo) { + return; + } + slot.tableInfo = null; + slot.plan = null; + slots.remove(tablePath, slot); + } + } + + long estimatedSize() { + return slots.size(); + } + + private CacheSlot getOrCreateSlot(TablePath tablePath) { + CacheSlot slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + synchronized (slotCreationLock) { + slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + if (slots.size() >= maximumSize) { + evictOneSlot(); + } + slot = new CacheSlot(); + slots.put(tablePath, slot); + return slot; + } + } + + private void evictOneSlot() { + for (Map.Entry entry : slots.entrySet()) { + CacheSlot slot = entry.getValue(); + synchronized (slot) { + if (slots.remove(entry.getKey(), slot)) { + return; + } + } + } + } + + private static final class CacheSlot { + private TableInfo tableInfo; + private KafkaTopicWritePlan plan; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java index 9bcc465b2a2..063728b77fa 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -19,6 +19,7 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; @@ -27,10 +28,17 @@ /** Converts copied Kafka records into the native Fluss log representation. */ @Internal public interface KafkaRecordTranscoder { - /** - * Transcodes records from the requested Kafka partition according to the target Fluss table - * schema and log format. - */ - BytesView transcode(int partitionId, List records, TableInfo tableInfo) - throws Exception; + /** Validates the table contract and prepares a shareable plan once per topic write. */ + default KafkaTopicWritePlan prepare(TableInfo tableInfo) { + return new KafkaTopicWritePlan( + tableInfo, new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor())); + } + + /** Transcodes one requested partition using an already prepared plan. */ + default BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { + return transcode(partitionId, records, plan.tableInfo()); + } + + /** Transcodes records from the requested Kafka partition using the target table schema. */ + BytesView transcode(int partitionId, List records, TableInfo tableInfo) throws Exception; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java new file mode 100644 index 00000000000..1f9d744afb5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.TableInfo; + +import javax.annotation.concurrent.ThreadSafe; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Immutable binary/string conversion plan; mutable Arrow resources are owned by each conversion. */ +@Internal +@ThreadSafe +public final class KafkaTopicWritePlan { + private final TableInfo tableInfo; + private final KafkaTableSchema topicSchema; + private final KafkaRowAssembler rowAssembler; + + KafkaTopicWritePlan(TableInfo tableInfo, KafkaTableSchema topicSchema) { + this.tableInfo = checkNotNull(tableInfo); + this.topicSchema = checkNotNull(topicSchema); + this.rowAssembler = new KafkaRowAssembler(topicSchema); + } + + /** Returns the authoritative table snapshot used to compile this plan. */ + public TableInfo tableInfo() { + return tableInfo; + } + + KafkaTableSchema topicSchema() { + return topicSchema; + } + + KafkaRowAssembler rowAssembler() { + return rowAssembler; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 8f6fab3a2e5..fbc35bc0bf8 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -49,10 +49,13 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -156,6 +159,51 @@ void testSameNamedTablesAcrossDatabases() throws Exception { } } + @Test + void testPipelinedProducePreservesAppendOrder() throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_pipelined"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("raw"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + assertThat(producer.partitionsFor(path.toString())).hasSize(1); + List values = new ArrayList<>(); + List> writes = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + byte[] value = new byte[i % 2 == 0 ? 16384 : 1]; + Arrays.fill(value, (byte) i); + values.add(value); + writes.add( + producer.send( + new ProducerRecord<>( + path.toString(), 0, (long) i, KEY, value))); + } + for (int i = 0; i < writes.size(); i++) { + assertThat(writes.get(i).get(30, TimeUnit.SECONDS).offset()).isEqualTo(i); + } + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < values.size() && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.getRow().getBytes(0)).isEqualTo(values.get(count)); + assertThat(record.getRow().getTimestampLtz(1, 3).getEpochMillisecond()) + .isEqualTo(count); + count++; + } + } + assertThat(count).isEqualTo(values.size()); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + private static void assertReadback( Connection connection, TablePath path, String format, byte[] expectedValue) throws Exception { @@ -216,6 +264,10 @@ private static TableDescriptor descriptor(String format) { } private static KafkaProducer producer(String acks) { + return producer(acks, 16384); + } + + private static KafkaProducer producer(String acks, int batchSize) { ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); Map config = new HashMap<>(); config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); @@ -223,6 +275,7 @@ private static KafkaProducer producer(String acks) { config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10000); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index d556d910db7..fb4f999d6e3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -24,10 +24,14 @@ import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.messages.GetTableInfoRequest; import org.apache.fluss.rpc.messages.GetTableInfoResponse; @@ -45,6 +49,8 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -253,6 +259,131 @@ void testInvalidQualifiedNamesDoNotReachGatewayOrSuppressValidTopic() throws Exc assertThat(service.appends).hasSize(1); } + @Test + void testReusesPlansButAlwaysRefreshesMetadataAndValidatesChangedContract() throws Exception { + TestingProduceService service = new TestingProduceService(); + ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); + List prepared = new ArrayList<>(); + List converted = new ArrayList<>(); + KafkaRecordTranscoder transcoder = + new KafkaRecordTranscoder() { + @Override + public KafkaTopicWritePlan prepare(TableInfo tableInfo) { + KafkaTopicWritePlan plan = arrow.prepare(tableInfo); + prepared.add(plan); + return plan; + } + + @Override + public BytesView transcode(List records, TableInfo tableInfo) + throws Exception { + return arrow.transcode(records, tableInfo); + } + + @Override + public BytesView transcode(List records, KafkaTopicWritePlan plan) + throws Exception { + converted.add(plan); + return arrow.transcode(records, plan); + } + }; + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, transcoder); + backend.write(command((short) 1, topic("kafka.topic", good(0), good(1)))).join(); + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(); + assertThat(service.metadataPaths).hasSize(2); + assertThat(prepared).hasSize(2); + assertThat(prepared.get(1)).isSameAs(prepared.get(0)); + assertThat(converted).containsExactly(prepared.get(0), prepared.get(0), prepared.get(0)); + + service.pendingMetadata = CompletableFuture.completedFuture(metadata(true)); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.metadataPaths).hasSize(3); + assertThat(service.appends).hasSize(2); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingMetadata.completeExceptionally( + new org.apache.fluss.exception.AuthorizationException("revoked")); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.metadataPaths).hasSize(4); + assertThat(service.appends).hasSize(2); + } + + @Test + void testOffloadedRequestsAppendInOrderWithoutWaitingForAcknowledgement() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(2, 4); + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingAppend = new CompletableFuture<>(); + service.metadataEntered = new CountDownLatch(1); + service.appended = new CountDownLatch(2); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, new ArrowKafkaRecordTranscoder(), executor); + try { + CompletableFuture first = + backend.write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.metadataEntered.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture second = + backend.write(command((short) -1, topic("kafka.topic", good(1)))); + assertThat(service.metadataThread).isNotSameAs(Thread.currentThread()); + CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))) + .join(); + assertThat(service.appended.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(service.appends) + .extracting(request -> request.getBucketsReqsList().get(0).getBucketId()) + .containsExactly(0, 1); + assertThat(first).isNotDone(); + assertThat(second).isNotDone(); + service.pendingAppend.complete( + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setBaseOffset(17L), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setBaseOffset(18L)))); + assertErrors(first.get(10, TimeUnit.SECONDS), Errors.NONE); + assertErrors(second.get(10, TimeUnit.SECONDS), Errors.NONE); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + service.metadataEntered = new CountDownLatch(1); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, new ArrowKafkaRecordTranscoder(), executor); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); + try { + CompletableFuture running = backend.write(command); + assertThat(service.metadataEntered.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture queued = backend.write(command); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + executor.closeAsync().get(10, TimeUnit.SECONDS); + assertErrors(running.get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + assertErrors(queued.get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + service.pendingMetadata.complete(metadata(false)); + assertThat(service.appends).isEmpty(); + assertThat(service.metadataPaths).hasSize(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + private static GatewayKafkaProduceBackend backend(TestingProduceService service) { return new GatewayKafkaProduceBackend(service, service, new ArrowKafkaRecordTranscoder()); } @@ -318,6 +449,9 @@ private static class TestingProduceService extends TestingTabletGatewayService { private boolean throwAppend; private boolean asyncMissing; private int drains; + private CountDownLatch metadataEntered; + private CountDownLatch appended; + private Thread metadataThread; @Override public CompletableFuture getTableInfo(GetTableInfoRequest request) { @@ -325,6 +459,10 @@ public CompletableFuture getTableInfo(GetTableInfoRequest String database = request.getTablePath().getDatabaseName(); String name = request.getTablePath().getTableName(); metadataPaths.add(TablePath.of(database, name)); + metadataThread = Thread.currentThread(); + if (metadataEntered != null) { + metadataEntered.countDown(); + } if (name.equals("missing")) { if (!asyncMissing) { throw new TableNotExistException("missing"); @@ -360,6 +498,9 @@ public CompletableFuture produceLog(ProduceLogRequest reques public void tryCompleteActions() { assertThat(append).isNotNull(); drains++; + if (appended != null) { + appended.countDown(); + } } } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java new file mode 100644 index 00000000000..88e95f420b6 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests bounded admission, ordering, independent lanes and executor shutdown. */ +class KafkaProduceConversionExecutorTest { + @Test + void testBoundedLanePreservesOrderAndDoesNotBlockAnotherLane() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(2, 2); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + List order = new ArrayList<>(); + AtomicReference worker = new AtomicReference<>(); + AtomicBoolean rejectedRan = new AtomicBoolean(); + try { + CompletableFuture first = + executor.submit( + 0, + () -> { + worker.set(Thread.currentThread()); + started.countDown(); + await(release); + order.add(1); + return CompletableFuture.completedFuture(1); + }); + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture second = + executor.submit( + 0, + () -> { + order.add(2); + return CompletableFuture.completedFuture(2); + }); + CompletableFuture rejected = + executor.submit( + 0, + () -> { + rejectedRan.set(true); + return CompletableFuture.completedFuture(3); + }); + assertThatThrownBy(rejected::join).hasCauseInstanceOf(RejectedExecutionException.class); + assertThat(rejectedRan).isFalse(); + assertThat(worker.get()).isNotSameAs(Thread.currentThread()); + assertThat(second).isNotDone(); + assertThat( + executor.submit(1, () -> CompletableFuture.completedFuture(4)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(4); + release.countDown(); + assertThat(first.get(10, TimeUnit.SECONDS)).isEqualTo(1); + assertThat(second.get(10, TimeUnit.SECONDS)).isEqualTo(2); + assertThat(order).containsExactly(1, 2); + } finally { + release.countDown(); + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testPendingAcknowledgementDoesNotOccupyWorker() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + CompletableFuture acknowledgement = new CompletableFuture<>(); + CountDownLatch submitted = new CountDownLatch(1); + try { + CompletableFuture first = + executor.submit( + 0, + () -> { + submitted.countDown(); + return acknowledgement; + }); + assertThat(submitted.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat( + executor.submit(0, () -> CompletableFuture.completedFuture(2)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(2); + assertThat(first).isNotDone(); + acknowledgement.complete(1); + assertThat(first.get(10, TimeUnit.SECONDS)).isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testCloseInterruptsWorkerCompletesQueuedRequestsAndRejectsNewWork() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + CountDownLatch started = new CountDownLatch(1); + AtomicBoolean queuedRan = new AtomicBoolean(); + CompletableFuture running = + executor.submit( + 0, + () -> { + started.countDown(); + await(new CountDownLatch(1)); + return CompletableFuture.completedFuture(null); + }); + try { + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture queued = + executor.submit( + 0, + () -> { + queuedRan.set(true); + return CompletableFuture.completedFuture(null); + }); + CompletableFuture closed = executor.closeAsync(); + assertThat(executor.closeAsync()).isSameAs(closed); + closed.get(10, TimeUnit.SECONDS); + assertThatThrownBy(running::join).hasRootCauseInstanceOf(InterruptedException.class); + assertThatThrownBy(queued::join).hasCauseInstanceOf(RejectedExecutionException.class); + assertThat(queuedRan).isFalse(); + assertThatThrownBy( + () -> + executor.submit( + 0, + () -> CompletableFuture.completedFuture(null)) + .join()) + .hasCauseInstanceOf(RejectedExecutionException.class); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testFailedOperationDoesNotStopLane() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + try { + CompletableFuture failed = + executor.submit( + 0, + () -> { + throw new IllegalStateException("conversion failed"); + }); + assertThatThrownBy(failed::join).hasCauseInstanceOf(IllegalStateException.class); + assertThat( + executor.submit(0, () -> CompletableFuture.completedFuture(1)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Test worker was not released."); + } + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(failure); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java new file mode 100644 index 00000000000..73c68ec725d --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class KafkaTableInfoCacheTest { + + private static final TablePath TABLE_PATH = TablePath.of("kafka", "orders"); + + @Test + void testReusesUnchangedResponseAndReloadsPropertyOnlyChange() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loadCount = new AtomicInteger(); + GetTableInfoResponse firstResponse = response(1L, 1, descriptor("v1")); + + TableInfo first = + cache.getOrLoad( + TABLE_PATH, + firstResponse, + () -> load(TABLE_PATH, firstResponse, loadCount)); + GetTableInfoResponse equivalentResponse = response(1L, 1, descriptor("v1")); + TableInfo reused = + cache.getOrLoad( + TABLE_PATH, + equivalentResponse, + () -> load(TABLE_PATH, equivalentResponse, loadCount)); + + assertThat(reused).isSameAs(first); + assertThat(loadCount).hasValue(1); + + // A Kafka contract property may change without a schema ID or timestamp change. + GetTableInfoResponse changedResponse = response(1L, 1, descriptor("v2")); + TableInfo changed = + cache.getOrLoad( + TABLE_PATH, + changedResponse, + () -> load(TABLE_PATH, changedResponse, loadCount)); + + assertThat(changed).isNotSameAs(first); + assertThat(loadCount).hasValue(2); + } + + @Test + void testReloadsDeleteAndRecreateAndSupportsExplicitInvalidation() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loadCount = new AtomicInteger(); + GetTableInfoResponse original = response(1L, 1, descriptor("v1")); + TableInfo first = + cache.getOrLoad(TABLE_PATH, original, () -> load(TABLE_PATH, original, loadCount)); + + GetTableInfoResponse recreated = response(2L, 1, descriptor("v1")); + TableInfo second = + cache.getOrLoad( + TABLE_PATH, recreated, () -> load(TABLE_PATH, recreated, loadCount)); + assertThat(second).isNotSameAs(first); + + cache.invalidate(TABLE_PATH); + TableInfo third = + cache.getOrLoad( + TABLE_PATH, recreated, () -> load(TABLE_PATH, recreated, loadCount)); + assertThat(third).isNotSameAs(second); + assertThat(loadCount).hasValue(3); + } + + @Test + void testReloadsWhenOnlyBucketCountEpochChanges() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loads = new AtomicInteger(); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + TableInfo first = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, loads)); + response.setBucketCountEpoch(7L); + TableInfo changed = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, loads)); + assertThat(changed).isNotSameAs(first); + assertThat(changed.getBucketCountEpoch()).isEqualTo(7L); + assertThat(loads).hasValue(2); + } + + @Test + void testLoadFailureIsNotCached() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger attempts = new AtomicInteger(); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + + assertThatThrownBy( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + attempts.incrementAndGet(); + throw new IllegalStateException("broken descriptor"); + })) + .isInstanceOf(IllegalStateException.class) + .hasMessage("broken descriptor"); + + TableInfo loaded = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, attempts)); + assertThat(loaded.getTableId()).isEqualTo(1L); + assertThat(attempts).hasValue(2); + } + + @Test + void testConcurrentColdLoadIsSingleFlight() throws Exception { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + futures.add( + executor.submit( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + loadCount.incrementAndGet(); + loaderStarted.countDown(); + await(releaseLoader); + return toTableInfo(TABLE_PATH, response); + }))); + } + loaderStarted.await(); + releaseLoader.countDown(); + + TableInfo first = futures.get(0).get(); + for (Future future : futures) { + assertThat(future.get()).isSameAs(first); + } + assertThat(loadCount).hasValue(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testInvalidationDoesNotAllowInFlightLoadToResurrect() throws Exception { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future loading = + executor.submit( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + loadCount.incrementAndGet(); + loaderStarted.countDown(); + await(releaseLoader); + return toTableInfo(TABLE_PATH, response); + })); + loaderStarted.await(); + Future invalidating = executor.submit(() -> cache.invalidate(TABLE_PATH)); + releaseLoader.countDown(); + + TableInfo loadedBeforeInvalidation = loading.get(); + invalidating.get(); + TableInfo loadedAfterInvalidation = + cache.getOrLoad( + TABLE_PATH, response, () -> load(TABLE_PATH, response, loadCount)); + + assertThat(loadedAfterInvalidation).isNotSameAs(loadedBeforeInvalidation); + assertThat(loadCount).hasValue(2); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCacheSizeIsBounded() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(2); + for (int i = 0; i < 10; i++) { + TablePath path = TablePath.of("kafka", "topic_" + i); + GetTableInfoResponse response = response(i + 1L, 1, descriptor("v1")); + cache.getOrLoad(path, response, () -> toTableInfo(path, response)); + } + + assertThat(cache.estimatedSize()).isLessThanOrEqualTo(2); + } + + private static TableInfo load( + TablePath tablePath, GetTableInfoResponse response, AtomicInteger loadCount) { + loadCount.incrementAndGet(); + return toTableInfo(tablePath, response); + } + + private static TableInfo toTableInfo(TablePath tablePath, GetTableInfoResponse response) { + return TableInfo.of( + tablePath, + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + null, + response.getCreatedTime(), + response.getModifiedTime(), + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L); + } + + private static GetTableInfoResponse response( + long tableId, int schemaId, TableDescriptor descriptor) { + return new GetTableInfoResponse() + .setTableId(tableId) + .setSchemaId(schemaId) + .setTableJson(descriptor.toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L); + } + + private static TableDescriptor descriptor(String version) { + TableDescriptor.Builder builder = + TableDescriptor.builder() + .schema(Schema.newBuilder().column("value", DataTypes.BYTES()).build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .customProperty("kafka.value.format", "raw"); + if ("v2".equals(version)) { + builder.customProperty("application.version", "changed"); + } + return builder.build(); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for test loader.", e); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java new file mode 100644 index 00000000000..e3026f40685 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class KafkaCompiledWritePlanCacheTest { + + @Test + void testReusesSameIdentityAndSeparatesEquivalentAndChangedInstances() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo original = tableInfo("orders", 1L, 1, "v1"); + + KafkaTopicWritePlan first = + cache.getOrCompile(original, () -> compile(original, compileCount)); + KafkaTopicWritePlan reused = + cache.getOrCompile(original, () -> compile(original, compileCount)); + assertThat(reused).isSameAs(first); + assertThat(cache.getIfPresent(original)).isSameAs(first); + + // The authoritative TableInfo cache owns canonical instances. A separately constructed, + // structurally equivalent object is conservatively treated as a new metadata snapshot. + TableInfo equivalent = tableInfo("orders", 1L, 1, "v1"); + assertThat(cache.getIfPresent(equivalent)).isNull(); + KafkaTopicWritePlan equivalentPlan = + cache.getOrCompile(equivalent, () -> compile(equivalent, compileCount)); + assertThat(equivalentPlan).isNotSameAs(first); + + TableInfo propertyChange = tableInfo("orders", 1L, 1, "v2"); + KafkaTopicWritePlan propertyPlan = + cache.getOrCompile(propertyChange, () -> compile(propertyChange, compileCount)); + TableInfo schemaChange = tableInfo("orders", 1L, 2, "v2"); + KafkaTopicWritePlan schemaPlan = + cache.getOrCompile(schemaChange, () -> compile(schemaChange, compileCount)); + TableInfo recreated = tableInfo("orders", 2L, 2, "v2"); + KafkaTopicWritePlan recreatedPlan = + cache.getOrCompile(recreated, () -> compile(recreated, compileCount)); + + assertThat(propertyPlan).isNotSameAs(first); + assertThat(schemaPlan).isNotSameAs(propertyPlan); + assertThat(recreatedPlan).isNotSameAs(schemaPlan); + assertThat(compileCount).hasValue(5); + } + + @Test + void testEquivalentInstanceCannotInvalidateCurrentIdentity() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo current = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan currentPlan = + cache.getOrCompile(current, () -> compile(current, compileCount)); + + cache.invalidate(tableInfo("orders", 1L, 1, "v1")); + + assertThat(cache.getOrCompile(current, () -> compile(current, compileCount))) + .isSameAs(currentPlan); + assertThat(compileCount).hasValue(1); + } + + @Test + void testInvalidationAndCompileFailureRecovery() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan first = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + + cache.invalidate(tableInfo); + KafkaTopicWritePlan second = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + assertThat(second).isNotSameAs(first); + + TableInfo failingTable = tableInfo("payments", 2L, 1, "v1"); + assertThatThrownBy( + () -> + cache.getOrCompile( + failingTable, + () -> { + compileCount.incrementAndGet(); + throw new IllegalStateException("compile failed"); + })) + .isInstanceOf(IllegalStateException.class) + .hasMessage("compile failed"); + assertThat(cache.getOrCompile(failingTable, () -> compile(failingTable, compileCount))) + .isNotNull(); + assertThat(compileCount).hasValue(4); + } + + @Test + void testConcurrentColdCompileIsSingleFlight() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + AtomicInteger compileCount = new AtomicInteger(); + CountDownLatch compilerStarted = new CountDownLatch(1); + CountDownLatch releaseCompiler = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + futures.add( + executor.submit( + () -> + cache.getOrCompile( + tableInfo, + () -> { + compileCount.incrementAndGet(); + compilerStarted.countDown(); + await(releaseCompiler); + return compile(tableInfo); + }))); + } + compilerStarted.await(); + releaseCompiler.countDown(); + + KafkaTopicWritePlan first = futures.get(0).get(); + for (Future future : futures) { + assertThat(future.get()).isSameAs(first); + } + assertThat(compileCount).hasValue(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testInvalidationDoesNotAllowInFlightCompileToResurrect() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + AtomicInteger compileCount = new AtomicInteger(); + CountDownLatch compilerStarted = new CountDownLatch(1); + CountDownLatch releaseCompiler = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future compiling = + executor.submit( + () -> + cache.getOrCompile( + tableInfo, + () -> { + compileCount.incrementAndGet(); + compilerStarted.countDown(); + await(releaseCompiler); + return compile(tableInfo); + })); + compilerStarted.await(); + Future invalidating = executor.submit(() -> cache.invalidate(tableInfo)); + releaseCompiler.countDown(); + + KafkaTopicWritePlan planBeforeInvalidation = compiling.get(); + invalidating.get(); + KafkaTopicWritePlan planAfterInvalidation = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + + assertThat(planAfterInvalidation).isNotSameAs(planBeforeInvalidation); + assertThat(compileCount).hasValue(2); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCompiledAssemblerIsSafeToShare() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan plan = cache.getOrCompile(tableInfo, () -> compile(tableInfo)); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + final int id = i; + futures.add( + executor.submit( + () -> + plan.rowAssembler() + .assemble( + new Object[0], + new Object[] { + Integer.toString(id) + .getBytes( + StandardCharsets.UTF_8) + }, + id, + new ArrayList<>()) + .getField(0))); + } + for (int i = 0; i < futures.size(); i++) { + assertThat((byte[]) futures.get(i).get()) + .isEqualTo(Integer.toString(i).getBytes(StandardCharsets.UTF_8)); + } + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCacheSizeIsBounded() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(2); + for (int i = 0; i < 10; i++) { + TableInfo tableInfo = tableInfo("topic_" + i, i + 1L, 1, "v1"); + cache.getOrCompile(tableInfo, () -> compile(tableInfo)); + } + + assertThat(cache.estimatedSize()).isLessThanOrEqualTo(2); + } + + private static KafkaTopicWritePlan compile(TableInfo tableInfo, AtomicInteger compileCount) { + compileCount.incrementAndGet(); + return compile(tableInfo); + } + + private static KafkaTopicWritePlan compile(TableInfo tableInfo) { + KafkaTopicSchema schema = + new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor()); + return new KafkaTopicWritePlan(tableInfo, schema); + } + + private static TableInfo tableInfo( + String tableName, long tableId, int schemaId, String propertyValue) { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(Schema.newBuilder().column("payload", DataTypes.BYTES()).build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty("application.version", propertyValue) + .build(); + return TableInfo.of( + TablePath.of("kafka", tableName), tableId, schemaId, descriptor, null, 1L, 1L); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for test compiler.", e); + } + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java index df2256985c2..37705f2e075 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java @@ -80,6 +80,7 @@ public final class NettyServer implements RpcServer { private EventLoopGroup selectorGroup; private volatile boolean isRunning; + private CompletableFuture closeFuture; public NettyServer( Configuration conf, @@ -104,7 +105,8 @@ public NettyServer( } @Override - public void start() throws IOException { + public synchronized void start() throws IOException { + checkState(closeFuture == null, "Netty server has already been closed."); checkState(bindChannels.isEmpty(), "Netty server has already been initialized."); int numNetworkThreads = conf.getInt(ConfigOptions.NETTY_SERVER_NUM_NETWORK_THREADS); int numWorkerThreads = conf.getInt(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS); @@ -298,30 +300,40 @@ public ScheduledExecutorService getScheduledExecutor() { @Override public CompletableFuture closeAsync() { - if (!isRunning) { - return CompletableFuture.completedFuture(null); - } - - isRunning = false; - - CompletableFuture acceptorShutdownFuture = shutdownGroup(acceptorGroup); - CompletableFuture selectorShutdownFuture = shutdownGroup(selectorGroup); - CompletableFuture channelShutdownFuture = - FutureUtils.completeAll( - bindChannels.stream() - .map(NettyUtils::shutdownChannel) - .collect(Collectors.toList())); - CompletableFuture workerShutdownFuture; - if (workerPool != null) { - workerShutdownFuture = workerPool.closeAsync(); - } else { - workerShutdownFuture = CompletableFuture.completedFuture(null); + synchronized (this) { + if (closeFuture != null) { + return closeFuture; + } + isRunning = false; + + CompletableFuture transportShutdownFuture = + CompletableFuture.allOf( + shutdownGroup(acceptorGroup), + shutdownGroup(selectorGroup), + FutureUtils.completeAll( + bindChannels.stream() + .map(NettyUtils::shutdownChannel) + .collect(Collectors.toList())), + workerPool.closeAsync()); + closeFuture = closeProtocolsAfter(transportShutdownFuture, protocols); + return closeFuture; } + } - return CompletableFuture.allOf( - acceptorShutdownFuture, - selectorShutdownFuture, - channelShutdownFuture, - workerShutdownFuture); + static CompletableFuture closeProtocolsAfter( + CompletableFuture shutdownFuture, List protocols) { + return FutureUtils.composeAfterwards( + shutdownFuture, + () -> { + List> closeFutures = new ArrayList<>(); + for (NetworkProtocolPlugin protocol : protocols) { + try { + closeFutures.add(protocol.closeAsync()); + } catch (Throwable failure) { + closeFutures.add(FutureUtils.completedExceptionally(failure)); + } + } + return FutureUtils.completeAll(closeFutures); + }); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java index f60202228ef..c8921473fad 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java @@ -25,6 +25,7 @@ import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import java.util.List; +import java.util.concurrent.CompletableFuture; /** A network protocol plugin that provides the server side implementation of a network protocol. */ public interface NetworkProtocolPlugin extends Plugin { @@ -52,4 +53,9 @@ public interface NetworkProtocolPlugin extends Plugin { * protocol. */ RequestHandler createRequestHandler(RpcGatewayService service); + + /** Closes protocol-owned resources after transport and request processors have stopped. */ + default CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } } diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java index c16f996d713..72032ecd5ef 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java @@ -25,18 +25,25 @@ import org.apache.fluss.config.MemorySize; import org.apache.fluss.metrics.groups.MetricGroup; import org.apache.fluss.metrics.util.NOPMetricsGroup; +import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.RpcServer; import org.apache.fluss.rpc.TestingGatewayService; import org.apache.fluss.rpc.messages.ApiVersionsRequest; import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; import org.apache.fluss.rpc.netty.client.NettyClient; import org.apache.fluss.rpc.protocol.ApiKeys; +import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; +import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import org.apache.fluss.utils.NetUtils; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import static org.apache.fluss.utils.NetUtils.getAvailablePort; import static org.assertj.core.api.Assertions.assertThat; @@ -45,6 +52,24 @@ /** Test for {@link NettyServer}. */ public class NettyServerTest { + @Test + void testCannotStartAfterClose() { + MetricGroup metricGroup = NOPMetricsGroup.newInstance(); + NettyServer server = + new NettyServer( + new Configuration(), + Endpoint.fromListenersString("FLUSS://localhost:0"), + new TestingGatewayService(), + metricGroup, + RequestsMetrics.createCoordinatorServerRequestMetrics(metricGroup)); + + server.closeAsync().join(); + + assertThatThrownBy(server::start) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already been closed"); + } + @Test void testPortAsZero() throws Exception { List endpoints = Endpoint.fromListenersString("FLUSS://localhost:0"); @@ -111,4 +136,94 @@ void testOversizedRequestRejected() throws Exception { } } } + + @Test + void testProtocolsCloseOnlyAfterTransportShutdown() { + CompletableFuture transportShutdown = new CompletableFuture<>(); + AtomicInteger closeCalls = new AtomicInteger(); + NetworkProtocolPlugin protocol = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + return CompletableFuture.completedFuture(null); + }); + + CompletableFuture result = + NettyServer.closeProtocolsAfter( + transportShutdown, Collections.singletonList(protocol)); + assertThat(closeCalls).hasValue(0); + assertThat(result).isNotDone(); + + transportShutdown.complete(null); + result.join(); + assertThat(closeCalls).hasValue(1); + } + + @Test + void testAllProtocolsCloseAfterExceptionalTransportShutdown() { + CompletableFuture transportShutdown = new CompletableFuture<>(); + AtomicInteger closeCalls = new AtomicInteger(); + RuntimeException asyncFailure = new RuntimeException("async protocol close failure"); + RuntimeException synchronousFailure = + new RuntimeException("synchronous protocol close failure"); + NetworkProtocolPlugin asyncFailing = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(asyncFailure); + return failed; + }); + NetworkProtocolPlugin synchronouslyFailing = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + throw synchronousFailure; + }); + CompletableFuture result = + NettyServer.closeProtocolsAfter( + transportShutdown, Arrays.asList(asyncFailing, synchronouslyFailing)); + + transportShutdown.completeExceptionally(new RuntimeException("transport failure")); + + assertThatThrownBy(result::join) + .hasStackTraceContaining("transport failure") + .hasStackTraceContaining("async protocol close failure") + .hasStackTraceContaining("synchronous protocol close failure"); + assertThat(closeCalls).hasValue(2); + } + + private static NetworkProtocolPlugin testingProtocol( + Supplier> closeAction) { + return new NetworkProtocolPlugin() { + @Override + public String name() { + return "testing"; + } + + @Override + public void setup(Configuration conf) {} + + @Override + public List listenerNames() { + return Collections.emptyList(); + } + + @Override + public ChannelHandler createChannelHandler( + RequestChannel[] requestChannels, String listenerName) { + return null; + } + + @Override + public RequestHandler createRequestHandler(RpcGatewayService service) { + return null; + } + + @Override + public CompletableFuture closeAsync() { + return closeAction.get(); + } + }; + } } From 433813ea6847fc66d728c20e92b52eb58fae4954 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:36:18 +0800 Subject: [PATCH 35/48] [kafka] Skip null Produce values with write authorization Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 50/50 AI-Contributed/UT: 99/99 --- .../produce/GatewayKafkaProduceBackend.java | 14 ++- .../backend/produce/KafkaProduceCommand.java | 21 +++++ .../fluss/kafka/KafkaProduceAppendITCase.java | 6 +- .../GatewayKafkaProduceBackendTest.java | 91 +++++++++++++++++++ .../apache/fluss/rpc/RpcGatewayService.java | 8 ++ .../fluss/server/tablet/TabletService.java | 7 +- 6 files changed, 141 insertions(+), 6 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index 2b3755408b6..313da86ad7d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -177,7 +177,15 @@ private CompletableFuture produceTopic( continue; } try { - BytesView records = transcoder.transcode(partition.records(), writePlan); + if (partition.hasNullValues() && partition.nonNullRecords().isEmpty()) { + setCurrentSession(command); + service.authorizeTableWrite(tableInfo.getTableId()); + failures.put( + partition.partitionId(), + new PartitionResult(partition.partitionId(), Errors.NONE, -1L, null)); + continue; + } + BytesView records = transcoder.transcode(partition.nonNullRecords(), writePlan); request.addBucketsReq() .setBucketId(partition.partitionId()) .setRecordsBytesView(records); @@ -272,7 +280,9 @@ private static TopicResult toTopicResult( new PartitionResult( partition.partitionId(), Errors.NONE, - bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, + !partition.hasNullValues() && bucket.hasBaseOffset() + ? bucket.getBaseOffset() + : -1L, null)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index db13f63b314..c359c6f8b21 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -109,11 +109,22 @@ public List partitions() { public static final class PartitionWrite { private final int partitionId; private final List records; + private final List nonNullRecords; /** Creates the writes for one partition. */ public PartitionWrite(int partitionId, List records) { this.partitionId = partitionId; this.records = immutableCopy(records); + List retained = new ArrayList<>(); + for (Record record : this.records) { + if (record.borrowedValue() != null) { + retained.add(record); + } + } + this.nonNullRecords = + retained.size() == records.size() + ? this.records + : Collections.unmodifiableList(retained); } /** Returns the Kafka partition ID. */ @@ -121,6 +132,16 @@ public int partitionId() { return partitionId; } + /** Returns records with non-null values, preserving their original order. */ + public List nonNullRecords() { + return nonNullRecords; + } + + /** Whether filtering changed the correspondence between Kafka and native offsets. */ + public boolean hasNullValues() { + return records.size() != nonNullRecords.size(); + } + /** Returns copied records in append order. */ public List records() { return records; diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index fbc35bc0bf8..2fb0fa4c01f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -191,7 +191,7 @@ void testPipelinedProducePreservesAppendOrder() throws Exception { while (count < values.size() && System.nanoTime() < deadline) { for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { assertThat(record.getRow().getBytes(0)).isEqualTo(values.get(count)); - assertThat(record.getRow().getTimestampLtz(1, 3).getEpochMillisecond()) + assertThat(record.getRow().getTimestampNtz(1, 3).getMillisecond()) .isEqualTo(count); count++; } @@ -224,7 +224,7 @@ private static void assertReadback( .isEqualTo(new String(expectedValue, StandardCharsets.UTF_8)); assertThat(row.getString(3).toString()).isEqualTo("key"); } - assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(123L); assertThat(row.getArray(2).size()).isEqualTo(2); assertThat(row.getArray(2).getRow(0, 2).getString(0).toString()) .isEqualTo("source"); @@ -242,7 +242,7 @@ private static TableDescriptor descriptor(String format) { .schema( Schema.newBuilder() .column("body", raw ? DataTypes.BYTES() : DataTypes.STRING()) - .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) .column( "attributes", DataTypes.ARRAY( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index fb4f999d6e3..842a75456b7 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.kafka.backend.produce; +import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; @@ -57,6 +58,84 @@ /** Tests append admission, error isolation, session propagation and delayed-fetch completion. */ class GatewayKafkaProduceBackendTest { + @Test + void testAllNullValuesRequireAuthorizationWithoutAppend() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service) + .write(command((short) 1, topic("kafka.topic", nullValue(0), nullValue(8)))) + .join(); + assertErrors(result, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); + assertThat(service.writeChecks).isEqualTo(1); + assertThat(service.append).isNull(); + assertThat(service.drains).isZero(); + service.denyWrite = true; + assertErrors( + backend(service) + .write(command((short) 1, topic("kafka.topic", nullValue(0)))) + .join(), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.append).isNull(); + } + + @Test + void testNullValuesCannotBypassTableAndSchemaValidation() throws Exception { + TestingProduceService service = new TestingProduceService(); + for (String name : new String[] {"invalid", "missing"}) { + assertErrors( + backend(service) + .write(command((short) 1, topic("kafka." + name, nullValue(0)))) + .join(), + name.equals("invalid") + ? Errors.INVALID_TOPIC_EXCEPTION + : Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + assertThat(service.writeChecks).isZero(); + assertThat(service.append).isNull(); + } + + @Test + void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exception { + TestingProduceService service = new TestingProduceService(); + List seen = new ArrayList<>(); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + (records, table) -> { + seen.addAll(records); + return transcoder.transcode(records, table); + }); + Record empty = new Record(1L, null, new byte[0], Collections.emptyList()); + Record valid = good(0).records().get(0); + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic( + "kafka.topic", + new PartitionWrite( + 0, + Arrays.asList( + nullValue(0).records().get(0), + empty, + valid))))) + .join(); + assertErrors(result, Errors.NONE); + assertThat(seen).containsExactly(empty, valid); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); + assertThat(service.drains).isEqualTo(1); + } + + private static PartitionWrite nullValue(int partition) { + return new PartitionWrite( + partition, + Collections.singletonList( + new Record(-1L, new byte[] {(byte) 0xff}, null, Collections.emptyList()))); + } + @Test void testDrainsImmediatelyWhileAcksAllResponseIsPending() throws Exception { TestingProduceService service = new TestingProduceService(); @@ -449,6 +528,8 @@ private static class TestingProduceService extends TestingTabletGatewayService { private boolean throwAppend; private boolean asyncMissing; private int drains; + private int writeChecks; + private boolean denyWrite; private CountDownLatch metadataEntered; private CountDownLatch appended; private Thread metadataThread; @@ -478,6 +559,16 @@ public CompletableFuture getTableInfo(GetTableInfoRequest : pendingMetadata; } + @Override + public void authorizeTableWrite(long tableId) { + assertThat(tableId).isEqualTo(42L); + assertThat(currentListenerName()).isEqualTo("KAFKA"); + writeChecks++; + if (denyWrite) { + throw new AuthorizationException("Write denied"); + } + } + @Override public CompletableFuture produceLog(ProduceLogRequest request) { assertThat(currentListenerName()).isEqualTo("KAFKA"); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java index 8f2365add70..1dc4a0da4c9 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java @@ -77,6 +77,14 @@ public String currentListenerName() { } } + /** + * Authorizes a table write that may not append any records, using the current request session. + * Services must explicitly support this operation; the default never grants write access. + */ + public void authorizeTableWrite(long tableId) { + throw new UnsupportedOperationException("This service cannot authorize table writes."); + } + /** Returns the provider type of this RPC gateway service. */ public abstract ServerType providerType(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index ae19ae90eb6..d7a637ff6a8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -222,9 +222,14 @@ public String name() { @Override public void shutdown() {} + @Override + public void authorizeTableWrite(long tableId) { + authorizeTable(WRITE, tableId); + } + @Override public CompletableFuture produceLog(ProduceLogRequest request) { - authorizeTable(WRITE, request.getTableId()); + authorizeTableWrite(request.getTableId()); long tableId = request.getTableId(); Map routingErrors = new HashMap<>(); collectRoutingErrors( From 7a9d08ac7edbd8e71a9e7233becf2967a17de2e4 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 18:46:25 +0800 Subject: [PATCH 36/48] [kafka] Verify Produce boundaries on the updated FIP base Cover WRITE revocation after metadata reuse, offloaded metadata identity and schema refresh, and shutdown with an already submitted native append. Preserve Record timestamp provenance and the existing null-value filtering behavior, including native offsets and real Kafka producer readback. The FIP null-value contract and new Core dependencies remain deferred. Validated on PR06 b6f31fc84baf91c030b296701eb319644578056c with Java 11: 206 Kafka unit tests, 10 Kafka integration tests, 8 RPC regression tests and 7 delayed-fetch regression tests passed. Checkstyle, Spotless, RAT and git diff --check passed. This commit changes tests only. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 191/191 --- .../fluss/kafka/KafkaProduceAppendITCase.java | 36 ++++++ .../GatewayKafkaProduceBackendTest.java | 114 +++++++++++++++++- .../produce/KafkaProduceCommandTest.java | 41 +++++++ 3 files changed, 190 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 2fb0fa4c01f..cc41dda5932 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -112,6 +112,42 @@ void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception } } + @Test + void testNullValuesRemainFilteredWithoutAdvancingNativeOffsets() throws Exception { + // Preserve the existing filtering behavior until the FIP defines its null-value contract. + TablePath path = TablePath.of(DATABASE, "produce_filtered_nulls"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("raw"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + ProducerRecord filtered = + new ProducerRecord(path.toString(), 0, 123L, KEY, null); + ProducerRecord retained = + new ProducerRecord<>( + path.toString(), + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + assertThat(producer.send(filtered).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(-1L); + assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()).isZero(); + assertThat(producer.send(filtered).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(-1L); + assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(1L); + assertReadback(connection, path, "raw", VALUE); + } finally { + admin.dropTable(path, true).get(); + } + } + } + @Test void testSameNamedTablesAcrossDatabases() throws Exception { Map tables = new LinkedHashMap<>(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index 842a75456b7..b3f1502396e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -43,6 +43,8 @@ import org.apache.kafka.common.protocol.Errors; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.net.InetAddress; import java.util.ArrayList; @@ -79,6 +81,42 @@ void testAllNullValuesRequireAuthorizationWithoutAppend() throws Exception { assertThat(service.append).isNull(); } + @ParameterizedTest + @ValueSource(strings = {"non-null", "all-null", "mixed"}) + void testCachedMetadataDoesNotBypassRevokedWritePermission(String recordsKind) + throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, new ArrowKafkaRecordTranscoder(), executor); + PartitionWrite partition = + recordsKind.equals("non-null") + ? good(0) + : recordsKind.equals("all-null") + ? nullValue(0) + : new PartitionWrite( + 0, + Arrays.asList( + nullValue(0).records().get(0), + good(0).records().get(0))); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", partition)); + try { + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + int appendedBeforeRevocation = service.appends.size(); + service.denyWrite = true; + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.metadataPaths).hasSize(2); + assertThat(service.writeChecks).isEqualTo(2); + assertThat(service.appends).hasSize(appendedBeforeRevocation); + assertThat(service.metadataThread).isNotSameAs(Thread.currentThread()); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + @Test void testNullValuesCannotBypassTableAndSchemaValidation() throws Exception { TestingProduceService service = new TestingProduceService(); @@ -434,6 +472,77 @@ void testOffloadedRequestsAppendInOrderWithoutWaitingForAcknowledgement() throws } } + @Test + void testOffloadedMetadataRefreshUsesNewTableIdentityAndSchema() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + List snapshots = new ArrayList<>(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + (records, table) -> { + snapshots.add(table); + return transcoder.transcode(records, table); + }, + executor); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); + try { + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableId(84L).setSchemaId(2).setBucketCountEpoch(7L)); + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + assertThat(snapshots).hasSize(3); + assertThat(snapshots.get(1)).isSameAs(snapshots.get(0)); + assertThat(snapshots.get(2)).isNotSameAs(snapshots.get(0)); + assertThat(snapshots.get(2).getTableId()).isEqualTo(84L); + assertThat(snapshots.get(2).getSchemaId()).isEqualTo(2); + assertThat(snapshots.get(2).getBucketCountEpoch()).isEqualTo(7L); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L, 84L); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingMetadata.completeExceptionally(new TableNotExistException("deleted")); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), + Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.metadataPaths).hasSize(4); + assertThat(service.appends).hasSize(3); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testShutdownPreservesAlreadySubmittedAppendResult() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = new CompletableFuture<>(); + service.appended = new CountDownLatch(1); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, new ArrowKafkaRecordTranscoder(), executor); + try { + CompletableFuture result = + backend.write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.appended.await(10, TimeUnit.SECONDS)).isTrue(); + executor.closeAsync().get(10, TimeUnit.SECONDS); + assertThat(result).isNotDone(); + assertThat(service.pendingAppend).isNotCancelled(); + service.pendingAppend.complete(success(0)); + KafkaProduceResult completed = result.get(10, TimeUnit.SECONDS); + assertErrors(completed, Errors.NONE); + assertThat(completed.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.appends).hasSize(1); + assertThat(service.drains).isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + @Test void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Exception { KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); @@ -561,8 +670,10 @@ public CompletableFuture getTableInfo(GetTableInfoRequest @Override public void authorizeTableWrite(long tableId) { - assertThat(tableId).isEqualTo(42L); + assertThat(tableId).isPositive(); assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(currentSession().getInetAddress()) + .isEqualTo(InetAddress.getLoopbackAddress()); writeChecks++; if (denyWrite) { throw new AuthorizationException("Write denied"); @@ -571,6 +682,7 @@ public void authorizeTableWrite(long tableId) { @Override public CompletableFuture produceLog(ProduceLogRequest request) { + authorizeTableWrite(request.getTableId()); assertThat(currentListenerName()).isEqualTo("KAFKA"); assertThat(currentSession().getInetAddress()) .isEqualTo(InetAddress.getLoopbackAddress()); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java index 3b9c8f15ec0..e3624f5535b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -162,6 +162,47 @@ void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { assertThatThrownBy(copied::clear).isInstanceOf(UnsupportedOperationException.class); } + @Test + void testNullFilteringPreservesRecordIdentityAndTimestampProvenance() { + Record filtered = + new Record( + 123L, + TimestampType.CREATE_TIME, + new byte[] {1}, + null, + Collections.emptyList()); + Record retained = + new Record( + 456L, + TimestampType.LOG_APPEND_TIME, + new byte[] {2}, + new byte[] {3}, + Collections.emptyList()); + Record empty = + new Record( + Record.NO_TIMESTAMP, + TimestampType.CREATE_TIME, + null, + new byte[0], + Collections.emptyList()); + List source = new ArrayList<>(Arrays.asList(filtered, retained, empty)); + PartitionWrite partition = new PartitionWrite(0, source); + source.clear(); + + assertThat(partition.records()).containsExactly(filtered, retained, empty); + assertThat(partition.hasNullValues()).isTrue(); + assertThat(partition.nonNullRecords()).containsExactly(retained, empty); + assertThat(partition.nonNullRecords().get(0)).isSameAs(retained); + assertThat(partition.nonNullRecords().get(0).timestampType()) + .isEqualTo(TimestampType.LOG_APPEND_TIME); + assertThat(partition.nonNullRecords().get(0).timestamp()).isEqualTo(456L); + assertThat(partition.nonNullRecords().get(1).timestampType()) + .isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.nonNullRecords().get(1).hasTimestamp()).isFalse(); + assertThatThrownBy(() -> partition.nonNullRecords().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + @Test void testBorrowedPayloadsReuseOwnedArraysWhileCopyAccessorsRemainDefensive() { byte[] key = {1}; From 2beeb9f01868bf30969fae1020fb4c45fd348baa Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 18:45:21 +0800 Subject: [PATCH 37/48] [kafka] Apply persisted table contracts to Produce append Inject the configured topic mapper into Produce, check authoritative table enablement before plan preparation, and use the shared fixed-column schema. Reject conversion failures as INVALID_RECORD and missing databases as UNKNOWN_TOPIC_OR_PARTITION without alias fallback. Cover shared alias plans and offsets, disable/reset and property/schema changes, null filtering, raw keys, last-wins headers and null rescue values. Preserve bounded conversion, authorization and atomic partition conversion. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 34/34 AI-Contributed/UT: 656/656 --- .../fluss/kafka/KafkaRequestHandler.java | 1 + .../produce/GatewayKafkaProduceBackend.java | 22 +- .../transcode/ArrowKafkaRecordTranscoder.java | 10 +- .../transcode/KafkaRecordTranscoder.java | 6 +- .../kafka/transcode/KafkaTopicWritePlan.java | 11 +- .../fluss/kafka/KafkaProduceAppendITCase.java | 301 ++++++++++++-- .../GatewayKafkaProduceBackendTest.java | 366 ++++++++++++++++-- .../produce/KafkaTableInfoCacheTest.java | 10 +- .../KafkaCompiledWritePlanCacheTest.java | 25 +- 9 files changed, 663 insertions(+), 89 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 2b48e91bbed..dc34e6aaaca 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -68,6 +68,7 @@ public KafkaRequestHandler( new GatewayKafkaProduceBackend( service, (TabletServerGateway) service, + topicMapper, new ArrowKafkaRecordTranscoder(), conversionExecutor)))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index 313da86ad7d..fa5daca2e2d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -18,12 +18,14 @@ package org.apache.fluss.kafka.backend.produce; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; @@ -66,7 +68,8 @@ public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { private final RpcGatewayService service; private final TabletServerGateway gateway; - private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); private final KafkaRecordTranscoder transcoder; private final KafkaTableInfoCache tableInfoCache = new KafkaTableInfoCache(); private final @Nullable KafkaProduceConversionExecutor conversionExecutor; @@ -75,18 +78,21 @@ public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { public GatewayKafkaProduceBackend( RpcGatewayService service, TabletServerGateway gateway, + KafkaTopicMapper topicMapper, KafkaRecordTranscoder transcoder) { - this(service, gateway, transcoder, null); + this(service, gateway, topicMapper, transcoder, null); } /** Creates a backend that submits work to a caller-owned bounded conversion executor. */ public GatewayKafkaProduceBackend( RpcGatewayService service, TabletServerGateway gateway, + KafkaTopicMapper topicMapper, KafkaRecordTranscoder transcoder, @Nullable KafkaProduceConversionExecutor conversionExecutor) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); + this.topicMapper = checkNotNull(topicMapper); this.transcoder = checkNotNull(transcoder); this.conversionExecutor = conversionExecutor; } @@ -157,6 +163,11 @@ private CompletableFuture writeTopic( private CompletableFuture produceTopic( KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { // Admission must match Metadata, including when the schema changes between requests. + // Even a cached snapshot is selected only after fetching the authoritative descriptor. + if (!schemaResolver.isEnabled(tableInfo.toTableDescriptor())) { + throw new TableNotExistException( + "Table " + tableInfo.getTablePath() + " is not enabled for Kafka."); + } KafkaTopicWritePlan writePlan = transcoder.prepare(tableInfo); ProduceLogRequest request = new ProduceLogRequest() @@ -185,7 +196,9 @@ private CompletableFuture produceTopic( new PartitionResult(partition.partitionId(), Errors.NONE, -1L, null)); continue; } - BytesView records = transcoder.transcode(partition.nonNullRecords(), writePlan); + BytesView records = + transcoder.transcode( + partition.partitionId(), partition.nonNullRecords(), writePlan); request.addBucketsReq() .setBucketId(partition.partitionId()) .setRecordsBytesView(records); @@ -309,7 +322,7 @@ private static PartitionResult failedPartition(int partitionId, Throwable failur || cause instanceof InvalidTopicException ? Errors.INVALID_TOPIC_EXCEPTION : cause instanceof KafkaRecordEncodingException - ? Errors.CORRUPT_MESSAGE + ? Errors.INVALID_RECORD : cause instanceof IllegalArgumentException ? Errors.INVALID_REQUEST : toKafkaError( @@ -323,6 +336,7 @@ private static Errors toKafkaError(org.apache.fluss.rpc.protocol.Errors error) { case NONE: return Errors.NONE; case TABLE_NOT_EXIST: + case DATABASE_NOT_EXIST: case UNKNOWN_TABLE_OR_BUCKET_EXCEPTION: return Errors.UNKNOWN_TOPIC_OR_PARTITION; case NOT_LEADER_OR_FOLLOWER: diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 7852657b3ba..e1597ca8985 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -53,19 +53,17 @@ public KafkaTopicWritePlan prepare(TableInfo tableInfo) { } @Override - public BytesView transcode(int partitionId, List records, TableInfo tableInfo) throws Exception { + public BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception { return transcode(partitionId, records, prepare(tableInfo)); } @Override - public BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { + public BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTableSchema schema = plan.topicSchema(); - if (schema.valueType() != ValueType.BINARY && schema.valueType() != ValueType.STRING) { - throw new KafkaRecordEncodingException( - "Unsupported Kafka value conversion type: " + schema.valueType()); - } KafkaRowAssembler assembler = plan.rowAssembler(); return arrowRecordEncoder.encodeStreaming( consumer -> { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java index 063728b77fa..6c87d80e273 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -35,10 +35,12 @@ default KafkaTopicWritePlan prepare(TableInfo tableInfo) { } /** Transcodes one requested partition using an already prepared plan. */ - default BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { + default BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { return transcode(partitionId, records, plan.tableInfo()); } /** Transcodes records from the requested Kafka partition using the target table schema. */ - BytesView transcode(int partitionId, List records, TableInfo tableInfo) throws Exception; + BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java index 1f9d744afb5..c68736e4233 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -19,13 +19,16 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; import org.apache.fluss.metadata.TableInfo; import javax.annotation.concurrent.ThreadSafe; import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Immutable binary/string conversion plan; mutable Arrow resources are owned by each conversion. */ +/** + * Immutable binary/string conversion plan; mutable Arrow resources are owned by each conversion. + */ @Internal @ThreadSafe public final class KafkaTopicWritePlan { @@ -36,6 +39,12 @@ public final class KafkaTopicWritePlan { KafkaTopicWritePlan(TableInfo tableInfo, KafkaTableSchema topicSchema) { this.tableInfo = checkNotNull(tableInfo); this.topicSchema = checkNotNull(topicSchema); + // Validate support during preparation as filtered partitions may never call transcode. + if (topicSchema.valueType() != ValueType.BINARY + && topicSchema.valueType() != ValueType.STRING) { + throw new KafkaRecordEncodingException( + "Unsupported Kafka value conversion type: " + topicSchema.valueType()); + } this.rowAssembler = new KafkaRowAssembler(topicSchema); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index cc41dda5932..ddbd1acf806 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -26,7 +26,6 @@ import org.apache.fluss.cluster.ServerNode; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; @@ -40,17 +39,37 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -58,11 +77,13 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; import static org.assertj.core.api.Assertions.assertThat; /** Exercises native DDL, Kafka discovery and Produce, replication, and native Fluss readback. */ class KafkaProduceAppendITCase { - private static final String DATABASE = "kafka"; + private static final String DATABASE = "produce_default"; private static final byte[] KEY = "key".getBytes(StandardCharsets.UTF_8); private static final byte[] VALUE = "value".getBytes(StandardCharsets.UTF_8); @@ -75,7 +96,7 @@ class KafkaProduceAppendITCase { .build(); @ParameterizedTest - @ValueSource(strings = {"raw", "string"}) + @ValueSource(strings = {"binary", "string"}) void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception { try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { @@ -98,7 +119,18 @@ void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception new RecordHeader("source", VALUE), new RecordHeader("source", null))); RecordMetadata first = producer.send(record).get(30, TimeUnit.SECONDS); - RecordMetadata second = producer.send(record).get(30, TimeUnit.SECONDS); + RecordMetadata second = + producer.send( + new ProducerRecord<>( + path.getTableName(), + 0, + 123L, + KEY, + VALUE, + record.headers())) + .get(30, TimeUnit.SECONDS); + assertThat(first.topic()).isEqualTo(topic); + assertThat(second.topic()).isEqualTo(path.getTableName()); if (!acks.equals("0")) { assertThat(first.offset()).isZero(); assertThat(second.offset()).isEqualTo(1L); @@ -119,7 +151,7 @@ void testNullValuesRemainFilteredWithoutAdvancingNativeOffsets() throws Exceptio try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - admin.createTable(path, descriptor("raw"), false).get(); + admin.createTable(path, descriptor("binary"), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); try (KafkaProducer producer = producer("all", 0)) { ProducerRecord filtered = @@ -141,7 +173,7 @@ void testNullValuesRemainFilteredWithoutAdvancingNativeOffsets() throws Exceptio .isEqualTo(-1L); assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()) .isEqualTo(1L); - assertReadback(connection, path, "raw", VALUE); + assertReadback(connection, path, "binary", VALUE); } finally { admin.dropTable(path, true).get(); } @@ -161,7 +193,7 @@ void testSameNamedTablesAcrossDatabases() throws Exception { for (TablePath path : tables.keySet()) { admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true) .get(); - admin.createTable(path, descriptor("raw"), false).get(); + admin.createTable(path, descriptor("binary"), false).get(); } CLUSTER.waitUntilAllGatewayHasSameMetadata(); try (KafkaProducer producer = producer("all")) { @@ -184,7 +216,7 @@ void testSameNamedTablesAcrossDatabases() throws Exception { .isEqualTo(1L); } for (Map.Entry entry : tables.entrySet()) { - assertReadback(connection, entry.getKey(), "raw", entry.getValue()); + assertReadback(connection, entry.getKey(), "binary", entry.getValue()); } } } finally { @@ -201,7 +233,7 @@ void testPipelinedProducePreservesAppendOrder() throws Exception { try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - admin.createTable(path, descriptor("raw"), false).get(); + admin.createTable(path, descriptor("binary"), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); try (KafkaProducer producer = producer("all", 0)) { assertThat(producer.partitionsFor(path.toString())).hasSize(1); @@ -227,8 +259,6 @@ void testPipelinedProducePreservesAppendOrder() throws Exception { while (count < values.size() && System.nanoTime() < deadline) { for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { assertThat(record.getRow().getBytes(0)).isEqualTo(values.get(count)); - assertThat(record.getRow().getTimestampNtz(1, 3).getMillisecond()) - .isEqualTo(count); count++; } } @@ -240,62 +270,260 @@ void testPipelinedProducePreservesAppendOrder() throws Exception { } } + @Test + void testAliasesShareOffsetsAndObservePersistedEnablementOnOneConnection() throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_enablement"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("binary"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.getTableName(), path.toString()}; + assertProduceOffsets(query(socket, names), names, 0L, 1L); + setEnabled(path, false); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setEnabled(path, true); + assertProduceOffsets(query(socket, names), names, 2L, 3L); + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList( + ConfigOptions.TABLE_KAFKA_ENABLED.key()), + Collections.emptyList(), + false)) + .get(); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertReadback(connection, path, "binary", VALUE, 4); + admin.dropTable(path, true).get(); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testDeletedDefaultDatabaseDoesNotFallbackOrRecreateDatabase() throws Exception { + Configuration config = clusterConfig(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "produce_missing_db"); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 1); + FlussClusterExtension cluster = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(config) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + cluster.start(); + try (Connection connection = ConnectionFactory.createConnection(cluster.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + // Startup creates the configured default; Produce must not recreate a deleted one. + assertThat(admin.databaseExists("produce_missing_db").get()).isTrue(); + admin.dropDatabase("produce_missing_db", false, false).get(); + assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); + TablePath existing = TablePath.of("fluss", "events"); + long tableId = createTable(cluster, existing, descriptor("binary")); + cluster.waitUntilTableReady(tableId); + ServerNode node = cluster.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port())) { + String[] missing = { + "events", + "produce_missing_db.events", + "another_missing_db.events", + "fluss.missing" + }; + assertProduceErrors( + query(socket, missing), missing, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertProduceOffsets( + query(socket, existing.toString()), new String[] {existing.toString()}, 0L); + assertProduceOffsets( + query(socket, existing.toString()), new String[] {existing.toString()}, 1L); + assertReadback(connection, existing, "binary", VALUE); + assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); + assertThat(admin.databaseExists("another_missing_db").get()).isFalse(); + } + } finally { + cluster.close(); + } + } + + private static void setEnabled(TablePath path, boolean enabled) throws Exception { + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled)), + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private static ProduceResponse query(Socket socket, String... names) throws Exception { + List topics = new ArrayList<>(); + for (String name : names) { + topics.add( + new TopicProduceData() + .setName(name) + .setPartitionData( + Collections.singletonList( + new PartitionProduceData() + .setIndex(0) + .setRecords( + MemoryRecords.withRecords( + Compression.NONE, + new SimpleRecord( + 123L, + KEY, + VALUE, + new Header[] { + new RecordHeader( + "source", + VALUE), + new RecordHeader( + "source", + null) + })))))); + } + ProduceRequest request = + new ProduceRequest( + new ProduceRequestData() + .setAcks((short) -1) + .setTimeoutMs(10000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + topics.iterator())), + (short) 9); + RequestHeader header = + new RequestHeader(ApiKeys.PRODUCE, request.version(), "produce-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] bytes = new byte[encoded.remaining()]; + encoded.get(bytes); + socket.setSoTimeout(15000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(bytes.length); + output.write(bytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + byte[] response = new byte[input.readInt()]; + input.readFully(response); + return (ProduceResponse) AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static void assertProduceOffsets( + ProduceResponse response, String[] names, long... offsets) { + assertProduceErrors(response, names, Errors.NONE); + for (int i = 0; i < names.length; i++) { + assertThat( + response.data() + .responses() + .find(names[i]) + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(offsets[i]); + } + } + + private static void assertProduceErrors( + ProduceResponse response, String[] names, Errors error) { + assertThat(response.data().responses()) + .extracting(topic -> topic.name()) + .containsExactly(names); + for (String name : names) { + List partitions = + response.data().responses().find(name).partitionResponses(); + assertThat(partitions).hasSize(1); + assertThat(partitions.get(0).index()).isZero(); + assertThat(partitions.get(0).errorCode()).isEqualTo(error.code()); + if (error != Errors.NONE) { + assertThat(partitions.get(0).baseOffset()).isEqualTo(-1L); + } + } + } + private static void assertReadback( Connection connection, TablePath path, String format, byte[] expectedValue) throws Exception { + assertReadback(connection, path, format, expectedValue, 2); + } + + private static void assertReadback( + Connection connection, + TablePath path, + String format, + byte[] expectedValue, + int expectedCount) + throws Exception { try (Table table = connection.getTable(path); LogScanner scanner = table.newScan().createLogScanner()) { scanner.subscribeFromBeginning(0); int count = 0; long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - while (count < 2 && System.nanoTime() < deadline) { + while (count < expectedCount && System.nanoTime() < deadline) { ScanRecords records = scanner.poll(Duration.ofSeconds(1)); for (ScanRecord record : records) { InternalRow row = record.getRow(); - if (format.equals("raw")) { + if (format.equals("binary")) { assertThat(row.getBytes(0)).isEqualTo(expectedValue); - assertThat(row.getBytes(3)).isEqualTo(KEY); } else { assertThat(row.getString(0).toString()) .isEqualTo(new String(expectedValue, StandardCharsets.UTF_8)); - assertThat(row.getString(3).toString()).isEqualTo("key"); } - assertThat(row.getTimestampNtz(1, 3).getMillisecond()).isEqualTo(123L); - assertThat(row.getArray(2).size()).isEqualTo(2); - assertThat(row.getArray(2).getRow(0, 2).getString(0).toString()) + assertThat(row.getBytes(3)).isEqualTo(KEY); + assertThat(row.getMap(1).size()).isEqualTo(1); + assertThat(row.getMap(1).keyArray().getString(0).toString()) .isEqualTo("source"); - assertThat(row.getArray(2).getRow(1, 2).isNullAt(1)).isTrue(); + assertThat(row.getMap(1).valueArray().isNullAt(0)).isTrue(); + assertThat(row.isNullAt(2)).isTrue(); + InternalRow metadata = row.getRow(4, 3); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()) + .isEqualTo(123L); + assertThat(metadata.getInt(1)).isZero(); + assertThat(metadata.getInt(2)).isZero(); count++; } } - assertThat(count).isEqualTo(2); + assertThat(count).isEqualTo(expectedCount); } } private static TableDescriptor descriptor(String format) { - boolean raw = format.equals("raw"); + boolean binary = format.equals("binary"); return TableDescriptor.builder() .schema( Schema.newBuilder() - .column("body", raw ? DataTypes.BYTES() : DataTypes.STRING()) - .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) .column( - "attributes", - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD( - "value", DataTypes.BYTES())))) - .column("message_key", raw ? DataTypes.BYTES() : DataTypes.STRING()) + "_kafka_value", + binary ? DataTypes.BYTES() : DataTypes.STRING()) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))) .build()) .distributedBy(1) .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format) - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format) - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) .build(); } @@ -322,6 +550,7 @@ private static KafkaProducer producer(String acks, int batchSize private static Configuration clusterConfig() { Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, DATABASE); config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); return config; diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index b3f1502396e..9acf78edddf 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -17,13 +17,16 @@ package org.apache.fluss.kafka.backend.produce; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.AuthorizationException; +import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; -import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; @@ -89,7 +92,7 @@ void testCachedMetadataDoesNotBypassRevokedWritePermission(String recordsKind) KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); GatewayKafkaProduceBackend backend = new GatewayKafkaProduceBackend( - service, service, new ArrowKafkaRecordTranscoder(), executor); + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); PartitionWrite partition = recordsKind.equals("non-null") ? good(0) @@ -142,9 +145,10 @@ void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exceptio new GatewayKafkaProduceBackend( service, service, - (records, table) -> { + mapper(), + (partitionId, records, table) -> { seen.addAll(records); - return transcoder.transcode(records, table); + return transcoder.transcode(partitionId, records, table); }); Record empty = new Record(1L, null, new byte[0], Collections.emptyList()); Record valid = good(0).records().get(0); @@ -207,7 +211,8 @@ void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception PartitionWrite malformed = new PartitionWrite( 0, - Collections.singletonList( + Arrays.asList( + good(0).records().get(0), new Record( 1L, null, @@ -220,8 +225,7 @@ void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception (short) 1, topic("kafka.topic", malformed, good(1), good(8)))) .join(); - assertErrors( - result, Errors.CORRUPT_MESSAGE, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertErrors(result, Errors.INVALID_RECORD, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); assertThat(service.append.getBucketsReqsList()).hasSize(1); assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); assertThat(service.drains).isEqualTo(1); @@ -267,13 +271,15 @@ void testMapsReplicationFailuresAndMissingResponsesPerPartition() throws Excepti org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_EXCEPTION, org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION, org.apache.fluss.rpc.protocol.Errors.REQUEST_TIME_OUT, - org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER + org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER, + org.apache.fluss.rpc.protocol.Errors.CORRUPT_RECORD_EXCEPTION }; Errors[] kafkaErrors = { Errors.NOT_ENOUGH_REPLICAS, Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, Errors.REQUEST_TIMED_OUT, - Errors.NOT_LEADER_OR_FOLLOWER + Errors.NOT_LEADER_OR_FOLLOWER, + Errors.CORRUPT_MESSAGE }; for (int i = 0; i < flussErrors.length; i++) { TestingProduceService service = new TestingProduceService(); @@ -322,9 +328,10 @@ void testQualifiedTopicsPreserveDatabaseAndTranscoderTablePath() throws Exceptio new GatewayKafkaProduceBackend( service, service, - (records, tableInfo) -> { + mapper(), + (partitionId, records, tableInfo) -> { transcodedPaths.add(tableInfo.getTablePath()); - return transcoder.transcode(records, tableInfo); + return transcoder.transcode(partitionId, records, tableInfo); }); KafkaProduceResult result = @@ -359,7 +366,7 @@ void testInvalidQualifiedNamesDoNotReachGatewayOrSuppressValidTopic() throws Exc .write( command( (short) 1, - topic("topic", good(0)), + topic(".", good(0)), topic("kafka.topic.extra", good(0)), topic("kafka.__internal", good(0)), topic("kafka.topic", good(0)))) @@ -382,6 +389,7 @@ void testReusesPlansButAlwaysRefreshesMetadataAndValidatesChangedContract() thro ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); List prepared = new ArrayList<>(); List converted = new ArrayList<>(); + List convertedPartitions = new ArrayList<>(); KafkaRecordTranscoder transcoder = new KafkaRecordTranscoder() { @Override @@ -392,26 +400,30 @@ public KafkaTopicWritePlan prepare(TableInfo tableInfo) { } @Override - public BytesView transcode(List records, TableInfo tableInfo) + public BytesView transcode( + int partitionId, List records, TableInfo tableInfo) throws Exception { - return arrow.transcode(records, tableInfo); + return arrow.transcode(partitionId, records, tableInfo); } @Override - public BytesView transcode(List records, KafkaTopicWritePlan plan) + public BytesView transcode( + int partitionId, List records, KafkaTopicWritePlan plan) throws Exception { converted.add(plan); - return arrow.transcode(records, plan); + convertedPartitions.add(partitionId); + return arrow.transcode(partitionId, records, plan); } }; GatewayKafkaProduceBackend backend = - new GatewayKafkaProduceBackend(service, service, transcoder); + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); backend.write(command((short) 1, topic("kafka.topic", good(0), good(1)))).join(); backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(); assertThat(service.metadataPaths).hasSize(2); assertThat(prepared).hasSize(2); assertThat(prepared.get(1)).isSameAs(prepared.get(0)); assertThat(converted).containsExactly(prepared.get(0), prepared.get(0), prepared.get(0)); + assertThat(convertedPartitions).containsExactly(0, 1, 0); service.pendingMetadata = CompletableFuture.completedFuture(metadata(true)); assertErrors( @@ -439,7 +451,7 @@ void testOffloadedRequestsAppendInOrderWithoutWaitingForAcknowledgement() throws service.appended = new CountDownLatch(2); GatewayKafkaProduceBackend backend = new GatewayKafkaProduceBackend( - service, service, new ArrowKafkaRecordTranscoder(), executor); + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); try { CompletableFuture first = backend.write(command((short) -1, topic("kafka.topic", good(0)))); @@ -482,24 +494,41 @@ void testOffloadedMetadataRefreshUsesNewTableIdentityAndSchema() throws Exceptio new GatewayKafkaProduceBackend( service, service, - (records, table) -> { + mapper(), + (partitionId, records, table) -> { snapshots.add(table); - return transcoder.transcode(records, table); + return transcoder.transcode(partitionId, records, table); }, executor); KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); try { assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + TableDescriptor changedSchema = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(3) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); service.pendingMetadata = CompletableFuture.completedFuture( - metadata(false).setTableId(84L).setSchemaId(2).setBucketCountEpoch(7L)); + metadata(false) + .setTableId(84L) + .setSchemaId(2) + .setBucketCountEpoch(7L) + .setTableJson(changedSchema.toJsonBytes())); assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); assertThat(snapshots).hasSize(3); assertThat(snapshots.get(1)).isSameAs(snapshots.get(0)); assertThat(snapshots.get(2)).isNotSameAs(snapshots.get(0)); assertThat(snapshots.get(2).getTableId()).isEqualTo(84L); assertThat(snapshots.get(2).getSchemaId()).isEqualTo(2); + assertThat(snapshots.get(2).getRowType().getTypeAt(0)).isEqualTo(DataTypes.BYTES()); assertThat(snapshots.get(2).getBucketCountEpoch()).isEqualTo(7L); assertThat(service.appends) .extracting(ProduceLogRequest::getTableId) @@ -524,7 +553,7 @@ void testShutdownPreservesAlreadySubmittedAppendResult() throws Exception { KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); GatewayKafkaProduceBackend backend = new GatewayKafkaProduceBackend( - service, service, new ArrowKafkaRecordTranscoder(), executor); + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); try { CompletableFuture result = backend.write(command((short) -1, topic("kafka.topic", good(0)))); @@ -551,7 +580,7 @@ void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Except service.metadataEntered = new CountDownLatch(1); GatewayKafkaProduceBackend backend = new GatewayKafkaProduceBackend( - service, service, new ArrowKafkaRecordTranscoder(), executor); + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); try { CompletableFuture running = backend.write(command); @@ -572,8 +601,283 @@ void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Except } } + @Test + void testAliasesSharePlansAndAuthorizationButRetainRequestNames() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 4); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder, executor); + try { + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("topic", good(0)), + topic("kafka.topic", good(0)))) + .get(10, TimeUnit.SECONDS); + assertThat(result.topics()) + .extracting(KafkaProduceResult.TopicResult::topicName) + .containsExactly("topic", "kafka.topic"); + assertThat(result.topics()) + .allSatisfy( + topic -> + assertThat(topic.partitions().get(0).error()) + .isEqualTo(Errors.NONE)); + assertThat(service.metadataPaths) + .containsExactly( + TablePath.of("kafka", "topic"), TablePath.of("kafka", "topic")); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L); + assertThat(service.writeChecks).isEqualTo(2); + assertThat(transcoder.prepared).hasSize(2); + assertThat(transcoder.prepared.get(0)).isSameAs(transcoder.prepared.get(1)); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDisabledAndResetFlagsRejectBothNamesBeforePrepare(boolean allNull) throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + PartitionWrite partition = allNull ? nullValue(0) : good(0); + assertErrors( + backend.write(command((short) 1, topic("topic", partition))).join(), Errors.NONE); + for (boolean reset : new boolean[] {false, true}) { + TableDescriptor enabled = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + java.util.Map properties = + new java.util.HashMap<>(enabled.getProperties()); + if (reset) { + properties.remove(ConfigOptions.TABLE_KAFKA_ENABLED.key()); + } else { + properties.put(ConfigOptions.TABLE_KAFKA_ENABLED.key(), "false"); + } + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + enabled.withProperties(properties).toJsonBytes())); + int prepared = transcoder.prepared.size(); + int appended = service.appends.size(); + int authorized = service.writeChecks; + for (String name : Arrays.asList("topic", "kafka.topic")) { + assertErrors( + backend.write(command((short) 1, topic(name, partition))).join(), + Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + assertThat(transcoder.prepared).hasSize(prepared); + assertThat(service.appends).hasSize(appended); + assertThat(service.writeChecks).isEqualTo(authorized); + service.pendingMetadata = CompletableFuture.completedFuture(metadata(false)); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", partition))).join(), + Errors.NONE); + assertThat(transcoder.prepared.get(prepared)) + .isNotSameAs(transcoder.prepared.get(prepared - 1)); + } + assertThat(service.metadataPaths).hasSize(7); + } + + @Test + void testPropertyOnlyChangesRejectInvalidContractsAndRecompileRecoveredPlan() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + assertErrors( + backend.write(command((short) 1, topic("topic", good(0)))).join(), Errors.NONE); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + for (TableDescriptor invalid : + Arrays.asList( + TableDescriptor.builder(original) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(), + TableDescriptor.builder(original) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + TableDescriptor.builder(original) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "string") + .build())) { + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(invalid.toJsonBytes())); + for (String name : Arrays.asList("topic", "kafka.topic")) { + assertErrors( + backend.write(command((short) 1, topic(name, nullValue(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + } + } + assertThat(service.appends).hasSize(1); + assertThat(transcoder.prepared).hasSize(1); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE, + "STRING") + .build() + .toJsonBytes())); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.NONE); + assertThat(transcoder.prepared).hasSize(2); + assertThat(transcoder.prepared.get(1)).isNotSameAs(transcoder.prepared.get(0)); + assertThat(transcoder.prepared.get(1).tableInfo().getSchemaId()).isEqualTo(1); + assertThat(service.appends).hasSize(2); + assertThat(service.metadataPaths).hasSize(8); + } + + @Test + void testMissingDefaultAndExplicitDatabasesDoNotFallback() throws Exception { + TestingProduceService service = new TestingProduceService(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_db"); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + new KafkaTopicMapper(config), + new ArrowKafkaRecordTranscoder()); + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("topic", good(0)), + topic("missing_db.topic", good(0)), + topic("other.topic", good(0)))) + .join(); + assertThat(result.topics()) + .extracting(topic -> topic.partitions().get(0).error()) + .containsExactly( + Errors.UNKNOWN_TOPIC_OR_PARTITION, + Errors.UNKNOWN_TOPIC_OR_PARTITION, + Errors.NONE); + assertThat(service.metadataPaths) + .containsExactly( + TablePath.of("missing_db", "topic"), + TablePath.of("missing_db", "topic"), + TablePath.of("other", "topic")); + assertThat(service.appends).extracting(ProduceLogRequest::getTableId).containsExactly(43L); + } + + @Test + void testRecordNullabilityFailureDoesNotAppendValidPrefix() throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.STRING()) + .column( + "_kafka_key", + DataTypes.BYTES() + .copy(false)) + .build()) + .build() + .toJsonBytes())); + Record valid = new Record(1L, new byte[] {1}, new byte[] {65}, Collections.emptyList()); + assertErrors( + backend(service) + .write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 0, + Arrays.asList( + valid, good(0).records().get(0)))))) + .join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + assertThat(service.drains).isZero(); + } + + @Test + void testMetadataOffsetsAndJsonCannotBeActivatedByFilteredRecords() throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + TableDescriptor unsupportedMetadata = + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()), + DataTypes.FIELD( + "offset", DataTypes.BIGINT()))) + .build()) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(unsupportedMetadata.toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + TableDescriptor json = + TableDescriptor.builder(original) + .schema(Schema.newBuilder().column("business", DataTypes.STRING()).build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture(metadata(false).setTableJson(json.toJsonBytes())); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + assertThat(service.writeChecks).isZero(); + } + + private static final class RecordingTranscoder implements KafkaRecordTranscoder { + private final ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); + private final List prepared = new ArrayList<>(); + + @Override + public KafkaTopicWritePlan prepare(TableInfo table) { + KafkaTopicWritePlan plan = arrow.prepare(table); + prepared.add(plan); + return plan; + } + + @Override + public BytesView transcode(int partitionId, List records, TableInfo table) + throws Exception { + return arrow.transcode(partitionId, records, table); + } + } + + private static KafkaTopicMapper mapper() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "kafka"); + return new KafkaTopicMapper(config); + } + private static GatewayKafkaProduceBackend backend(TestingProduceService service) { - return new GatewayKafkaProduceBackend(service, service, new ArrowKafkaRecordTranscoder()); + return new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder()); } private static KafkaProduceCommand command(short acks, TopicWrite... topics) throws Exception { @@ -595,11 +899,15 @@ private static PartitionWrite good(int id) { private static GetTableInfoResponse metadata(boolean invalid) { TableDescriptor.Builder descriptor = TableDescriptor.builder() - .schema(Schema.newBuilder().column("value", DataTypes.STRING()).build()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .build()) .distributedBy(3) - .logFormat(LogFormat.ARROW); + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true); if (!invalid) { - descriptor.customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"); + descriptor.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "string"); } return new GetTableInfoResponse() .setTableId(42L) @@ -653,6 +961,9 @@ public CompletableFuture getTableInfo(GetTableInfoRequest if (metadataEntered != null) { metadataEntered.countDown(); } + if (database.equals("missing_db")) { + throw new DatabaseNotExistException("missing_db"); + } if (name.equals("missing")) { if (!asyncMissing) { throw new TableNotExistException("missing"); @@ -699,7 +1010,6 @@ public CompletableFuture produceLog(ProduceLogRequest reques @Override public void tryCompleteActions() { - assertThat(append).isNotNull(); drains++; if (appended != null) { appended.countDown(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java index 73c68ec725d..3ee3a125495 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java @@ -255,12 +255,16 @@ private static GetTableInfoResponse response( private static TableDescriptor descriptor(String version) { TableDescriptor.Builder builder = TableDescriptor.builder() - .schema(Schema.newBuilder().column("value", DataTypes.BYTES()).build()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) .distributedBy(1) .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) - .customProperty("kafka.value.format", "raw"); + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); if ("v2".equals(version)) { - builder.customProperty("application.version", "changed"); + builder.property(ConfigOptions.TABLE_KAFKA_ENABLED, false); } return builder.build(); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java index e3026f40685..ac950afeeeb 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java @@ -18,9 +18,9 @@ package org.apache.fluss.kafka.transcode; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; @@ -212,14 +212,14 @@ void testCompiledAssemblerIsSafeToShare() throws Exception { () -> plan.rowAssembler() .assemble( - new Object[0], new Object[] { Integer.toString(id) .getBytes( StandardCharsets.UTF_8) }, - id, - new ArrayList<>()) + new Record( + id, null, null, new ArrayList<>()), + 0) .getField(0))); } for (int i = 0; i < futures.size(); i++) { @@ -248,7 +248,7 @@ private static KafkaTopicWritePlan compile(TableInfo tableInfo, AtomicInteger co } private static KafkaTopicWritePlan compile(TableInfo tableInfo) { - KafkaTopicSchema schema = + KafkaTableSchema schema = new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor()); return new KafkaTopicWritePlan(tableInfo, schema); } @@ -257,11 +257,18 @@ private static TableInfo tableInfo( String tableName, long tableId, int schemaId, String propertyValue) { TableDescriptor descriptor = TableDescriptor.builder() - .schema(Schema.newBuilder().column("payload", DataTypes.BYTES()).build()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .build()) .distributedBy(1) .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") - .customProperty("application.version", propertyValue) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + propertyValue.equals("v1") ? "binary" : "BINARY") .build(); return TableInfo.of( TablePath.of("kafka", tableName), tableId, schemaId, descriptor, null, 1L, 1L); From 5eca8c0701a93029a69b792ce6e41851cfb8a65e Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 16:15:53 +0800 Subject: [PATCH 38/48] [kafka] Wait for replicas before no-ack Produce tests Wait for table replica metadata and leader readiness before round-trip writes. Node discovery alone does not ensure a fresh table can accept acks=0 records, whose failures cannot be observed or retried by the producer. Include the table path in readback failures to identify format/acks cases. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 7/7 --- .../org/apache/fluss/kafka/KafkaProduceAppendITCase.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index ddbd1acf806..b331878c165 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -29,6 +29,7 @@ import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.row.InternalRow; @@ -106,6 +107,10 @@ void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception String topic = path.toString(); admin.createTable(path, descriptor(format), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); + // Node discovery does not wait for table replicas. With acks=0 the producer + // cannot observe or retry a write rejected before the leader is ready. + CLUSTER.waitUntilAllReplicaReady( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); try (KafkaProducer producer = producer(acks)) { assertThat(producer.partitionsFor(topic)).hasSize(1); ProducerRecord record = @@ -495,7 +500,7 @@ private static void assertReadback( count++; } } - assertThat(count).isEqualTo(expectedCount); + assertThat(count).as("native readback for %s", path).isEqualTo(expectedCount); } } From 29d90936f163c19bd6f133ffd262f3b0c461202b Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 12:04:12 +0800 Subject: [PATCH 39/48] [kafka] Decouple Produce append from delayed fetch wakeups Remove the Kafka drain-hook dependency after separating the core delayed fetch optimization into apache/fluss#3412. Continue to use native append responses for acknowledgements, including pending acks=all completions. Retain append, authorization, error mapping, ordering and shutdown coverage; move the test append signal to native submission instead of the drain hook. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 9/9 AI-Contributed/UT: 33/33 --- .../produce/GatewayKafkaProduceBackend.java | 9 +---- .../GatewayKafkaProduceBackendTest.java | 33 +++++-------------- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index fa5daca2e2d..8ab07a19de3 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -214,14 +214,7 @@ private CompletableFuture produceTopic( try { checkInterrupted(); setCurrentSession(command); - CompletableFuture appended; - try { - appended = gateway.produceLog(request); - } finally { - // produceLog appends synchronously, but acks=-1 can finish later. Wake delayed - // fetches now so replicas can fetch the append needed to complete that response. - service.tryCompleteActions(); - } + CompletableFuture appended = gateway.produceLog(request); return appended.handle( (response, failure) -> failure == null diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index 9acf78edddf..2d66b743601 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -60,7 +60,9 @@ import static org.assertj.core.api.Assertions.assertThat; -/** Tests append admission, error isolation, session propagation and delayed-fetch completion. */ +/** + * Tests append admission, error isolation, session propagation and asynchronous acknowledgements. + */ class GatewayKafkaProduceBackendTest { @Test @@ -74,7 +76,6 @@ void testAllNullValuesRequireAuthorizationWithoutAppend() throws Exception { assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); assertThat(service.writeChecks).isEqualTo(1); assertThat(service.append).isNull(); - assertThat(service.drains).isZero(); service.denyWrite = true; assertErrors( backend(service) @@ -168,7 +169,6 @@ void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exceptio assertErrors(result, Errors.NONE); assertThat(seen).containsExactly(empty, valid); assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); - assertThat(service.drains).isEqualTo(1); } private static PartitionWrite nullValue(int partition) { @@ -179,7 +179,7 @@ private static PartitionWrite nullValue(int partition) { } @Test - void testDrainsImmediatelyWhileAcksAllResponseIsPending() throws Exception { + void testAcksAllWaitsForNativeAppendResponse() throws Exception { TestingProduceService service = new TestingProduceService(); service.pendingAppend = new CompletableFuture<>(); CompletableFuture result = @@ -187,22 +187,19 @@ void testDrainsImmediatelyWhileAcksAllResponseIsPending() throws Exception { assertThat(service.append.getAcks()).isEqualTo(-1); assertThat(service.append.getTimeoutMs()).isEqualTo(4321); assertThat(service.append.getTableId()).isEqualTo(42L); - assertThat(service.drains).isEqualTo(1); assertThat(result).isNotDone(); assertThat(service.pendingAppend).isNotDone(); service.pendingAppend.complete(success(0)); assertThat(result.join().topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); - assertThat(service.drains).isEqualTo(1); } @Test - void testDrainsAfterSynchronousAppendFailure() throws Exception { + void testMapsSynchronousAppendFailure() throws Exception { TestingProduceService service = new TestingProduceService(); service.throwAppend = true; assertErrors( backend(service).write(command((short) 1, topic("kafka.topic", good(0)))).join(), Errors.UNKNOWN_SERVER_ERROR); - assertThat(service.drains).isEqualTo(1); } @Test @@ -228,7 +225,6 @@ void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception assertErrors(result, Errors.INVALID_RECORD, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); assertThat(service.append.getBucketsReqsList()).hasSize(1); assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); - assertThat(service.drains).isEqualTo(1); } @Test @@ -251,18 +247,16 @@ void testInvalidAndMissingTablesDoNotSuppressOtherTopics() throws Exception { assertThat(result.topics().get(1).partitions().get(0).error()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION); assertThat(result.topics().get(2).partitions().get(0).error()).isEqualTo(Errors.NONE); - assertThat(service.drains).isEqualTo(1); } } @Test - void testNoAppendOrDrainWhenAllPartitionsFailConversion() throws Exception { + void testNoAppendWhenAllPartitionsFailConversion() throws Exception { TestingProduceService service = new TestingProduceService(); KafkaProduceResult result = backend(service).write(command((short) 1, topic("kafka.topic", good(8)))).join(); assertErrors(result, Errors.UNKNOWN_TOPIC_OR_PARTITION); assertThat(service.append).isNull(); - assertThat(service.drains).isZero(); } @Test @@ -316,7 +310,6 @@ void testAsyncMetadataCompletionRestoresSessionAndAcksZero() throws Exception { CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))).join(); assertErrors(result.join(), Errors.NONE); assertThat(service.append.getAcks()).isZero(); - assertThat(service.drains).isEqualTo(1); } @Test @@ -566,7 +559,6 @@ void testShutdownPreservesAlreadySubmittedAppendResult() throws Exception { assertErrors(completed, Errors.NONE); assertThat(completed.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); assertThat(service.appends).hasSize(1); - assertThat(service.drains).isEqualTo(1); } finally { executor.closeAsync().get(10, TimeUnit.SECONDS); } @@ -804,7 +796,6 @@ valid, good(0).records().get(0)))))) .join(), Errors.INVALID_RECORD); assertThat(service.appends).isEmpty(); - assertThat(service.drains).isZero(); } @Test @@ -944,7 +935,6 @@ private static class TestingProduceService extends TestingTabletGatewayService { private CompletableFuture pendingMetadata; private boolean throwAppend; private boolean asyncMissing; - private int drains; private int writeChecks; private boolean denyWrite; private CountDownLatch metadataEntered; @@ -1002,18 +992,13 @@ public CompletableFuture produceLog(ProduceLogRequest reques if (throwAppend) { throw new IllegalStateException("append failed"); } + if (appended != null) { + appended.countDown(); + } return pendingAppend == null ? CompletableFuture.completedFuture( success(request.getBucketsReqsList().get(0).getBucketId())) : pendingAppend; } - - @Override - public void tryCompleteActions() { - drains++; - if (appended != null) { - appended.countDown(); - } - } } } From af1ec7e52bfd4d6673ab16338f9452952921c8b3 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 17:30:26 +0800 Subject: [PATCH 40/48] [kafka] Persist null Produce values with native offsets Append null-valued records through the native Produce path, preserving record order and offsets. Remove the separate authorization shortcut for filtered records. Cover nullable and NOT NULL values with unit and native readback tests. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 54/54 AI-Contributed/UT: 243/271 --- .../produce/GatewayKafkaProduceBackend.java | 14 +- .../backend/produce/KafkaProduceCommand.java | 23 +-- .../kafka/transcode/KafkaTopicWritePlan.java | 2 +- .../fluss/kafka/KafkaProduceAppendITCase.java | 173 +++++++++++++----- .../GatewayKafkaProduceBackendTest.java | 70 +++++-- .../produce/KafkaProduceCommandTest.java | 27 +-- .../ArrowKafkaRecordTranscoderTest.java | 1 - .../apache/fluss/rpc/RpcGatewayService.java | 8 - .../fluss/server/tablet/TabletService.java | 7 +- 9 files changed, 203 insertions(+), 122 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index 8ab07a19de3..c5dd9638dea 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -188,17 +188,9 @@ private CompletableFuture produceTopic( continue; } try { - if (partition.hasNullValues() && partition.nonNullRecords().isEmpty()) { - setCurrentSession(command); - service.authorizeTableWrite(tableInfo.getTableId()); - failures.put( - partition.partitionId(), - new PartitionResult(partition.partitionId(), Errors.NONE, -1L, null)); - continue; - } BytesView records = transcoder.transcode( - partition.partitionId(), partition.nonNullRecords(), writePlan); + partition.partitionId(), partition.records(), writePlan); request.addBucketsReq() .setBucketId(partition.partitionId()) .setRecordsBytesView(records); @@ -286,9 +278,7 @@ private static TopicResult toTopicResult( new PartitionResult( partition.partitionId(), Errors.NONE, - !partition.hasNullValues() && bucket.hasBaseOffset() - ? bucket.getBaseOffset() - : -1L, + bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, null)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index c359c6f8b21..b0d83392d39 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -109,22 +109,11 @@ public List partitions() { public static final class PartitionWrite { private final int partitionId; private final List records; - private final List nonNullRecords; /** Creates the writes for one partition. */ public PartitionWrite(int partitionId, List records) { this.partitionId = partitionId; this.records = immutableCopy(records); - List retained = new ArrayList<>(); - for (Record record : this.records) { - if (record.borrowedValue() != null) { - retained.add(record); - } - } - this.nonNullRecords = - retained.size() == records.size() - ? this.records - : Collections.unmodifiableList(retained); } /** Returns the Kafka partition ID. */ @@ -132,17 +121,7 @@ public int partitionId() { return partitionId; } - /** Returns records with non-null values, preserving their original order. */ - public List nonNullRecords() { - return nonNullRecords; - } - - /** Whether filtering changed the correspondence between Kafka and native offsets. */ - public boolean hasNullValues() { - return records.size() != nonNullRecords.size(); - } - - /** Returns copied records in append order. */ + /** Returns copied records in append order, including records with null values. */ public List records() { return records; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java index c68736e4233..d297713d5ba 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -39,7 +39,7 @@ public final class KafkaTopicWritePlan { KafkaTopicWritePlan(TableInfo tableInfo, KafkaTableSchema topicSchema) { this.tableInfo = checkNotNull(tableInfo); this.topicSchema = checkNotNull(topicSchema); - // Validate support during preparation as filtered partitions may never call transcode. + // Validate supported value formats when compiling the plan. if (topicSchema.valueType() != ValueType.BINARY && topicSchema.valueType() != ValueType.STRING) { throw new KafkaRecordEncodingException( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index b331878c165..99614f0c138 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -149,36 +149,100 @@ void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception } } - @Test - void testNullValuesRemainFilteredWithoutAdvancingNativeOffsets() throws Exception { - // Preserve the existing filtering behavior until the FIP defines its null-value contract. - TablePath path = TablePath.of(DATABASE, "produce_filtered_nulls"); + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullValuesArePersistedForAllAckModes(String format) throws Exception { + List values = Arrays.asList(null, new byte[0], VALUE, null); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - admin.createTable(path, descriptor("binary"), false).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, "produce_null_" + format + "_" + acks); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + CLUSTER.waitUntilAllReplicaReady( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); + try (KafkaProducer producer = producer(acks, 0)) { + for (int i = 0; i < values.size(); i++) { + ProducerRecord record = + new ProducerRecord<>( + i % 2 == 0 ? path.toString() : path.getTableName(), + 0, + 123L, + KEY, + values.get(i), + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + RecordMetadata metadata = producer.send(record).get(30, TimeUnit.SECONDS); + if (!acks.equals("0")) { + assertThat(metadata.offset()).isEqualTo(i); + } + } + producer.flush(); + assertReadback(connection, path, format, values); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testAllNullAndMixedBatchesPreserveOffsets(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_null_batches_" + format); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.toString()}; + assertProduceOffsets(query(socket, recordBatch(null, null), names), names, 0L); + assertProduceOffsets( + query(socket, recordBatch(VALUE, null, new byte[0], null), names), + names, + 2L); + assertProduceOffsets(query(socket, names), names, 6L); + assertReadback( + connection, + path, + format, + Arrays.asList(null, null, VALUE, null, new byte[0], null, VALUE)); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNotNullValueRejectsWholeBatchWithoutAdvancingOffsets(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_not_null_" + format); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor(format, false), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); try (KafkaProducer producer = producer("all", 0)) { - ProducerRecord filtered = - new ProducerRecord(path.toString(), 0, 123L, KEY, null); - ProducerRecord retained = - new ProducerRecord<>( - path.toString(), - 0, - 123L, - KEY, - VALUE, - Arrays.asList( - new RecordHeader("source", VALUE), - new RecordHeader("source", null))); - assertThat(producer.send(filtered).get(30, TimeUnit.SECONDS).offset()) - .isEqualTo(-1L); - assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()).isZero(); - assertThat(producer.send(filtered).get(30, TimeUnit.SECONDS).offset()) - .isEqualTo(-1L); - assertThat(producer.send(retained).get(30, TimeUnit.SECONDS).offset()) - .isEqualTo(1L); - assertReadback(connection, path, "binary", VALUE); + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.toString()}; + assertProduceErrors( + query(socket, recordBatch(null, null), names), + names, + Errors.INVALID_RECORD); + assertProduceErrors( + query(socket, recordBatch(VALUE, null, VALUE), names), + names, + Errors.INVALID_RECORD); + assertProduceOffsets(query(socket, names), names, 0L); + assertReadback(connection, path, format, VALUE, 1); + } } finally { admin.dropTable(path, true).get(); } @@ -375,7 +439,27 @@ private static void setEnabled(TablePath path, boolean enabled) throws Exception .get(); } + private static MemoryRecords recordBatch(byte[]... values) { + SimpleRecord[] records = new SimpleRecord[values.length]; + for (int i = 0; i < values.length; i++) { + records[i] = + new SimpleRecord( + 123L, + KEY, + values[i], + new Header[] { + new RecordHeader("source", VALUE), new RecordHeader("source", null) + }); + } + return MemoryRecords.withRecords(Compression.NONE, records); + } + private static ProduceResponse query(Socket socket, String... names) throws Exception { + return query(socket, recordBatch(VALUE), names); + } + + private static ProduceResponse query(Socket socket, MemoryRecords records, String... names) + throws Exception { List topics = new ArrayList<>(); for (String name : names) { topics.add( @@ -385,21 +469,7 @@ private static ProduceResponse query(Socket socket, String... names) throws Exce Collections.singletonList( new PartitionProduceData() .setIndex(0) - .setRecords( - MemoryRecords.withRecords( - Compression.NONE, - new SimpleRecord( - 123L, - KEY, - VALUE, - new Header[] { - new RecordHeader( - "source", - VALUE), - new RecordHeader( - "source", - null) - })))))); + .setRecords(records)))); } ProduceRequest request = new ProduceRequest( @@ -471,16 +541,26 @@ private static void assertReadback( byte[] expectedValue, int expectedCount) throws Exception { + assertReadback(connection, path, format, Collections.nCopies(expectedCount, expectedValue)); + } + + private static void assertReadback( + Connection connection, TablePath path, String format, List expectedValues) + throws Exception { try (Table table = connection.getTable(path); LogScanner scanner = table.newScan().createLogScanner()) { scanner.subscribeFromBeginning(0); int count = 0; long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - while (count < expectedCount && System.nanoTime() < deadline) { + while (count < expectedValues.size() && System.nanoTime() < deadline) { ScanRecords records = scanner.poll(Duration.ofSeconds(1)); for (ScanRecord record : records) { + assertThat(record.logOffset()).isEqualTo(count); + byte[] expectedValue = expectedValues.get(count); InternalRow row = record.getRow(); - if (format.equals("binary")) { + if (expectedValue == null) { + assertThat(row.isNullAt(0)).isTrue(); + } else if (format.equals("binary")) { assertThat(row.getBytes(0)).isEqualTo(expectedValue); } else { assertThat(row.getString(0).toString()) @@ -500,18 +580,23 @@ private static void assertReadback( count++; } } - assertThat(count).as("native readback for %s", path).isEqualTo(expectedCount); + assertThat(count).as("native readback for %s", path).isEqualTo(expectedValues.size()); } } private static TableDescriptor descriptor(String format) { + return descriptor(format, true); + } + + private static TableDescriptor descriptor(String format, boolean nullableValue) { boolean binary = format.equals("binary"); return TableDescriptor.builder() .schema( Schema.newBuilder() .column( "_kafka_value", - binary ? DataTypes.BYTES() : DataTypes.STRING()) + (binary ? DataTypes.BYTES() : DataTypes.STRING()) + .copy(nullableValue)) .column( "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index 2d66b743601..d40c294f89c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -66,23 +66,26 @@ class GatewayKafkaProduceBackendTest { @Test - void testAllNullValuesRequireAuthorizationWithoutAppend() throws Exception { + void testAllNullValuesUseNativeAppendAndAuthorization() throws Exception { TestingProduceService service = new TestingProduceService(); KafkaProduceResult result = backend(service) .write(command((short) 1, topic("kafka.topic", nullValue(0), nullValue(8)))) .join(); assertErrors(result, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); - assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); assertThat(service.writeChecks).isEqualTo(1); - assertThat(service.append).isNull(); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isZero(); service.denyWrite = true; assertErrors( backend(service) .write(command((short) 1, topic("kafka.topic", nullValue(0)))) .join(), Errors.TOPIC_AUTHORIZATION_FAILED); - assertThat(service.append).isNull(); + assertThat(service.appends).hasSize(1); + assertThat(service.writeChecks).isEqualTo(2); } @ParameterizedTest @@ -138,7 +141,7 @@ void testNullValuesCannotBypassTableAndSchemaValidation() throws Exception { } @Test - void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exception { + void testMixedNullValuesPreserveRecordOrderAndNativeOffsets() throws Exception { TestingProduceService service = new TestingProduceService(); List seen = new ArrayList<>(); ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); @@ -151,6 +154,7 @@ void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exceptio seen.addAll(records); return transcoder.transcode(partitionId, records, table); }); + Record nullRecord = nullValue(0).records().get(0); Record empty = new Record(1L, null, new byte[0], Collections.emptyList()); Record valid = good(0).records().get(0); KafkaProduceResult result = @@ -161,14 +165,11 @@ void testMixedNullValuesAreRemovedBeforeDecodingAndHideOffsets() throws Exceptio "kafka.topic", new PartitionWrite( 0, - Arrays.asList( - nullValue(0).records().get(0), - empty, - valid))))) + Arrays.asList(nullRecord, empty, valid))))) .join(); assertErrors(result, Errors.NONE); - assertThat(seen).containsExactly(empty, valid); - assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(-1L); + assertThat(seen).containsExactly(nullRecord, empty, valid); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); } private static PartitionWrite nullValue(int partition) { @@ -798,8 +799,48 @@ valid, good(0).records().get(0)))))) assertThat(service.appends).isEmpty(); } + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullValueViolationRejectsWholePartitionWithoutAppendingPrefix(String format) + throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + TableDescriptor notNull = + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + (format.equals("binary") + ? DataTypes.BYTES() + : DataTypes.STRING()) + .copy(false)) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(notNull.toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + + Record valid = good(0).records().get(0); + PartitionWrite invalid = + new PartitionWrite(0, Arrays.asList(valid, nullValue(0).records().get(0), valid)); + assertErrors( + backend.write(command((short) 1, topic("topic", invalid, good(1)))).join(), + Errors.INVALID_RECORD, + Errors.NONE); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + } + @Test - void testMetadataOffsetsAndJsonCannotBeActivatedByFilteredRecords() throws Exception { + void testMetadataOffsetsAndJsonCannotBeActivatedByNullValues() throws Exception { TestingProduceService service = new TestingProduceService(); TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); TableDescriptor unsupportedMetadata = @@ -969,8 +1010,7 @@ public CompletableFuture getTableInfo(GetTableInfoRequest : pendingMetadata; } - @Override - public void authorizeTableWrite(long tableId) { + private void checkWriteAuthorization(long tableId) { assertThat(tableId).isPositive(); assertThat(currentListenerName()).isEqualTo("KAFKA"); assertThat(currentSession().getInetAddress()) @@ -983,7 +1023,7 @@ public void authorizeTableWrite(long tableId) { @Override public CompletableFuture produceLog(ProduceLogRequest request) { - authorizeTableWrite(request.getTableId()); + checkWriteAuthorization(request.getTableId()); assertThat(currentListenerName()).isEqualTo("KAFKA"); assertThat(currentSession().getInetAddress()) .isEqualTo(InetAddress.getLoopbackAddress()); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java index e3624f5535b..6243bb8a900 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -163,8 +163,8 @@ void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { } @Test - void testNullFilteringPreservesRecordIdentityAndTimestampProvenance() { - Record filtered = + void testNullValuesPreserveRecordIdentityAndTimestampProvenance() { + Record nullRecord = new Record( 123L, TimestampType.CREATE_TIME, @@ -185,21 +185,22 @@ void testNullFilteringPreservesRecordIdentityAndTimestampProvenance() { null, new byte[0], Collections.emptyList()); - List source = new ArrayList<>(Arrays.asList(filtered, retained, empty)); + List source = new ArrayList<>(Arrays.asList(nullRecord, retained, empty)); PartitionWrite partition = new PartitionWrite(0, source); source.clear(); - assertThat(partition.records()).containsExactly(filtered, retained, empty); - assertThat(partition.hasNullValues()).isTrue(); - assertThat(partition.nonNullRecords()).containsExactly(retained, empty); - assertThat(partition.nonNullRecords().get(0)).isSameAs(retained); - assertThat(partition.nonNullRecords().get(0).timestampType()) + assertThat(partition.records()).containsExactly(nullRecord, retained, empty); + assertThat(partition.records().get(0)).isSameAs(nullRecord); + assertThat(partition.records().get(0).value()).isNull(); + assertThat(partition.records().get(0).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.records().get(0).timestamp()).isEqualTo(123L); + assertThat(partition.records().get(1)).isSameAs(retained); + assertThat(partition.records().get(1).timestampType()) .isEqualTo(TimestampType.LOG_APPEND_TIME); - assertThat(partition.nonNullRecords().get(0).timestamp()).isEqualTo(456L); - assertThat(partition.nonNullRecords().get(1).timestampType()) - .isEqualTo(TimestampType.CREATE_TIME); - assertThat(partition.nonNullRecords().get(1).hasTimestamp()).isFalse(); - assertThatThrownBy(() -> partition.nonNullRecords().clear()) + assertThat(partition.records().get(1).timestamp()).isEqualTo(456L); + assertThat(partition.records().get(2).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.records().get(2).hasTimestamp()).isFalse(); + assertThatThrownBy(() -> partition.records().clear()) .isInstanceOf(UnsupportedOperationException.class); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 2ffa89093c3..7bc449ed4e4 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -130,7 +130,6 @@ void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) thro @ValueSource(booleans = {false, true}) void testNullAndEmptyPayloadsRemainDistinct(boolean string) throws Exception { TableInfo table = envelope(string, 7); - // This converter retains its existing nullability checks; ingress filtering is separate. List records = Arrays.asList( new Record(Record.NO_TIMESTAMP, null, null, Collections.emptyList()), diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java index 1dc4a0da4c9..8f2365add70 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java @@ -77,14 +77,6 @@ public String currentListenerName() { } } - /** - * Authorizes a table write that may not append any records, using the current request session. - * Services must explicitly support this operation; the default never grants write access. - */ - public void authorizeTableWrite(long tableId) { - throw new UnsupportedOperationException("This service cannot authorize table writes."); - } - /** Returns the provider type of this RPC gateway service. */ public abstract ServerType providerType(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index d7a637ff6a8..ae19ae90eb6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -222,14 +222,9 @@ public String name() { @Override public void shutdown() {} - @Override - public void authorizeTableWrite(long tableId) { - authorizeTable(WRITE, tableId); - } - @Override public CompletableFuture produceLog(ProduceLogRequest request) { - authorizeTableWrite(request.getTableId()); + authorizeTable(WRITE, request.getTableId()); long tableId = request.getTableId(); Map routingErrors = new HashMap<>(); collectRoutingErrors( From 7ec87847fff6b3865ca9de3059e25157cb9aa90c Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 11 Sep 2026 14:44:26 +0800 Subject: [PATCH 41/48] [kafka] Cover Produce lifecycle and replication regressions Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 364/364 --- .../kafka/KafkaProduceRegressionITCase.java | 364 ++++++++++++++++++ 1 file changed, 364 insertions(+) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java new file mode 100644 index 00000000000..0748bd81503 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.cluster.AlterConfig; +import org.apache.fluss.config.cluster.AlterConfigOpType; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.NotEnoughReplicasException; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.tuple; + +/** Regression coverage for native DDL lifecycle, metadata refresh, and Produce minimum ISR. */ +class KafkaProduceRegressionITCase { + private static final String DATABASE = "kafka"; + private static final short METADATA_VERSION = 11; + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { + String topic = "metadata_lifecycle"; + TablePath path = TablePath.of(DATABASE, topic); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, topic, 3); + MetadataResponse initial = waitForTopic(topic); + Uuid initialId = initial.data().topics().find(topic).topicId(); + assertThat(initialId).isNotEqualTo(Uuid.ZERO_UUID); + assertKafkaListenerEndpoints(initial); + assertThat(fetchMetadata(topic).data().topics().find(topic).topicId()) + .isEqualTo(initialId); + + admin.dropTable(path, false).get(); + retry( + Duration.ofMinutes(1), + () -> + assertThat(fetchMetadata(topic).errors()) + .containsEntry(topic, Errors.UNKNOWN_TOPIC_OR_PARTITION)); + + createTable(admin, topic, 3); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponseTopic recreated = + fetchMetadata(topic).data().topics().find(topic); + assertThat(recreated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(recreated.topicId()) + .isNotEqualTo(Uuid.ZERO_UUID) + .isNotEqualTo(initialId); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { + String topic = "metadata_failover"; + int stoppedLeader = -1; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, topic, 3); + MetadataResponsePartition initial = + waitForTopic(topic).data().topics().find(topic).partitions().get(0); + stoppedLeader = initial.leaderId(); + try (KafkaProducer producer = producer("1", true)) { + assertThat(send(producer, topic, "before_failover").offset()).isNotNegative(); + CLUSTER.stopTabletServer(stoppedLeader); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponseTopic metadata = + fetchMetadata(topic).data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = metadata.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(partition.leaderId()) + .isNotEqualTo(-1) + .isNotEqualTo(initial.leaderId()); + assertThat(partition.leaderEpoch()) + .isGreaterThan(initial.leaderEpoch()); + }); + // Reuse the producer's cached route so the failed request triggers a refresh. + // acks=1 permits data loss and offset reuse after failover; this verifies + // continued routing, not durability of the first acknowledged record. + assertThat(send(producer, topic, "after_failover").offset()).isNotNegative(); + } + } finally { + try { + if (stoppedLeader >= 0 && CLUSTER.getTabletServerById(stoppedLeader) == null) { + CLUSTER.startTabletServer(stoppedLeader); + CLUSTER.assertHasTabletServerNumber(3); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + } + } finally { + admin.dropTable(TablePath.of(DATABASE, topic), true).get(); + } + } + } + } + + @Test + void testClusterMinIsrPolicyAndDynamicReload() throws Exception { + String replicated = "acks_all_replicated"; + String underReplicated = "acks_all_under_replicated"; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, replicated, 3); + createTable(admin, underReplicated, 1); + try (KafkaProducer producer = producer("all", false)) { + assertThat(send(producer, replicated, "replicated").offset()).isZero(); + assertThatThrownBy(() -> send(producer, underReplicated, "rejected")) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(NotEnoughReplicasException.class); + + setMinIsr(admin, 1); + assertThat(send(producer, underReplicated, "accepted").offset()).isZero(); + // Offset zero and native readback also prove the rejected write was not stored. + assertSingleValue(connection, underReplicated, "accepted"); + } + } finally { + try { + setMinIsr(admin, 2); + } finally { + admin.dropTable(TablePath.of(DATABASE, replicated), true).get(); + admin.dropTable(TablePath.of(DATABASE, underReplicated), true).get(); + } + } + } + } + + private static void createTable(Admin admin, String topic, int replicas) throws Exception { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + TablePath path = TablePath.of(DATABASE, topic); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + } + + private static void setMinIsr(Admin admin, int minIsr) throws Exception { + admin.alterClusterConfigs( + Collections.singletonList( + new AlterConfig( + ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER.key(), + Integer.toString(minIsr), + AlterConfigOpType.SET))) + .get(30, TimeUnit.SECONDS); + retry( + Duration.ofMinutes(1), + () -> + assertThat(CLUSTER.getTabletServers()) + .allSatisfy( + server -> + assertThat( + server.getReplicaManager() + .getMinInSyncReplicas()) + .isEqualTo(minIsr))); + } + + private static void assertSingleValue(Connection connection, String topic, String expected) + throws Exception { + try (Table table = connection.getTable(TablePath.of(DATABASE, topic)); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + retry( + Duration.ofSeconds(30), + () -> { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + assertThat(records).hasSize(1); + assertThat(records.iterator().next().getRow().getBytes(0)) + .isEqualTo(expected.getBytes(StandardCharsets.UTF_8)); + }); + } + } + + private static MetadataResponse waitForTopic(String topic) { + AtomicReference result = new AtomicReference<>(); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponse response = fetchMetadata(topic); + MetadataResponseTopic metadata = response.data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(metadata.partitions()).hasSize(1); + assertThat(metadata.partitions().get(0).leaderId()).isNotEqualTo(-1); + result.set(response); + }); + return result.get(); + } + + private static void assertKafkaListenerEndpoints(MetadataResponse response) { + assertThat(response.brokers()) + .extracting( + org.apache.kafka.common.Node::id, + org.apache.kafka.common.Node::host, + org.apache.kafka.common.Node::port, + org.apache.kafka.common.Node::rack) + .containsExactlyInAnyOrderElementsOf( + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map( + node -> + tuple( + node.id(), + node.host(), + node.port(), + node.rack())) + .collect(Collectors.toList())); + } + + private static MetadataResponse fetchMetadata(String topic) throws Exception { + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList(topic), false) + .build(METADATA_VERSION); + RequestHeader header = + new RequestHeader(ApiKeys.METADATA, METADATA_VERSION, "regression-test", 1); + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + byte[] requestBytes = new byte[serialized.remaining()]; + serialized.get(requestBytes); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(node.host(), node.port()), 5000); + socket.setSoTimeout(5000); + try (DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream())) { + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + int size = input.readInt(); + assertThat(size).isBetween(1, 1024 * 1024); + byte[] response = new byte[size]; + input.readFully(response); + return (MetadataResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + } + } + + private static RecordMetadata send( + KafkaProducer producer, String topic, String value) throws Exception { + return producer.send( + new ProducerRecord<>( + topic, 0, null, value.getBytes(StandardCharsets.UTF_8))) + .get(60, TimeUnit.SECONDS); + } + + private static KafkaProducer producer(String acks, boolean retryOnFailure) { + Map config = new HashMap<>(); + config.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map(node -> node.host() + ":" + node.port()) + .collect(Collectors.joining(","))); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.RETRIES_CONFIG, retryOnFailure ? 20 : 0); + config.put(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, 200); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} From a9142d4ec8636d0784708202d4c6f6c432d6ed4c Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 18 Sep 2026 10:34:24 +0800 Subject: [PATCH 42/48] [kafka] Cover online leader handoff and qualified topics Adapt Produce regressions to database-qualified topic names from PR07. Exercise a real leader handoff while the old node stays online, verify NOT_LEADER_OR_FOLLOWER on the existing connection, and confirm the same producer refreshes its route without disconnecting. Check acks=all offsets and native readback across the handoff. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 285/285 --- .../kafka/KafkaProduceRegressionITCase.java | 285 +++++++++++++++--- 1 file changed, 238 insertions(+), 47 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java index 0748bd81503..32b0f2a2ced 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -21,9 +21,13 @@ import org.apache.fluss.client.ConnectionFactory; import org.apache.fluss.client.admin.Admin; import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; import org.apache.fluss.client.table.scanner.log.LogScanner; import org.apache.fluss.client.table.scanner.log.ScanRecords; import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.cluster.rebalance.RebalancePlanForBucket; +import org.apache.fluss.cluster.rebalance.RebalanceProgress; +import org.apache.fluss.cluster.rebalance.RebalanceStatus; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.cluster.AlterConfig; @@ -32,9 +36,12 @@ import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.coordinator.event.RecoverRebalanceEvent; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.data.RebalanceTask; import org.apache.fluss.types.DataTypes; import org.apache.kafka.clients.producer.KafkaProducer; @@ -42,14 +49,20 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.compress.Compression; import org.apache.kafka.common.errors.NotEnoughReplicasException; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -63,8 +76,10 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -91,12 +106,12 @@ class KafkaProduceRegressionITCase { @Test void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { - String topic = "metadata_lifecycle"; - TablePath path = TablePath.of(DATABASE, topic); + TablePath path = TablePath.of(DATABASE, "metadata_lifecycle"); + String topic = path.toString(); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { - createTable(admin, topic, 3); + createTable(admin, path, 3); MetadataResponse initial = waitForTopic(topic); Uuid initialId = initial.data().topics().find(topic).topicId(); assertThat(initialId).isNotEqualTo(Uuid.ZERO_UUID); @@ -111,7 +126,7 @@ void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { assertThat(fetchMetadata(topic).errors()) .containsEntry(topic, Errors.UNKNOWN_TOPIC_OR_PARTITION)); - createTable(admin, topic, 3); + createTable(admin, path, 3); retry( Duration.ofMinutes(1), () -> { @@ -130,12 +145,13 @@ void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { @Test void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { - String topic = "metadata_failover"; + TablePath path = TablePath.of(DATABASE, "metadata_failover"); + String topic = path.toString(); int stoppedLeader = -1; try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { - createTable(admin, topic, 3); + createTable(admin, path, 3); MetadataResponsePartition initial = waitForTopic(topic).data().topics().find(topic).partitions().get(0); stoppedLeader = initial.leaderId(); @@ -156,7 +172,7 @@ void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { assertThat(partition.leaderEpoch()) .isGreaterThan(initial.leaderEpoch()); }); - // Reuse the producer's cached route so the failed request triggers a refresh. + // Reuse the same producer to verify routing recovers after leader disconnect. // acks=1 permits data loss and offset reuse after failover; this verifies // continued routing, not durability of the first acknowledged record. assertThat(send(producer, topic, "after_failover").offset()).isNotNegative(); @@ -169,7 +185,104 @@ void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { CLUSTER.waitUntilAllGatewayHasSameMetadata(); } } finally { - admin.dropTable(TablePath.of(DATABASE, topic), true).get(); + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { + TablePath path = TablePath.of(DATABASE, "metadata_online_leader_change"); + String topic = path.toString(); + String rebalanceId = "kafka-online-leader-change"; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + TableBucket bucket = + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0); + MetadataResponsePartition initial = + waitForTopic(topic).data().topics().find(topic).partitions().get(0); + int oldLeader = initial.leaderId(); + int newLeader = + initial.replicaNodes().stream() + .filter(id -> id != oldLeader) + .findFirst() + .get(); + List nodes = CLUSTER.getTabletServerNodes("KAFKA"); + ServerNode oldNode = + nodes.stream().filter(node -> node.id() == oldLeader).findFirst().get(); + + // Keep this socket open across the handoff to verify the old leader stays + // reachable. + try (Socket oldLeaderSocket = connect(oldNode); + KafkaProducer producer = producer("all", true)) { + MetadataResponse before = + (MetadataResponse) sendRequest(oldLeaderSocket, metadataRequest(topic)); + assertThat(before.data().topics().find(topic).partitions().get(0).leaderId()) + .isEqualTo(oldLeader); + assertThat(send(producer, topic, "before_handoff").offset()).isZero(); + CLUSTER.waitUntilReplicaExpandToIsr(bucket, newLeader); + double retriesBefore = producerMetric(producer, "record-retry-total"); + double disconnectsBefore = producerMetric(producer, "connection-close-total"); + + rebalanceLeader(bucket, initial, newLeader, rebalanceId); + retry( + Duration.ofMinutes(1), + () -> { + assertThat(admin.listRebalanceProgress(rebalanceId).get()) + .isPresent() + .get() + .extracting(RebalanceProgress::status) + .isEqualTo(RebalanceStatus.COMPLETED); + MetadataResponsePartition updated = + fetchMetadata(topic) + .data() + .topics() + .find(topic) + .partitions() + .get(0); + assertThat(updated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(updated.leaderId()).isEqualTo(newLeader); + assertThat(updated.leaderEpoch()) + .isGreaterThan(initial.leaderEpoch()); + }); + CLUSTER.waitAndGetFollowerReplica(bucket, oldLeader); + assertThat(CLUSTER.getTabletServerNodes("KAFKA")) + .containsExactlyInAnyOrderElementsOf(nodes); + + ProduceResponse rejected = + (ProduceResponse) + sendRequest(oldLeaderSocket, produceRequest(topic, "rejected")); + assertThat(rejected.data().responses().find(topic).partitionResponses()) + .singleElement() + .satisfies( + partition -> { + assertThat(partition.index()).isZero(); + assertThat(partition.errorCode()) + .isEqualTo(Errors.NOT_LEADER_OR_FOLLOWER.code()); + assertThat(partition.baseOffset()).isEqualTo(-1L); + }); + + // partitionsFor uses the already populated producer cache. Separate raw + // Metadata requests above must not refresh this route before the next send. + assertThat(producer.partitionsFor(topic).get(0).leader().id()) + .isEqualTo(oldLeader); + assertThat(send(producer, topic, "after_handoff").offset()).isEqualTo(1L); + assertThat(producerMetric(producer, "record-retry-total")) + .isGreaterThan(retriesBefore); + assertThat(producerMetric(producer, "connection-close-total")) + .isEqualTo(disconnectsBefore); + assertThat(producer.partitionsFor(topic).get(0).leader().id()) + .isEqualTo(newLeader); + assertValues(connection, path, "before_handoff", "after_handoff"); + } + } finally { + try { + admin.cancelRebalance(rebalanceId).get(); + } finally { + admin.dropTable(path, true).get(); } } } @@ -177,38 +290,39 @@ void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { @Test void testClusterMinIsrPolicyAndDynamicReload() throws Exception { - String replicated = "acks_all_replicated"; - String underReplicated = "acks_all_under_replicated"; + TablePath replicated = TablePath.of(DATABASE, "acks_all_replicated"); + TablePath underReplicated = TablePath.of(DATABASE, "acks_all_under_replicated"); try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { createTable(admin, replicated, 3); createTable(admin, underReplicated, 1); try (KafkaProducer producer = producer("all", false)) { - assertThat(send(producer, replicated, "replicated").offset()).isZero(); - assertThatThrownBy(() -> send(producer, underReplicated, "rejected")) + assertThat(send(producer, replicated.toString(), "replicated").offset()) + .isZero(); + assertThatThrownBy(() -> send(producer, underReplicated.toString(), "rejected")) .isInstanceOf(ExecutionException.class) .hasCauseInstanceOf(NotEnoughReplicasException.class); setMinIsr(admin, 1); - assertThat(send(producer, underReplicated, "accepted").offset()).isZero(); + assertThat(send(producer, underReplicated.toString(), "accepted").offset()) + .isZero(); // Offset zero and native readback also prove the rejected write was not stored. - assertSingleValue(connection, underReplicated, "accepted"); + assertValues(connection, underReplicated, "accepted"); } } finally { try { setMinIsr(admin, 2); } finally { - admin.dropTable(TablePath.of(DATABASE, replicated), true).get(); - admin.dropTable(TablePath.of(DATABASE, underReplicated), true).get(); + admin.dropTable(replicated, true).get(); + admin.dropTable(underReplicated, true).get(); } } } } - private static void createTable(Admin admin, String topic, int replicas) throws Exception { - admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - TablePath path = TablePath.of(DATABASE, topic); + private static void createTable(Admin admin, TablePath path, int replicas) throws Exception { + admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); admin.createTable( path, TableDescriptor.builder() @@ -246,18 +360,22 @@ private static void setMinIsr(Admin admin, int minIsr) throws Exception { .isEqualTo(minIsr))); } - private static void assertSingleValue(Connection connection, String topic, String expected) + private static void assertValues(Connection connection, TablePath path, String... expected) throws Exception { - try (Table table = connection.getTable(TablePath.of(DATABASE, topic)); + try (Table table = connection.getTable(path); LogScanner scanner = table.newScan().createLogScanner()) { scanner.subscribeFromBeginning(0); + List values = new ArrayList<>(); retry( Duration.ofSeconds(30), () -> { ScanRecords records = scanner.poll(Duration.ofSeconds(1)); - assertThat(records).hasSize(1); - assertThat(records.iterator().next().getRow().getBytes(0)) - .isEqualTo(expected.getBytes(StandardCharsets.UTF_8)); + for (ScanRecord record : records) { + values.add( + new String( + record.getRow().getBytes(0), StandardCharsets.UTF_8)); + } + assertThat(values).containsExactly(expected); }); } } @@ -297,33 +415,106 @@ private static void assertKafkaListenerEndpoints(MetadataResponse response) { } private static MetadataResponse fetchMetadata(String topic) throws Exception { - MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList(topic), false) - .build(METADATA_VERSION); + try (Socket socket = connect(CLUSTER.getTabletServerNodes("KAFKA").get(0))) { + return (MetadataResponse) sendRequest(socket, metadataRequest(topic)); + } + } + + private static MetadataRequest metadataRequest(String topic) { + return new MetadataRequest.Builder(Collections.singletonList(topic), false) + .build(METADATA_VERSION); + } + + private static ProduceRequest produceRequest(String topic, String value) { + MemoryRecords records = + MemoryRecords.withRecords( + Compression.NONE, new SimpleRecord(value.getBytes(StandardCharsets.UTF_8))); + ProduceRequestData.TopicProduceData topicData = + new ProduceRequestData.TopicProduceData() + .setName(topic) + .setPartitionData( + Collections.singletonList( + new ProduceRequestData.PartitionProduceData() + .setIndex(0) + .setRecords(records))); + return new ProduceRequest( + new ProduceRequestData() + .setAcks((short) 1) + .setTimeoutMs(5000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topicData).iterator())), + (short) 11); + } + + private static Socket connect(ServerNode node) throws Exception { + Socket socket = new Socket(); + try { + socket.connect(new InetSocketAddress(node.host(), node.port()), 5000); + socket.setSoTimeout(5000); + return socket; + } catch (Exception failure) { + socket.close(); + throw failure; + } + } + + private static AbstractResponse sendRequest(Socket socket, AbstractRequest request) + throws Exception { RequestHeader header = - new RequestHeader(ApiKeys.METADATA, METADATA_VERSION, "regression-test", 1); + new RequestHeader(request.apiKey(), request.version(), "regression-test", 1); ByteBuffer serialized = RequestUtils.serialize( header.data(), header.headerVersion(), request.data(), request.version()); byte[] requestBytes = new byte[serialized.remaining()]; serialized.get(requestBytes); - ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(node.host(), node.port()), 5000); - socket.setSoTimeout(5000); - try (DataOutputStream output = new DataOutputStream(socket.getOutputStream()); - DataInputStream input = new DataInputStream(socket.getInputStream())) { - output.writeInt(requestBytes.length); - output.write(requestBytes); - output.flush(); - int size = input.readInt(); - assertThat(size).isBetween(1, 1024 * 1024); - byte[] response = new byte[size]; - input.readFully(response); - return (MetadataResponse) - AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); - } - } + // The caller owns the socket and may send another request on the same connection. + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + int size = input.readInt(); + assertThat(size).isBetween(1, 1024 * 1024); + byte[] response = new byte[size]; + input.readFully(response); + return AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static void rebalanceLeader( + TableBucket bucket, + MetadataResponsePartition initial, + int newLeader, + String rebalanceId) + throws Exception { + List replicas = new ArrayList<>(initial.replicaNodes()); + Collections.swap(replicas, 0, replicas.indexOf(newLeader)); + RebalancePlanForBucket plan = + new RebalancePlanForBucket( + bucket, initial.leaderId(), newLeader, initial.replicaNodes(), replicas); + RebalanceTask task = + new RebalanceTask( + rebalanceId, + RebalanceStatus.NOT_STARTED, + Collections.singletonMap(bucket, plan)); + // Submit a deterministic leader-only plan through the coordinator event thread. + // All replicas and their client connections remain online during the real handoff. + CLUSTER.getZooKeeperClient().registerRebalanceTask(task); + CLUSTER.getCoordinatorServer() + .getCoordinatorEventProcessor() + .getCoordinatorEventManager() + .put(new RecoverRebalanceEvent(task)); + } + + private static double producerMetric(KafkaProducer producer, String name) { + return producer.metrics().entrySet().stream() + .filter( + entry -> + entry.getKey().group().equals("producer-metrics") + && entry.getKey().name().equals(name)) + .mapToDouble(entry -> ((Number) entry.getValue().metricValue()).doubleValue()) + .findFirst() + .getAsDouble(); } private static RecordMetadata send( @@ -350,13 +541,13 @@ private static KafkaProducer producer(String acks, boolean retry config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, (int) Duration.ofHours(1).toMillis()); return new KafkaProducer<>(config); } private static Configuration clusterConfig() { Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_ENABLED, true); - config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); return config; From a7ce6a75aa51309ee41d023632d285950fb9d5c1 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:38:54 +0800 Subject: [PATCH 43/48] [kafka] Verify null value filtering with native raw and string readback Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 67/67 --- .../fluss/kafka/KafkaProduceAppendITCase.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 99614f0c138..0c4bf03d479 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -249,6 +249,73 @@ void testNotNullValueRejectsWholeBatchWithoutAdvancingOffsets(String format) thr } } + @Test + void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Exception { + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String format : new String[] {"raw", "string"}) { + TablePath path = TablePath.of(DATABASE, "null_values_" + format); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + String topic = path.toString(); + assertThat( + producer.send( + new ProducerRecord( + topic, + 0, + 123L, + new byte[] {(byte) 0xff}, + null)) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(-1L); + assertThat( + producer.send( + new ProducerRecord( + topic, 0, 123L, null, new byte[0])) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + assertThat( + producer.send( + new ProducerRecord( + topic, 0, 123L, null, VALUE)) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(1L); + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < 2 && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + InternalRow row = record.getRow(); + byte[] expected = count == 0 ? new byte[0] : VALUE; + if (format.equals("raw")) { + assertThat(row.getBytes(0)).isEqualTo(expected); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo( + new String(expected, StandardCharsets.UTF_8)); + } + assertThat(row.isNullAt(3)).isTrue(); + assertThat(row.getTimestampNtz(1, 3).getMillisecond()) + .isEqualTo(123L); + count++; + } + } + assertThat(count).isEqualTo(2); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + @Test void testSameNamedTablesAcrossDatabases() throws Exception { Map tables = new LinkedHashMap<>(); From ab1ada7740462a68236ed422b8f6c8dabe962348 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 19:08:21 +0800 Subject: [PATCH 44/48] [kafka] Wait for bucket readiness in Produce round trips Server metadata convergence only confirms node discovery, so a newly created table can still reject writes before its leader replica is ready. Wait for the real bucket leader before the all-ack-mode round trip, including acks=0 which cannot report or retry such a rejected write. Include the table and format in native readback assertion failures. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 7/7 --- .../java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 0c4bf03d479..e524f1faabf 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -647,7 +647,9 @@ private static void assertReadback( count++; } } - assertThat(count).as("native readback for %s", path).isEqualTo(expectedValues.size()); + assertThat(count) + .as("Native readback for %s (%s)", path, format) + .isEqualTo(expectedValues.size()); } } From cdfb7723a5493e22d50f5c507086e9ce766be01a Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 20:45:56 +0800 Subject: [PATCH 45/48] [kafka] Validate Produce contracts with isolated client matrix Exercise persisted table contracts, alias lifecycle, invalid-record atomicity, and replicated offsets across leader loss and restart using native readback. Keep null-value filtering and online leader-transfer regressions on the PR07 base. Add checksum-verified, isolated Java producer tests for all 72 formal releases from 0.11.0.0 through 4.3.1. Verify all ACK modes and both topic names before allowing clients to close. Include the provided Jackson 2.10.2 closure required by Kafka 2.6.0 and matching diagnostic logging bindings in test processes only. Validation: cumulative reactor regressions and both full matrix phases pass; 396 distinct passing cases, with two pre-existing disabled RPC cases. Spotless, Checkstyle and RAT pass. SASL and persisted per-row metadata offsets remain outside this stage. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 1027/1027 --- fluss-kafka/src/test/client-matrix/README.md | 109 +++++++ fluss-kafka/src/test/client-matrix/prepare.py | 162 ++++++++++ .../fluss/kafka/KafkaClientMatrixITCase.java | 306 ++++++++++++++++++ .../fluss/kafka/KafkaProduceAppendITCase.java | 35 +- .../kafka/KafkaProduceRegressionITCase.java | 304 ++++++++++++++++- .../fluss/kafka/matrix/KafkaClientProbe.java | 111 +++++++ 6 files changed, 1014 insertions(+), 13 deletions(-) create mode 100644 fluss-kafka/src/test/client-matrix/README.md create mode 100644 fluss-kafka/src/test/client-matrix/prepare.py create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java diff --git a/fluss-kafka/src/test/client-matrix/README.md b/fluss-kafka/src/test/client-matrix/README.md new file mode 100644 index 00000000000..3275dc7d843 --- /dev/null +++ b/fluss-kafka/src/test/client-matrix/README.md @@ -0,0 +1,109 @@ +# Isolated Kafka producer compatibility matrix + +This test facility runs real Kafka producers against the three-node Fluss test +cluster and reads their records through the native Fluss client. Each selected +client runs in a separate JVM whose classpath contains only a standalone probe, +one `kafka-clients` JAR and that release's runtime libraries. Prepared classpaths +also include a matching SLF4J Simple binding so client network-thread failures +appear in the log. The probe +checks `AppInfoParser.getVersion()` and records its code-source JAR, Java version, +classpath and results. No production dependency is replaced. + +## Requirements and preparation + +Use Java 11 for the reactor and child JVMs, Python 3.8+, curl and Maven. Initial +artifact preparation needs access to Maven Central; subsequent reactor runs can +be offline once the project's own dependencies have been cached. + +The FIP target is every formal Java client release from **0.11.0.0 through 4.3.1**. +[Apache Kafka downloads](https://kafka.apache.org/community/downloads/) records +both boundary releases. The preparation script reads the actual formal versions +from [official Maven metadata](https://repo.maven.apache.org/maven2/org/apache/kafka/kafka-clients/maven-metadata.xml), +fetches each client POM and its runtime JARs, verifies Central SHA-1 sidecars, and +records local SHA-256 hashes. Test dependencies from the published POM are excluded. +It fails explicitly on an unfamiliar runtime dependency instead of guessing its +transitive dependency closure. + +Kafka **2.6.0** needs one explicit exception: its generated ApiVersions messages +load Jackson `JsonNode`, although its published POM declares `jackson-databind` +as `provided`. This is recorded in +[KAFKA-10378](https://issues.apache.org/jira/browse/KAFKA-10378). The script supplies +the release's exact Jackson **2.10.2** databind, core and annotations JARs, all +checksum-verified. These and the diagnostic binding are listed separately under +`test_runtime_additions` in the preparation report. Other provided dependencies +and published test dependencies are not implicitly added. These additions affect +only the isolated test processes, not Fluss production dependencies. + +Start with the boundaries and representative cached releases: + +```sh +python3 fluss-kafka/src/test/client-matrix/prepare.py \ + --output /tmp/fluss-kafka-client-matrix \ + --versions 0.11.0.0,3.6.1,3.9.1,3.9.2,4.2.0,4.3.1 + +# Set JAVA_HOME to a Java 11 installation before running Maven. +./mvnw -o -pl fluss-kafka -am verify \ + -Dtest=KafkaClientMatrixITCase \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dkafka.client.matrix.file=/tmp/fluss-kafka-client-matrix/clients.properties +``` + +After the boundary run, prepare and execute the complete formal interval: + +```sh +python3 fluss-kafka/src/test/client-matrix/prepare.py \ + --output /tmp/fluss-kafka-client-matrix --versions all + +./mvnw -o -pl fluss-kafka -am verify \ + -Dtest=KafkaClientMatrixITCase \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dkafka.client.matrix.file=/tmp/fluss-kafka-client-matrix/clients.properties +``` + +Do not remove `-am`: server, client, RPC and test artifacts must come from this +worktree's reactor, not another branch's installed SNAPSHOTs. A normal build +without `kafka.client.matrix.file` still runs an isolated smoke test using the +module's actual declared client version; that single-version run is not a matrix. + +## Coverage and result interpretation + +For each selected version the probe uses PLAINTEXT and +`enable.idempotence=false`, negotiates APIs through the real producer, and tests +`acks=0`, `acks=1` and `acks=all`. The same producer alternates canonical and bare +names for the same table, checking acknowledged topic names and offsets. Native +readback verifies all 12 records, their real storage offsets, keys, last-header +wins semantics and null binary rescue fields. Bucket leaders are ready before +unacknowledged writes begin. Producer connections stay open until the parent +confirms native readback; a stdin handshake then permits the probe to close. This +avoids racing unacknowledged appends against connection-close cancellation. + +Preparation is not execution. `preparation.json` lists every requested version as +prepared, unavailable or not executed, including exact artifacts and any errors. +The script deletes the previous executable manifest at startup, including when +metadata retrieval or parsing fails. It only publishes `clients.properties` when +all requested versions are prepared. If downloads are blocked, keep that failure +report and make an explicit, separately recorded subset request for the available +versions; never call the subset a full interval pass. + +Runtime evidence lives under: + +- `fluss-kafka/target/client-matrix-results//command.txt` +- `fluss-kafka/target/client-matrix-results//client.log` +- `fluss-kafka/target/client-matrix-results//result.txt` +- `fluss-kafka/target/surefire-reports/TEST-org.apache.fluss.kafka.KafkaClientMatrixITCase.xml` + +Each invocation deletes the old result and log, writes `FAIL` with the error on +failure, and only writes `PASS` after successful native readback. A client failure +fails its parameterized test; other prepared versions still run. A missing JAR, +version mismatch, process timeout or unsupported JDK is a failure, not a skipped +compatibility result. Archive the preparation report, manifest, Maven log, runtime +logs and current Surefire XML together before `clean` or another matrix run. +The POM runs the selected tests in both Surefire phases; count distinct versions +once and require both executions to succeed. + +This is a bounded Produce compatibility acceptance flow, not proof of every +operation in the FIP. Leadership, lifecycle, min-ISR, invalid-record atomicity and +null-value filtering run separately in the cumulative integration tests using the +module client. They are not yet repeated for every matrix version. SASL is reserved +for PR19. Idempotence, other client SDKs, pre-releases, versions outside the target +interval, and persisted per-row `_kafka_metadata.offset` are not covered here. diff --git a/fluss-kafka/src/test/client-matrix/prepare.py b/fluss-kafka/src/test/client-matrix/prepare.py new file mode 100644 index 00000000000..448949bf003 --- /dev/null +++ b/fluss-kafka/src/test/client-matrix/prepare.py @@ -0,0 +1,162 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Prepare isolated, checksum-verified client classpaths from Maven Central.""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from pathlib import Path + +CENTRAL = "https://repo.maven.apache.org/maven2/" +NS = {"m": "http://maven.apache.org/POM/4.0.0"} +RUNTIME = {"slf4j-api", "lz4", "lz4-java", "snappy-java", "zstd-jni"} + + +def fetch(relative, destination, refresh=False): + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists() and not refresh: + return + temporary = destination.with_name(destination.name + ".download") + try: + subprocess.run(["curl", "--fail", "--location", "--silent", "--show-error", + "--connect-timeout", "10", "--max-time", "120", "--retry", "2", + CENTRAL + relative, "--output", str(temporary)], check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + + +def jar(cache, group, artifact, version): + relative = "/".join([group.replace(".", "/"), artifact, version, + artifact + "-" + version + ".jar"]) + target = cache / relative + fetch(relative, target) + checksum = target.with_suffix(".jar.sha1") + fetch(relative + ".sha1", checksum) + expected = checksum.read_text().strip().split()[0].lower() + actual = hashlib.sha1(target.read_bytes()).hexdigest() + if actual != expected: + raise ValueError("Maven Central checksum mismatch: " + str(target)) + return target.resolve() + + +def prepare(output, version): + cache = output / "artifacts" + relative = "org/apache/kafka/kafka-clients/" + version + "/kafka-clients-" + version + ".pom" + pom = cache / relative + fetch(relative, pom) + root = ET.parse(pom).getroot() + paths = [jar(cache, "org.apache.kafka", "kafka-clients", version)] + dependencies = [] + test_runtime = [] + for dep in root.findall("m:dependencies/m:dependency", NS): + scope = dep.findtext("m:scope", "compile", NS) + group = dep.findtext("m:groupId", namespaces=NS) + artifact = dep.findtext("m:artifactId", namespaces=NS) + dep_version = dep.findtext("m:version", namespaces=NS) + # Kafka 2.6.0 generated ApiVersions messages require JsonNode even for + # byte-array producers, but its POM marks databind provided (KAFKA-10378). + # Supply the release's exact Jackson version and its two runtime dependencies. + if version == "2.6.0" and group == "com.fasterxml.jackson.core" and artifact == "jackson-databind": + if scope != "provided" or dep_version != "2.10.2": + raise ValueError("Unexpected Kafka 2.6.0 Jackson dependency: " + ET.tostring(dep).decode()) + for jackson in ("jackson-databind", "jackson-core", "jackson-annotations"): + paths.append(jar(cache, group, jackson, dep_version)) + test_runtime.append({"coordinate": ":".join([group, jackson, dep_version]), + "reason": "Kafka 2.6.0 provided Jackson closure; KAFKA-10378"}) + continue + if scope not in ("compile", "runtime") or dep.findtext("m:optional", "false", NS) == "true": + continue + # These published Kafka runtime libraries are standalone JARs. Fail on a new + # dependency rather than silently constructing an incomplete classpath. + if artifact not in RUNTIME or not dep_version or "${" in dep_version: + raise ValueError("Unsupported dependency; resolve its runtime closure explicitly: " + ET.tostring(dep).decode()) + paths.append(jar(cache, group, artifact, dep_version)) + dependencies.append(":".join([group, artifact, dep_version])) + if group == "org.slf4j" and artifact == "slf4j-api": + # Without a binding, Kafka's network-thread failures disappear into NOP logging. + paths.append(jar(cache, group, "slf4j-simple", dep_version)) + test_runtime.append({"coordinate": ":".join([group, "slf4j-simple", dep_version]), + "reason": "Isolated client diagnostics; matches published slf4j-api"}) + return paths, dependencies, test_runtime + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--versions", default="0.11.0.0,3.6.1,3.9.1,3.9.2,4.2.0,4.3.1", + help="Comma-separated formal releases, or 'all' for the FIP interval") + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + manifest = output / "clients.properties" + manifest.unlink(missing_ok=True) + metadata_path = output / "maven-metadata.xml" + try: + fetch("org/apache/kafka/kafka-clients/maven-metadata.xml", metadata_path, refresh=True) + except Exception as exc: + (output / "preparation.json").write_text(json.dumps( + {"status": "METADATA_UNAVAILABLE_NOT_EXECUTED", "error": str(exc)}, indent=2) + "\n") + return 1 + root = ET.parse(metadata_path).getroot() + versions = [v.text for v in root.findall("versioning/versions/version") + if re.fullmatch(r"[0-9]+(?:\.[0-9]+){2,3}", v.text)] + def number(version): + return tuple(int(x) for x in version.split(".")) + eligible = sorted([v for v in versions if number("0.11.0.0") <= number(v) <= number("4.3.1")], key=number) + requested = eligible if args.versions == "all" else args.versions.split(",") + report = {"source": CENTRAL + "org/apache/kafka/kafka-clients/maven-metadata.xml", + "prepared_at": datetime.now(timezone.utc).isoformat(), + "metadata_updated": root.findtext("versioning/lastUpdated"), + "formal_releases_in_scope": eligible, "requested": requested, + "not_requested": [v for v in eligible if v not in requested], + "clients": {v: {"status": "NOT_EXECUTED"} for v in requested}} + entries = [] + for version in requested: + try: + if version not in eligible: + raise ValueError("Version is absent from the official formal-release interval") + paths, dependencies, test_runtime = prepare(output, version) + entries.append(version + "=" + os.pathsep.join(str(p) for p in paths)) + report["clients"][version] = {"status": "PREPARED_NOT_EXECUTED", "dependencies": dependencies, + "test_runtime_additions": test_runtime, + "jars": [{"path": str(p), "sha256": hashlib.sha256(p.read_bytes()).hexdigest()} for p in paths]} + except Exception as exc: + detail = getattr(exc, "stderr", b"") + report["clients"][version] = {"status": "UNAVAILABLE_NOT_EXECUTED", + "error": str(exc) + " " + (detail.decode(errors="replace") if isinstance(detail, bytes) else str(detail))} + print(version, report["clients"][version]["status"], flush=True) + (output / "preparation.json").write_text(json.dumps(report, indent=2) + "\n") + # Do not publish a partial executable matrix when a requested version is missing. + manifest.unlink(missing_ok=True) + if any(v["status"] == "UNAVAILABLE_NOT_EXECUTED" for v in report["clients"].values()): + return 1 + manifest.write_text("\n".join(entries) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java new file mode 100644 index 00000000000..dd378dece44 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaClientMatrixITCase.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.matrix.KafkaClientProbe; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.utils.AppInfoParser; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.File; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Runs actual, version-checked Kafka clients in isolated JVMs and verifies native Fluss storage. + */ +class KafkaClientMatrixITCase { + private static final String DATABASE = "client_matrix"; + private static final Path OUTPUT = + Paths.get(System.getProperty("project.basedir"), "target", "client-matrix-results"); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + static Stream clients() throws Exception { + return installations().stringPropertyNames().stream().sorted(); + } + + private static Properties installations() throws Exception { + Properties properties = new Properties(); + String manifest = System.getProperty("kafka.client.matrix.file"); + if (manifest != null) { + try (InputStream input = Files.newInputStream(Paths.get(manifest))) { + properties.load(input); + } + assertThat(properties).as("The requested matrix must not be empty").isNotEmpty(); + } else { + // Normal builds still exercise isolation. Other versions require an explicit manifest. + String classpath = + Arrays.stream( + System.getProperty("java.class.path") + .split(Pattern.quote(File.pathSeparator))) + .filter( + entry -> + Paths.get(entry) + .getFileName() + .toString() + .matches( + "(kafka-clients|slf4j-api|zstd-jni|snappy-java|lz4|lz4-java)-.+\\.jar")) + .collect(Collectors.joining(File.pathSeparator)); + properties.setProperty(AppInfoParser.getVersion(), classpath); + } + return properties; + } + + @ParameterizedTest(name = "Kafka client {0}") + @MethodSource("clients") + void testIsolatedClientWithAllAckModesAndBothNames(String version) throws Exception { + Path output = OUTPUT.resolve(version); + Files.createDirectories(output); + Files.deleteIfExists(output.resolve("result.txt")); + Files.deleteIfExists(output.resolve("client.log")); + Files.deleteIfExists(output.resolve("command.txt")); + try { + runClient(version, output); + } catch (Exception | AssertionError failure) { + Files.write( + output.resolve("result.txt"), + Arrays.asList("FAIL " + version, failure.toString()), + StandardCharsets.UTF_8); + throw failure; + } + } + + private static void runClient(String version, Path output) throws Exception { + String classpath = installations().getProperty(version); + List jars = Arrays.asList(classpath.split(Pattern.quote(File.pathSeparator))); + assertThat(jars).allSatisfy(jar -> assertThat(Paths.get(jar)).isRegularFile()); + assertThat( + jars.stream() + .filter( + jar -> + Paths.get(jar) + .getFileName() + .toString() + .startsWith("kafka-clients-")) + .collect(Collectors.toList())) + .singleElement() + .satisfies( + jar -> + assertThat(Paths.get(jar).getFileName().toString()) + .isEqualTo("kafka-clients-" + version + ".jar")); + // Export just the probe bytecode, never the server/test classpath, to the child JVM. + String resource = KafkaClientProbe.class.getName().replace('.', '/') + ".class"; + Path probe = output.resolve("probe").resolve(resource); + Files.createDirectories(probe.getParent()); + try (InputStream input = + KafkaClientProbe.class.getClassLoader().getResourceAsStream(resource)) { + assertThat(input).isNotNull(); + Files.copy(input, probe, StandardCopyOption.REPLACE_EXISTING); + } + String prefix = "client_" + version.replace('.', '_'); + List paths = new ArrayList<>(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, prefix + "_" + acks); + paths.add(path); + admin.createTable(path, descriptor(), false).get(); + CLUSTER.waitAndGetLeaderReplica( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); + } + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + List command = + Arrays.asList( + Paths.get(System.getProperty("java.home"), "bin", "java") + .toString(), + "-cp", + output.resolve("probe") + File.pathSeparator + classpath, + KafkaClientProbe.class.getName(), + version, + node.host() + ":" + node.port(), + DATABASE, + prefix); + Files.write(output.resolve("command.txt"), command, StandardCharsets.UTF_8); + Path log = output.resolve("client.log"); + Process process = + new ProcessBuilder(command) + .redirectErrorStream(true) + .redirectOutput(log.toFile()) + .start(); + try { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(120); + String text = ""; + while (process.isAlive() && System.nanoTime() < deadline) { + text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + if (text.contains("WRITES_COMPLETE")) { + break; + } + process.waitFor(100, TimeUnit.MILLISECONDS); + } + text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + assertThat(text) + .as("Client %s log: %s", version, log) + .contains( + "VERSION=" + version, + "JAR=", + "JAVA=", + "ACKS=0:SENT", + "ACKS=1:SENT", + "ACKS=all:SENT", + "WRITES_COMPLETE"); + assertThat(process.isAlive()) + .as("Client must stay connected during native readback") + .isTrue(); + for (String acks : Arrays.asList("0", "1", "all")) { + assertNativeReadback( + connection, + TablePath.of(DATABASE, prefix + "_" + acks), + version, + acks); + } + process.getOutputStream().write('\n'); + process.getOutputStream().flush(); + assertThat(process.waitFor(30, TimeUnit.SECONDS)) + .as("Client log: %s", log) + .isTrue(); + text = new String(Files.readAllBytes(log), StandardCharsets.UTF_8); + assertThat(process.exitValue()).as("Client %s: %s", version, text).isZero(); + assertThat(text).contains("NATIVE_READBACK_CONFIRMED"); + Files.write( + output.resolve("result.txt"), + Collections.singletonList( + "PASS " + + version + + " PLAINTEXT non-idempotent; acks=0,1,all; same producer full/alias; 12 native rows and offsets verified"), + StandardCharsets.UTF_8); + } finally { + if (process.isAlive()) { + process.destroyForcibly(); + process.waitFor(10, TimeUnit.SECONDS); + } + } + } finally { + for (TablePath path : paths) { + admin.dropTable(path, true).get(); + } + } + } + } + + private static void assertNativeReadback( + Connection connection, TablePath path, String version, String acks) throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < 4 && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + InternalRow row = record.getRow(); + assertThat(row.getBytes(0)) + .isEqualTo( + (version + ":" + acks + ":" + count) + .getBytes(StandardCharsets.UTF_8)); + assertThat(row.getBytes(1)).isEqualTo("key".getBytes(StandardCharsets.UTF_8)); + assertThat(row.getMap(2).size()).isEqualTo(1); + assertThat(row.getMap(2).keyArray().getString(0).toString()) + .isEqualTo("source"); + assertThat(row.getMap(2).valueArray().getBytes(0)).containsExactly((byte) 2); + assertThat(row.isNullAt(3)).isTrue(); + assertThat(record.logOffset()).isEqualTo(count); + count++; + } + } + assertThat(count) + .as("Native readback for client %s, acks %s", version, acks) + .isEqualTo(4); + } + } + + private static TableDescriptor descriptor() { + return TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index e524f1faabf..10c9dbc1d86 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -254,7 +254,7 @@ void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Excepti try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); - for (String format : new String[] {"raw", "string"}) { + for (String format : new String[] {"binary", "string"}) { TablePath path = TablePath.of(DATABASE, "null_values_" + format); admin.createTable(path, descriptor(format), false).get(); CLUSTER.waitUntilAllGatewayHasSameMetadata(); @@ -294,7 +294,7 @@ void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Excepti for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { InternalRow row = record.getRow(); byte[] expected = count == 0 ? new byte[0] : VALUE; - if (format.equals("raw")) { + if (format.equals("binary")) { assertThat(row.getBytes(0)).isEqualTo(expected); } else { assertThat(row.getString(0).toString()) @@ -302,8 +302,9 @@ void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Excepti new String(expected, StandardCharsets.UTF_8)); } assertThat(row.isNullAt(3)).isTrue(); - assertThat(row.getTimestampNtz(1, 3).getMillisecond()) - .isEqualTo(123L); + assertThat(row.getMap(1).size()).isZero(); + assertThat(row.isNullAt(2)).isTrue(); + assertThat(record.logOffset()).isEqualTo(count); count++; } } @@ -483,7 +484,25 @@ void testDeletedDefaultDatabaseDoesNotFallbackOrRecreateDatabase() throws Except query(socket, existing.toString()), new String[] {existing.toString()}, 0L); assertProduceOffsets( query(socket, existing.toString()), new String[] {existing.toString()}, 1L); - assertReadback(connection, existing, "binary", VALUE); + try (KafkaProducer producer = producer(node, "all", 0)) { + assertThat( + producer.send( + new ProducerRecord<>( + existing.toString(), + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader( + "source", VALUE), + new RecordHeader( + "source", null)))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(2L); + } + assertReadback(connection, existing, "binary", VALUE, 3); assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); assertThat(admin.databaseExists("another_missing_db").get()).isFalse(); } @@ -691,7 +710,11 @@ private static KafkaProducer producer(String acks) { } private static KafkaProducer producer(String acks, int batchSize) { - ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + return producer(CLUSTER.getTabletServerNodes("KAFKA").get(0), acks, batchSize); + } + + private static KafkaProducer producer( + ServerNode node, String acks, int batchSize) { Map config = new HashMap<>(); config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java index 32b0f2a2ced..8066f396991 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -32,11 +32,11 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.cluster.AlterConfig; import org.apache.fluss.config.cluster.AlterConfigOpType; -import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.server.coordinator.event.RecoverRebalanceEvent; @@ -93,7 +93,7 @@ /** Regression coverage for native DDL lifecycle, metadata refresh, and Produce minimum ISR. */ class KafkaProduceRegressionITCase { - private static final String DATABASE = "kafka"; + private static final String DATABASE = "fluss"; private static final short METADATA_VERSION = 11; @RegisterExtension @@ -321,6 +321,292 @@ void testClusterMinIsrPolicyAndDynamicReload() throws Exception { } } + @Test + void testSameProducerAliasesFollowEnablementPropertiesAndRecreation() throws Exception { + TablePath path = TablePath.of(DATABASE, "cached_lifecycle"); + String full = path.toString(); + String alias = path.getTableName(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + Uuid fullId = topicId(full); + Uuid aliasId = topicId(alias); + assertThat(aliasId).isNotEqualTo(Uuid.ZERO_UUID).isNotEqualTo(fullId); + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, full, "full").offset()).isZero(); + assertThat(send(producer, alias, "alias").offset()).isEqualTo(1L); + setTableEnabled(admin, path, false); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setTableEnabled(admin, path, true); + assertThat(topicId(full)).isEqualTo(fullId); + assertThat(topicId(alias)).isEqualTo(aliasId); + assertThat(send(producer, alias, "enabled").offset()).isEqualTo(2L); + Map properties = + admin.getTableInfo(path).get().getProperties().toMap(); + assertThatThrownBy( + () -> + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE + .key(), + "string")), + false) + .get()) + .isInstanceOf(ExecutionException.class); + assertThat(admin.getTableInfo(path).get().getProperties().toMap()) + .isEqualTo(properties); + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE + .key(), + "BINARY")), + false) + .get(); + assertThat(admin.getTableInfo(path).get().getProperties().toMap()) + .containsEntry( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "BINARY"); + assertThat(send(producer, full, "property").offset()).isEqualTo(3L); + assertValues(connection, path, "full", "alias", "enabled", "property"); + admin.dropTable(path, false).get(); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + createTable(admin, path, 3); + assertThat(topicId(full)).isNotEqualTo(fullId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(alias)).isNotEqualTo(aliasId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(full)).isNotEqualTo(topicId(alias)); + assertThat(send(producer, full, "recreated_full").offset()).isZero(); + assertThat(send(producer, alias, "recreated_alias").offset()).isEqualTo(1L); + assertValues(connection, path, "recreated_full", "recreated_alias"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testNeverEnabledTableRejectsBothNamesWithoutAppending() throws Exception { + TablePath path = TablePath.of(DATABASE, "never_enabled"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + "binary") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setTableEnabled(admin, path, true); + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, path.getTableName(), "accepted").offset()).isZero(); + assertValues(connection, path, "accepted"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testInvalidRecordRejectsWholePartitionBeforeNativeAppend() throws Exception { + TablePath path = TablePath.of(DATABASE, "atomic_conversion"); + byte[] key = new byte[] {1}; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_key", + DataTypes.BYTES().copy(false)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + "binary") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + try (KafkaProducer producer = producer("all", true)) { + int leader = producer.partitionsFor(path.toString()).get(0).leader().id(); + ServerNode node = + CLUSTER.getTabletServerNodes("KAFKA").stream() + .filter(n -> n.id() == leader) + .findFirst() + .get(); + try (Socket socket = connect(node)) { + ProduceResponse rejected = + (ProduceResponse) + sendRequest( + socket, + produceRequest( + path.toString(), + new SimpleRecord( + key, + "valid_prefix" + .getBytes( + StandardCharsets + .UTF_8)), + new SimpleRecord( + null, + "invalid_key" + .getBytes( + StandardCharsets + .UTF_8)))); + assertThat(rejected.errorCounts()).containsEntry(Errors.INVALID_RECORD, 1); + assertThat( + rejected.data() + .responses() + .find(path.toString()) + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(-1L); + } + assertThat( + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + key, + "accepted" + .getBytes( + StandardCharsets + .UTF_8))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + assertValues(connection, path, "accepted"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testReplicatedOffsetsSurviveLeaderLossAndRestart() throws Exception { + TablePath path = TablePath.of(DATABASE, "durable_restart"); + int stopped = -1; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + TableBucket bucket = + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0); + MetadataResponsePartition initial = + waitForTopic(path.toString()) + .data() + .topics() + .find(path.toString()) + .partitions() + .get(0); + for (int replica : initial.replicaNodes()) { + CLUSTER.waitUntilReplicaExpandToIsr(bucket, replica); + } + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, path.toString(), "before").offset()).isZero(); + stopped = initial.leaderId(); + CLUSTER.stopTabletServer(stopped); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + waitForTopic(path.toString()) + .data() + .topics() + .find(path.toString()) + .partitions() + .get(0) + .leaderId()) + .isNotEqualTo(initial.leaderId())); + assertThat(send(producer, path.toString(), "after_loss").offset()) + .isEqualTo(1L); + CLUSTER.startTabletServer(stopped); + CLUSTER.assertHasTabletServerNumber(3); + CLUSTER.waitUntilReplicaExpandToIsr(bucket, stopped); + assertThat(send(producer, path.getTableName(), "after_restart").offset()) + .isEqualTo(2L); + assertValues(connection, path, "before", "after_loss", "after_restart"); + } + } finally { + if (stopped >= 0 && CLUSTER.getTabletServerById(stopped) == null) { + CLUSTER.startTabletServer(stopped); + CLUSTER.assertHasTabletServerNumber(3); + } + admin.dropTable(path, true).get(); + } + } + } + + private static void setTableEnabled(Admin admin, TablePath path, boolean enabled) + throws Exception { + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled))), + false) + .get(); + } + + private static Uuid topicId(String topic) { + return waitForTopic(topic).data().topics().find(topic).topicId(); + } + + private static void assertRejectedNames(TablePath path, Errors error) throws Exception { + retry( + Duration.ofMinutes(1), + () -> { + for (String topic : new String[] {path.toString(), path.getTableName()}) { + for (ServerNode node : CLUSTER.getTabletServerNodes("KAFKA")) { + try (Socket socket = connect(node)) { + MetadataResponse response = + (MetadataResponse) + sendRequest(socket, metadataRequest(topic)); + assertThat(response.errors()).containsEntry(topic, error); + } + } + } + }); + for (String topic : new String[] {path.toString(), path.getTableName()}) { + for (ServerNode node : CLUSTER.getTabletServerNodes("KAFKA")) { + try (Socket socket = connect(node)) { + ProduceResponse response = + (ProduceResponse) + sendRequest(socket, produceRequest(topic, "rejected")); + assertThat(response.errorCounts()).containsEntry(error, 1); + } + } + } + } + private static void createTable(Admin admin, TablePath path, int replicas) throws Exception { admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); admin.createTable( @@ -328,12 +614,13 @@ private static void createTable(Admin admin, TablePath path, int replicas) throw TableDescriptor.builder() .schema( Schema.newBuilder() - .column("value", DataTypes.BYTES()) + .column("_kafka_value", DataTypes.BYTES()) .build()) .distributedBy(1) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") .build(), false) .get(); @@ -371,6 +658,7 @@ private static void assertValues(Connection connection, TablePath path, String.. () -> { ScanRecords records = scanner.poll(Duration.ofSeconds(1)); for (ScanRecord record : records) { + assertThat(record.logOffset()).isEqualTo(values.size()); values.add( new String( record.getRow().getBytes(0), StandardCharsets.UTF_8)); @@ -426,9 +714,11 @@ private static MetadataRequest metadataRequest(String topic) { } private static ProduceRequest produceRequest(String topic, String value) { - MemoryRecords records = - MemoryRecords.withRecords( - Compression.NONE, new SimpleRecord(value.getBytes(StandardCharsets.UTF_8))); + return produceRequest(topic, new SimpleRecord(value.getBytes(StandardCharsets.UTF_8))); + } + + private static ProduceRequest produceRequest(String topic, SimpleRecord... values) { + MemoryRecords records = MemoryRecords.withRecords(Compression.NONE, values); ProduceRequestData.TopicProduceData topicData = new ProduceRequestData.TopicProduceData() .setName(topic) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java new file mode 100644 index 00000000000..b0dd5a0e146 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/matrix/KafkaClientProbe.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.matrix; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.utils.AppInfoParser; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Standalone client process: deliberately depends only on the JDK and the selected Kafka client. + */ +public final class KafkaClientProbe { + private KafkaClientProbe() {} + + /** Writes through a real endpoint; a failed check terminates the isolated process. */ + public static void main(String[] args) throws Exception { + String expectedVersion = args[0]; + String actualVersion = AppInfoParser.getVersion(); + if (!expectedVersion.equals(actualVersion)) { + throw new IllegalStateException( + "Expected " + expectedVersion + " but loaded " + actualVersion); + } + System.out.println("VERSION=" + actualVersion); + System.out.println( + "JAR=" + KafkaProducer.class.getProtectionDomain().getCodeSource().getLocation()); + System.out.println("JAVA=" + System.getProperty("java.version")); + System.out.println("CLASSPATH=" + System.getProperty("java.class.path")); + List> producers = new ArrayList<>(); + try { + for (String acks : Arrays.asList("0", "1", "all")) { + Map config = new HashMap<>(); + config.put("bootstrap.servers", args[1]); + config.put("key.serializer", ByteArraySerializer.class); + config.put("value.serializer", ByteArraySerializer.class); + config.put("enable.idempotence", false); + config.put("security.protocol", "PLAINTEXT"); + config.put("acks", acks); + config.put("max.in.flight.requests.per.connection", 1); + config.put("request.timeout.ms", 10000); + config.put("max.block.ms", 30000); + config.put("retries", 3); + config.put("compression.type", "none"); + config.put("linger.ms", 0); + String alias = args[3] + "_" + acks; + KafkaProducer producer = new KafkaProducer<>(config); + producers.add(producer); + for (int i = 0; i < 4; i++) { + String topic = i % 2 == 0 ? args[2] + "." + alias : alias; + RecordMetadata result = + producer.send( + new ProducerRecord<>( + topic, + 0, + 123L, + "key".getBytes(StandardCharsets.UTF_8), + (actualVersion + ":" + acks + ":" + i) + .getBytes(StandardCharsets.UTF_8), + Arrays.asList( + new RecordHeader( + "source", new byte[] {1}), + new RecordHeader( + "source", new byte[] {2})))) + .get(30, TimeUnit.SECONDS); + if (!result.topic().equals(topic) + || (!acks.equals("0") && result.offset() != i)) { + throw new IllegalStateException("Unexpected acknowledgement " + result); + } + } + producer.flush(); + System.out.println("ACKS=" + acks + ":SENT"); + } + // acks=0 only confirms a socket write. Keep connections open until the + // parent verifies native storage, rather than racing cancellation on close. + System.out.println("WRITES_COMPLETE"); + if (System.in.read() != '\n') { + throw new IllegalStateException("Native readback was not confirmed by the parent"); + } + System.out.println("NATIVE_READBACK_CONFIRMED"); + } finally { + for (KafkaProducer producer : producers) { + producer.close(); + } + } + } +} From 48d1d0f944d42f21d3267d2b89dbece546e1a6c7 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 15 Oct 2026 22:40:15 +0800 Subject: [PATCH 46/48] [kafka] Verify metadata across schema changes and leader recovery Verify that a cached producer using both topic names picks up an added metadata column. Read back the schema ID and all three metadata fields through the native client after online leader transfer, leader loss, and restart. Transfer leadership back to the recovered replica to prove it retains records written while it was offline. The source matches the verified PR07-based snapshot byte for byte: 325 distinct tests pass, with two pre-existing disabled RPC cases. All nine Produce regression cases pass in both execution phases; Spotless, Checkstyle, and RAT pass. The isolated client check used 3.9.2; the full 72-version matrix was not rerun for this snapshot. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 325/325 --- .../kafka/KafkaProduceRegressionITCase.java | 325 +++++++++++++++--- 1 file changed, 277 insertions(+), 48 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java index 8066f396991..5d016ac5c9b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -38,11 +38,14 @@ import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; import org.apache.fluss.server.coordinator.event.RecoverRebalanceEvent; import org.apache.fluss.server.testutils.FlussClusterExtension; import org.apache.fluss.server.zk.data.RebalanceTask; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; @@ -66,6 +69,7 @@ import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.assertj.core.groups.Tuple; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -199,9 +203,9 @@ void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { - createTable(admin, path, 3); - TableBucket bucket = - new TableBucket(admin.getTableInfo(path).get().getTableId(), 0); + createTable(admin, path, 3, metadataSchema(), 1); + TableInfo tableInfo = admin.getTableInfo(path).get(); + TableBucket bucket = new TableBucket(tableInfo.getTableId(), 0); MetadataResponsePartition initial = waitForTopic(topic).data().topics().find(topic).partitions().get(0); int oldLeader = initial.leaderId(); @@ -222,32 +226,21 @@ void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { (MetadataResponse) sendRequest(oldLeaderSocket, metadataRequest(topic)); assertThat(before.data().topics().find(topic).partitions().get(0).leaderId()) .isEqualTo(oldLeader); - assertThat(send(producer, topic, "before_handoff").offset()).isZero(); + assertThat(send(producer, topic, 0, 123L, "before_handoff").offset()).isZero(); + assertRecordsWithMetadata( + connection, + path, + 0, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before_handoff", 123L, 0, 0)); CLUSTER.waitUntilReplicaExpandToIsr(bucket, newLeader); double retriesBefore = producerMetric(producer, "record-retry-total"); double disconnectsBefore = producerMetric(producer, "connection-close-total"); rebalanceLeader(bucket, initial, newLeader, rebalanceId); - retry( - Duration.ofMinutes(1), - () -> { - assertThat(admin.listRebalanceProgress(rebalanceId).get()) - .isPresent() - .get() - .extracting(RebalanceProgress::status) - .isEqualTo(RebalanceStatus.COMPLETED); - MetadataResponsePartition updated = - fetchMetadata(topic) - .data() - .topics() - .find(topic) - .partitions() - .get(0); - assertThat(updated.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(updated.leaderId()).isEqualTo(newLeader); - assertThat(updated.leaderEpoch()) - .isGreaterThan(initial.leaderEpoch()); - }); + waitForLeaderChange( + admin, topic, 0, newLeader, initial.leaderEpoch(), rebalanceId); CLUSTER.waitAndGetFollowerReplica(bucket, oldLeader); assertThat(CLUSTER.getTabletServerNodes("KAFKA")) .containsExactlyInAnyOrderElementsOf(nodes); @@ -269,14 +262,22 @@ void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { // Metadata requests above must not refresh this route before the next send. assertThat(producer.partitionsFor(topic).get(0).leader().id()) .isEqualTo(oldLeader); - assertThat(send(producer, topic, "after_handoff").offset()).isEqualTo(1L); + assertThat(send(producer, topic, 0, 456L, "after_handoff").offset()) + .isEqualTo(1L); assertThat(producerMetric(producer, "record-retry-total")) .isGreaterThan(retriesBefore); assertThat(producerMetric(producer, "connection-close-total")) .isEqualTo(disconnectsBefore); assertThat(producer.partitionsFor(topic).get(0).leader().id()) .isEqualTo(newLeader); - assertValues(connection, path, "before_handoff", "after_handoff"); + assertRecordsWithMetadata( + connection, + path, + 0, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before_handoff", 123L, 0, 0), + tuple(1L, "after_handoff", 456L, 0, 0)); } } finally { try { @@ -389,6 +390,68 @@ void testSameProducerAliasesFollowEnablementPropertiesAndRecreation() throws Exc } } + @Test + void testSameProducerAliasesUseAddedMetadataColumn() throws Exception { + TablePath path = TablePath.of(DATABASE, "cached_schema_alter"); + String full = path.toString(); + String alias = path.getTableName(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + TableInfo before = admin.getTableInfo(path).get(); + Uuid fullId = topicId(full); + Uuid aliasId = topicId(alias); + try (KafkaProducer producer = producer("all", true)) { + // Populate both name mappings and the write plan before changing the schema. + assertThat(send(producer, full, "old_full").offset()).isZero(); + assertThat(send(producer, alias, "old_alias").offset()).isEqualTo(1L); + assertValues(connection, path, "old_full", "old_alias"); + + admin.alterTable( + path, + Collections.singletonList( + TableChange.addColumn( + "_kafka_metadata", + metadataType(), + null, + TableChange.ColumnPosition.last())), + false) + .get(); + TableInfo after = admin.getTableInfo(path).get(); + assertThat(after.getTableId()).isEqualTo(before.getTableId()); + assertThat(after.getSchemaId()).isGreaterThan(before.getSchemaId()); + assertThat(after.getProperties().toMap()) + .isEqualTo(before.getProperties().toMap()); + assertThat(after.getSchema().getColumns()).hasSize(2); + assertThat(after.getSchema().getColumn("_kafka_metadata").getDataType()) + .isEqualTo(metadataType()); + + // Neither producer recreation nor a Kafka Metadata refresh should be needed. + assertThat(send(producer, full, 0, 789L, "new_full").offset()).isEqualTo(2L); + assertThat(send(producer, alias, 0, 987L, "new_alias").offset()).isEqualTo(3L); + assertThat(topicId(full)).isEqualTo(fullId); + assertThat(topicId(alias)).isEqualTo(aliasId); + // Read the new batches using the new native schema, independently of the + // native client's earlier cached table handle. + try (Connection reader = + ConnectionFactory.createConnection(CLUSTER.getClientConfig())) { + assertRecordsWithMetadata( + reader, + path, + 0, + 2L, + after.getSchemaId(), + tuple(2L, "new_full", 789L, 0, 0), + tuple(3L, "new_alias", 987L, 0, 0)); + } + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + @Test void testNeverEnabledTableRejectsBothNamesWithoutAppending() throws Exception { TablePath path = TablePath.of(DATABASE, "never_enabled"); @@ -510,56 +573,108 @@ void testInvalidRecordRejectsWholePartitionBeforeNativeAppend() throws Exception } @Test - void testReplicatedOffsetsSurviveLeaderLossAndRestart() throws Exception { + void testReplicatedOffsetsAndMetadataSurviveLeaderLossAndRestart() throws Exception { TablePath path = TablePath.of(DATABASE, "durable_restart"); + String topic = path.toString(); + String rebalanceId = "kafka-recovered-leader"; + int partition = 1; int stopped = -1; try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { try { - createTable(admin, path, 3); - TableBucket bucket = - new TableBucket(admin.getTableInfo(path).get().getTableId(), 0); + createTable(admin, path, 3, metadataSchema(), 2); + TableInfo tableInfo = admin.getTableInfo(path).get(); + TableBucket bucket = new TableBucket(tableInfo.getTableId(), partition); MetadataResponsePartition initial = - waitForTopic(path.toString()) + waitForTopic(topic, 2) .data() .topics() - .find(path.toString()) + .find(topic) .partitions() - .get(0); + .get(partition); for (int replica : initial.replicaNodes()) { CLUSTER.waitUntilReplicaExpandToIsr(bucket, replica); } try (KafkaProducer producer = producer("all", true)) { - assertThat(send(producer, path.toString(), "before").offset()).isZero(); + assertThat(send(producer, topic, partition, 123L, "before").offset()).isZero(); + assertRecordsWithMetadata( + connection, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition)); stopped = initial.leaderId(); CLUSTER.stopTabletServer(stopped); retry( Duration.ofMinutes(1), () -> assertThat( - waitForTopic(path.toString()) + waitForTopic(topic, 2) .data() .topics() - .find(path.toString()) + .find(topic) .partitions() - .get(0) + .get(partition) .leaderId()) .isNotEqualTo(initial.leaderId())); - assertThat(send(producer, path.toString(), "after_loss").offset()) + assertThat(send(producer, topic, partition, 456L, "after_loss").offset()) .isEqualTo(1L); + assertRecordsWithMetadata( + connection, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition), + tuple(1L, "after_loss", 456L, 0, partition)); CLUSTER.startTabletServer(stopped); CLUSTER.assertHasTabletServerNumber(3); CLUSTER.waitUntilReplicaExpandToIsr(bucket, stopped); - assertThat(send(producer, path.getTableName(), "after_restart").offset()) + // Make the recovered replica serve reads and writes, including the record + // produced while it was offline. ISR membership alone does not prove readback. + MetadataResponsePartition current = + waitForTopic(topic, 2) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + rebalanceLeader(bucket, current, stopped, rebalanceId); + waitForLeaderChange( + admin, topic, partition, stopped, current.leaderEpoch(), rebalanceId); + assertThat( + send( + producer, + path.getTableName(), + partition, + 789L, + "after_restart") + .offset()) .isEqualTo(2L); - assertValues(connection, path, "before", "after_loss", "after_restart"); + try (Connection reader = + ConnectionFactory.createConnection(CLUSTER.getClientConfig())) { + assertRecordsWithMetadata( + reader, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition), + tuple(1L, "after_loss", 456L, 0, partition), + tuple(2L, "after_restart", 789L, 0, partition)); + } } } finally { if (stopped >= 0 && CLUSTER.getTabletServerById(stopped) == null) { CLUSTER.startTabletServer(stopped); CLUSTER.assertHasTabletServerNumber(3); } - admin.dropTable(path, true).get(); + try { + admin.cancelRebalance(rebalanceId).get(); + } finally { + admin.dropTable(path, true).get(); + } } } } @@ -608,15 +723,23 @@ private static void assertRejectedNames(TablePath path, Errors error) throws Exc } private static void createTable(Admin admin, TablePath path, int replicas) throws Exception { + createTable( + admin, + path, + replicas, + Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build(), + 1); + } + + private static void createTable( + Admin admin, TablePath path, int replicas, Schema schema, int buckets) + throws Exception { admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); admin.createTable( path, TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .distributedBy(1) + .schema(schema) + .distributedBy(buckets) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) @@ -627,6 +750,20 @@ private static void createTable(Admin admin, TablePath path, int replicas) throw CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); } + private static Schema metadataSchema() { + return Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_metadata", metadataType()) + .build(); + } + + private static RowType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + private static void setMinIsr(Admin admin, int minIsr) throws Exception { admin.alterClusterConfigs( Collections.singletonList( @@ -668,7 +805,52 @@ private static void assertValues(Connection connection, TablePath path, String.. } } + private static void assertRecordsWithMetadata( + Connection connection, + TablePath path, + int partition, + long startOffset, + int schemaId, + Tuple... expected) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribe(partition, startOffset); + List actual = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (actual.size() < expected.length && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.getSchemaId()) + .as("Schema at offset %s", record.logOffset()) + .isEqualTo(schemaId); + InternalRow row = record.getRow(); + assertThat(row.getFieldCount()).isEqualTo(2); + assertThat(row.isNullAt(1)).isFalse(); + InternalRow metadata = row.getRow(1, 3); + assertThat(metadata.getFieldCount()).isEqualTo(3); + assertThat(metadata.isNullAt(0)).isFalse(); + assertThat(metadata.isNullAt(1)).isFalse(); + assertThat(metadata.isNullAt(2)).isFalse(); + actual.add( + tuple( + record.logOffset(), + new String(row.getBytes(0), StandardCharsets.UTF_8), + metadata.getTimestampLtz(0, 3).getEpochMillisecond(), + metadata.getInt(1), + metadata.getInt(2))); + } + } + assertThat(actual) + .as("Native metadata readback for %s bucket %s", path, partition) + .containsExactly(expected); + } + } + private static MetadataResponse waitForTopic(String topic) { + return waitForTopic(topic, 1); + } + + private static MetadataResponse waitForTopic(String topic, int partitions) { AtomicReference result = new AtomicReference<>(); retry( Duration.ofMinutes(1), @@ -676,8 +858,10 @@ private static MetadataResponse waitForTopic(String topic) { MetadataResponse response = fetchMetadata(topic); MetadataResponseTopic metadata = response.data().topics().find(topic); assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(metadata.partitions()).hasSize(1); - assertThat(metadata.partitions().get(0).leaderId()).isNotEqualTo(-1); + assertThat(metadata.partitions()) + .hasSize(partitions) + .allSatisfy( + partition -> assertThat(partition.leaderId()).isNotEqualTo(-1)); result.set(response); }); return result.get(); @@ -796,6 +980,34 @@ private static void rebalanceLeader( .put(new RecoverRebalanceEvent(task)); } + private static void waitForLeaderChange( + Admin admin, + String topic, + int partition, + int newLeader, + int oldEpoch, + String rebalanceId) { + retry( + Duration.ofMinutes(1), + () -> { + assertThat(admin.listRebalanceProgress(rebalanceId).get()) + .isPresent() + .get() + .extracting(RebalanceProgress::status) + .isEqualTo(RebalanceStatus.COMPLETED); + MetadataResponsePartition updated = + fetchMetadata(topic) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + assertThat(updated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(updated.leaderId()).isEqualTo(newLeader); + assertThat(updated.leaderEpoch()).isGreaterThan(oldEpoch); + }); + } + private static double producerMetric(KafkaProducer producer, String name) { return producer.metrics().entrySet().stream() .filter( @@ -815,6 +1027,23 @@ private static RecordMetadata send( .get(60, TimeUnit.SECONDS); } + private static RecordMetadata send( + KafkaProducer producer, + String topic, + int partition, + long timestamp, + String value) + throws Exception { + return producer.send( + new ProducerRecord<>( + topic, + partition, + timestamp, + null, + value.getBytes(StandardCharsets.UTF_8))) + .get(60, TimeUnit.SECONDS); + } + private static KafkaProducer producer(String acks, boolean retryOnFailure) { Map config = new HashMap<>(); config.put( From 6da9e21b637255620e8d4c94c3f5d466439b90bc Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 16 Oct 2026 19:18:34 +0800 Subject: [PATCH 47/48] [kafka] Verify null-value persistence and native offsets Update the Produce regression to expect null values to be persisted in binary and string tables. Verify contiguous offsets and native readback for the null record with a binary key, followed by empty and nonempty values with null keys. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 32/32 --- .../fluss/kafka/KafkaProduceAppendITCase.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java index 10c9dbc1d86..d32fa77059b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -250,7 +250,7 @@ void testNotNullValueRejectsWholeBatchWithoutAdvancingOffsets(String format) thr } @Test - void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Exception { + void testNullAndEmptyValuesPreserveKeysAndOffsets() throws Exception { try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); @@ -270,45 +270,51 @@ void testNullValuesAreSkippedWhileEmptyValuesAndNullKeysSurvive() throws Excepti null)) .get(30, TimeUnit.SECONDS) .offset()) - .isEqualTo(-1L); + .isZero(); assertThat( producer.send( new ProducerRecord( topic, 0, 123L, null, new byte[0])) .get(30, TimeUnit.SECONDS) .offset()) - .isZero(); + .isEqualTo(1L); assertThat( producer.send( new ProducerRecord( topic, 0, 123L, null, VALUE)) .get(30, TimeUnit.SECONDS) .offset()) - .isEqualTo(1L); + .isEqualTo(2L); try (Table table = connection.getTable(path); LogScanner scanner = table.newScan().createLogScanner()) { scanner.subscribeFromBeginning(0); int count = 0; long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - while (count < 2 && System.nanoTime() < deadline) { + while (count < 3 && System.nanoTime() < deadline) { for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { InternalRow row = record.getRow(); - byte[] expected = count == 0 ? new byte[0] : VALUE; - if (format.equals("binary")) { - assertThat(row.getBytes(0)).isEqualTo(expected); + if (count == 0) { + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.getBytes(3)).isEqualTo(new byte[] {(byte) 0xff}); } else { - assertThat(row.getString(0).toString()) - .isEqualTo( - new String(expected, StandardCharsets.UTF_8)); + byte[] expected = count == 1 ? new byte[0] : VALUE; + if (format.equals("binary")) { + assertThat(row.getBytes(0)).isEqualTo(expected); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo( + new String( + expected, StandardCharsets.UTF_8)); + } + assertThat(row.isNullAt(3)).isTrue(); } - assertThat(row.isNullAt(3)).isTrue(); assertThat(row.getMap(1).size()).isZero(); assertThat(row.isNullAt(2)).isTrue(); assertThat(record.logOffset()).isEqualTo(count); count++; } } - assertThat(count).isEqualTo(2); + assertThat(count).isEqualTo(3); } } finally { admin.dropTable(path, true).get(); From c715c6a794f893543860a3270c1ab5ed4bfde502 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 11 Sep 2026 15:42:27 +0800 Subject: [PATCH 48/48] [kafka] Separate value decoding and preserve record metadata Use the shared table contract for binary and strict UTF-8 string value decoders. Preserve the partition-aware transcoder API and pass each full record to row assembly so optional key, headers and three-field metadata retain their input semantics. Binary values keep borrowed array storage. Cover invalid UTF-8 and nullability failures with declared metadata: failed partitions append no prefix, and reused plans preserve missing timestamps, batch types and the next request's partition after failure. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 274/274 AI-Contributed/UT: 268/268 --- .../format/BinaryKafkaFormatFactory.java | 46 +++++ .../fluss/kafka/format/KafkaFieldDecoder.java | 35 ++++ .../kafka/format/KafkaFormatFactory.java | 33 ++++ .../format/KafkaFormatFactoryRegistry.java | 54 ++++++ .../format/StringKafkaFormatFactory.java | 72 +++++++ .../transcode/ArrowKafkaRecordTranscoder.java | 34 +--- .../GatewayKafkaProduceBackendTest.java | 55 ++++-- .../KafkaFormatFactoryRegistryTest.java | 178 ++++++++++++++++++ .../ArrowKafkaRecordTranscoderTest.java | 35 +++- 9 files changed, 489 insertions(+), 53 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java new file mode 100644 index 00000000000..bce333819de --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.types.BytesType; + +/** Factory for the Kafka binary byte format. */ +@Internal +public final class BinaryKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public ValueType format() { + return ValueType.BINARY; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof BytesType)) { + throw new KafkaTopicSchemaException( + "Kafka binary format requires exactly one _kafka_value BYTES field."); + } + return bytes -> new Object[] {bytes}; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java new file mode 100644 index 00000000000..b70c85f17d4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +/** Decodes one nullable Kafka record value into its ordered Fluss field projection. */ +@Internal +public interface KafkaFieldDecoder { + + /** + * Decodes Kafka value bytes into internal Fluss field values. + * + *

The input is borrowed and must not be modified. Returned values may borrow its storage; + * callers must encode them synchronously before the input can be reused. + */ + Object[] decode(@Nullable byte[] bytes); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java new file mode 100644 index 00000000000..71c5db5f6ad --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; + +/** Creates a decoder for one Kafka value format and Fluss field projection. */ +@Internal +public interface KafkaFormatFactory { + + /** Returns the format implemented by this factory. */ + ValueType format(); + + /** Creates and validates a decoder for the projection. */ + KafkaFieldDecoder createDecoder(KafkaFieldProjection projection); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java new file mode 100644 index 00000000000..3ac61403a70 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; + +import java.util.EnumMap; +import java.util.Map; + +/** Registry of built-in Kafka record format factories. */ +@Internal +public final class KafkaFormatFactoryRegistry { + + private final Map factories; + + /** Creates a registry containing all built-in formats. */ + public KafkaFormatFactoryRegistry() { + factories = new EnumMap<>(ValueType.class); + register(new BinaryKafkaFormatFactory()); + register(new StringKafkaFormatFactory()); + } + + /** Creates a decoder for the requested format and projection. */ + public KafkaFieldDecoder createDecoder(ValueType format, KafkaFieldProjection projection) { + KafkaFormatFactory factory = factories.get(format); + if (factory == null) { + throw new KafkaTopicSchemaException( + "No Kafka format factory is registered for '" + format + "'."); + } + return factory.createDecoder(projection); + } + + private void register(KafkaFormatFactory factory) { + factories.put(factory.format(), factory); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java new file mode 100644 index 00000000000..c004c968e3a --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.types.StringType; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** Factory for strict UTF-8 Kafka string decoding. */ +@Internal +public final class StringKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public ValueType format() { + return ValueType.STRING; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof StringType)) { + throw new KafkaTopicSchemaException( + "Kafka string format requires exactly one _kafka_value STRING field."); + } + final String fieldName = projection.nameAt(0); + return bytes -> new Object[] {decode(bytes, fieldName)}; + } + + private static BinaryString decode(byte[] bytes, String fieldName) { + if (bytes == null) { + return null; + } + try { + return BinaryString.fromString( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString()); + } catch (CharacterCodingException e) { + throw new KafkaRecordEncodingException( + "Kafka field '" + fieldName + "' is not valid UTF-8.", e); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index e1597ca8985..767095b5902 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -19,19 +19,13 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.format.KafkaFieldDecoder; +import org.apache.fluss.kafka.format.KafkaFormatFactoryRegistry; import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; import org.apache.fluss.metadata.KafkaTableSchema; -import org.apache.fluss.metadata.KafkaTableSchema.ValueType; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; -import org.apache.fluss.row.BinaryString; -import javax.annotation.Nullable; - -import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.StandardCharsets; import java.util.List; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -39,6 +33,7 @@ /** Converts binary/string Kafka values and fixed system columns into owned Arrow log bytes. */ @Internal public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { + private final KafkaFormatFactoryRegistry formats = new KafkaFormatFactoryRegistry(); private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); private final KafkaCompiledWritePlanCache writePlanCache = new KafkaCompiledWritePlanCache(); @@ -64,36 +59,19 @@ public BytesView transcode(int partitionId, List records, KafkaTopicWrit checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTableSchema schema = plan.topicSchema(); + KafkaFieldDecoder valueDecoder = + formats.createDecoder(schema.valueType(), schema.valueProjection()); KafkaRowAssembler assembler = plan.rowAssembler(); return arrowRecordEncoder.encodeStreaming( consumer -> { for (Record record : records) { consumer.append( assembler.assemble( - new Object[] { - decode(schema.valueType(), record.borrowedValue()) - }, + valueDecoder.decode(record.borrowedValue()), record, partitionId)); } }, plan.tableInfo()); } - - private static @Nullable Object decode(ValueType format, @Nullable byte[] bytes) { - if (bytes == null || format == ValueType.BINARY) { - return bytes; - } - try { - return BinaryString.fromString( - StandardCharsets.UTF_8 - .newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(bytes)) - .toString()); - } catch (CharacterCodingException e) { - throw new KafkaRecordEncodingException("Kafka string field is not valid UTF-8.", e); - } - } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java index d40c294f89c..039f0ebcfd5 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -761,42 +761,63 @@ void testMissingDefaultAndExplicitDatabasesDoNotFallback() throws Exception { assertThat(service.appends).extracting(ProduceLogRequest::getTableId).containsExactly(43L); } - @Test - void testRecordNullabilityFailureDoesNotAppendValidPrefix() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testInvalidRecordDoesNotAppendValidPrefixWithMetadata(boolean malformedUtf8) + throws Exception { TestingProduceService service = new TestingProduceService(); TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES().copy(false)) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))) + .build(); service.pendingMetadata = CompletableFuture.completedFuture( metadata(false) .setTableJson( TableDescriptor.builder(original) - .schema( - Schema.newBuilder() - .column( - "_kafka_value", - DataTypes.STRING()) - .column( - "_kafka_key", - DataTypes.BYTES() - .copy(false)) - .build()) + .schema(schema) .build() .toJsonBytes())); Record valid = new Record(1L, new byte[] {1}, new byte[] {65}, Collections.emptyList()); + Record invalid = + malformedUtf8 + ? new Record( + 2L, + new byte[] {1}, + new byte[] {(byte) 0xc3, 0x28}, + Collections.emptyList()) + : good(0).records().get(0); + GatewayKafkaProduceBackend backend = backend(service); assertErrors( - backend(service) - .write( + backend.write( command( (short) 1, topic( "topic", new PartitionWrite( - 0, - Arrays.asList( - valid, good(0).records().get(0)))))) + 2, Arrays.asList(valid, invalid))))) .join(), Errors.INVALID_RECORD); assertThat(service.appends).isEmpty(); + assertErrors( + backend.write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 1, Collections.singletonList(valid))))) + .join(), + Errors.NONE); + assertThat(service.appends).hasSize(1); } @ParameterizedTest diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java new file mode 100644 index 00000000000..b4b028ba0c5 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests value decoders against the shared fixed-column table contract. */ +class KafkaFormatFactoryRegistryTest { + private final KafkaFormatFactoryRegistry formats = new KafkaFormatFactoryRegistry(); + + @Test + void testBinaryBorrowsOpaqueBytesAndPreservesNullAndEmpty() { + KafkaFieldDecoder decoder = + formats.createDecoder(ValueType.BINARY, projection(DataTypes.BYTES())); + byte[] opaque = new byte[] {(byte) 0xff, 0}; + assertThat(decoder.decode(opaque)[0]).isSameAs(opaque); + assertThat(opaque).containsExactly((byte) 0xff, (byte) 0); + assertThat(decoder.decode(null)).containsExactly((Object) null); + byte[] empty = new byte[0]; + assertThat(decoder.decode(empty)[0]).isSameAs(empty); + } + + @Test + void testStringRejectsMalformedUtf8AndCanBeReused() { + KafkaFieldDecoder decoder = + formats.createDecoder(ValueType.STRING, projection(DataTypes.STRING())); + for (byte[] invalid : + new byte[][] { + {(byte) 0xc3, 0x28}, + {(byte) 0xc0, (byte) 0x80}, + {(byte) 0xed, (byte) 0xa0, (byte) 0x80}, + {(byte) 0xe2, (byte) 0x82} + }) { + byte[] original = invalid.clone(); + assertThatThrownBy(() -> decoder.decode(invalid)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("_kafka_value") + .hasMessageContaining("UTF-8"); + assertThat(invalid).isEqualTo(original); + byte[] valid = "键".getBytes(StandardCharsets.UTF_8); + assertThat(decoder.decode(valid)).containsExactly(BinaryString.fromString("键")); + assertThat(valid).isEqualTo("键".getBytes(StandardCharsets.UTF_8)); + } + assertThat(decoder.decode(null)).containsExactly((Object) null); + assertThat(decoder.decode(new byte[0])).containsExactly(BinaryString.fromString("")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesUseThePublicValueProjection(boolean string) { + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + TableDescriptor table = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES()) + .column(KafkaTableSchema.VALUE_COLUMN, type) + .column( + KafkaTableSchema.HEADERS_COLUMN, + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + string ? "string" : "binary") + .build(); + KafkaTableSchema schema = KafkaTableSchema.resolve(table); + assertThat(schema.valueProjection().positions()).containsExactly(1); + KafkaFieldDecoder decoder = + formats.createDecoder(schema.valueType(), schema.valueProjection()); + byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + assertThat(decoder.decode(payload)) + .containsExactly(string ? BinaryString.fromString("payload") : payload); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesRejectIncompatibleProjections(boolean string) { + ValueType format = string ? ValueType.STRING : ValueType.BINARY; + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + assertThatThrownBy( + () -> + formats.createDecoder( + format, + projection( + string ? DataTypes.BYTES() : DataTypes.STRING()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining(string ? "STRING" : "BYTES"); + KafkaFieldProjection multiple = + new KafkaFieldProjection( + DataTypes.ROW( + DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type), + DataTypes.FIELD(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES())), + Arrays.asList(0, 1)); + KafkaFieldProjection empty = + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type)), + Collections.emptyList()); + for (KafkaFieldProjection invalid : Arrays.asList(multiple, empty)) { + assertThatThrownBy(() -> formats.createDecoder(format, invalid)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("exactly one"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesRejectKeyAndArbitraryValueColumnNames(boolean string) { + ValueType format = string ? ValueType.STRING : ValueType.BINARY; + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + for (String name : Arrays.asList(KafkaTableSchema.KEY_COLUMN, "body")) { + KafkaFieldProjection invalid = + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(name, type)), + Collections.singletonList(0)); + assertThatThrownBy(() -> formats.createDecoder(format, invalid)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value"); + } + } + + @Test + void testJsonHasNoDecoderUntilItsImplementationIsRegistered() { + assertThatThrownBy( + () -> + formats.createDecoder( + ValueType.PARSE_JSON, projection(DataTypes.STRING()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("PARSE_JSON"); + } + + private static KafkaFieldProjection projection(DataType type) { + return new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type)), + Collections.singletonList(0)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 7bc449ed4e4..6d476a5a63b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -679,10 +679,24 @@ void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exce }); } - @Test - void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { - TableInfo table = valueTable(true); - Record valid = new Record(1L, null, bytes("valid"), Collections.emptyList()); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable(boolean metadata) + throws Exception { + Schema.Builder schema = + Schema.newBuilder().column("_kafka_value", DataTypes.STRING().copy(false)); + if (metadata) { + schema.column("_kafka_metadata", metadataType(true)); + } + TableInfo table = table(descriptor(schema.build(), true).build()); + KafkaTopicWritePlan plan = transcoder.prepare(table); + Record valid = + new Record( + Record.NO_TIMESTAMP, + TimestampType.LOG_APPEND_TIME, + null, + bytes("valid"), + Collections.emptyList()); for (byte[] invalid : new byte[][] {null, {(byte) 0xc3, 0x28}, {(byte) 0xff}}) { AtomicReference output = new AtomicReference<>(); Throwable failure = @@ -690,7 +704,7 @@ void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw () -> output.set( transcoder.transcode( - 0, + 7, Arrays.asList( valid, new Record( @@ -698,16 +712,21 @@ void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throw null, invalid, Collections.emptyList())), - table))); + plan))); assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); assertThat(failure.getSuppressed()).isEmpty(); assertThat(output).hasNullValue(); } read( - transcoder.transcode(0, Collections.singletonList(valid), table), + transcoder.transcode(3, Collections.singletonList(valid), plan), table, 1, - row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); + row -> { + assertThat(row.getString(0).toString()).isEqualTo("valid"); + if (metadata) { + assertMetadata(row, 1, Record.NO_TIMESTAMP, 1, 3); + } + }); } @Test