diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index b39739c6b01..8938993fa5a 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -19,8 +19,10 @@ import org.apache.fluss.client.Connection; import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.FlussConnection; import org.apache.fluss.client.metadata.KvSnapshotMetadata; import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.MetadataUpdater; import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.writer.AppendWriter; @@ -1211,19 +1213,24 @@ void testKvSnapshotLeaseAfterCoordinatorServerRestart() throws Exception { assertThat(zkClient.getKvSnapshotLeaseMetadata(lease.leaseId())).isPresent(); restartCoordinatorServer(zkClient); - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); - lease.dropLease().get(); assertThat(zkClient.getKvSnapshotLeaseMetadata(lease.leaseId())).isNotPresent(); } private void restartCoordinatorServer(ZooKeeperClient zkClient) throws Exception { + MetadataUpdater metadataUpdater = ((FlussConnection) conn).getMetadataUpdater(); + ServerNode cachedCoordinator = metadataUpdater.getCoordinatorServer(); FLUSS_CLUSTER_EXTENSION.stopCoordinatorServer(); waitUntil( () -> !zkClient.getCoordinatorLeaderAddress().isPresent(), Duration.ofMinutes(1), "Coordinator server node still exists in ZooKeeper"); FLUSS_CLUSTER_EXTENSION.startCoordinatorServer(); + // Election completes before the new coordinator address reaches every tablet server. + // Wait for server metadata to converge so that the client's retry can discover it. + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + // Keep the client cache stale to exercise the lease's metadata refresh and retry. + assertThat(metadataUpdater.getCoordinatorServer()).isEqualTo(cachedCoordinator); } @Test diff --git a/fluss-common/src/main/java/org/apache/fluss/row/TimestampNtz.java b/fluss-common/src/main/java/org/apache/fluss/row/TimestampNtz.java index c58141da950..c7dd14b9387 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/TimestampNtz.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/TimestampNtz.java @@ -137,12 +137,8 @@ public long toEpochMicros() { /** Converts this {@link TimestampNtz} object to a {@link LocalDateTime}. */ public LocalDateTime toLocalDateTime() { - int date = (int) (millisecond / MILLIS_PER_DAY); - long time = millisecond % MILLIS_PER_DAY; - if (time < 0) { - --date; - time += MILLIS_PER_DAY; - } + long date = Math.floorDiv(millisecond, MILLIS_PER_DAY); + long time = Math.floorMod(millisecond, MILLIS_PER_DAY); long nanoOfDay = time * 1_000_000L + nanoOfMillisecond; LocalDate localDate = LocalDate.ofEpochDay(date); LocalTime localTime = LocalTime.ofNanoOfDay(nanoOfDay); diff --git a/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampLtzColumnVector.java b/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampLtzColumnVector.java index 9b870cdf1ff..9af6601bc53 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampLtzColumnVector.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampLtzColumnVector.java @@ -57,10 +57,12 @@ public TimestampLtz getTimestampLtz(int i, int precision) { return TimestampLtz.fromEpochMillis(((TimeStampMilliVector) valueVector).get(i)); } else if (valueVector instanceof TimeStampMicroVector) { long micros = ((TimeStampMicroVector) valueVector).get(i); - return TimestampLtz.fromEpochMillis(micros / 1000, (int) (micros % 1000) * 1000); + return TimestampLtz.fromEpochMillis( + Math.floorDiv(micros, 1_000L), (int) Math.floorMod(micros, 1_000L) * 1_000); } else { long nanos = ((TimeStampNanoVector) valueVector).get(i); - return TimestampLtz.fromEpochMillis(nanos / 1_000_000, (int) (nanos % 1_000_000)); + return TimestampLtz.fromEpochMillis( + Math.floorDiv(nanos, 1_000_000L), (int) Math.floorMod(nanos, 1_000_000L)); } } diff --git a/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampNtzColumnVector.java b/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampNtzColumnVector.java index 37bc9f7b9df..19ac10479c1 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampNtzColumnVector.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/arrow/vectors/ArrowTimestampNtzColumnVector.java @@ -56,10 +56,12 @@ public TimestampNtz getTimestampNtz(int i, int precision) { return TimestampNtz.fromMillis(((TimeStampMilliVector) valueVector).get(i)); } else if (valueVector instanceof TimeStampMicroVector) { long micros = ((TimeStampMicroVector) valueVector).get(i); - return TimestampNtz.fromMillis(micros / 1000, (int) (micros % 1000) * 1000); + return TimestampNtz.fromMillis( + Math.floorDiv(micros, 1_000L), (int) Math.floorMod(micros, 1_000L) * 1_000); } else { long nanos = ((TimeStampNanoVector) valueVector).get(i); - return TimestampNtz.fromMillis(nanos / 1_000_000, (int) (nanos % 1_000_000)); + return TimestampNtz.fromMillis( + Math.floorDiv(nanos, 1_000_000L), (int) Math.floorMod(nanos, 1_000_000L)); } } diff --git a/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampLtzWriter.java b/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampLtzWriter.java index fe10b374e34..42a4bbc582f 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampLtzWriter.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampLtzWriter.java @@ -27,6 +27,8 @@ import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.TimeStampVector; import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.types.pojo.ArrowType; +import static org.apache.fluss.utils.DateTimeUtils.timestampToMicros; +import static org.apache.fluss.utils.DateTimeUtils.timestampToNanos; import static org.apache.fluss.utils.Preconditions.checkState; /** {@link ArrowFieldWriter} for TimestampLtz. */ @@ -64,8 +66,8 @@ public void doWrite(int rowIndex, DataGetters row, int ordinal, boolean handleSa } } else if (vector instanceof TimeStampMicroVector) { long microSec = - timestamp.getEpochMillisecond() * 1000 - + timestamp.getNanoOfMillisecond() / 1000; + timestampToMicros( + timestamp.getEpochMillisecond(), timestamp.getNanoOfMillisecond()); if (handleSafe) { vector.setSafe(rowIndex, microSec); } else { @@ -73,7 +75,8 @@ public void doWrite(int rowIndex, DataGetters row, int ordinal, boolean handleSa } } else { long nanoSec = - timestamp.getEpochMillisecond() * 1_000_000 + timestamp.getNanoOfMillisecond(); + timestampToNanos( + timestamp.getEpochMillisecond(), timestamp.getNanoOfMillisecond()); if (handleSafe) { vector.setSafe(rowIndex, nanoSec); } else { diff --git a/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampNtzWriter.java b/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampNtzWriter.java index 89f965ad362..5d79bff0d1b 100644 --- a/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampNtzWriter.java +++ b/fluss-common/src/main/java/org/apache/fluss/row/arrow/writers/ArrowTimestampNtzWriter.java @@ -27,6 +27,8 @@ import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.TimeStampVector; import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.types.pojo.ArrowType; +import static org.apache.fluss.utils.DateTimeUtils.timestampToMicros; +import static org.apache.fluss.utils.DateTimeUtils.timestampToNanos; import static org.apache.fluss.utils.Preconditions.checkState; /** {@link ArrowFieldWriter} for TimestampNtz. */ @@ -64,7 +66,7 @@ public void doWrite(int rowIndex, DataGetters row, int ordinal, boolean handleSa } } else if (vector instanceof TimeStampMicroVector) { long microSec = - timestamp.getMillisecond() * 1000 + timestamp.getNanoOfMillisecond() / 1000; + timestampToMicros(timestamp.getMillisecond(), timestamp.getNanoOfMillisecond()); if (handleSafe) { vector.setSafe(rowIndex, microSec); } else { @@ -72,7 +74,7 @@ public void doWrite(int rowIndex, DataGetters row, int ordinal, boolean handleSa } } else { long nanoSec = - timestamp.getMillisecond() * 1_000_000 + timestamp.getNanoOfMillisecond(); + timestampToNanos(timestamp.getMillisecond(), timestamp.getNanoOfMillisecond()); if (handleSafe) { vector.setSafe(rowIndex, nanoSec); } else { diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/DateTimeUtils.java b/fluss-common/src/main/java/org/apache/fluss/utils/DateTimeUtils.java index 3019df4e33c..1140c8c1165 100644 --- a/fluss-common/src/main/java/org/apache/fluss/utils/DateTimeUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/utils/DateTimeUtils.java @@ -71,6 +71,35 @@ public class DateTimeUtils { .optionalEnd() .toFormatter(); + /** + * Converts a timestamp's milliseconds and nanoseconds within the millisecond to microseconds. + * Sub-microsecond digits are discarded. + * + * @throws ArithmeticException if the result does not fit in a long + */ + public static long timestampToMicros(long millisecond, int nanoOfMillisecond) { + return scaleTimestamp(millisecond, MICROS_PER_MILLIS, nanoOfMillisecond / NANOS_PER_MICROS); + } + + /** + * Converts a timestamp's milliseconds and nanoseconds within the millisecond to nanoseconds. + * + * @throws ArithmeticException if the result does not fit in a long + */ + public static long timestampToNanos(long millisecond, int nanoOfMillisecond) { + return scaleTimestamp(millisecond, 1_000_000L, nanoOfMillisecond); + } + + private static long scaleTimestamp(long millisecond, long unitsPerMillis, long fraction) { + if (millisecond < 0) { + // The non-negative fraction can bring the final value back into range even when + // multiplying the floor-rounded millisecond would underflow at Long.MIN_VALUE. + return Math.addExact( + Math.multiplyExact(millisecond + 1, unitsPerMillis), fraction - unitsPerMillis); + } + return Math.addExact(Math.multiplyExact(millisecond, unitsPerMillis), fraction); + } + public static LocalDate toLocalDate(int date) { return julianToLocalDate(date + EPOCH_JULIAN); } diff --git a/fluss-common/src/test/java/org/apache/fluss/row/TimestampNtzTest.java b/fluss-common/src/test/java/org/apache/fluss/row/TimestampNtzTest.java index 1a939842450..dcd4df406b9 100644 --- a/fluss-common/src/test/java/org/apache/fluss/row/TimestampNtzTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/row/TimestampNtzTest.java @@ -19,7 +19,9 @@ import org.junit.jupiter.api.Test; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.TimeZone; import static org.assertj.core.api.Assertions.assertThat; @@ -46,6 +48,29 @@ public void testNormal() { assertThat(TimestampNtz.fromLocalDateTime(ldt3).toLocalDateTime()).isEqualTo(ldt3); } + @Test + void testToLocalDateTimeAcrossMillisecondRange() { + long[] milliseconds = { + Long.MIN_VALUE, + (long) Integer.MIN_VALUE * 86_400_000 - 1, + -86_400_001, + -1, + 0, + 86_400_000, + ((long) Integer.MAX_VALUE + 1) * 86_400_000, + Long.MAX_VALUE + }; + for (long millis : milliseconds) { + for (int nanos : new int[] {0, 999_999}) { + LocalDateTime expected = + LocalDateTime.ofInstant( + Instant.ofEpochMilli(millis).plusNanos(nanos), ZoneOffset.UTC); + assertThat(TimestampNtz.fromMillis(millis, nanos).toLocalDateTime()) + .isEqualTo(expected); + } + } + } + @Test public void testDaylightSavingTime() { TimeZone tz = TimeZone.getDefault(); diff --git a/fluss-common/src/test/java/org/apache/fluss/row/arrow/ArrowTimestampWriterTest.java b/fluss-common/src/test/java/org/apache/fluss/row/arrow/ArrowTimestampWriterTest.java new file mode 100644 index 00000000000..5c5becd1a8e --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/row/arrow/ArrowTimestampWriterTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.row.arrow; + +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.row.arrow.writers.ArrowFieldWriter; +import org.apache.fluss.row.columnar.ColumnVector; +import org.apache.fluss.row.columnar.ColumnarRow; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator; +import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.TimeStampVector; +import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.ArrowUtils; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies timestamp range checks and round trips through Arrow vectors. */ +class ArrowTimestampWriterTest { + + @ParameterizedTest + @MethodSource("timestampWriters") + void testRoundTripsTimestampBoundaries(int precision, boolean withZone, boolean handleSafe) { + DataType type = + withZone ? DataTypes.TIMESTAMP_LTZ(precision) : DataTypes.TIMESTAMP(precision); + RowType rowType = DataTypes.ROW(type); + long unitsPerMilli = precision == 6 ? 1_000 : 1_000_000; + int nanosPerUnit = precision == 6 ? 1_000 : 1; + try (RootAllocator allocator = new RootAllocator(1024 * 1024); + VectorSchemaRoot root = + VectorSchemaRoot.create(ArrowUtils.toArrowSchema(rowType), allocator)) { + root.allocateNew(); + TimeStampVector vector = (TimeStampVector) root.getVector(0); + ArrowFieldWriter writer = ArrowUtils.createArrowFieldWriter(vector, type); + long[] values = { + Long.MIN_VALUE, + -1_000_001, + -unitsPerMilli, + -unitsPerMilli + 1, + -1, + 0, + 1, + 1_000_001, + Long.MAX_VALUE + }; + for (int i = 0; i < values.length; i++) { + long millis = Math.floorDiv(values[i], unitsPerMilli); + int nanos = (int) Math.floorMod(values[i], unitsPerMilli) * nanosPerUnit; + writer.write(i, timestampRow(millis, nanos, withZone), 0, handleSafe); + assertThat(vector.get(i)).isEqualTo(values[i]); + } + root.setRowCount(values.length); + ArrowReader reader = + new ArrowReader( + new ColumnVector[] {ArrowUtils.createArrowColumnVector(vector, type)}, + root.getRowCount()); + for (int i = 0; i < values.length; i++) { + long millis = Math.floorDiv(values[i], unitsPerMilli); + int nanos = (int) Math.floorMod(values[i], unitsPerMilli) * nanosPerUnit; + ColumnarRow row = reader.read(i); + if (withZone) { + assertThat(row.getTimestampLtz(0, precision)) + .isEqualTo(TimestampLtz.fromEpochMillis(millis, nanos)); + } else { + assertThat(row.getTimestampNtz(0, precision)) + .isEqualTo(TimestampNtz.fromMillis(millis, nanos)); + } + } + } + } + + @ParameterizedTest + @MethodSource("timestampWriters") + void testRejectsOverflowBeforeChangingVector( + int precision, boolean withZone, boolean handleSafe) { + DataType type = + withZone ? DataTypes.TIMESTAMP_LTZ(precision) : DataTypes.TIMESTAMP(precision); + RowType rowType = DataTypes.ROW(type); + long unitsPerMilli = precision == 6 ? 1_000 : 1_000_000; + int nanosPerUnit = precision == 6 ? 1_000 : 1; + try (RootAllocator allocator = new RootAllocator(1024 * 1024); + VectorSchemaRoot root = + VectorSchemaRoot.create(ArrowUtils.toArrowSchema(rowType), allocator)) { + root.allocateNew(); + TimeStampVector vector = (TimeStampVector) root.getVector(0); + ArrowFieldWriter writer = ArrowUtils.createArrowFieldWriter(vector, type); + vector.set(0, 42); + for (long limit : new long[] {Long.MIN_VALUE, Long.MAX_VALUE}) { + long millis = Math.floorDiv(limit, unitsPerMilli); + int nanos = (int) Math.floorMod(limit, unitsPerMilli) * nanosPerUnit; + int outsideNanos = nanos + (limit < 0 ? -nanosPerUnit : nanosPerUnit); + assertThatThrownBy( + () -> + writer.write( + 0, + timestampRow(millis, outsideNanos, withZone), + 0, + handleSafe)) + .isInstanceOf(ArithmeticException.class); + long outsideMillis = millis + (limit < 0 ? -1 : 1); + assertThatThrownBy( + () -> + writer.write( + 0, + timestampRow(outsideMillis, nanos, withZone), + 0, + handleSafe)) + .isInstanceOf(ArithmeticException.class); + assertThat(vector.get(0)).isEqualTo(42); + } + } + } + + private static GenericRow timestampRow(long millis, int nanos, boolean withZone) { + return GenericRow.of( + withZone + ? TimestampLtz.fromEpochMillis(millis, nanos) + : TimestampNtz.fromMillis(millis, nanos)); + } + + private static Stream timestampWriters() { + return Stream.of( + Arguments.of(6, false, false), Arguments.of(6, false, true), + Arguments.of(6, true, false), Arguments.of(6, true, true), + Arguments.of(9, false, false), Arguments.of(9, false, true), + Arguments.of(9, true, false), Arguments.of(9, true, true)); + } +} diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index b77695848ca..e2ed458a1a3 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -63,7 +63,18 @@ ${project.version} + + org.apache.fluss + fluss-shaded-jackson + + + + ${project.groupId} + fluss-client + ${project.version} + test + org.apache.curator curator-test diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 9a9e2e9ff6e..07bdbbfe86b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -19,19 +19,24 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.backend.produce.KafkaProduceConversionExecutor; import org.apache.fluss.rpc.RpcGatewayService; -import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkState; /** The Kafka protocol plugin. */ public class KafkaProtocolPlugin implements NetworkProtocolPlugin { private Configuration conf; + private KafkaProduceConversionExecutor conversionExecutor; + private CompletableFuture closeFuture; @Override public String name() { @@ -40,7 +45,7 @@ public String name() { @Override public void setup(Configuration conf) { - this.conf = conf; + this.conf = new Configuration(conf); } @Override @@ -60,12 +65,35 @@ public ChannelHandler createChannelHandler( } @Override - public RequestHandler createRequestHandler(RpcGatewayService service) { - if (!(service instanceof TabletServerGateway)) { - throw new IllegalArgumentException( - "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " - + service.getClass().getSimpleName()); + public synchronized RequestHandler createRequestHandler(RpcGatewayService service) { + checkState(closeFuture == null, "Kafka protocol plugin has already been closed."); + if (conversionExecutor == null) { + // Mirror the RPC worker concurrency, with a separate queue capped at 1024 requests. + int queueCapacity = + Math.max( + 1, + Math.min( + 1024, + conf.get(ConfigOptions.NETTY_SERVER_MAX_QUEUED_REQUESTS))); + int threads = + Math.max( + 1, + Math.min( + queueCapacity, + conf.get(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS))); + conversionExecutor = new KafkaProduceConversionExecutor(threads, queueCapacity); + } + return new KafkaRequestHandler(conf, service, conversionExecutor); + } + + @Override + public synchronized CompletableFuture closeAsync() { + if (closeFuture == null) { + closeFuture = + conversionExecutor == null + ? CompletableFuture.completedFuture(null) + : conversionExecutor.closeAsync(); } - return new KafkaRequestHandler(); + return closeFuture; } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index b3b555a8f9a..dc34e6aaaca 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,14 +17,28 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.api.metadata.MetadataHandler; +import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; +import org.apache.fluss.kafka.backend.produce.GatewayKafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceConversionExecutor; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import java.util.Collections; +import javax.annotation.Nullable; + +import java.util.Arrays; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { @@ -32,11 +46,31 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaApiRegistry registry; private final KafkaRequestDispatcher dispatcher; - /** Creates a Kafka request handler with the implemented server capabilities. */ - public KafkaRequestHandler() { + /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ + public KafkaRequestHandler(Configuration conf, RpcGatewayService service) { + this(conf, service, null); + } + + /** Creates a handler with a plugin-owned executor for blocking Produce work. */ + public KafkaRequestHandler( + Configuration conf, + RpcGatewayService service, + @Nullable KafkaProduceConversionExecutor conversionExecutor) { + KafkaTopicMapper topicMapper = new KafkaTopicMapper(checkNotNull(conf)); this.registry = new KafkaApiRegistry( - Collections.singletonList(new ApiVersionsHandler(this::apiRegistry))); + Arrays.asList( + new ApiVersionsHandler(this::apiRegistry), + new MetadataHandler( + new GatewayKafkaMetadataBackend(service, topicMapper), + topicMapper), + new ProduceHandler( + new GatewayKafkaProduceBackend( + service, + (TabletServerGateway) service, + topicMapper, + new ArrowKafkaRecordTranscoder(), + conversionExecutor)))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java new file mode 100644 index 00000000000..bffc28d6de2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataBackend; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements Kafka Metadata versions 0 through 11 using a narrow Fluss metadata backend. */ +@Internal +public final class MetadataHandler implements KafkaApiHandler { + + private static final short MAX_SUPPORTED_VERSION = 11; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.METADATA, + ApiKeys.METADATA.oldestVersion(), + (short) Math.min(ApiKeys.METADATA.latestVersion(), MAX_SUPPORTED_VERSION), + true); + + private final KafkaMetadataBackend backend; + private final KafkaTopicMapper topicMapper; + + /** Creates a Metadata handler. */ + public MetadataHandler(KafkaMetadataBackend backend, KafkaTopicMapper topicMapper) { + this.backend = checkNotNull(backend); + this.topicMapper = checkNotNull(topicMapper); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, MetadataRequest request) { + List validTopics = new ArrayList<>(); + List invalidTopics = new ArrayList<>(); + if (!request.isAllTopics()) { + for (MetadataRequestTopic topic : request.data().topics()) { + if (topic.name() == null) { + throw new InvalidRequestException( + "Topic name must be set because topic ID lookup is not supported by " + + "Metadata versions 10 and 11."); + } else if (!isValidTopic(topic.name())) { + invalidTopics.add( + new KafkaClusterMetadata.Topic( + topic.name(), + topic.topicId(), + TopicError.INVALID_TOPIC, + Collections.emptyList())); + } else { + // Kafka added the topic ID fields in v10, but ID-based Metadata lookup was not + // implemented until v12. This handler intentionally stops at v11. + validTopics.add(new TopicReference(topic.name(), Uuid.ZERO_UUID)); + } + } + } + + KafkaMetadataQuery query = + new KafkaMetadataQuery( + request.isAllTopics(), + validTopics, + context.listenerName(), + clientAddress(context.remoteAddress())); + return backend.getMetadata(query) + .thenApply( + metadata -> { + List topics = + new ArrayList<>(metadata.topics()); + topics.addAll(invalidTopics); + return toResponse( + request.version(), + new KafkaClusterMetadata(metadata.brokers(), topics)); + }); + } + + private boolean isValidTopic(String name) { + try { + // Validate the resolved path too: a bare name's canonical form includes the configured + // default database and must fit the same Kafka name limit as an explicit name. + topicMapper.toTablePath(name); + return true; + } catch (InvalidTopicException e) { + return false; + } + } + + private MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { + MetadataResponseData data = + new MetadataResponseData() + .setThrottleTimeMs(0) + .setControllerId(MetadataResponse.NO_CONTROLLER_ID) + .setClusterAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Broker broker : metadata.brokers()) { + MetadataResponseData.MetadataResponseBroker responseBroker = + new MetadataResponseData.MetadataResponseBroker() + .setNodeId(broker.id()) + .setHost(broker.host()) + .setPort(broker.port()); + if (broker.rack() != null) { + responseBroker.setRack(broker.rack()); + } + data.brokers().add(responseBroker); + } + for (KafkaClusterMetadata.Topic topic : metadata.topics()) { + MetadataResponseData.MetadataResponseTopic responseTopic = + new MetadataResponseData.MetadataResponseTopic() + .setName(topic.name()) + // The backend retains the shared table identity. Distinguish names + // only on the wire so clients can cache both and detect recreation. + .setTopicId( + topic.name() != null + && topic.name().indexOf('.') < 0 + && topicMapper.isMappedTopicId(topic.topicId()) + ? topicMapper.toAliasTopicId( + topicMapper.toTableId(topic.topicId())) + : topic.topicId()) + .setErrorCode(toKafkaError(topic.error()).code()) + .setIsInternal(topic.name() != null && Topic.isInternal(topic.name())) + .setTopicAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Partition partition : topic.partitions()) { + responseTopic + .partitions() + .add( + new MetadataResponseData.MetadataResponsePartition() + .setErrorCode( + partition.leaderAvailable() + ? Errors.NONE.code() + : Errors.LEADER_NOT_AVAILABLE.code()) + .setPartitionIndex(partition.partitionId()) + .setLeaderId(partition.leaderId()) + .setLeaderEpoch(partition.leaderEpoch()) + .setReplicaNodes(partition.replicas()) + .setIsrNodes(partition.isr()) + .setOfflineReplicas(partition.offlineReplicas())); + } + data.topics().add(responseTopic); + } + return new MetadataResponse(data, version); + } + + private static Errors toKafkaError(TopicError error) { + switch (error) { + case NONE: + return Errors.NONE; + case UNKNOWN_TOPIC_OR_PARTITION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case UNKNOWN_TOPIC_ID: + return Errors.UNKNOWN_TOPIC_ID; + case INVALID_TOPIC: + return Errors.INVALID_TOPIC_EXCEPTION; + default: + throw new IllegalArgumentException("Unsupported metadata error " + error); + } + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java new file mode 100644 index 00000000000..c485e618152 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.InvalidRecordException; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidRequiredAcksException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.BaseRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.CloseableIterator; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Implements non-idempotent Kafka Produce versions 3 through 11. + * + *

Topic names cross the backend boundary unchanged. The backend resolves bare and qualified + * names and returns results keyed by the original request names. + */ +@Internal +public final class ProduceHandler implements KafkaApiHandler { + + private static final short MIN_SUPPORTED_VERSION = 3; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec(ApiKeys.PRODUCE, MIN_SUPPORTED_VERSION, (short) 11, true); + + private final KafkaProduceBackend backend; + + /** Creates a non-idempotent Produce handler. */ + public ProduceHandler(KafkaProduceBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ProduceRequest request) { + validateRequest(request); + List topics = new ArrayList<>(); + Map failures = new HashMap<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + try { + if (!Topic.isValid(topic.name())) { + throw new InvalidTopicException("Invalid Kafka topic name " + topic.name()); + } + if (partition.index() < 0) { + throw new InvalidRequestException("Negative Kafka partition ID."); + } + partitions.add( + new PartitionWrite( + partition.index(), + copyRecords(request.version(), partition.records()))); + } catch (RuntimeException failure) { + failures.put( + new TopicPartition(topic.name(), partition.index()), + failedPartition(partition.index(), failure)); + } + } + if (!partitions.isEmpty()) { + topics.add(new TopicWrite(topic.name(), partitions)); + } + } + KafkaProduceCommand command = + new KafkaProduceCommand( + request.acks(), + request.timeout(), + topics, + context.listenerName(), + clientAddress(context.remoteAddress())); + CompletableFuture result; + try { + result = + topics.isEmpty() + ? CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList())) + : checkNotNull(backend.write(command)); + } catch (RuntimeException failure) { + result = new CompletableFuture<>(); + result.completeExceptionally(failure); + } + return result.handle( + (written, failure) -> { + Map results = new HashMap<>(failures); + if (failure == null && written != null) { + for (TopicResult topic : written.topics()) { + for (PartitionResult partition : topic.partitions()) { + results.putIfAbsent( + new TopicPartition( + topic.topicName(), partition.partitionId()), + partition); + } + } + } + List ordered = new ArrayList<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + TopicPartition key = + new TopicPartition(topic.name(), partition.index()); + PartitionResult value = results.get(key); + if (value == null) { + value = + failedPartition( + partition.index(), + failure == null + ? new IllegalStateException( + "Produce backend omitted this partition.") + : failure); + } + partitions.add(value); + } + ordered.add(new TopicResult(topic.name(), partitions)); + } + return toResponse(new KafkaProduceResult(ordered)); + }); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + while (failure instanceof CompletionException && failure.getCause() != null) { + failure = failure.getCause(); + } + return new PartitionResult( + partitionId, Errors.forException(failure), -1L, failure.getMessage()); + } + + private static void validateRequest(ProduceRequest request) { + Set names = new HashSet<>(); + for (TopicProduceData topic : request.data().topicData()) { + if (!names.add(topic.name())) { + throw new InvalidRequestException("Duplicate Kafka topic in Produce request."); + } + Set partitions = new HashSet<>(); + for (PartitionProduceData partition : topic.partitionData()) { + if (!partitions.add(partition.index())) { + throw new InvalidRequestException( + "Duplicate Kafka partition in Produce request."); + } + } + } + if (request.transactionalId() != null) { + throw new InvalidRequestException( + "Transactional Produce is not supported by the Fluss Kafka compatibility layer."); + } + if (request.acks() != -1 && request.acks() != 0 && request.acks() != 1) { + throw new InvalidRequiredAcksException("Invalid required acks " + request.acks()); + } + } + + private static List copyRecords( + short version, BaseRecords baseRecords) { + if (!(baseRecords instanceof Records)) { + throw new InvalidRequestException("Unsupported Kafka records representation."); + } + ProduceRequest.validateRecords(version, baseRecords); + Records records = (Records) baseRecords; + if (records.sizeInBytes() == 0) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + List copied = new ArrayList<>(); + int validBytes = 0; + for (RecordBatch batch : records.batches()) { + validBytes += batch.sizeInBytes(); + batch.ensureValid(); + if (batch.hasProducerId() || batch.isTransactional() || batch.isControlBatch()) { + throw new InvalidRequestException( + "Idempotent, transactional, and control record batches are not supported."); + } + // Validate the incoming Kafka batch before copying away its offset metadata. Fluss + // assigns its own storage offsets later, independently of this input validation. + long countFromOffsets = batch.lastOffset() - batch.baseOffset() + 1; + Integer recordCount = batch.countOrNull(); + if (countFromOffsets <= 0 + || recordCount == null + || recordCount <= 0 + || countFromOffsets != recordCount) { + throw new InvalidRecordException( + "Invalid Kafka record batch offset range [" + + batch.baseOffset() + + ", " + + batch.lastOffset() + + "] for record count " + + recordCount + + "."); + } + TimestampType timestampType = timestampType(batch); + // The ordinary compressed iterator preallocates a list using the untrusted record + // count. Stream records instead and close the decompressor on success and failure. + try (CloseableIterator iterator = + batch.streamingIterator(BufferSupplier.NO_CACHING)) { + while (iterator.hasNext()) { + org.apache.kafka.common.record.Record record = iterator.next(); + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + timestampType, + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } + } + } + if (copied.isEmpty() || validBytes != records.sizeInBytes()) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + return copied; + } + + private static TimestampType timestampType(RecordBatch batch) { + switch (batch.timestampType()) { + case CREATE_TIME: + return TimestampType.CREATE_TIME; + case LOG_APPEND_TIME: + return TimestampType.LOG_APPEND_TIME; + case NO_TIMESTAMP_TYPE: + return TimestampType.UNKNOWN; + default: + throw new InvalidRecordException("Unknown batch timestamp type."); + } + } + + private static ProduceResponse toResponse(KafkaProduceResult result) { + ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0); + for (TopicResult topic : result.topics()) { + ProduceResponseData.TopicProduceResponse topicResponse = + new ProduceResponseData.TopicProduceResponse().setName(topic.topicName()); + for (PartitionResult partition : topic.partitions()) { + topicResponse + .partitionResponses() + .add( + new ProduceResponseData.PartitionProduceResponse() + .setIndex(partition.partitionId()) + .setErrorCode(partition.error().code()) + .setBaseOffset(partition.baseOffset()) + .setLogAppendTimeMs(-1L) + .setLogStartOffset(-1L) + .setErrorMessage(partition.errorMessage())); + } + data.responses().add(topicResponse); + } + return new ProduceResponse(data); + } + + private static List copyHeaders(Header[] headers) { + List copied = new ArrayList<>(headers.length); + for (Header header : headers) { + copied.add(new RecordHeader(header.key(), header.value())); + } + return copied; + } + + private static byte[] copyBuffer(ByteBuffer buffer) { + if (buffer == null) { + return null; + } + ByteBuffer duplicate = buffer.duplicate(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return bytes; + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java new file mode 100644 index 00000000000..2c2ff42e117 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -0,0 +1,399 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.MetadataResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.utils.concurrent.FutureUtils; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */ +@Internal +public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { + + private static final Logger LOG = LoggerFactory.getLogger(GatewayKafkaMetadataBackend.class); + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + + /** + * Creates a metadata backend backed by the local TabletServer gateway. + * + * @throws IllegalArgumentException if the service does not implement {@link + * TabletServerGateway} + */ + public GatewayKafkaMetadataBackend(RpcGatewayService service, KafkaTopicMapper topicMapper) { + this.service = checkNotNull(service); + checkArgument( + service instanceof TabletServerGateway, + "Kafka protocol endpoints can only be enabled on TabletServers, but the service is %s", + service.getClass().getSimpleName()); + this.gateway = (TabletServerGateway) service; + this.topicMapper = checkNotNull(topicMapper); + } + + @Override + public CompletableFuture getMetadata(KafkaMetadataQuery query) { + if (query.allTopics() || containsTopicId(query.topics())) { + return currentTablePaths(query) + .thenCompose(tablePaths -> requestFlussMetadata(query, tablePaths)); + } + + Set tablePaths = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + tablePaths.add(topicMapper.toTablePath(topic.topicName())); + } + } + return requestFlussMetadata(query, tablePaths); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set tablePaths) { + return requestFlussMetadata(query, tablePaths, true); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set tablePaths, boolean refreshAndRetry) { + MetadataRequest request = new MetadataRequest(); + for (TablePath tablePath : tablePaths) { + request.addAllTablePaths( + Collections.singletonList( + new PbTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()))); + } + setCurrentSession(query); + try { + return gateway.metadata(request) + .handle( + (response, failure) -> + failure == null + ? CompletableFuture.completedFuture( + toKafkaMetadata(query, response)) + : recoverMetadataFailure( + query, failure, refreshAndRetry)) + .thenCompose(future -> future); + } catch (Throwable failure) { + return recoverMetadataFailure(query, failure, refreshAndRetry); + } + } + + private CompletableFuture recoverMetadataFailure( + KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) { + if (refreshAndRetry) { + return currentTablePaths(query) + .thenCompose(currentPaths -> requestFlussMetadata(query, currentPaths, false)); + } + LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure)); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(unwrap(failure)); + return failed; + } + + private CompletableFuture> currentTablePaths(KafkaMetadataQuery query) { + CompletableFuture> databasesFuture; + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + databasesFuture = + gateway.listDatabases(new ListDatabasesRequest()) + .thenApply( + response -> + new LinkedHashSet<>(response.getDatabaseNamesList())); + } else { + Set databases = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + databases.add(topicMapper.toTablePath(topic.topicName()).getDatabaseName()); + } + } + databasesFuture = CompletableFuture.completedFuture(databases); + } + return databasesFuture + .thenCompose(databases -> listTablePaths(query, databases)) + .thenApply( + currentPaths -> { + if (!query.allTopics() && !containsTopicId(query.topics())) { + Set requestedTables = new HashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + requestedTables.add( + topicMapper.toTablePath(topic.topicName())); + } + } + // Canonical names and aliases share one table identity, including + // when a missing database triggers this discovery retry. + currentPaths.retainAll(requestedTables); + } + return currentPaths; + }); + } + + private CompletableFuture> listTablePaths( + KafkaMetadataQuery query, Set databases) { + CompletableFuture> tablesFuture = + CompletableFuture.completedFuture(new LinkedHashSet<>()); + for (String database : databases) { + tablesFuture = + tablesFuture.thenCompose( + tablePaths -> + listTables(query, database) + .thenApply( + response -> { + for (String tableName : + response.getTableNamesList()) { + tablePaths.add( + TablePath.of( + database, tableName)); + } + return tablePaths; + })); + } + return tablesFuture; + } + + private CompletableFuture listTables( + KafkaMetadataQuery query, String database) { + setCurrentSession(query); + CompletableFuture tablesFuture; + try { + tablesFuture = gateway.listTables(new ListTablesRequest().setDatabaseName(database)); + } catch (Exception failure) { + tablesFuture = FutureUtils.completedExceptionally(failure); + } + return tablesFuture.exceptionally( + failure -> { + Throwable cause = unwrap(failure); + if (cause instanceof DatabaseNotExistException) { + // A database may be missing or deleted after listDatabases. Keep loading + // the remaining topics and brokers through the normal metadata request. + return new ListTablesResponse(); + } + throw new CompletionException(cause); + }); + } + + private KafkaClusterMetadata toKafkaMetadata( + KafkaMetadataQuery query, MetadataResponse response) { + List brokers = new ArrayList<>(); + Set aliveBrokerIds = new HashSet<>(); + for (PbServerNode server : response.getTabletServersList()) { + brokers.add( + new Broker( + server.getNodeId(), + server.getHost(), + server.getPort(), + server.hasRack() ? server.getRack() : null)); + aliveBrokerIds.add(server.getNodeId()); + } + Collections.sort(brokers, Comparator.comparingInt(Broker::id)); + + Map topicsByTable = new HashMap<>(); + Map topicsById = new HashMap<>(); + for (PbTableMetadata table : response.getTableMetadatasList()) { + TablePath tablePath = + TablePath.of( + table.getTablePath().getDatabaseName(), + table.getTablePath().getTableName()); + Topic topic = toKafkaTopic(table, tablePath, aliveBrokerIds); + if (topic != null) { + topicsByTable.put(tablePath, topic); + topicsById.put(topic.topicId(), topic); + } + } + + List topics = new ArrayList<>(); + if (query.allTopics()) { + // Invalid enabled tables remain visible as errors. Only disabled tables are omitted. + topics.addAll(topicsByTable.values()); + Collections.sort(topics, Comparator.comparing(Topic::name)); + } else { + for (TopicReference reference : query.topics()) { + Topic topic = + reference.hasTopicId() + ? topicsById.get(reference.topicId()) + : topicsByTable.get(topicMapper.toTablePath(reference.topicName())); + if (topic != null && matches(reference, topic)) { + topics.add( + new Topic( + reference.topicName() == null + ? topic.name() + : reference.topicName(), + topic.topicId(), + topic.error(), + topic.partitions())); + } else { + topics.add(missingTopic(reference)); + } + } + } + return new KafkaClusterMetadata(brokers, topics); + } + + @Nullable + private Topic toKafkaTopic( + PbTableMetadata table, TablePath tablePath, Set aliveBrokerIds) { + String topicName = tablePath.toString(); + try { + TableDescriptor descriptor = TableDescriptor.fromJsonBytes(table.getTableJson()); + if (!schemaResolver.isEnabled(descriptor)) { + return null; + } + topicMapper.toTopicName(tablePath); + schemaResolver.resolve(descriptor); + } catch (IllegalArgumentException | InvalidTopicException e) { + LOG.debug( + "Enabled table {} has an invalid Kafka contract: {}", + topicName, + e.getMessage()); + return new Topic( + topicName, + topicMapper.toTopicId(table.getTableId()), + TopicError.INVALID_TOPIC, + Collections.emptyList()); + } + List partitions = new ArrayList<>(); + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + boolean leaderAvailable = + bucket.hasLeaderId() && aliveBrokerIds.contains(bucket.getLeaderId()); + List replicas = new ArrayList<>(); + List isr = new ArrayList<>(); + List offlineReplicas = new ArrayList<>(); + Set isrIds = new HashSet<>(); + if (bucket.hasBucketEpoch()) { + for (int isrId : bucket.getIsrs()) { + isrIds.add(isrId); + } + } else if (leaderAvailable) { + // Legacy metadata has no authoritative ISR. Report only the available leader + // as a conservative fallback; live followers may still be out of sync. + isrIds.add(bucket.getLeaderId()); + } + for (int replicaId : bucket.getReplicaIds()) { + replicas.add(replicaId); + if (isrIds.contains(replicaId)) { + isr.add(replicaId); + } + if (!aliveBrokerIds.contains(replicaId)) { + offlineReplicas.add(replicaId); + } + } + partitions.add( + new Partition( + bucket.getBucketId(), + leaderAvailable ? bucket.getLeaderId() : -1, + bucket.hasLeaderEpoch() ? bucket.getLeaderEpoch() : -1, + replicas, + isr, + offlineReplicas, + leaderAvailable)); + } + Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); + return new Topic( + topicName, topicMapper.toTopicId(table.getTableId()), TopicError.NONE, partitions); + } + + private static Topic missingTopic(TopicReference reference) { + TopicError error = + reference.hasTopicId() + ? TopicError.UNKNOWN_TOPIC_ID + : TopicError.UNKNOWN_TOPIC_OR_PARTITION; + return new Topic( + reference.topicName(), reference.topicId(), error, Collections.emptyList()); + } + + private boolean matches(TopicReference reference, Topic topic) { + return (reference.topicName() == null + || topicMapper + .toTablePath(reference.topicName()) + .equals(topicMapper.toTablePath(topic.name()))) + && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId())); + } + + private void setCurrentSession(KafkaMetadataQuery query) { + service.setCurrentSession( + new Session( + (short) 0, + query.listenerName(), + false, + query.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static boolean containsTopicId(List topics) { + for (TopicReference topic : topics) { + if (topic.hasTopicId()) { + return true; + } + } + return false; + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java new file mode 100644 index 00000000000..bbc634af6f0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Kafka-domain cluster metadata returned by a Fluss metadata backend. */ +@Internal +public final class KafkaClusterMetadata { + + private final List brokers; + private final List topics; + + /** Creates cluster metadata. */ + public KafkaClusterMetadata(List brokers, List topics) { + this.brokers = immutableCopy(brokers); + this.topics = immutableCopy(topics); + } + + /** Returns Kafka-reachable brokers. */ + public List brokers() { + return brokers; + } + + /** Returns topic metadata and topic-level errors. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Kafka-reachable broker information. */ + @Internal + public static final class Broker { + + private final int id; + private final String host; + private final int port; + private final @Nullable String rack; + + /** Creates broker information. */ + public Broker(int id, String host, int port, @Nullable String rack) { + this.id = id; + this.host = checkNotNull(host); + this.port = port; + this.rack = rack; + } + + /** Returns the Kafka broker ID. */ + public int id() { + return id; + } + + /** Returns the Kafka listener host. */ + public String host() { + return host; + } + + /** Returns the Kafka listener port. */ + public int port() { + return port; + } + + /** Returns the broker rack, if configured. */ + public @Nullable String rack() { + return rack; + } + } + + /** Topic-level error independent of a Kafka response schema version. */ + @Internal + public enum TopicError { + NONE, + UNKNOWN_TOPIC_OR_PARTITION, + UNKNOWN_TOPIC_ID, + INVALID_TOPIC + } + + /** Metadata for one Kafka topic. */ + @Internal + public static final class Topic { + + private final @Nullable String name; + private final Uuid topicId; + private final TopicError error; + private final List partitions; + + /** Creates topic metadata. */ + public Topic( + @Nullable String name, Uuid topicId, TopicError error, List partitions) { + this.name = name; + this.topicId = checkNotNull(topicId); + this.error = checkNotNull(error); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name, if known. */ + public @Nullable String name() { + return name; + } + + /** Returns the stable Kafka topic ID. */ + public Uuid topicId() { + return topicId; + } + + /** Returns the topic-level domain error. */ + public TopicError error() { + return error; + } + + /** Returns the topic partitions. */ + public List partitions() { + return partitions; + } + } + + /** Metadata for one Kafka partition backed by a Fluss bucket. */ + @Internal + public static final class Partition { + + private final int partitionId; + private final int leaderId; + private final int leaderEpoch; + private final List replicas; + private final List isr; + private final List offlineReplicas; + private final boolean leaderAvailable; + + /** Creates partition metadata. */ + public Partition( + int partitionId, + int leaderId, + int leaderEpoch, + List replicas, + List isr, + List offlineReplicas, + boolean leaderAvailable) { + this.partitionId = partitionId; + this.leaderId = leaderId; + this.leaderEpoch = leaderEpoch; + this.replicas = immutableCopy(replicas); + this.isr = immutableCopy(isr); + this.offlineReplicas = immutableCopy(offlineReplicas); + this.leaderAvailable = leaderAvailable; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the current leader ID, or {@code -1} when unavailable. */ + public int leaderId() { + return leaderId; + } + + /** Returns the leader epoch, or {@code -1} when unavailable. */ + public int leaderEpoch() { + return leaderEpoch; + } + + /** Returns assigned replica IDs. */ + public List replicas() { + return replicas; + } + + /** Returns replica IDs currently visible as in-sync. */ + public List isr() { + return isr; + } + + /** Returns assigned replicas whose TabletServers are unavailable. */ + public List offlineReplicas() { + return offlineReplicas; + } + + /** Returns whether the partition has a reachable leader. */ + public boolean leaderAvailable() { + return leaderAvailable; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java new file mode 100644 index 00000000000..abb02a920ed --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Metadata API. */ +@Internal +public interface KafkaMetadataBackend { + + /** Resolves Kafka-domain metadata asynchronously. */ + CompletableFuture getMetadata(KafkaMetadataQuery query); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java new file mode 100644 index 00000000000..01e21fa4440 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Domain query used by the Metadata API to access the Fluss adapter layer. */ +@Internal +public final class KafkaMetadataQuery { + + private final boolean allTopics; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a metadata query. */ + public KafkaMetadataQuery( + boolean allTopics, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.allTopics = allTopics; + this.topics = Collections.unmodifiableList(new ArrayList<>(checkNotNull(topics))); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns whether all Kafka topics should be returned. */ + public boolean allTopics() { + return allTopics; + } + + /** Returns the explicitly requested topic identities. */ + public List topics() { + return topics; + } + + /** Returns the Kafka listener used by the client connection. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client address when it is available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + /** Kafka topic name and ID supplied by a Metadata request. */ + @Internal + public static final class TopicReference { + + private final @Nullable String topicName; + private final Uuid topicId; + + /** Creates a topic reference. */ + public TopicReference(@Nullable String topicName, Uuid topicId) { + this.topicName = topicName; + this.topicId = checkNotNull(topicId); + } + + /** Returns the requested topic name, if present. */ + public @Nullable String topicName() { + return topicName; + } + + /** Returns the requested topic ID, or {@link Uuid#ZERO_UUID} when absent. */ + public Uuid topicId() { + return topicId; + } + + /** Returns whether this reference identifies a topic by ID. */ + public boolean hasTopicId() { + return !Uuid.ZERO_UUID.equals(topicId); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java new file mode 100644 index 00000000000..c5dd9638dea --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the local TabletServer write gateway to the Kafka Produce backend contract. */ +@Internal +public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final KafkaRecordTranscoder transcoder; + private final KafkaTableInfoCache tableInfoCache = new KafkaTableInfoCache(); + private final @Nullable KafkaProduceConversionExecutor conversionExecutor; + + /** Creates a Produce backend backed by the local TabletServer gateway. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + KafkaTopicMapper topicMapper, + KafkaRecordTranscoder transcoder) { + this(service, gateway, topicMapper, transcoder, null); + } + + /** Creates a backend that submits work to a caller-owned bounded conversion executor. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + KafkaTopicMapper topicMapper, + KafkaRecordTranscoder transcoder, + @Nullable KafkaProduceConversionExecutor conversionExecutor) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.topicMapper = checkNotNull(topicMapper); + this.transcoder = checkNotNull(transcoder); + this.conversionExecutor = conversionExecutor; + } + + @Override + public CompletableFuture write(KafkaProduceCommand command) { + if (conversionExecutor != null) { + return conversionExecutor + .submit(Thread.currentThread().getId(), () -> writeTopics(command)) + .exceptionally( + failure -> { + List results = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + results.add(failedTopic(topic, failure)); + } + return new KafkaProduceResult(results); + }); + } + return writeTopics(command); + } + + private CompletableFuture writeTopics(KafkaProduceCommand command) { + List> futures = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + futures.add(writeTopic(command, topic)); + } + CompletableFuture all = + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + return all.thenApply( + ignored -> { + List results = new ArrayList<>(); + for (CompletableFuture future : futures) { + results.add(future.join()); + } + return new KafkaProduceResult(results); + }); + } + + private CompletableFuture writeTopic( + KafkaProduceCommand command, TopicWrite topic) { + try { + checkInterrupted(); + TablePath path = topicMapper.toTablePath(topic.topicName()); + setCurrentSession(command); + GetTableInfoRequest request = new GetTableInfoRequest(); + request.setTablePath() + .setDatabaseName(path.getDatabaseName()) + .setTableName(path.getTableName()); + CompletableFuture metadata = gateway.getTableInfo(request); + if (conversionExecutor != null) { + // The local gateway is synchronous. Waiting here also preserves FIFO append + // order if an alternative gateway returns asynchronously; only this worker waits. + GetTableInfoResponse response = + metadata.get(Math.max(0, command.timeoutMs()), TimeUnit.MILLISECONDS); + return produceTopic(command, topic, toTableInfo(path, response)); + } + return metadata.thenCompose( + response -> produceTopic(command, topic, toTableInfo(path, response))) + .exceptionally(failure -> failedTopic(topic, failure)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + return CompletableFuture.completedFuture(failedTopic(topic, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedTopic(topic, failure)); + } + } + + private CompletableFuture produceTopic( + KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { + // Admission must match Metadata, including when the schema changes between requests. + // Even a cached snapshot is selected only after fetching the authoritative descriptor. + if (!schemaResolver.isEnabled(tableInfo.toTableDescriptor())) { + throw new TableNotExistException( + "Table " + tableInfo.getTablePath() + " is not enabled for Kafka."); + } + KafkaTopicWritePlan writePlan = transcoder.prepare(tableInfo); + ProduceLogRequest request = + new ProduceLogRequest() + .setTableId(tableInfo.getTableId()) + .setAcks(command.acks()) + .setTimeoutMs(command.timeoutMs()); + Map failures = new HashMap<>(); + for (PartitionWrite partition : topic.partitions()) { + if (partition.partitionId() < 0 + || partition.partitionId() >= tableInfo.getNumBuckets()) { + failures.put( + partition.partitionId(), + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_TOPIC_OR_PARTITION, + -1L, + "Partition is outside the table bucket range.")); + continue; + } + try { + BytesView records = + transcoder.transcode( + partition.partitionId(), partition.records(), writePlan); + request.addBucketsReq() + .setBucketId(partition.partitionId()) + .setRecordsBytesView(records); + } catch (Exception failure) { + failures.put( + partition.partitionId(), failedPartition(partition.partitionId(), failure)); + } + } + if (request.getBucketsReqsCount() == 0) { + return CompletableFuture.completedFuture( + toTopicResult(topic, new ProduceLogResponse(), failures)); + } + try { + checkInterrupted(); + setCurrentSession(command); + CompletableFuture appended = gateway.produceLog(request); + return appended.handle( + (response, failure) -> + failure == null + ? toTopicResult(topic, response, failures) + : failedAppend(topic, failures, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedAppend(topic, failures, failure)); + } + } + + private static TopicResult failedAppend( + TopicWrite topic, Map failures, Throwable failure) { + List results = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult local = failures.get(partition.partitionId()); + results.add(local == null ? failedPartition(partition.partitionId(), failure) : local); + } + return new TopicResult(topic.topicName(), results); + } + + private TableInfo toTableInfo(TablePath path, GetTableInfoResponse response) { + return tableInfoCache.getOrLoad(path, response, () -> parseTableInfo(path, response)); + } + + private static TableInfo parseTableInfo(TablePath path, GetTableInfoResponse response) { + return TableInfo.of( + path, + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null, + response.getCreatedTime(), + response.getModifiedTime(), + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L); + } + + private static TopicResult toTopicResult( + TopicWrite topic, ProduceLogResponse response, Map failures) { + Map responses = new HashMap<>(); + for (PbProduceLogRespForBucket bucket : response.getBucketsRespsList()) { + responses.put(bucket.getBucketId(), bucket); + } + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult localFailure = failures.get(partition.partitionId()); + if (localFailure != null) { + partitions.add(localFailure); + continue; + } + PbProduceLogRespForBucket bucket = responses.get(partition.partitionId()); + if (bucket == null) { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_SERVER_ERROR, + -1L, + "Fluss Produce response omitted this bucket.")); + } else if (bucket.hasErrorCode() && bucket.getErrorCode() != 0) { + partitions.add( + new PartitionResult( + partition.partitionId(), + toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forCode( + bucket.getErrorCode())), + -1L, + bucket.hasErrorMessage() ? bucket.getErrorMessage() : null)); + } else { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.NONE, + bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, + null)); + } + } + return new TopicResult(topic.topicName(), partitions); + } + + private static TopicResult failedTopic(TopicWrite topic, Throwable failure) { + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + partitions.add(failedPartition(partition.partitionId(), failure)); + } + return new TopicResult(topic.topicName(), partitions); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + Throwable cause = unwrap(failure); + Errors kafkaError = + cause instanceof RejectedExecutionException + || cause instanceof TimeoutException + || cause instanceof InterruptedException + || cause instanceof CancellationException + ? Errors.REQUEST_TIMED_OUT + : cause instanceof KafkaTopicSchemaException + || cause instanceof InvalidTopicException + ? Errors.INVALID_TOPIC_EXCEPTION + : cause instanceof KafkaRecordEncodingException + ? Errors.INVALID_RECORD + : cause instanceof IllegalArgumentException + ? Errors.INVALID_REQUEST + : toKafkaError( + org.apache.fluss.rpc.protocol.Errors + .forException(cause)); + return new PartitionResult(partitionId, kafkaError, -1L, cause.getMessage()); + } + + private static Errors toKafkaError(org.apache.fluss.rpc.protocol.Errors error) { + switch (error) { + case NONE: + return Errors.NONE; + case TABLE_NOT_EXIST: + case DATABASE_NOT_EXIST: + case UNKNOWN_TABLE_OR_BUCKET_EXCEPTION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case NOT_LEADER_OR_FOLLOWER: + return Errors.NOT_LEADER_OR_FOLLOWER; + case LEADER_NOT_AVAILABLE_EXCEPTION: + return Errors.LEADER_NOT_AVAILABLE; + case RECORD_TOO_LARGE_EXCEPTION: + return Errors.MESSAGE_TOO_LARGE; + case CORRUPT_MESSAGE: + case CORRUPT_RECORD_EXCEPTION: + return Errors.CORRUPT_MESSAGE; + case INVALID_REQUIRED_ACKS: + return Errors.INVALID_REQUIRED_ACKS; + case REQUEST_TIME_OUT: + return Errors.REQUEST_TIMED_OUT; + case NOT_ENOUGH_REPLICAS_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS; + case NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND; + case AUTHORIZATION_EXCEPTION: + return Errors.TOPIC_AUTHORIZATION_FAILED; + case LOG_STORAGE_EXCEPTION: + case STORAGE_EXCEPTION: + case DISK_WRITE_LOCKED: + return Errors.KAFKA_STORAGE_ERROR; + default: + return Errors.UNKNOWN_SERVER_ERROR; + } + } + + private void setCurrentSession(KafkaProduceCommand command) { + service.setCurrentSession( + new Session( + (short) 0, + command.listenerName(), + false, + command.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while ((current instanceof CompletionException || current instanceof ExecutionException) + && current.getCause() != null) { + current = current.getCause(); + } + return current; + } + + private void checkInterrupted() { + if (conversionExecutor != null && Thread.currentThread().isInterrupted()) { + throw new CancellationException("Kafka conversion worker was interrupted."); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java new file mode 100644 index 00000000000..3b8428c5670 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Produce API. */ +@Internal +public interface KafkaProduceBackend { + /** Writes copied Kafka records through the native Fluss write path. */ + CompletableFuture write(KafkaProduceCommand command); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java new file mode 100644 index 00000000000..b0d83392d39 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Protocol-independent write command used by the Kafka Produce backend. */ +@Internal +public final class KafkaProduceCommand { + + private final short acks; + private final int timeoutMs; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a Kafka write command. */ + public KafkaProduceCommand( + short acks, + int timeoutMs, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.acks = acks; + this.timeoutMs = timeoutMs; + this.topics = immutableCopy(topics); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns Kafka required acknowledgements. */ + public short acks() { + return acks; + } + + /** Returns the Produce timeout in milliseconds. */ + public int timeoutMs() { + return timeoutMs; + } + + /** Returns the topic writes in request order. */ + public List topics() { + return topics; + } + + /** Returns the listener that received the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client network address when available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Records addressed to one Kafka topic. */ + @Internal + public static final class TopicWrite { + private final String topicName; + private final List partitions; + + /** Creates the writes for one topic. */ + public TopicWrite(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the original request topic name, including any default-database alias. */ + public String topicName() { + return topicName; + } + + /** Returns partition writes in request order. */ + public List partitions() { + return partitions; + } + } + + /** Records addressed to one Kafka partition. */ + @Internal + public static final class PartitionWrite { + private final int partitionId; + private final List records; + + /** Creates the writes for one partition. */ + public PartitionWrite(int partitionId, List records) { + this.partitionId = partitionId; + this.records = immutableCopy(records); + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns copied records in append order, including records with null values. */ + public List records() { + return records; + } + } + + /** + * Timestamp provenance supplied by a batch, independent of whether a record has a timestamp. + */ + @Internal + public enum TimestampType { + /** No batch timestamp type was supplied, including by the legacy record constructor. */ + UNKNOWN, + /** The timestamp was assigned when the record was created. */ + CREATE_TIME, + /** The timestamp was assigned when the batch was appended to a log. */ + LOG_APPEND_TIME + } + + /** A copied Kafka record whose lifetime is independent of the network request buffer. */ + @Internal + public static final class Record { + /** Sentinel used when the input record has no timestamp. */ + public static final long NO_TIMESTAMP = -1L; + + private final long timestamp; + private final TimestampType timestampType; + private final @Nullable byte[] key; + private final @Nullable byte[] value; + private final List headers; + + /** + * Creates a copied record without batch timestamp provenance. + * + *

The type is {@link TimestampType#UNKNOWN} even when a timestamp is present. Callers + * that know the batch type should use the explicit constructor instead of inferring it from + * the timestamp value. + */ + public Record( + long timestamp, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { + this(timestamp, TimestampType.UNKNOWN, key, value, headers); + } + + /** Creates a copied record with the timestamp type supplied by its source batch. */ + public Record( + long timestamp, + TimestampType timestampType, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { + this.timestamp = timestamp; + this.timestampType = checkNotNull(timestampType); + this.key = copyNullable(key); + this.value = copyNullable(value); + this.headers = immutableCopy(headers); + } + + /** Returns the record timestamp in milliseconds, or {@link #NO_TIMESTAMP} when absent. */ + public long timestamp() { + return timestamp; + } + + /** Returns whether the input record has a timestamp, independently of its batch type. */ + public boolean hasTimestamp() { + return timestamp != NO_TIMESTAMP; + } + + /** Returns the source batch's timestamp type, never inferred from the timestamp value. */ + public TimestampType timestampType() { + return timestampType; + } + + /** Returns a copy of the nullable Kafka record key. */ + public @Nullable byte[] key() { + return copyNullable(key); + } + + /** + * Borrows the nullable key owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #key()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedKey() { + return key; + } + + /** Returns a copy of the nullable Kafka record value. */ + public @Nullable byte[] value() { + return copyNullable(value); + } + + /** + * Borrows the nullable value owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } + + /** Returns the copied Kafka headers in record order. */ + public List headers() { + return headers; + } + + private static @Nullable byte[] copyNullable(@Nullable byte[] value) { + return value == null ? null : value.clone(); + } + } + + /** A copied Kafka record header. */ + @Internal + public static final class RecordHeader { + private final String name; + private final @Nullable byte[] value; + + /** Creates a copied Kafka record header. */ + public RecordHeader(String name, @Nullable byte[] value) { + this.name = checkNotNull(name); + this.value = value == null ? null : value.clone(); + } + + /** Returns the header name. */ + public String name() { + return name; + } + + /** Returns a copy of the nullable header value. */ + public @Nullable byte[] value() { + return value == null ? null : value.clone(); + } + + /** + * Borrows the nullable header value for synchronous conversion. + * + *

The returned array is owned by this header and must not be modified or retained after + * conversion. Use {@link #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java new file mode 100644 index 00000000000..e3953f001e0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutor.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; + +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Dedicated, bounded FIFO lanes for Kafka metadata lookup, conversion, and native submission. + * + *

The caller uses its RPC processor identity as the ordering key. The server routes each + * connection to one processor, so its requests remain ordered when moved off that processor. An + * operation must invoke native append before returning, but need not wait for acknowledgements. + */ +@Internal +public final class KafkaProduceConversionExecutor { + private final ThreadPoolExecutor[] workers; + private volatile boolean closed; + private CompletableFuture closeFuture; + + /** Creates fixed single-worker lanes whose combined queue capacity is bounded. */ + public KafkaProduceConversionExecutor(int threads, int queueCapacity) { + checkArgument(threads > 0, "Conversion thread count must be positive."); + checkArgument( + queueCapacity >= threads, "Queue capacity must be at least the thread count."); + workers = new ThreadPoolExecutor[threads]; + for (int i = 0; i < threads; i++) { + int capacity = queueCapacity / threads + (i < queueCapacity % threads ? 1 : 0); + workers[i] = + new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(capacity), + new ExecutorThreadFactory("kafka-produce-conversion-" + i), + new ThreadPoolExecutor.AbortPolicy()); + } + } + + /** Submits ordered work without ever running rejected work on the caller's thread. */ + public CompletableFuture submit( + long orderingKey, Supplier> operation) { + ConversionTask task = new ConversionTask<>(checkNotNull(operation)); + try { + if (closed) { + throw new RejectedExecutionException("Kafka conversion executor is closed."); + } + workers[(int) Math.floorMod(orderingKey, (long) workers.length)].execute(task); + } catch (RejectedExecutionException failure) { + task.cancel(failure); + } + return task.result; + } + + /** + * Stops new work, rejects queued requests, interrupts workers, and awaits their termination. + */ + public synchronized CompletableFuture closeAsync() { + if (closeFuture != null) { + return closeFuture; + } + closed = true; + RejectedExecutionException failure = + new RejectedExecutionException("Kafka conversion executor is shutting down."); + for (ThreadPoolExecutor worker : workers) { + List queued = worker.shutdownNow(); + for (Runnable task : queued) { + ((ConversionTask) task).cancel(failure); + } + } + closeFuture = + CompletableFuture.runAsync( + () -> { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + try { + for (ThreadPoolExecutor worker : workers) { + if (!worker.awaitTermination( + Math.max(0L, deadline - System.nanoTime()), + TimeUnit.NANOSECONDS)) { + throw new IllegalStateException( + "Kafka conversion worker did not terminate."); + } + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted closing Kafka conversion workers.", + interrupted); + } + }); + return closeFuture; + } + + private static final class ConversionTask implements Runnable { + private final AtomicReference>> operation; + private final CompletableFuture result = new CompletableFuture<>(); + private final AtomicBoolean claimed = new AtomicBoolean(); + + private ConversionTask(Supplier> operation) { + this.operation = new AtomicReference<>(operation); + } + + @Override + public void run() { + if (!claimed.compareAndSet(false, true)) { + return; + } + Supplier> action = operation.getAndSet(null); + if (result.isDone()) { + return; + } + try { + checkNotNull(action.get(), "conversion result") + .whenComplete( + (value, failure) -> { + if (failure == null) { + result.complete(value); + } else { + result.completeExceptionally(failure); + } + }); + } catch (Throwable failure) { + result.completeExceptionally(failure); + } + } + + private void cancel(Throwable failure) { + if (claimed.compareAndSet(false, true)) { + operation.set(null); + result.completeExceptionally(failure); + } + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java new file mode 100644 index 00000000000..8f41864f85b --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Result of a Kafka Produce backend invocation. */ +@Internal +public final class KafkaProduceResult { + private final List topics; + + /** Creates a Produce result. */ + public KafkaProduceResult(List topics) { + this.topics = immutableCopy(topics); + } + + /** Returns topic results in request order. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Results for one topic. */ + @Internal + public static final class TopicResult { + private final String topicName; + private final List partitions; + + /** Creates the result for one topic. */ + public TopicResult(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the original request topic name, rather than a resolved table path. */ + public String topicName() { + return topicName; + } + + /** Returns the partition results in request order. */ + public List partitions() { + return partitions; + } + } + + /** Result for one partition. */ + @Internal + public static final class PartitionResult { + private final int partitionId; + private final Errors error; + private final long baseOffset; + private final @Nullable String errorMessage; + + /** Creates the result for one partition. */ + public PartitionResult( + int partitionId, Errors error, long baseOffset, @Nullable String errorMessage) { + this.partitionId = partitionId; + this.error = checkNotNull(error); + this.baseOffset = baseOffset; + this.errorMessage = errorMessage; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the Kafka protocol error. */ + public Errors error() { + return error; + } + + /** Returns the first appended offset, or {@code -1} on failure. */ + public long baseOffset() { + return baseOffset; + } + + /** Returns an optional diagnostic error message. */ + public @Nullable String errorMessage() { + return errorMessage; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java new file mode 100644 index 00000000000..1d53a5eed8a --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCache.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded cache for parsed Kafka topic {@link TableInfo} snapshots. + * + *

The cache never replaces the authoritative {@code GetTableInfo} call. Every Produce request + * still obtains a fresh response (and therefore retains its authorization and freshness semantics). + * This cache only avoids parsing an unchanged descriptor JSON response again. + * + *

An entry is reused only when all response identity fields and the complete descriptor bytes + * match. Comparing the complete descriptor is intentional: a {@code kafka.*} property change may + * alter decoding semantics without changing the schema ID. Cache slots serialize parsing for one + * table, while independent topics remain concurrent. + */ +@Internal +@ThreadSafe +final class KafkaTableInfoCache { + + static final long DEFAULT_MAXIMUM_SIZE = 1_000L; + + private final long maximumSize; + private final ConcurrentMap slots; + private final Object slotCreationLock = new Object(); + + KafkaTableInfoCache() { + this(DEFAULT_MAXIMUM_SIZE); + } + + KafkaTableInfoCache(long maximumSize) { + checkArgument(maximumSize > 0, "Kafka TableInfo cache size must be greater than zero."); + this.maximumSize = maximumSize; + this.slots = new ConcurrentHashMap<>(); + } + + TableInfo getOrLoad( + TablePath tablePath, + GetTableInfoResponse response, + Supplier tableInfoLoader) { + checkNotNull(tablePath); + checkNotNull(response); + checkNotNull(tableInfoLoader); + + while (true) { + CacheSlot slot = getOrCreateSlot(tablePath); + synchronized (slot) { + if (slots.get(tablePath) != slot) { + continue; + } + if (slot.entry != null && slot.entry.matches(response)) { + return slot.entry.tableInfo; + } + TableInfo tableInfo = checkNotNull(tableInfoLoader.get()); + slot.entry = new CacheEntry(response, tableInfo); + return tableInfo; + } + } + } + + void invalidate(TablePath tablePath) { + CacheSlot slot = slots.get(checkNotNull(tablePath)); + if (slot == null) { + return; + } + synchronized (slot) { + slot.entry = null; + slots.remove(tablePath, slot); + } + } + + long estimatedSize() { + return slots.size(); + } + + private CacheSlot getOrCreateSlot(TablePath tablePath) { + CacheSlot slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + synchronized (slotCreationLock) { + slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + if (slots.size() >= maximumSize) { + evictOneSlot(); + } + slot = new CacheSlot(); + slots.put(tablePath, slot); + return slot; + } + } + + private void evictOneSlot() { + for (Map.Entry entry : slots.entrySet()) { + CacheSlot slot = entry.getValue(); + synchronized (slot) { + if (slots.remove(entry.getKey(), slot)) { + return; + } + } + } + } + + private static final class CacheSlot { + private CacheEntry entry; + } + + private static final class CacheEntry { + private final long tableId; + private final int schemaId; + private final byte[] tableJson; + private final long createdTime; + private final long modifiedTime; + private final long bucketCountEpoch; + private final String remoteDataDir; + private final TableInfo tableInfo; + + private CacheEntry(GetTableInfoResponse response, TableInfo tableInfo) { + this.tableId = response.getTableId(); + this.schemaId = response.getSchemaId(); + this.tableJson = response.getTableJson().clone(); + this.createdTime = response.getCreatedTime(); + this.modifiedTime = response.getModifiedTime(); + this.bucketCountEpoch = + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L; + this.remoteDataDir = response.hasRemoteDataDir() ? response.getRemoteDataDir() : null; + this.tableInfo = tableInfo; + } + + private boolean matches(GetTableInfoResponse response) { + return tableId == response.getTableId() + && schemaId == response.getSchemaId() + && createdTime == response.getCreatedTime() + && modifiedTime == response.getModifiedTime() + && bucketCountEpoch + == (response.hasBucketCountEpoch() + ? response.getBucketCountEpoch() + : 0L) + && Objects.equals( + remoteDataDir, + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null) + && Arrays.equals(tableJson, response.getTableJson()); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java new file mode 100644 index 00000000000..bfa8e1288c7 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/BinaryKafkaFormatFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.types.BytesType; + +/** Factory for the Kafka binary byte format. */ +@Internal +public final class BinaryKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public KafkaValueConversionType format() { + return KafkaValueConversionType.BINARY; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof BytesType)) { + throw new KafkaTopicSchemaException( + "Kafka binary format requires exactly one _kafka_value BYTES field."); + } + return bytes -> new Object[] {bytes}; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java new file mode 100644 index 00000000000..6b22c566e14 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFieldDecoder.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +/** Decodes one nullable Kafka record value into its ordered Fluss field projection. */ +@Internal +public interface KafkaFieldDecoder { + + /** + * Decodes Kafka value bytes into internal Fluss field values. + * + *

A JSON decoder created from a table schema with a declared {@code _kafka_rescue} appends + * that column's nullable STRING value after the ordered value projection. Binary/string + * decoders return only the value projection. + * + *

The input is borrowed and must not be modified. Returned values may borrow its storage; + * callers must encode them synchronously before the input can be reused. + */ + Object[] decode(@Nullable byte[] bytes); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java new file mode 100644 index 00000000000..6fcf4cd8bac --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactory.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; + +import javax.annotation.Nullable; + +/** Creates a decoder for one Kafka value format and Fluss field projection. */ +@Internal +public interface KafkaFormatFactory { + + /** Returns the format implemented by this factory. */ + KafkaValueConversionType format(); + + /** Creates and validates a decoder for the projection. */ + KafkaFieldDecoder createDecoder(KafkaFieldProjection projection); + + /** Creates a decoder from the validated fixed-column contract. */ + default KafkaFieldDecoder createDecoder(KafkaTableSchema schema) { + return createDecoder(schema.valueProjection(), schema.valueConversionLayout()); + } + + /** Creates a decoder with the JSON layout, or no layout for binary/string values. */ + default KafkaFieldDecoder createDecoder( + KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { + return createDecoder(projection); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java new file mode 100644 index 00000000000..7288bb61f7b --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistry.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.format.json.JsonKafkaFormatFactory; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; + +import javax.annotation.Nullable; + +import java.util.EnumMap; +import java.util.Map; + +/** Registry of built-in Kafka record format factories. */ +@Internal +public final class KafkaFormatFactoryRegistry { + + private final Map factories; + + /** Creates a registry containing all built-in formats. */ + public KafkaFormatFactoryRegistry() { + factories = new EnumMap<>(KafkaValueConversionType.class); + register(new BinaryKafkaFormatFactory()); + register(new StringKafkaFormatFactory()); + register(new JsonKafkaFormatFactory()); + } + + /** Creates a decoder for the requested format and projection. */ + public KafkaFieldDecoder createDecoder( + KafkaValueConversionType format, KafkaFieldProjection projection) { + return createDecoder(format, projection, null); + } + + /** Creates and validates a decoder using the table's physical projection and value layout. */ + public KafkaFieldDecoder createDecoder( + KafkaValueConversionType format, + KafkaFieldProjection projection, + @Nullable KafkaValueConversionLayout layout) { + return factory(format).createDecoder(projection, layout); + } + + /** Creates a decoder with fixed rescue-column support from the validated table schema. */ + public KafkaFieldDecoder createDecoder(KafkaTableSchema schema) { + return factory(schema.valueConversionType()).createDecoder(schema); + } + + private KafkaFormatFactory factory(KafkaValueConversionType format) { + KafkaFormatFactory factory = factories.get(format); + if (factory == null) { + throw new KafkaTopicSchemaException( + "No Kafka format factory is registered for '" + format + "'."); + } + return factory; + } + + private void register(KafkaFormatFactory factory) { + factories.put(factory.format(), factory); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java new file mode 100644 index 00000000000..ff6fd544397 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/StringKafkaFormatFactory.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.types.StringType; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** Factory for strict UTF-8 Kafka string decoding. */ +@Internal +public final class StringKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public KafkaValueConversionType format() { + return KafkaValueConversionType.STRING; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof StringType)) { + throw new KafkaTopicSchemaException( + "Kafka string format requires exactly one _kafka_value STRING field."); + } + final String fieldName = projection.nameAt(0); + return bytes -> new Object[] {decode(bytes, fieldName)}; + } + + private static BinaryString decode(byte[] bytes, String fieldName) { + if (bytes == null) { + return null; + } + try { + return BinaryString.fromString( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString()); + } catch (CharacterCodingException e) { + throw new KafkaRecordEncodingException( + "Kafka field '" + fieldName + "' is not valid UTF-8.", e); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java new file mode 100644 index 00000000000..37094fb55d1 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoder.java @@ -0,0 +1,462 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.kafka.format.KafkaFieldDecoder; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonParser; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonToken; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.StreamReadConstraints; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DoubleNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.FloatNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.DataField; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; + +import javax.annotation.Nullable; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** Decodes a strict UTF-8 JSON object according to the projected Fluss fields. */ +@Internal +public final class JsonKafkaFieldDecoder implements KafkaFieldDecoder { + // Jackson counts both the JSON root and a container at business depth zero. An empty ROW + // at the maximum business depth is still a container. The nested physical ROW is the root. + static final int MAX_JSON_NESTING_DEPTH = KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH + 2; + static final int MAX_CONTAINER_ELEMENTS = JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; + private static final ObjectMapper OBJECT_MAPPER = createObjectMapper(); + + private static final DataType JSON_STRING_TYPE = DataTypes.STRING(); + + private final KafkaFieldProjection projection; + private final boolean rescueEnabled; + private final @Nullable RowType nestedType; + private final Map projectedFields = new HashMap<>(); + private final JsonToFlussConverter[] converters; + + /** Creates a JSON decoder and validates every projected field type. */ + public JsonKafkaFieldDecoder(KafkaFieldProjection projection) { + this(projection, KafkaValueConversionLayout.FLAT); + } + + /** Creates a decoder whose nested physical ROW represents the JSON object root. */ + public JsonKafkaFieldDecoder( + KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { + this(projection, layout, false); + } + + /** Creates a decoder with rescue enabled only by the validated fixed system column. */ + public JsonKafkaFieldDecoder(KafkaTableSchema schema) { + this( + schema.valueProjection(), + schema.valueConversionLayout(), + schema.rescuePosition() >= 0); + } + + private JsonKafkaFieldDecoder( + KafkaFieldProjection projection, + @Nullable KafkaValueConversionLayout layout, + boolean rescueEnabled) { + this.rescueEnabled = rescueEnabled; + if (layout == KafkaValueConversionLayout.NESTED) { + if (projection.size() != 1 + || !KafkaTableSchema.VALUE_COLUMN.equals(projection.nameAt(0)) + || !(projection.dataTypeAt(0) instanceof RowType)) { + throw new KafkaTopicSchemaException( + "Nested Kafka JSON requires exactly one _kafka_value ROW."); + } + nestedType = (RowType) projection.dataTypeAt(0); + List positions = new ArrayList<>(); + for (int i = 0; i < nestedType.getFieldCount(); i++) { + positions.add(i); + } + this.projection = new KafkaFieldProjection(nestedType, positions); + } else { + nestedType = null; + this.projection = projection; + } + converters = new JsonToFlussConverter[this.projection.size()]; + for (int i = 0; i < this.projection.size(); i++) { + String name = this.projection.nameAt(i); + if (KafkaTableSchema.isReservedName(name)) { + throw new KafkaTopicSchemaException( + "Reserved name '" + name + "' cannot be a Kafka JSON business field."); + } + if (projectedFields.put(name, i) != null) { + throw new KafkaTopicSchemaException("Duplicate Kafka JSON field '" + name + "'."); + } + converters[i] = JsonToFlussConverters.create(this.projection.dataTypeAt(i)); + } + } + + @Override + public Object[] decode(@Nullable byte[] bytes) { + if (bytes == null) { + return nullValues(); + } + try (Reader reader = + new InputStreamReader( + new ByteArrayInputStream(bytes), + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT)); + JsonParser parser = OBJECT_MAPPER.getFactory().createParser(reader)) { + if (parser.nextToken() != JsonToken.START_OBJECT) { + throw new KafkaRecordEncodingException( + "Kafka JSON record value must have an object root."); + } + Object[] values = + new Object[projection.size() + (rescueEnabled && nestedType == null ? 1 : 0)]; + ObjectNode rescuedFields = rescueEnabled ? OBJECT_MAPPER.createObjectNode() : null; + JsonNode[] nestedRescues = rescueEnabled ? new JsonNode[projection.size()] : null; + boolean[] present = new boolean[projection.size()]; + int fieldCount = 0; + while (parser.nextToken() != JsonToken.END_OBJECT) { + if (++fieldCount > MAX_CONTAINER_ELEMENTS) { + throw new KafkaRecordEncodingException( + "Kafka JSON object exceeds the maximum field count of " + + MAX_CONTAINER_ELEMENTS + + "."); + } + String fieldName = parser.currentName(); + String path = JsonPath.field(JsonPath.ROOT, fieldName); + validateUnicode(fieldName, path); + Integer position = projectedFields.get(fieldName); + if (position == null || KafkaTableSchema.isReservedName(fieldName)) { + if (!rescueEnabled) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + (KafkaTableSchema.isReservedName(fieldName) + ? ": reserved field." + : ": unknown field.")); + } + parser.nextToken(); + rescuedFields.set(fieldName, readValue(parser, null, path)); + continue; + } + parser.nextToken(); + JsonNode value = readValue(parser, projection.dataTypeAt(position), path); + if (rescueEnabled) { + nestedRescues[position] = + extractNestedUnknownFields( + value, projection.dataTypeAt(position), path); + } + values[position] = converters[position].convert(value, path); + present[position] = true; + } + if (parser.nextToken() != null) { + throw new KafkaRecordEncodingException( + "Kafka record value is not valid strict UTF-8 JSON: trailing content."); + } + for (int i = 0; i < projection.size(); i++) { + if (!present[i]) { + values[i] = + converters[i].convert( + null, JsonPath.field(JsonPath.ROOT, projection.nameAt(i))); + } + } + Object[] decoded = values; + if (nestedType != null) { + decoded = new Object[rescueEnabled ? 2 : 1]; + decoded[0] = GenericRow.of(values); + } + if (rescueEnabled) { + // Root members retain input order; nested unknowns follow projection order. + for (int i = 0; i < projection.size(); i++) { + if (nestedRescues[i] != null) { + rescuedFields.set(projection.nameAt(i), nestedRescues[i]); + } + } + decoded[decoded.length - 1] = + rescuedFields.isEmpty() + ? null + : BinaryString.fromString(rescuedFields.toString()); + } + return decoded; + } catch (KafkaRecordEncodingException e) { + throw e; + } catch (IOException | RuntimeException e) { + throw new KafkaRecordEncodingException( + "Kafka record value is not valid strict UTF-8 JSON.", e); + } + } + + private JsonNode readValue(JsonParser parser, @Nullable DataType dataType, String path) + throws IOException { + if (dataType == null) { + return readBoundedTree(parser, null, path); + } + if (parser.currentToken() == JsonToken.START_OBJECT + || parser.currentToken() == JsonToken.START_ARRAY) { + switch (dataType.getTypeRoot()) { + case ROW: + case ARRAY: + case MAP: + break; + default: + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + path + ": expected a JSON scalar."); + } + } + if (parser.currentToken().isNumeric()) { + // Parse floating-point targets from the original token: BigDecimal loses negative + // zero, and parsing FLOAT through a double can round twice. Other numeric targets + // retain exact Jackson integer/BigDecimal nodes and their strict conversion rules. + switch (dataType.getTypeRoot()) { + case FLOAT: + return FloatNode.valueOf(Float.parseFloat(parser.getText())); + case DOUBLE: + return DoubleNode.valueOf(Double.parseDouble(parser.getText())); + default: + break; + } + } + return readBoundedTree(parser, dataType, path); + } + + private JsonNode readBoundedTree(JsonParser parser, @Nullable DataType dataType, String path) + throws IOException { + // Bound every container while consuming tokens, before allocating excess children. + // Read each child with its schema so numeric tokens retain their original precision + // and floating-point sign. The parser also bounds depth and rejects duplicate fields. + switch (parser.currentToken()) { + case START_ARRAY: + if (dataType != null && !(dataType instanceof ArrayType)) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + path + ": expected a JSON object."); + } + DataType elementType = + dataType == null ? null : ((ArrayType) dataType).getElementType(); + ArrayNode array = OBJECT_MAPPER.createArrayNode(); + while (parser.nextToken() != JsonToken.END_ARRAY) { + checkContainerSize(array.size() + 1, path); + array.add(readValue(parser, elementType, JsonPath.index(path, array.size()))); + } + return array; + case START_OBJECT: + if (dataType instanceof ArrayType) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + path + ": expected a JSON array."); + } + Map rowFields = new HashMap<>(); + if (dataType instanceof RowType) { + for (DataField field : ((RowType) dataType).getFields()) { + rowFields.put(field.getName(), field.getType()); + } + } + ObjectNode object = OBJECT_MAPPER.createObjectNode(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + checkContainerSize(object.size() + 1, path); + String fieldName = parser.currentName(); + String fieldPath = JsonPath.field(path, fieldName); + validateUnicode(fieldName, fieldPath); + DataType fieldType = + dataType instanceof MapType + ? ((MapType) dataType).getValueType() + : rowFields.get(fieldName); + if (fieldType == null && dataType != null && !rescueEnabled) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + fieldPath + ": unknown field."); + } + parser.nextToken(); + object.set(fieldName, readValue(parser, fieldType, fieldPath)); + } + return object; + default: + JsonNode value = OBJECT_MAPPER.readTree(parser); + if (value.isTextual()) { + validateUnicode(value.textValue(), path); + } + // Jackson can produce a DoubleNode on overflow even in BigDecimal mode. + // Never turn a rescued numeric token into the JSON string "Infinity". + if ((value.isDouble() || value.isFloat()) + && !Double.isFinite(value.doubleValue())) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + ": non-finite or overflowing number."); + } + return value; + } + } + + private static void validateUnicode(String value, String path) { + JsonToFlussConverters.validateUnicode(value, path, JSON_STRING_TYPE); + } + + private static void checkContainerSize(int size, String path) { + if (size > MAX_CONTAINER_ELEMENTS) { + throw new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + ": container size exceeds " + + MAX_CONTAINER_ELEMENTS + + "."); + } + } + + private static JsonNode extractNestedUnknownFields( + JsonNode node, DataType dataType, String path) { + if (node == null || node.isNull()) { + return null; + } + if (dataType instanceof RowType && node.isObject()) { + return extractRowUnknownFields((ObjectNode) node, (RowType) dataType, path); + } + if (dataType instanceof ArrayType && node.isArray()) { + return extractArrayUnknownFields((ArrayNode) node, (ArrayType) dataType, path); + } + if (dataType instanceof MapType && node.isObject()) { + return extractMapValueUnknownFields((ObjectNode) node, (MapType) dataType, path); + } + return null; + } + + private static JsonNode extractRowUnknownFields(ObjectNode node, RowType rowType, String path) { + checkContainerSize(node.size(), path); + ObjectNode rescuedFields = OBJECT_MAPPER.createObjectNode(); + List fieldsToRemove = new ArrayList<>(); + Iterator> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + int fieldPosition = rowType.getFieldIndex(field.getKey()); + if (fieldPosition < 0) { + rescuedFields.set(field.getKey(), field.getValue()); + fieldsToRemove.add(field.getKey()); + } else { + JsonNode nestedRescue = + extractNestedUnknownFields( + field.getValue(), + rowType.getTypeAt(fieldPosition), + JsonPath.field(path, field.getKey())); + if (nestedRescue != null) { + rescuedFields.set(field.getKey(), nestedRescue); + } + } + } + for (String fieldName : fieldsToRemove) { + node.remove(fieldName); + } + return rescuedFields.isEmpty() ? null : rescuedFields; + } + + private static JsonNode extractArrayUnknownFields( + ArrayNode node, ArrayType arrayType, String path) { + checkContainerSize(node.size(), path); + ArrayNode rescuedElements = OBJECT_MAPPER.createArrayNode(); + boolean hasRescuedElement = false; + int elementPosition = 0; + for (JsonNode element : node) { + JsonNode rescuedElement = + extractNestedUnknownFields( + element, + arrayType.getElementType(), + JsonPath.index(path, elementPosition)); + if (rescuedElement == null) { + rescuedElements.addNull(); + } else { + rescuedElements.add(rescuedElement); + hasRescuedElement = true; + } + elementPosition++; + } + return hasRescuedElement ? rescuedElements : null; + } + + private static JsonNode extractMapValueUnknownFields( + ObjectNode node, MapType mapType, String path) { + checkContainerSize(node.size(), path); + ObjectNode rescuedValues = OBJECT_MAPPER.createObjectNode(); + Iterator> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + JsonNode rescuedValue = + extractNestedUnknownFields( + field.getValue(), + mapType.getValueType(), + JsonPath.field(path, field.getKey())); + if (rescuedValue != null) { + rescuedValues.set(field.getKey(), rescuedValue); + } + } + return rescuedValues.isEmpty() ? null : rescuedValues; + } + + private Object[] nullValues() { + if (nestedType != null) { + if (!nestedType.isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka null value cannot populate NOT NULL field '_kafka_value'."); + } + return new Object[rescueEnabled ? 2 : 1]; + } + Object[] values = new Object[projection.size() + (rescueEnabled ? 1 : 0)]; + for (int i = 0; i < projection.size(); i++) { + DataType dataType = projection.dataTypeAt(i); + if (!dataType.isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka null value cannot populate NOT NULL field '" + + projection.nameAt(i) + + "'."); + } + } + return values; + } + + private static ObjectMapper createObjectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.getFactory() + .setStreamReadConstraints( + StreamReadConstraints.builder() + .maxNestingDepth(MAX_JSON_NESTING_DEPTH) + .maxNumberLength(StreamReadConstraints.DEFAULT_MAX_NUM_LEN) + .maxStringLength(StreamReadConstraints.DEFAULT_MAX_STRING_LEN) + .build()); + mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + mapper.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); + // Trailing content is checked after the root object, not after individual field values. + return mapper; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java new file mode 100644 index 00000000000..7a9628966b7 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonKafkaFormatFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.format.KafkaFieldDecoder; +import org.apache.fluss.kafka.format.KafkaFormatFactory; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; + +import javax.annotation.Nullable; + +/** Factory for schema-aware Kafka JSON value decoding. */ +@Internal +public final class JsonKafkaFormatFactory implements KafkaFormatFactory { + + @Override + public KafkaValueConversionType format() { + return KafkaValueConversionType.PARSE_JSON; + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaFieldProjection projection) { + return new JsonKafkaFieldDecoder(projection); + } + + @Override + public KafkaFieldDecoder createDecoder(KafkaTableSchema schema) { + return new JsonKafkaFieldDecoder(schema); + } + + @Override + public KafkaFieldDecoder createDecoder( + KafkaFieldProjection projection, @Nullable KafkaValueConversionLayout layout) { + return new JsonKafkaFieldDecoder(projection, layout); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonPath.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonPath.java new file mode 100644 index 00000000000..c6a8eb5b958 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonPath.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +/** Builds bounded, escaped JSON paths for Kafka record errors. */ +final class JsonPath { + + static final String ROOT = "$"; + + private static final int MAX_SEGMENT_LENGTH = 128; + private static final int MAX_PATH_LENGTH = 512; + private static final String TRUNCATED = "..."; + + private JsonPath() {} + + static String field(String path, String fieldName) { + return append(path, "[\"" + escapeAndTruncate(fieldName) + "\"]"); + } + + static String index(String path, int index) { + return append(path, "[" + index + "]"); + } + + private static String append(String path, String segment) { + if (path.length() + segment.length() <= MAX_PATH_LENGTH) { + return path + segment; + } + int prefixLength = MAX_PATH_LENGTH - TRUNCATED.length() - segment.length(); + if (prefixLength <= ROOT.length()) { + return (ROOT + TRUNCATED + segment).substring(0, MAX_PATH_LENGTH); + } + return path.substring(0, Math.min(path.length(), prefixLength)) + TRUNCATED + segment; + } + + private static String escapeAndTruncate(String value) { + StringBuilder escaped = new StringBuilder(MAX_SEGMENT_LENGTH); + int contentLimit = MAX_SEGMENT_LENGTH - TRUNCATED.length(); + int index = 0; + while (index < value.length()) { + String encodedCharacter; + char character = value.charAt(index); + switch (character) { + case '\\': + encodedCharacter = "\\\\"; + break; + case '"': + encodedCharacter = "\\\""; + break; + case '\b': + encodedCharacter = "\\b"; + break; + case '\f': + encodedCharacter = "\\f"; + break; + case '\n': + encodedCharacter = "\\n"; + break; + case '\r': + encodedCharacter = "\\r"; + break; + case '\t': + encodedCharacter = "\\t"; + break; + default: + if (character < 0x20) { + encodedCharacter = String.format("\\u%04x", (int) character); + } else { + encodedCharacter = String.valueOf(character); + } + } + if (escaped.length() + encodedCharacter.length() > contentLimit) { + break; + } + escaped.append(encodedCharacter); + index++; + } + if (index < value.length()) { + escaped.append(TRUNCATED); + } + return escaped.toString(); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java new file mode 100644 index 00000000000..e1bbf8ae52b --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverter.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; + +import javax.annotation.Nullable; + +/** Converts one JSON node into the internal representation of one Fluss field. */ +@Internal +interface JsonToFlussConverter { + + /** Converts the node, which is null when the JSON property is missing. */ + @Nullable + Object convert(@Nullable JsonNode node, String path); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java new file mode 100644 index 00000000000..c5fce8b4d13 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/json/JsonToFlussConverters.java @@ -0,0 +1,505 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BinaryType; +import org.apache.fluss.types.CharType; +import org.apache.fluss.types.DataField; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DecimalType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.TimeType; +import org.apache.fluss.types.TimestampType; + +import javax.annotation.Nullable; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; +import java.util.Base64; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.fluss.utils.DateTimeUtils.timestampToMicros; +import static org.apache.fluss.utils.DateTimeUtils.timestampToNanos; + +/** Strict recursive JSON-to-Fluss converter construction. */ +final class JsonToFlussConverters { + + static final int MAX_SCHEMA_NESTING_DEPTH = KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; + static final int MAX_CONTAINER_ELEMENTS = 10_000; + + private JsonToFlussConverters() {} + + /** + * Creates a converter for one top-level JSON business field, excluding the JSON root container. + * The shared schema contract validates the entire type before recursive converter construction. + */ + static JsonToFlussConverter create(DataType dataType) { + try { + KafkaTableSchema.validateJsonType(dataType); + } catch (InvalidTableException e) { + throw new KafkaTopicSchemaException(e.getMessage()); + } + return createValidated(dataType); + } + + private static JsonToFlussConverter createValidated(DataType dataType) { + final JsonToFlussConverter notNullConverter = createNotNull(dataType); + return (node, path) -> { + if (node == null || node.isNull()) { + if (!dataType.isNullable()) { + throw invalid(path, dataType, "field is missing or null"); + } + return null; + } + try { + return notNullConverter.convert(node, path); + } catch (KafkaRecordEncodingException e) { + throw e; + } catch (RuntimeException e) { + throw invalid(path, dataType, "value cannot be converted", e); + } + }; + } + + private static JsonToFlussConverter createNotNull(DataType dataType) { + switch (dataType.getTypeRoot()) { + case BOOLEAN: + return (node, path) -> { + require(node.isBoolean(), path, dataType, "expected a JSON boolean"); + return node.booleanValue(); + }; + case TINYINT: + return (node, path) -> { + long value = integralValue(node, path, dataType); + require( + value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE, + path, + dataType, + "integer overflow"); + return (byte) value; + }; + case SMALLINT: + return (node, path) -> { + long value = integralValue(node, path, dataType); + require( + value >= Short.MIN_VALUE && value <= Short.MAX_VALUE, + path, + dataType, + "integer overflow"); + return (short) value; + }; + case INTEGER: + return (node, path) -> { + long value = integralValue(node, path, dataType); + require( + value >= Integer.MIN_VALUE && value <= Integer.MAX_VALUE, + path, + dataType, + "integer overflow"); + return (int) value; + }; + case BIGINT: + return (node, path) -> integralValue(node, path, dataType); + case FLOAT: + return (node, path) -> { + require(node.isNumber(), path, dataType, "expected a JSON number"); + float value = node.floatValue(); + require( + Float.isFinite(value), + path, + dataType, + "non-finite or overflowing number"); + return value; + }; + case DOUBLE: + return (node, path) -> { + require(node.isNumber(), path, dataType, "expected a JSON number"); + double value = node.doubleValue(); + require( + Double.isFinite(value), + path, + dataType, + "non-finite or overflowing number"); + return value; + }; + case DECIMAL: + return decimalConverter((DecimalType) dataType); + case CHAR: + return charConverter((CharType) dataType); + case STRING: + return (node, path) -> BinaryString.fromString(textualValue(node, path, dataType)); + case BINARY: + return binaryConverter((BinaryType) dataType); + case BYTES: + return (node, path) -> decodeBase64(node, path, dataType); + case DATE: + return (node, path) -> { + String value = textualValue(node, path, dataType); + try { + return Math.toIntExact(LocalDate.parse(value).toEpochDay()); + } catch (DateTimeException | ArithmeticException e) { + throw invalid(path, dataType, "invalid ISO-8601 date", e); + } + }; + case TIME_WITHOUT_TIME_ZONE: + return timeConverter((TimeType) dataType); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return timestampConverter((TimestampType) dataType); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return timestampLtzConverter((LocalZonedTimestampType) dataType); + case ROW: + return rowConverter((RowType) dataType); + case ARRAY: + return arrayConverter((ArrayType) dataType); + case MAP: + return mapConverter((MapType) dataType); + default: + throw new KafkaTopicSchemaException( + "Kafka JSON format does not support Fluss type " + + dataType.asSummaryString() + + "."); + } + } + + private static JsonToFlussConverter decimalConverter(DecimalType dataType) { + return (node, path) -> { + require(node.isNumber(), path, dataType, "expected a JSON number"); + BigDecimal value = node.decimalValue(); + if (value.signum() == 0) { + return Decimal.zero(dataType.getPrecision(), dataType.getScale()); + } + // Check the resulting digits before setScale can expand a large JSON exponent. + // Use long arithmetic because BigDecimal scales may span the entire int range. + long scaledPrecision = (long) value.precision() - value.scale() + dataType.getScale(); + require( + scaledPrecision <= dataType.getPrecision(), + path, + dataType, + "decimal precision exceeds target"); + require( + value.stripTrailingZeros().scale() <= dataType.getScale(), + path, + dataType, + "decimal scale exceeds target"); + final BigDecimal scaled; + try { + scaled = value.setScale(dataType.getScale(), RoundingMode.UNNECESSARY); + } catch (ArithmeticException e) { + throw invalid(path, dataType, "decimal scale exceeds target", e); + } + Decimal decimal = + Decimal.fromBigDecimal(scaled, dataType.getPrecision(), dataType.getScale()); + if (decimal == null) { + throw invalid(path, dataType, "decimal precision exceeds target"); + } + return decimal; + }; + } + + private static JsonToFlussConverter charConverter(CharType dataType) { + return (node, path) -> { + String value = textualValue(node, path, dataType); + int codePoints = value.codePointCount(0, value.length()); + require( + codePoints <= dataType.getLength(), + path, + dataType, + "character length exceeds target"); + return BinaryString.fromString(value); + }; + } + + private static JsonToFlussConverter binaryConverter(BinaryType dataType) { + return (node, path) -> { + byte[] value = decodeBase64(node, path, dataType); + require( + value.length == dataType.getLength(), + path, + dataType, + "decoded binary length must equal " + dataType.getLength()); + return value; + }; + } + + private static JsonToFlussConverter timeConverter(TimeType dataType) { + return (node, path) -> { + String value = textualValue(node, path, dataType); + final LocalTime time; + try { + time = LocalTime.parse(value); + } catch (DateTimeParseException e) { + throw invalid(path, dataType, "invalid ISO-8601 local time", e); + } + validatePrecision(time.getNano(), dataType.getPrecision(), path, dataType); + require( + time.getNano() % 1_000_000 == 0, + path, + dataType, + "Fluss TIME storage currently requires millisecond precision"); + return Math.toIntExact(time.toNanoOfDay() / 1_000_000); + }; + } + + private static JsonToFlussConverter timestampConverter(TimestampType dataType) { + return (node, path) -> { + String value = textualValue(node, path, dataType); + final LocalDateTime timestamp; + try { + timestamp = LocalDateTime.parse(value); + } catch (DateTimeParseException e) { + throw invalid(path, dataType, "invalid ISO-8601 local timestamp", e); + } + validatePrecision(timestamp.getNano(), dataType.getPrecision(), path, dataType); + long millis = + timestampMillis( + timestamp.toInstant(ZoneOffset.UTC), + dataType.getPrecision(), + path, + dataType); + return TimestampNtz.fromMillis(millis, timestamp.getNano() % 1_000_000); + }; + } + + private static JsonToFlussConverter timestampLtzConverter(LocalZonedTimestampType dataType) { + return (node, path) -> { + String value = textualValue(node, path, dataType); + final OffsetDateTime timestamp; + try { + timestamp = OffsetDateTime.parse(value); + } catch (DateTimeParseException e) { + throw invalid( + path, + dataType, + "invalid ISO-8601 timestamp; an offset or Z is required", + e); + } + validatePrecision(timestamp.getNano(), dataType.getPrecision(), path, dataType); + long millis = + timestampMillis(timestamp.toInstant(), dataType.getPrecision(), path, dataType); + return TimestampLtz.fromEpochMillis(millis, timestamp.getNano() % 1_000_000); + }; + } + + private static long timestampMillis( + Instant timestamp, int precision, String path, DataType dataType) { + try { + long millis = timestamp.toEpochMilli(); + int nanos = timestamp.getNano() % 1_000_000; + // Arrow stores precision 4-6 in microseconds and precision 7-9 in nanoseconds. + // Check here as well so Kafka reports the offending JSON field before encoding. + if (precision > 6) { + timestampToNanos(millis, nanos); + } else if (precision > 3) { + timestampToMicros(millis, nanos); + } + return millis; + } catch (ArithmeticException e) { + throw invalid(path, dataType, "timestamp exceeds storage range", e); + } + } + + private static JsonToFlussConverter rowConverter(RowType dataType) { + List fields = dataType.getFields(); + String[] fieldNames = new String[fields.size()]; + Set declaredFieldNames = new HashSet<>(); + JsonToFlussConverter[] fieldConverters = new JsonToFlussConverter[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + DataField field = fields.get(i); + fieldNames[i] = field.getName(); + if (!declaredFieldNames.add(field.getName())) { + throw new KafkaTopicSchemaException( + "Duplicate Kafka JSON row field '" + field.getName() + "'."); + } + fieldConverters[i] = createValidated(field.getType()); + } + return (node, path) -> { + require(node.isObject(), path, dataType, "expected a JSON object"); + requireContainerSize(node.size(), path, dataType); + Iterator inputFieldNames = node.fieldNames(); + while (inputFieldNames.hasNext()) { + String inputFieldName = inputFieldNames.next(); + require( + declaredFieldNames.contains(inputFieldName), + JsonPath.field(path, inputFieldName), + dataType, + "unknown field"); + } + GenericRow row = new GenericRow(fieldNames.length); + for (int i = 0; i < fieldNames.length; i++) { + String fieldPath = JsonPath.field(path, fieldNames[i]); + row.setField(i, fieldConverters[i].convert(node.get(fieldNames[i]), fieldPath)); + } + return row; + }; + } + + private static JsonToFlussConverter arrayConverter(ArrayType dataType) { + JsonToFlussConverter elementConverter = createValidated(dataType.getElementType()); + return (node, path) -> { + require(node.isArray(), path, dataType, "expected a JSON array"); + requireContainerSize(node.size(), path, dataType); + Object[] elements = new Object[node.size()]; + for (int i = 0; i < node.size(); i++) { + elements[i] = elementConverter.convert(node.get(i), JsonPath.index(path, i)); + } + return new GenericArray(elements); + }; + } + + private static JsonToFlussConverter mapConverter(MapType dataType) { + JsonToFlussConverter valueConverter = createValidated(dataType.getValueType()); + return (node, path) -> { + require(node.isObject(), path, dataType, "expected a JSON object"); + requireContainerSize(node.size(), path, dataType); + Map values = new LinkedHashMap<>(); + Iterator> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + String fieldPath = JsonPath.field(path, field.getKey()); + validateUnicode(field.getKey(), fieldPath, dataType.getKeyType()); + values.put( + BinaryString.fromString(field.getKey()), + valueConverter.convert(field.getValue(), fieldPath)); + } + return new GenericMap(values); + }; + } + + private static void requireContainerSize(int size, String path, DataType dataType) { + require( + size <= MAX_CONTAINER_ELEMENTS, + path, + dataType, + "container size exceeds " + MAX_CONTAINER_ELEMENTS); + } + + private static long integralValue(JsonNode node, String path, DataType dataType) { + require(node.isIntegralNumber(), path, dataType, "expected a JSON integer"); + require(node.canConvertToLong(), path, dataType, "integer overflow"); + return node.longValue(); + } + + private static String textualValue(JsonNode node, String path, DataType dataType) { + require(node.isTextual(), path, dataType, "expected a JSON string"); + String value = node.textValue(); + validateUnicode(value, path, dataType); + return value; + } + + static void validateUnicode(String value, String path, DataType dataType) { + // Jackson can retain unpaired JSON escapes in Java strings. Reject them before + // BinaryString's UTF-8 encoding silently replaces them with a question mark. + for (int i = 0; i < value.length(); i++) { + char character = value.charAt(i); + if (Character.isHighSurrogate(character)) { + require( + i + 1 < value.length() && Character.isLowSurrogate(value.charAt(i + 1)), + path, + dataType, + "unpaired UTF-16 surrogate"); + i++; + } else { + require( + !Character.isLowSurrogate(character), + path, + dataType, + "unpaired UTF-16 surrogate"); + } + } + } + + private static byte[] decodeBase64(JsonNode node, String path, DataType dataType) { + String value = textualValue(node, path, dataType); + try { + return Base64.getDecoder().decode(value); + } catch (IllegalArgumentException e) { + throw invalid(path, dataType, "invalid Base64 string", e); + } + } + + private static void validatePrecision( + int nanos, int precision, String path, DataType dataType) { + if (precision >= 9) { + return; + } + int unit = 1; + for (int i = precision; i < 9; i++) { + unit *= 10; + } + require(nanos % unit == 0, path, dataType, "fractional seconds exceed target precision"); + } + + private static void require(boolean condition, String path, DataType dataType, String reason) { + if (!condition) { + throw invalid(path, dataType, reason); + } + } + + private static KafkaRecordEncodingException invalid( + String path, DataType dataType, String reason) { + return new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + " for " + + dataType.asSummaryString() + + ": " + + reason + + "."); + } + + private static KafkaRecordEncodingException invalid( + String path, DataType dataType, String reason, @Nullable Throwable cause) { + return new KafkaRecordEncodingException( + "Invalid Kafka record value at " + + path + + " for " + + dataType.asSummaryString() + + ": " + + reason + + ".", + cause); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index aa31d618986..e0f0d8d23fb 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -35,6 +35,8 @@ public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + // ASCII "FlussA": distinguishes default-database aliases from canonical topic names. + private static final long DEFAULT_DATABASE_TOPIC_ID_NAMESPACE = 0x466c757373410000L; private final String defaultDatabase; @@ -104,16 +106,29 @@ public static boolean isValidTopic(String topicName) { topicName.substring(0, separator), topicName.substring(separator + 1))); } - /** Maps a Fluss table ID to a stable Kafka topic ID. */ + /** Maps a Fluss table ID to the canonical name's stable Kafka topic ID. */ public Uuid toTopicId(long tableId) { checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); return new Uuid(TOPIC_ID_NAMESPACE, tableId); } + /** + * Maps a Fluss table ID to the default-database alias's stable Kafka topic ID. + * + *

The wire ID must differ from the canonical name's ID because Kafka clients maintain a + * one-to-one ID-to-name map. It must also change on recreation so a client can accept a new + * table's lower leader epoch without first observing a missing-topic response. + */ + public Uuid toAliasTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(DEFAULT_DATABASE_TOPIC_ID_NAMESPACE, tableId); + } + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ public boolean isMappedTopicId(Uuid topicId) { return topicId != null - && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && (topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + || topicId.getMostSignificantBits() == DEFAULT_DATABASE_TOPIC_ID_NAMESPACE) && topicId.getLeastSignificantBits() >= 0L; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index c02879e922a..8d5465ca6a7 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -18,7 +18,9 @@ package org.apache.fluss.kafka.schema; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.kafka.format.json.JsonKafkaFieldDecoder; import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TableDescriptor; @@ -38,7 +40,12 @@ public boolean isEnabled(TableDescriptor table) { /** Resolves one enabled table's contract; invalid enabled tables remain visible as errors. */ public KafkaTableSchema resolve(TableDescriptor table) { try { - return KafkaTableSchema.resolve(table); + KafkaTableSchema schema = KafkaTableSchema.resolve(table); + if (schema.valueConversionType() == KafkaValueConversionType.PARSE_JSON) { + // Metadata and Produce must reject schemas beyond the installed decoder support. + new JsonKafkaFieldDecoder(schema); + } + return schema; } catch (InvalidTableException | IllegalArgumentException e) { throw new KafkaTopicSchemaException(e.getMessage()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java new file mode 100644 index 00000000000..e990e70c4e4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.format.KafkaFieldDecoder; +import org.apache.fluss.kafka.format.KafkaFormatFactoryRegistry; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; + +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Converts schema-mapped Kafka values and fixed system columns into owned Arrow log bytes. */ +@Internal +public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { + private final KafkaFormatFactoryRegistry formats = new KafkaFormatFactoryRegistry(); + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); + private final KafkaCompiledWritePlanCache writePlanCache = new KafkaCompiledWritePlanCache(); + + @Override + public KafkaTopicWritePlan prepare(TableInfo tableInfo) { + return writePlanCache.getOrCompile( + tableInfo, + () -> + new KafkaTopicWritePlan( + tableInfo, schemaResolver.resolve(tableInfo.toTableDescriptor()))); + } + + @Override + public BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception { + return transcode(partitionId, records, prepare(tableInfo)); + } + + @Override + public BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { + checkArgument(partitionId >= 0, "Kafka partition ID must be non-negative."); + checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); + KafkaTableSchema schema = plan.topicSchema(); + KafkaFieldDecoder valueDecoder = formats.createDecoder(schema); + KafkaRowAssembler assembler = plan.rowAssembler(); + return arrowRecordEncoder.encodeStreaming( + consumer -> { + for (Record record : records) { + consumer.append( + assembler.assemble( + valueDecoder.decode(record.borrowedValue()), + record, + partitionId)); + } + }, + plan.tableInfo()); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java new file mode 100644 index 00000000000..c84dd20d690 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.memory.UnmanagedPagedOutputView; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.MemoryLogRecordsArrowBuilder; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.arrow.ArrowWriter; +import org.apache.fluss.row.arrow.ArrowWriterPool; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator; + +import java.util.List; + +/** Encodes assembled physical Fluss rows into one native Arrow log batch. */ +@Internal +public final class FlussArrowRecordEncoder { + + private static final int INITIAL_PAGE_SIZE = 4096; + + /** Encodes all rows using the table's current schema ID and Arrow compression settings. */ + public BytesView encode(List rows, TableInfo tableInfo) throws Exception { + return encodeStreaming( + consumer -> { + for (GenericRow row : rows) { + consumer.append(row); + } + }, + tableInfo); + } + + BytesView encodeStreaming(RowSource rowSource, TableInfo tableInfo) throws Exception { + try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + ArrowWriterPool provider = new ArrowWriterPool(allocator)) { + ArrowWriter writer = + provider.getOrCreateWriter( + tableInfo.getTableId(), + tableInfo.getSchemaId(), + Integer.MAX_VALUE, + tableInfo.getRowType(), + tableInfo.getTableConfig().getArrowCompressionInfo()); + long epoch = writer.getEpoch(); + try { + MemoryLogRecordsArrowBuilder builder = + MemoryLogRecordsArrowBuilder.builder( + tableInfo.getSchemaId(), + writer, + new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), + true, + null); + rowSource.produce(row -> builder.append(ChangeType.APPEND_ONLY, row)); + // The output view owns heap pages. The result remains valid after the Arrow + // writer and allocator close, including across asynchronous append completion. + return builder.build(); + } finally { + writer.recycle(epoch); + } + } + } + + @FunctionalInterface + interface RowSource { + /** Produces rows synchronously and never retains the supplied consumer. */ + void produce(RowConsumer consumer) throws Exception; + } + + @FunctionalInterface + interface RowConsumer { + /** Serializes one row before returning and never retains the row. */ + void append(GenericRow row) throws Exception; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java new file mode 100644 index 00000000000..258e8c557b8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCache.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Bounded, single-flight cache for immutable Kafka topic write plans. + * + *

The bounded map is keyed by {@link TablePath}, while each slot fences its plan with the exact + * {@link TableInfo} instance from the authoritative metadata cache. This avoids the recursive and + * allocation-heavy {@link TableInfo#hashCode()} on every Produce request. An equivalent but + * separately constructed {@code TableInfo} is deliberately treated as a cache miss. In the + * production path, {@code KafkaTableInfoCache} returns the same instance only when the complete + * {@code GetTableInfo} response is unchanged. + * + *

Consequently a delete/recreate, schema change, or property-only Kafka contract change receives + * a different instance and cannot reuse an older plan. Slots are synchronized individually so a + * concurrent cold start compiles once per table version. + */ +@Internal +@ThreadSafe +final class KafkaCompiledWritePlanCache { + + static final long DEFAULT_MAXIMUM_SIZE = 1_000L; + + private final long maximumSize; + private final ConcurrentMap slots; + private final Object slotCreationLock = new Object(); + + KafkaCompiledWritePlanCache() { + this(DEFAULT_MAXIMUM_SIZE); + } + + KafkaCompiledWritePlanCache(long maximumSize) { + checkArgument( + maximumSize > 0, "Kafka compiled write plan cache size must be greater than zero."); + this.maximumSize = maximumSize; + this.slots = new ConcurrentHashMap<>(); + } + + KafkaTopicWritePlan getIfPresent(TableInfo tableInfo) { + checkNotNull(tableInfo); + CacheSlot slot = slots.get(tableInfo.getTablePath()); + if (slot == null) { + return null; + } + synchronized (slot) { + if (slots.get(tableInfo.getTablePath()) != slot) { + return null; + } + return slot.tableInfo == tableInfo ? slot.plan : null; + } + } + + KafkaTopicWritePlan getOrCompile( + TableInfo tableInfo, Supplier planCompiler) { + checkNotNull(tableInfo); + checkNotNull(planCompiler); + + TablePath tablePath = tableInfo.getTablePath(); + while (true) { + CacheSlot slot = getOrCreateSlot(tablePath); + synchronized (slot) { + if (slots.get(tablePath) != slot) { + continue; + } + if (slot.tableInfo == tableInfo && slot.plan != null) { + return slot.plan; + } + KafkaTopicWritePlan plan = checkNotNull(planCompiler.get()); + slot.tableInfo = tableInfo; + slot.plan = plan; + return plan; + } + } + } + + void invalidate(TableInfo tableInfo) { + checkNotNull(tableInfo); + TablePath tablePath = tableInfo.getTablePath(); + CacheSlot slot = slots.get(tablePath); + if (slot == null) { + return; + } + synchronized (slot) { + if (slot.tableInfo != tableInfo) { + return; + } + slot.tableInfo = null; + slot.plan = null; + slots.remove(tablePath, slot); + } + } + + long estimatedSize() { + return slots.size(); + } + + private CacheSlot getOrCreateSlot(TablePath tablePath) { + CacheSlot slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + synchronized (slotCreationLock) { + slot = slots.get(tablePath); + if (slot != null) { + return slot; + } + if (slots.size() >= maximumSize) { + evictOneSlot(); + } + slot = new CacheSlot(); + slots.put(tablePath, slot); + return slot; + } + } + + private void evictOneSlot() { + for (Map.Entry entry : slots.entrySet()) { + CacheSlot slot = entry.getValue(); + synchronized (slot) { + if (slots.remove(entry.getKey(), slot)) { + return; + } + } + } + } + + private static final class CacheSlot { + private TableInfo tableInfo; + private KafkaTopicWritePlan plan; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java new file mode 100644 index 00000000000..b48dd9d894f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that Kafka record bytes cannot be decoded using the configured data format. */ +@Internal +public final class KafkaRecordEncodingException extends IllegalArgumentException { + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message) { + super(message); + } + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java new file mode 100644 index 00000000000..6c87d80e273 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; + +import java.util.List; + +/** Converts copied Kafka records into the native Fluss log representation. */ +@Internal +public interface KafkaRecordTranscoder { + /** Validates the table contract and prepares a shareable plan once per topic write. */ + default KafkaTopicWritePlan prepare(TableInfo tableInfo) { + return new KafkaTopicWritePlan( + tableInfo, new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor())); + } + + /** Transcodes one requested partition using an already prepared plan. */ + default BytesView transcode(int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { + return transcode(partitionId, records, plan.tableInfo()); + } + + /** Transcodes records from the requested Kafka partition using the target table schema. */ + BytesView transcode(int partitionId, List records, TableInfo tableInfo) + throws Exception; +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java new file mode 100644 index 00000000000..289b5bd7177 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Assembles decoded values and system columns using the shared fixed-column contract. */ +@Internal +public final class KafkaRowAssembler { + + private final KafkaTableSchema topicSchema; + + /** Creates an assembler for the resolved topic schema. */ + public KafkaRowAssembler(KafkaTableSchema topicSchema) { + this.topicSchema = checkNotNull(topicSchema); + } + + /** + * Assembles one Fluss row from a record and its requested partition for immediate synchronous + * encoding. + * + *

The row borrows decoded values, keys and header payloads; it must not modify them or + * retain them after conversion. + */ + public GenericRow assemble(Object[] valueValues, Record record, int partitionId) { + GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); + boolean withRescue = + topicSchema.valueConversionType() == KafkaValueConversionType.PARSE_JSON + && topicSchema.rescuePosition() >= 0; + setProjectedFields(row, topicSchema.valueProjection(), valueValues, withRescue); + if (withRescue) { + row.setField(topicSchema.rescuePosition(), valueValues[valueValues.length - 1]); + } + if (topicSchema.keyPosition() >= 0) { + int position = topicSchema.keyPosition(); + byte[] key = record.borrowedKey(); + if (key == null && !topicSchema.rowType().getTypeAt(position).isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka key cannot populate NOT NULL _kafka_key."); + } + row.setField(position, key); + } + if (topicSchema.headersPosition() >= 0) { + int position = topicSchema.headersPosition(); + row.setField( + position, + toHeaders( + record.headers(), (MapType) topicSchema.rowType().getTypeAt(position))); + } + if (topicSchema.metadataPosition() >= 0) { + int position = topicSchema.metadataPosition(); + RowType type = (RowType) topicSchema.rowType().getTypeAt(position); + if (!record.hasTimestamp() && !type.getTypeAt(0).isNullable()) { + throw new KafkaRecordEncodingException( + "Missing Kafka timestamp cannot populate NOT NULL _kafka_metadata.timestamp."); + } + row.setField( + position, + GenericRow.of( + record.hasTimestamp() + ? TimestampLtz.fromEpochMillis(record.timestamp()) + : null, + timestampTypeId(record.timestampType()), + partitionId)); + } + // A declared _kafka_rescue stays null for binary/string; omitted columns do not exist. + return row; + } + + private static int timestampTypeId(TimestampType type) { + // Preserve batch provenance even when a record's timestamp is absent. The legacy record + // constructor supplies UNKNOWN, so a present timestamp alone cannot imply CreateTime. + switch (type) { + case UNKNOWN: + return -1; + case CREATE_TIME: + return 0; + case LOG_APPEND_TIME: + return 1; + default: + throw new KafkaRecordEncodingException("Unsupported Kafka timestamp type: " + type); + } + } + + private static void setProjectedFields( + GenericRow row, KafkaFieldProjection projection, Object[] values, boolean withRescue) { + int expected = projection.size() + (withRescue ? 1 : 0); + if (values.length != expected) { + throw new IllegalArgumentException( + "Kafka decoder returned " + + values.length + + " fields but the value/rescue contract requires " + + expected + + "."); + } + for (int i = 0; i < projection.size(); i++) { + Object value = values[i]; + if (value == null && !projection.dataTypeAt(i).isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka record cannot populate NOT NULL field '" + + projection.nameAt(i) + + "' with null."); + } + row.setField(projection.positionAt(i), value); + } + } + + private static GenericMap toHeaders(List headers, MapType type) { + Map values = new LinkedHashMap<>(); + for (RecordHeader header : headers) { + values.put(BinaryString.fromString(header.name()), header.borrowedValue()); + } + // Validate the final map: an overwritten null must not reject a non-null last value. + for (Map.Entry entry : values.entrySet()) { + if (entry.getKey() == null && !type.getKeyType().isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka header name cannot populate NOT NULL _kafka_headers map keys."); + } + if (entry.getValue() == null && !type.getValueType().isNullable()) { + throw new KafkaRecordEncodingException( + "Null Kafka header value cannot populate NOT NULL _kafka_headers map values."); + } + } + return new GenericMap(values); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java new file mode 100644 index 00000000000..1a494192660 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaTopicWritePlan.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.format.KafkaFormatFactoryRegistry; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.TableInfo; + +import javax.annotation.concurrent.ThreadSafe; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Immutable value conversion plan; mutable Arrow resources are owned by each conversion. */ +@Internal +@ThreadSafe +public final class KafkaTopicWritePlan { + private final TableInfo tableInfo; + private final KafkaTableSchema topicSchema; + private final KafkaRowAssembler rowAssembler; + + KafkaTopicWritePlan(TableInfo tableInfo, KafkaTableSchema topicSchema) { + this.tableInfo = checkNotNull(tableInfo); + this.topicSchema = checkNotNull(topicSchema); + // Validate supported value formats and projections when compiling the plan. + new KafkaFormatFactoryRegistry().createDecoder(topicSchema); + this.rowAssembler = new KafkaRowAssembler(topicSchema); + } + + /** Returns the authoritative table snapshot used to compile this plan. */ + public TableInfo tableInfo() { + return tableInfo; + } + + KafkaTableSchema topicSchema() { + return topicSchema; + } + + KafkaRowAssembler rowAssembler() { + return rowAssembler; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index dda2e52bde5..f745fe7fe8b 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -17,6 +17,8 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; @@ -70,7 +72,8 @@ public void testUnsupportedApiVersionsRequestReturnsVersionZeroErrorResponse() { assertThat(parsedRequest.hasUnsupportedRequestVersion()).isTrue(); assertThat(parsedRequest.version()).isZero(); - new KafkaRequestHandler().processRequest(request); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + new KafkaRequestHandler(new Configuration(), service).processRequest(request); channel.runPendingTasks(); responseBuffer = channel.readOutbound(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java new file mode 100644 index 00000000000..8f4f753cf1a --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaJsonProduceITCase.java @@ -0,0 +1,1371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalMap; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.MemoryRecordsBuilder; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.record.TimestampType; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.math.BigDecimal; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies DDL-discovered JSON writes and native Arrow readback. */ +class KafkaJsonProduceITCase { + private static final String DATABASE = "fluss"; + private static final byte[] KEY = new byte[] {(byte) 0xff, 1}; + private static final String FORGED_METADATA = + "\"_kafka_metadata\":{\"timestamp\":0,\"timestamp_type\":-1,\"partition\":99}"; + private static final RowType SCALAR_TYPE = + RowType.builder() + .field("id", DataTypes.INT()) + .field("amount", DataTypes.DECIMAL(30, 10)) + .field("received_at", DataTypes.TIMESTAMP(6)) + .field("emitted_at", DataTypes.TIMESTAMP_LTZ(6)) + .field("float_value", DataTypes.FLOAT()) + .field("double_value", DataTypes.DOUBLE()) + .field("optional", DataTypes.STRING()) + .field("rounded_float", DataTypes.FLOAT()) + .build(); + + private static final RowType NUMBER_TYPE = + RowType.builder() + .field("float_value", DataTypes.FLOAT()) + .field("double_value", DataTypes.DOUBLE()) + .field("rounded_float", DataTypes.FLOAT()) + .field("amount", DataTypes.DECIMAL(30, 10)) + .build(); + private static final RowType CUSTOMER_TYPE = + RowType.builder() + .field("name", DataTypes.STRING()) + .field(KafkaTableSchema.KEY_COLUMN, DataTypes.INT()) + .field(KafkaTableSchema.VALUE_COLUMN, DataTypes.INT()) + .field(KafkaTableSchema.HEADERS_COLUMN, DataTypes.INT()) + .field(KafkaTableSchema.METADATA_COLUMN, DataTypes.INT()) + .field(KafkaTableSchema.RESCUE_COLUMN, DataTypes.INT()) + .build(); + private static final RowType COMPLEX_TYPE = + RowType.builder() + .field("id", DataTypes.INT()) + .field("customer", CUSTOMER_TYPE) + .field("measurements", NUMBER_TYPE) + .field("float_values", DataTypes.ARRAY(DataTypes.FLOAT())) + .field("double_values", DataTypes.ARRAY(DataTypes.DOUBLE())) + .field("float_attributes", DataTypes.MAP(DataTypes.STRING(), DataTypes.FLOAT())) + .field( + "double_attributes", + DataTypes.MAP(DataTypes.STRING(), DataTypes.DOUBLE())) + .field("items", DataTypes.ARRAY(NUMBER_TYPE)) + .field("attributes", DataTypes.MAP(DataTypes.STRING(), NUMBER_TYPE)) + .build(); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @ParameterizedTest + @CsvSource({"flat", "nested"}) + void testComplexJsonRoundTripPreservesNumericValuesAndMetadata(String layout) throws Exception { + TablePath path = TablePath.of(DATABASE, "json_complex_" + layout); + int metadataPosition = layout.equals("nested") ? 3 : COMPLEX_TYPE.getFieldCount() + 2; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(COMPLEX_TYPE, layout, true, false, 2)); + try (KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(2); + String[] negativeZeros = {"-0", "-0.0", "-0e0"}; + for (int index = 0; index < negativeZeros.length; index++) { + assertThat( + producer.send( + new ProducerRecord<>( + path.toString(), + 1, + 123L + index, + KEY, + bytes( + complexJson( + index, + negativeZeros[index])))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(index); + } + read( + connection, + path, + 1, + negativeZeros.length, + (row, index) -> { + assertThat(row.getBytes(0)).isEqualTo(KEY); + assertHeaders(row.getMap(metadataPosition - 1), false); + assertMetadata(row, metadataPosition, 123L + index, 0, 1); + InternalRow value = + layout.equals("nested") + ? row.getRow(1, COMPLEX_TYPE.getFieldCount()) + : row; + int offset = layout.equals("nested") ? 0 : 1; + assertThat(value.getInt(offset)).isEqualTo(index); + InternalRow customer = + value.getRow(offset + 1, CUSTOMER_TYPE.getFieldCount()); + assertThat(customer.getString(0).toString()).isEqualTo("Alice"); + // Reserved system names are ordinary declared business fields below + // root. + for (int field = 1; field < CUSTOMER_TYPE.getFieldCount(); field++) { + assertThat(customer.getInt(field)).isEqualTo(field); + } + assertNumbers(value.getRow(offset + 2, NUMBER_TYPE.getFieldCount())); + assertThat(value.getArray(offset + 3).size()).isEqualTo(2); + assertNegativeZero(value.getArray(offset + 3).getFloat(0)); + assertThat(value.getArray(offset + 3).getFloat(1)) + .isEqualTo(1.0000001F); + assertThat(value.getArray(offset + 4).size()).isEqualTo(1); + assertNegativeZero(value.getArray(offset + 4).getDouble(0)); + InternalMap floats = value.getMap(offset + 5); + assertThat(floats.size()).isEqualTo(2); + assertThat(floats.keyArray().getString(0).toString()).isEqualTo("zero"); + assertNegativeZero(floats.valueArray().getFloat(0)); + assertThat(floats.keyArray().getString(1).toString()) + .isEqualTo("rounded"); + assertThat(floats.valueArray().getFloat(1)).isEqualTo(1.0000001F); + InternalMap doubles = value.getMap(offset + 6); + assertThat(doubles.size()).isEqualTo(1); + assertNegativeZero(doubles.valueArray().getDouble(0)); + assertThat(value.getArray(offset + 7).size()).isEqualTo(1); + assertNumbers( + value.getArray(offset + 7) + .getRow(0, NUMBER_TYPE.getFieldCount())); + InternalMap attributes = value.getMap(offset + 8); + assertThat(attributes.size()).isEqualTo(1); + assertThat(attributes.keyArray().getString(0).toString()) + .isEqualTo("dynamic"); + assertNumbers( + attributes.valueArray().getRow(0, NUMBER_TYPE.getFieldCount())); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testInvalidComplexPartitionIsAtomicAndDoesNotBlockOtherPartitions( + String layout, boolean withRescue) throws Exception { + TablePath path = + TablePath.of(DATABASE, "json_complex_rejections_" + layout + "_" + withRescue); + int metadataPosition = layout.equals("nested") ? 3 : COMPLEX_TYPE.getFieldCount() + 2; + List invalidValues = + new ArrayList<>( + Arrays.asList( + "{\"id\":1,\"customer\":{\"name\":\"Alice\",\"unknown\":1}}", + "{\"id\":1,\"customer\":{\"name\":[]}}", + "{\"id\":1,\"customer\":{\"name\":true}}", + "{\"id\":1,\"customer\":{\"name\":1}}", + "{\"id\":1,\"customer\":[]}", + "{\"id\":1,\"float_values\":[1,\"bad\"]}", + "{\"id\":1,\"double_values\":[1,\"bad\"]}", + "{\"id\":1,\"float_values\":{}}", + "{\"id\":1,\"float_attributes\":[]}", + "{\"id\":1,\"double_attributes\":{\"bad\":\"bad\"}}", + "{\"id\":1,\"items\":[{\"float_value\":\"bad\"}]}", + "{\"id\":1,\"items\":[{\"unknown\":1}]}", + "{\"id\":1,\"attributes\":{\"bad\":{\"amount\":\"bad\"}}}", + "{\"id\":1,\"attributes\":{\"bad\":{\"unknown\":1}}}", + // The escaped surrogate must not be encoded as '?' and overwrite + // that key. + "{\"id\":1,\"float_attributes\":{\"?\":1,\"\\ud800\":2}}", + "{\"id\":1,\"double_attributes\":{\"?\":1,\"\\udc00\":2}}")); + if (withRescue) { + invalidValues.removeIf(value -> value.contains("\"unknown\"")); + } + invalidValues.add("{\"id\":1,\"measurements\":{\"float_value\":1e40}}"); + invalidValues.add( + "{\"id\":1,\"measurements\":{\"float_value\":0,\"double_value\":0,\"rounded_float\":0,\"amount\":1.12345678901}}"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(COMPLEX_TYPE, layout, true, withRescue, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port()); + KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(2); + for (int index = 0; index < invalidValues.size(); index++) { + String name = index % 2 == 0 ? path.toString() : path.getTableName(); + ProduceResponse response = + produce( + socket, + name, + MemoryRecords.withRecords( + Compression.NONE, + record(complexJson(1000, "-0")), + record(invalidValues.get(index))), + MemoryRecords.withRecords( + Compression.NONE, record(complexJson(index, "-0e0")))); + PartitionProduceResponse rejected = partition(response, name, 0); + assertThat(rejected.errorCode()) + .as( + "rejection for %s with rescue=%s", + invalidValues.get(index), withRescue) + .isEqualTo(Errors.INVALID_RECORD.code()); + assertThat(rejected.baseOffset()).isEqualTo(-1L); + PartitionProduceResponse accepted = partition(response, name, 1); + assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(accepted.baseOffset()).isEqualTo(index); + } + assertThat( + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + 789L, + KEY, + bytes(complexJson(-1, "-0.0")))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + read( + connection, + path, + 0, + 1, + (row, index) -> { + assertThat(valueId(row, layout, COMPLEX_TYPE)).isEqualTo(-1); + assertMetadata(row, metadataPosition, 789L, 0, 0); + if (withRescue) { + assertThat(row.isNullAt(metadataPosition + 1)).isTrue(); + } + }); + read( + connection, + path, + 1, + invalidValues.size(), + (row, index) -> { + assertThat(valueId(row, layout, COMPLEX_TYPE)).isEqualTo(index); + assertMetadata(row, metadataPosition, 123L, 0, 1); + if (withRescue) { + assertThat(row.isNullAt(metadataPosition + 1)).isTrue(); + } + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testScalarJsonAliasesAndOptionalSystemColumnsRoundTrip( + String layout, boolean withSystemColumns) throws Exception { + TablePath path = TablePath.of(DATABASE, "json_scalars_" + layout + "_" + withSystemColumns); + TableDescriptor descriptor = + descriptor(SCALAR_TYPE, layout, withSystemColumns, withSystemColumns, 1); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor); + try (KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(1); + assertThat(producer.partitionsFor(path.getTableName())).hasSize(1); + RecordMetadata first = + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + 123L, + KEY, + bytes(scalarJson(7, false)), + Arrays.asList( + new RecordHeader("source", bytes("old")), + new RecordHeader( + "source", new byte[] {1, 2}), + new RecordHeader("nullable", null)))) + .get(30, TimeUnit.SECONDS); + assertThat(first.topic()).isEqualTo(path.toString()); + assertThat(first.offset()).isZero(); + // A bare alias and the full name append to the same native table and bucket. + RecordMetadata second = + producer.send( + new ProducerRecord( + path.getTableName(), + 0, + 456L, + null, + bytes(scalarJson(8, true)))) + .get(30, TimeUnit.SECONDS); + assertThat(second.topic()).isEqualTo(path.getTableName()); + assertThat(second.offset()).isEqualTo(1L); + read( + connection, + path, + 0, + 2, + (row, index) -> { + int valuePosition = withSystemColumns ? 1 : 0; + InternalRow value = + layout.equals("nested") + ? row.getRow(valuePosition, SCALAR_TYPE.getFieldCount()) + : row; + int fieldOffset = layout.equals("nested") ? 0 : valuePosition; + assertScalars(value, fieldOffset, 7 + index); + if (withSystemColumns) { + if (index == 0) { + assertThat(row.getBytes(0)).isEqualTo(KEY); + } else { + assertThat(row.isNullAt(0)).isTrue(); + } + int headersPosition = + valuePosition + + (layout.equals("nested") + ? 1 + : SCALAR_TYPE.getFieldCount()); + assertHeaders(row.getMap(headersPosition), index == 0); + assertMetadata( + row, headersPosition + 1, index == 0 ? 123L : 456L, 0, 0); + assertThat(row.isNullAt(headersPosition + 2)).isTrue(); + } + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @CsvSource({ + "flat,scalar", + "nested,scalar", + "flat,row", + "nested,row", + "flat,array", + "nested,array", + "flat,map", + "nested,map" + }) + void testNullValuesPersistWithSystemColumnsAndContinuousOffsets(String layout, String fieldType) + throws Exception { + TablePath path = TablePath.of(DATABASE, "json_null_values_" + layout + "_" + fieldType); + // Null Kafka values populate nullable flat fields or the nullable nested value container. + RowType valueType = + RowType.builder() + .field("id", nullValueBusinessType(fieldType).copy(true)) + .field("label", DataTypes.STRING()) + .build(); + Header[] headers = { + new RecordHeader("source", bytes("old")), + new RecordHeader("source", new byte[] {1, 2}), + new RecordHeader("nullable", null) + }; + // Nullable flat fields also accept explicit JSON null and missing fields. + SimpleRecord[] records = new SimpleRecord[layout.equals("flat") ? 7 : 5]; + for (int index = 0; index < records.length; index++) { + String json = + index == 2 || index == 4 + ? "{\"id\":" + + nullValueBusinessJson(fieldType, index) + + ",\"label\":\"v" + + index + + "\"}" + : null; + if (index == 5) { + json = "{\"id\":null,\"label\":null,\"extra\":{\"index\":5}}"; + } else if (index == 6) { + json = "{\"extra\":{\"index\":6}}"; + } else if (index == 2) { + json = "{\"extra\":{\"index\":2}," + json.substring(1); + } + records[index] = + new SimpleRecord( + (index + 1) * 10L, + index % 2 == 0 ? KEY : null, + json == null ? null : bytes(json), + headers); + } + MemoryRecords[] batches = { + MemoryRecords.withRecords(Compression.NONE, records[0], records[1]), + MemoryRecords.withRecords( + Compression.NONE, Arrays.copyOfRange(records, 2, records.length)) + }; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(valueType, layout, true, true, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port())) { + for (int batch = 0; batch < batches.length; batch++) { + String topic = batch == 0 ? path.toString() : path.getTableName(); + ProduceResponse response = + produce(socket, topic, batches[batch], batches[batch]); + for (int partition = 0; partition < 2; partition++) { + PartitionProduceResponse result = partition(response, topic, partition); + assertThat(result.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(result.baseOffset()).isEqualTo(batch == 0 ? 0L : 2L); + } + } + for (int partition = 0; partition < 2; partition++) { + int expectedPartition = partition; + read( + connection, + path, + partition, + records.length, + (row, index) -> { + if (index % 2 == 0) { + assertThat(row.getBytes(0)).isEqualTo(KEY); + } else { + assertThat(row.isNullAt(0)).isTrue(); + } + if (index == 2 || index == 4) { + InternalRow value = + layout.equals("nested") ? row.getRow(1, 2) : row; + int start = layout.equals("nested") ? 0 : 1; + assertNullValueBusinessId(value, start, fieldType, index); + assertThat(value.getString(start + 1).toString()) + .isEqualTo("v" + index); + } else { + assertThat(row.isNullAt(1)).isTrue(); + if (layout.equals("flat")) { + assertThat(row.isNullAt(2)).isTrue(); + } + } + int headersPosition = layout.equals("nested") ? 2 : 3; + assertHeaders(row.getMap(headersPosition), true); + assertMetadata( + row, + headersPosition + 1, + (index + 1) * 10L, + 0, + expectedPartition); + if (index == 2 || index == 5 || index == 6) { + assertThat(row.getString(headersPosition + 2).toString()) + .isEqualTo("{\"extra\":{\"index\":" + index + "}}"); + } else { + // A following null or known-only value must not reuse rescue. + assertThat(row.isNullAt(headersPosition + 2)).isTrue(); + } + }); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @CsvSource({ + "flat,scalar,false", + "nested,scalar,false", + "flat,row,false", + "nested,row,false", + "flat,array,false", + "nested,array,false", + "flat,map,false", + "nested,map,false", + "flat,scalar,true", + "nested,scalar,true", + "flat,row,true", + "nested,row,true", + "flat,array,true", + "nested,array,true", + "flat,map,true", + "nested,map,true" + }) + void testNotNullBusinessFieldsAreRejectedAtTableCreation( + String layout, String fieldType, boolean withRescue) throws Exception { + TablePath path = + TablePath.of( + DATABASE, + "json_required_value_" + layout + "_" + fieldType + "_" + withRescue); + RowType valueType = + RowType.builder(false) + .field("id", nullValueBusinessType(fieldType).copy(false)) + .build(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + assertThatThrownBy( + () -> + create( + admin, + path, + descriptor(valueType, layout, true, withRescue, 2))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("nullable fields"); + } + } + + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testInvalidPartitionIsAtomicAndDoesNotBlockOtherPartitions( + String layout, boolean withRescue) throws Exception { + TablePath path = TablePath.of(DATABASE, "json_rejections_" + layout + "_" + withRescue); + RowType valueType = RowType.builder().field("id", DataTypes.INT()).build(); + List invalidValues = + new ArrayList<>( + Arrays.asList( + "", + "{", + "null", + "{\"id\":\"bad\"}", + "{\"id\":[]}", + "{\"id\":1} {}", + "{\"id\":1,\"id\":2}")); + if (!withRescue) { + invalidValues.add("{\"id\":1,\"unknown\":2}"); + for (String reserved : + new String[] { + KafkaTableSchema.KEY_COLUMN, + KafkaTableSchema.HEADERS_COLUMN, + KafkaTableSchema.METADATA_COLUMN, + KafkaTableSchema.RESCUE_COLUMN, + KafkaTableSchema.VALUE_COLUMN + }) { + // This also rejects the erroneous nested payload wrapper {"_kafka_value":{...}}. + String value = + reserved.equals(KafkaTableSchema.METADATA_COLUMN) + ? "{\"timestamp\":0,\"timestamp_type\":1,\"partition\":99}" + : "{\"id\":2}"; + invalidValues.add("{\"id\":1,\"" + reserved + "\":" + value + "}"); + } + } + invalidValues.add("{" + FORGED_METADATA + ",\"id\":\"bad\"}"); + invalidValues.add("{\"id\":2147483648}"); + invalidValues.add("{\"id\":1,\"_kafka_rescue\":\"\\ud800\"}"); + invalidValues.add("{\"id\":1,\"extra\":{\"x\":1,\"x\":2}}"); + StringBuilder oversized = new StringBuilder("{\"id\":1,\"extra\":["); + for (int i = 0; i < 10000; i++) { + if (i > 0) { + oversized.append(','); + } + oversized.append('0'); + } + invalidValues.add(oversized.append(",0]}").toString()); + String deep = "0"; + for (int i = 0; i < 67; i++) { + deep = "[" + deep + "]"; + } + invalidValues.add("{\"id\":1,\"extra\":" + deep + "}"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(valueType, layout, true, withRescue, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port()); + KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(2); + for (int index = 0; index < invalidValues.size(); index++) { + String name = index % 2 == 0 ? path.toString() : path.getTableName(); + ProduceResponse response = + produce( + socket, + name, + MemoryRecords.withRecords( + Compression.NONE, + record("{\"id\":1000}"), + record(invalidValues.get(index))), + MemoryRecords.withRecords( + Compression.NONE, record("{\"id\":" + index + "}"))); + assertThat(response.data().responses()).hasSize(1); + assertThat(response.data().responses().find(name).partitionResponses()) + .hasSize(2); + PartitionProduceResponse rejected = partition(response, name, 0); + assertThat(rejected.errorCode()) + .as( + "rejection for %s with rescue=%s", + invalidValues.get(index), withRescue) + .isEqualTo(Errors.INVALID_RECORD.code()); + assertThat(rejected.baseOffset()).isEqualTo(-1L); + PartitionProduceResponse accepted = partition(response, name, 1); + assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(accepted.baseOffset()).isEqualTo(index); + } + // Known-field and JSON validation failures remain atomic with rescue enabled. + // A later successful write must be the first actual append to the rejected bucket. + assertThat( + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + 123L, + KEY, + bytes("{\"id\":-1}"))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isZero(); + read( + connection, + path, + 0, + 1, + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(-1); + assertMetadata(row, 3, 123L, 0, 0); + }); + read( + connection, + path, + 1, + invalidValues.size(), + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(index); + assertMetadata(row, 3, 123L, 0, 1); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testBatchMetadataRoundTripWithAndWithoutTimestamps(String layout, boolean withRescue) + throws Exception { + TablePath path = TablePath.of(DATABASE, "json_metadata_" + layout + "_" + withRescue); + RowType valueType = RowType.builder().field("id", DataTypes.INT()).build(); + // Magic v2 batches carry CREATE_TIME or LOG_APPEND_TIME even without a timestamp. + TimestampType[] types = { + TimestampType.CREATE_TIME, + TimestampType.LOG_APPEND_TIME, + TimestampType.CREATE_TIME, + TimestampType.LOG_APPEND_TIME + }; + long[] expectedTimestamps = {0L, 456L, 123L, 123L, -1L, -1L, -1L, -1L}; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(valueType, layout, true, withRescue, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port())) { + for (int batch = 0; batch < types.length; batch++) { + long firstTimestamp = batch < 2 ? 0L : RecordBatch.NO_TIMESTAMP; + long secondTimestamp = batch < 2 ? 456L : RecordBatch.NO_TIMESTAMP; + long appendTimestamp = batch == 1 ? 123L : RecordBatch.NO_TIMESTAMP; + MemoryRecords records = + timestampedRecords( + types[batch], + appendTimestamp, + new SimpleRecord( + firstTimestamp, + KEY, + bytes(idJson(batch * 2, withRescue))), + new SimpleRecord( + secondTimestamp, + KEY, + bytes(idJson(batch * 2 + 1, withRescue)))); + String topic = batch % 2 == 0 ? path.toString() : path.getTableName(); + ProduceResponse response = produce(socket, topic, records, records); + for (int partition = 0; partition < 2; partition++) { + PartitionProduceResponse result = partition(response, topic, partition); + assertThat(result.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(result.baseOffset()).isEqualTo(batch * 2L); + } + } + for (int partition = 0; partition < 2; partition++) { + int expectedPartition = partition; + read( + connection, + path, + partition, + expectedTimestamps.length, + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(index); + assertMetadata( + row, + 3, + expectedTimestamps[index], + (index / 2) % 2, + expectedPartition); + if (withRescue) { + assertThat(row.getString(4).toString()) + .isEqualTo("{" + FORGED_METADATA + "}"); + } + }); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testMissingMetadataTimestampPersistsAsNull(String layout, boolean withRescue) + throws Exception { + TablePath path = + TablePath.of(DATABASE, "json_required_timestamp_" + layout + "_" + withRescue); + RowType valueType = RowType.builder().field("id", DataTypes.INT()).build(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor(valueType, layout, true, withRescue, 2)); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port()); + KafkaProducer producer = producer()) { + ProduceResponse response = + produce( + socket, + path.toString(), + timestampedRecords( + TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + record(idJson(1000, withRescue)), + new SimpleRecord( + RecordBatch.NO_TIMESTAMP, + KEY, + bytes(idJson(1001, withRescue)))), + MemoryRecords.withRecords(Compression.NONE, record("{\"id\":1}"))); + PartitionProduceResponse missingTimestamp = partition(response, path.toString(), 0); + assertThat(missingTimestamp.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(missingTimestamp.baseOffset()).isZero(); + PartitionProduceResponse accepted = partition(response, path.toString(), 1); + assertThat(accepted.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(accepted.baseOffset()).isZero(); + assertThat( + producer.send( + new ProducerRecord<>( + path.getTableName(), + 0, + 789L, + KEY, + bytes("{\"id\":-1}"))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(2L); + read( + connection, + path, + 0, + 3, + (row, index) -> { + if (index == 0) { + assertThat(valueId(row, layout)).isEqualTo(1000); + assertMetadata(row, 3, 123L, 0, 0); + } else if (index == 1) { + assertThat(valueId(row, layout)).isEqualTo(1001); + assertMetadata(row, 3, RecordBatch.NO_TIMESTAMP, 0, 0); + } else { + assertThat(valueId(row, layout)).isEqualTo(-1); + assertMetadata(row, 3, 789L, 0, 0); + } + }); + read( + connection, + path, + 1, + 1, + (row, index) -> { + assertThat(valueId(row, layout)).isEqualTo(1); + assertMetadata(row, 3, 123L, 0, 1); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + private static String idJson(int id, boolean forgedMetadata) { + return "{\"id\":" + id + (forgedMetadata ? "," + FORGED_METADATA : "") + "}"; + } + + private static void assertMetadata( + InternalRow row, int position, long timestamp, int timestampType, int partition) { + assertThat(row.isNullAt(position)).isFalse(); + InternalRow metadata = row.getRow(position, 3); + if (timestamp == RecordBatch.NO_TIMESTAMP) { + assertThat(metadata.isNullAt(0)).isTrue(); + } else { + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(timestamp); + } + assertThat(metadata.getInt(1)).isEqualTo(timestampType); + assertThat(metadata.getInt(2)).isEqualTo(partition); + } + + @ParameterizedTest + @CsvSource({ + "flat,false,false", + "flat,false,true", + "flat,true,false", + "flat,true,true", + "nested,false,false", + "nested,false,true", + "nested,true,false", + "nested,true,true" + }) + void testRescueRoundTripWithReorderedSystemColumns( + String layout, boolean withSystemColumns, boolean rescueFirst) throws Exception { + TablePath path = + TablePath.of( + DATABASE, + "json_rescue_" + layout + "_" + withSystemColumns + "_" + rescueFirst); + TableDescriptor descriptor = + reorderedRescueDescriptor(layout, withSystemColumns, rescueFirst); + KafkaTableSchema contract = KafkaTableSchema.resolve(descriptor); + ObjectMapper mapper = new ObjectMapper(); + List payloads = new ArrayList<>(); + List expected = new ArrayList<>(); + String extra = "{\"values\":[1,null,2]}"; + String[] rescueValues = {"null", "123", "\"text\"", "{\"a\":[1,null]}"}; + String[] zeros = {"-0", "-0.0", "-0e0", "-0"}; + for (int i = 0; i < rescueValues.length; i++) { + String json = + complexJson(i, zeros[i]) + .replace( + "\"amount\":12345678901234567890.1234567890}", + "\"amount\":12345678901234567890.1234567890,\"extra\":" + + extra + + "}"); + String collisions = + "\"_kafka_key\":{\"id\":2},\"_kafka_headers\":null," + + FORGED_METADATA + + ",\"_kafka_value\":{\"id\":2}," + + "\"_kafka_rescue\":" + + rescueValues[i]; + payloads.add(json.substring(0, json.length() - 1) + "," + collisions + "}"); + expected.add( + mapper.readTree( + "{" + + collisions + + ",\"measurements\":{\"extra\":" + + extra + + "},\"items\":[{\"extra\":" + + extra + + "}],\"attributes\":{\"dynamic\":{\"extra\":" + + extra + + "}}}")); + } + payloads.add(complexJson(4, "-0e0")); + expected.add(null); + // These were rejection cases in the inherited no-capture implementation. + payloads.add("{\"id\":1,\"unknown\":2}"); + expected.add(mapper.readTree("{\"unknown\":2}")); + payloads.add("{\"id\":1,\"customer\":{\"name\":\"Alice\",\"unknown\":1}}"); + expected.add(mapper.readTree("{\"customer\":{\"unknown\":1}}")); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + create(admin, path, descriptor); + try (KafkaProducer producer = producer()) { + assertThat(producer.partitionsFor(path.toString())).hasSize(2); + for (int i = 0; i < payloads.size(); i++) { + assertThat( + producer.send( + new ProducerRecord<>( + path.toString(), + 1, + 123L, + KEY, + bytes(payloads.get(i)), + Arrays.asList( + new RecordHeader( + "source", + new byte[] {1, 2}), + new RecordHeader( + "nullable", null)))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(i); + } + read( + connection, + path, + 1, + payloads.size(), + (physical, index) -> { + InternalRow value = jsonBusinessRow(physical, contract); + assertThat(value.getInt(0)).isEqualTo(index < 5 ? index : 1); + if (index < 5) { + InternalRow customer = + value.getRow(1, CUSTOMER_TYPE.getFieldCount()); + assertThat(customer.getString(0).toString()).isEqualTo("Alice"); + for (int field = 1; + field < CUSTOMER_TYPE.getFieldCount(); + field++) { + assertThat(customer.getInt(field)).isEqualTo(field); + } + assertNumbers(value.getRow(2, NUMBER_TYPE.getFieldCount())); + assertNumbers( + value.getArray(7).getRow(0, NUMBER_TYPE.getFieldCount())); + assertNumbers( + value.getMap(8) + .valueArray() + .getRow(0, NUMBER_TYPE.getFieldCount())); + } + if (expected.get(index) == null) { + assertThat(physical.isNullAt(contract.rescuePosition())).isTrue(); + } else { + try { + assertThat( + mapper.readTree( + physical.getString( + contract + .rescuePosition()) + .toString())) + .isEqualTo(expected.get(index)); + } catch (java.io.IOException e) { + throw new AssertionError(e); + } + } + if (withSystemColumns) { + assertThat(physical.getBytes(contract.keyPosition())) + .isEqualTo(KEY); + assertHeaders(physical.getMap(contract.headersPosition()), true); + assertMetadata(physical, contract.metadataPosition(), 123L, 0, 1); + } + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + private static InternalRow jsonBusinessRow(InternalRow row, KafkaTableSchema contract) { + if (contract.valueConversionLayout() == KafkaValueConversionLayout.NESTED) { + return row.getRow( + contract.valueProjection().positionAt(0), COMPLEX_TYPE.getFieldCount()); + } + GenericRow business = new GenericRow(COMPLEX_TYPE.getFieldCount()); + for (int i = 0; i < business.getFieldCount(); i++) { + business.setField( + i, + InternalRow.createFieldGetter( + COMPLEX_TYPE.getTypeAt(i), + contract.valueProjection().positionAt(i)) + .getFieldOrNull(row)); + } + return business; + } + + private static TableDescriptor reorderedRescueDescriptor( + String layout, boolean withSystemColumns, boolean rescueFirst) { + List columns = + new ArrayList<>( + descriptor(COMPLEX_TYPE, layout, withSystemColumns, false, 2) + .getSchema() + .getColumns()); + Schema.Builder schema = Schema.newBuilder(); + int rescuePosition = rescueFirst ? 0 : columns.size() / 2 + columns.size() % 2; + for (int i = 0; i <= columns.size(); i++) { + if (i == rescuePosition) { + schema.column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()); + } + if (i < columns.size()) { + schema.column(columns.get(i).getName(), columns.get(i).getDataType()); + } + } + return TableDescriptor.builder() + .schema(schema.build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), layout) + .build(); + } + + private static void assertNegativeZero(float value) { + assertThat(Float.floatToRawIntBits(value)).isEqualTo(Float.floatToRawIntBits(-0.0F)); + } + + private static void assertNegativeZero(double value) { + assertThat(Double.doubleToRawLongBits(value)).isEqualTo(Double.doubleToRawLongBits(-0.0D)); + } + + private static void assertNumbers(InternalRow row) { + assertNegativeZero(row.getFloat(0)); + assertNegativeZero(row.getDouble(1)); + assertThat(row.getFloat(2)).isEqualTo(1.0000001F); + assertThat(row.getDecimal(3, 30, 10).toBigDecimal()) + .isEqualByComparingTo(new BigDecimal("12345678901234567890.1234567890")); + } + + private static String complexJson(int id, String negativeZero) { + String numbers = + "{\"float_value\":" + + negativeZero + + ",\"double_value\":" + + negativeZero + + ",\"rounded_float\":1.0000000596046448," + + "\"amount\":12345678901234567890.1234567890}"; + return "{\"id\":" + + id + + ",\"customer\":{\"name\":\"Alice\",\"_kafka_key\":1,\"_kafka_value\":2," + + "\"_kafka_headers\":3,\"_kafka_metadata\":4,\"_kafka_rescue\":5}," + + "\"measurements\":" + + numbers + + ",\"float_values\":[" + + negativeZero + + ",1.0000000596046448],\"double_values\":[" + + negativeZero + + "],\"float_attributes\":{\"zero\":" + + negativeZero + + ",\"rounded\":1.0000000596046448},\"double_attributes\":{\"zero\":" + + negativeZero + + "},\"items\":[" + + numbers + + "],\"attributes\":{\"dynamic\":" + + numbers + + "}}"; + } + + private static int valueId(InternalRow row, String layout, RowType valueType) { + return layout.equals("nested") + ? row.getRow(1, valueType.getFieldCount()).getInt(0) + : row.getInt(1); + } + + private static int valueId(InternalRow row, String layout) { + return layout.equals("nested") ? row.getRow(1, 1).getInt(0) : row.getInt(1); + } + + private static void assertScalars(InternalRow row, int start, int id) { + assertThat(row.getInt(start)).isEqualTo(id); + assertThat(row.getDecimal(start + 1, 30, 10).toBigDecimal()) + .isEqualByComparingTo(new BigDecimal("12345678901234567890.1234567890")); + assertThat(row.getTimestampNtz(start + 2, 6).toLocalDateTime()) + .isEqualTo(LocalDateTime.parse("1969-12-31T23:59:59.999999")); + assertThat(row.getTimestampLtz(start + 3, 6).toInstant()) + .isEqualTo(Instant.parse("1969-12-31T23:59:59.999999Z")); + assertThat(Float.floatToRawIntBits(row.getFloat(start + 4))) + .isEqualTo(Float.floatToRawIntBits(-0.0F)); + assertThat(Double.doubleToRawLongBits(row.getDouble(start + 5))) + .isEqualTo(Double.doubleToRawLongBits(-0.0D)); + assertThat(row.isNullAt(start + 6)).isTrue(); + assertThat(row.getFloat(start + 7)).isEqualTo(1.0000001F); + } + + private static void assertHeaders(InternalMap headers, boolean populated) { + if (!populated) { + assertThat(headers.size()).isZero(); + return; + } + Map values = new HashMap<>(); + for (int index = 0; index < headers.size(); index++) { + values.put( + headers.keyArray().getString(index).toString(), + headers.valueArray().isNullAt(index) + ? null + : headers.valueArray().getBytes(index)); + } + assertThat(values).containsOnlyKeys("source", "nullable"); + assertThat(values.get("source")).containsExactly(1, 2); + assertThat(values.get("nullable")).isNull(); + } + + private static DataType nullValueBusinessType(String fieldType) { + switch (fieldType) { + case "scalar": + return DataTypes.INT(); + case "row": + return DataTypes.ROW(DataTypes.FIELD("value", DataTypes.INT())); + case "array": + return DataTypes.ARRAY(DataTypes.INT()); + case "map": + return DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()); + default: + throw new IllegalArgumentException("Unexpected business field type: " + fieldType); + } + } + + private static String nullValueBusinessJson(String fieldType, int id) { + switch (fieldType) { + case "scalar": + return Integer.toString(id); + case "row": + case "map": + return "{\"value\":" + id + "}"; + case "array": + return "[" + id + "]"; + default: + throw new IllegalArgumentException("Unexpected business field type: " + fieldType); + } + } + + private static void assertNullValueBusinessId( + InternalRow row, int position, String fieldType, int id) { + switch (fieldType) { + case "scalar": + assertThat(row.getInt(position)).isEqualTo(id); + break; + case "row": + assertThat(row.getRow(position, 1).getInt(0)).isEqualTo(id); + break; + case "array": + assertThat(row.getArray(position).size()).isEqualTo(1); + assertThat(row.getArray(position).getInt(0)).isEqualTo(id); + break; + case "map": + InternalMap map = row.getMap(position); + assertThat(map.size()).isEqualTo(1); + assertThat(map.keyArray().getString(0).toString()).isEqualTo("value"); + assertThat(map.valueArray().getInt(0)).isEqualTo(id); + break; + default: + throw new IllegalArgumentException("Unexpected business field type: " + fieldType); + } + } + + private static String scalarJson(int id, boolean explicitNull) { + // Both layouts consume this same root object; nested does not require a value wrapper. + return "{\"id\":" + + id + + ",\"amount\":12345678901234567890.1234567890," + + "\"received_at\":\"1969-12-31T23:59:59.999999\"," + + "\"emitted_at\":\"1970-01-01T07:59:59.999999+08:00\"," + + "\"float_value\":-0.0,\"double_value\":-0.0," + + "\"rounded_float\":1.0000000596046448" + + (explicitNull ? ",\"optional\":null" : "") + + "}"; + } + + private static void create(Admin admin, TablePath path, TableDescriptor descriptor) + throws Exception { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor, false).get(); + TableInfo table = admin.getTableInfo(path).get(); + CLUSTER.waitUntilTableReady(table.getTableId()); + for (int bucket = 0; bucket < table.getNumBuckets(); bucket++) { + CLUSTER.waitAndGetLeaderReplica(new TableBucket(table.getTableId(), bucket)); + } + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + } + + private static TableDescriptor descriptor( + RowType valueType, + String layout, + boolean withSystemColumns, + boolean withRescue, + int buckets) { + Schema.Builder schema = Schema.newBuilder(); + if (withSystemColumns) { + schema.column(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES()); + } + if (layout.equals("nested")) { + schema.column(KafkaTableSchema.VALUE_COLUMN, valueType); + } else { + for (int index = 0; index < valueType.getFieldCount(); index++) { + schema.column(valueType.getFieldNames().get(index), valueType.getTypeAt(index)); + } + } + if (withSystemColumns) { + schema.column( + KafkaTableSchema.HEADERS_COLUMN, + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + schema.column( + KafkaTableSchema.METADATA_COLUMN, + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))); + } + if (withRescue) { + schema.column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()); + } + return TableDescriptor.builder() + .schema(schema.build()) + .distributedBy(buckets) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), layout) + .build(); + } + + private static MemoryRecords timestampedRecords( + TimestampType timestampType, long logAppendTime, SimpleRecord... records) { + try (MemoryRecordsBuilder builder = + MemoryRecords.builder( + ByteBuffer.allocate(1024), + RecordBatch.MAGIC_VALUE_V2, + Compression.NONE, + timestampType, + 0L, + logAppendTime)) { + for (SimpleRecord record : records) { + builder.append(record); + } + return builder.build(); + } + } + + private static SimpleRecord record(String json) { + return new SimpleRecord(123L, KEY, json == null ? null : bytes(json), new Header[0]); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static ProduceResponse produce( + Socket socket, String topic, MemoryRecords rejected, MemoryRecords accepted) + throws Exception { + TopicProduceData topicData = + new TopicProduceData() + .setName(topic) + .setPartitionData( + Arrays.asList( + new PartitionProduceData().setIndex(0).setRecords(rejected), + new PartitionProduceData() + .setIndex(1) + .setRecords(accepted))); + ProduceRequest request = + new ProduceRequest( + new ProduceRequestData() + .setAcks((short) -1) + .setTimeoutMs(10000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topicData).iterator())), + (short) 9); + RequestHeader header = + new RequestHeader(ApiKeys.PRODUCE, request.version(), "json-produce-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] requestBytes = new byte[encoded.remaining()]; + encoded.get(requestBytes); + socket.setSoTimeout(15000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + int responseSize = input.readInt(); + assertThat(responseSize).isBetween(1, 1024 * 1024); + byte[] response = new byte[responseSize]; + input.readFully(response); + return (ProduceResponse) AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static PartitionProduceResponse partition( + ProduceResponse response, String topic, int index) { + return response.data().responses().find(topic).partitionResponses().stream() + .filter(partition -> partition.index() == index) + .findFirst() + .get(); + } + + private static void read( + Connection connection, + TablePath path, + int bucket, + int expectedCount, + BiConsumer check) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(bucket); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + int count = 0; + while (count < expectedCount && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.logOffset()).isEqualTo(count); + check.accept(record.getRow(), count); + count++; + } + } + assertThat(count).isEqualTo(expectedCount); + assertThat(scanner.poll(Duration.ofMillis(100))).isEmpty(); + } + } + + private static KafkaProducer producer() { + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, "all"); + config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 1); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java new file mode 100644 index 00000000000..3b452fb28e0 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -0,0 +1,1119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; +import org.apache.fluss.rpc.messages.ListDatabasesResponse; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.concurrent.FutureUtils; + +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.utils.LogContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol compatibility tests for the Kafka Metadata API. */ +public class KafkaMetadataHandlerTest { + + private static final Uuid TOPIC_ID = new Uuid(0x466c757373000000L, 123L); + + @Test + public void testNamedTopicForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = topicsRequest(version, "kafka.topic"); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(100, new byte[] {1, 2, 3})); + request.data() + .topics() + .get(0) + .unknownTaggedFields() + .add(new RawTaggedField(101, new byte[] {4, 5, 6})); + } + MetadataResponse response = handle(service, request, version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.controller()).isNull(); + Node broker = response.brokers().iterator().next(); + assertThat(broker.host()).isEqualTo("broker-1"); + assertThat(broker.port()).isEqualTo(9092); + assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.partitionIndex()).isZero(); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).isEmpty(); + } + assertThat(service.lastListenerName).isEqualTo("KAFKA"); + } + + @Test + public void testAllTopicsForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = allTopicsRequest(version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(102, new byte[] {7, 8, 9})); + } + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic"); + } + } + + @Test + public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName("kafka.topic") + .setTopicId( + new Uuid( + 0x466c757373000000L, + 999L)))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(response.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); + } + } + + @Test + public void testV10AndV11RejectTopicIdOnlyLookup() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Uuid requestedTopicId = new Uuid(0x466c757373000000L, 999L); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(null) + .setTopicId(requestedTopicId))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()).hasSize(1); + MetadataResponseTopic responseTopic = response.data().topics().iterator().next(); + assertThat(responseTopic.name()).isEmpty(); + assertThat(responseTopic.topicId()).isEqualTo(requestedTopicId); + assertThat(responseTopic.errorCode()).isEqualTo(Errors.INVALID_REQUEST.code()); + } + } + + @Test + public void testDeleteRaceBecomesUnknownTopicResult() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.removeTable("kafka.topic"); + service.failNextMetadataAsMissing = true; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + } + + @Test + public void testUnavailableLeaderUsesPartitionError() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicLeaderAvailable = false; + MetadataRequest request = namedTopicRequest("kafka.topic"); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.leaderId()).isEqualTo(-1); + assertThat(partition.replicaNodes()).containsExactly(1, 2, 3); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).containsExactly(3); + } + + @Test + public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicIsr = new int[] {1}; + MetadataRequest request = namedTopicRequest("kafka.topic"); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("kafka.topic").partitions().get(0); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataUsesLeaderOnlyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("kafka.topic").partitions().get(0); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + service.topicLeaderAvailable = false; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("kafka.topic").partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.isrNodes()).isEmpty(); + } + + @Test + public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { + for (int bucketEpoch : new int[] {7, -1}) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = bucketEpoch; + service.topicIsr = new int[0]; + + MetadataResponse response = + handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("kafka.topic").partitions().get(0); + assertThat(partition.isrNodes()).isEmpty(); + } + } + + @Test + public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failMetadata = true; + MetadataRequest request = namedTopicRequest("kafka.topic"); + + MetadataResponse response = handle(service, request, (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 1)); + assertThat(response.brokers()).isEmpty(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testJsonTableIsDiscoverableForEveryMetadataVersion(boolean nested) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Schema schema = + nested + ? Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT()), + DataTypes.FIELD("name", DataTypes.STRING()))) + .column("_kafka_key", DataTypes.BYTES()) + .build() + : Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("_kafka_key", DataTypes.BYTES()) + .column("name", DataTypes.STRING()) + .build(); + service.putTable( + "fluss.json_topic", + 140L, + TableDescriptor.builder(defaultDescriptor()) + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + nested ? "nested" : "flat") + .build()); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + handle( + service, + topicsRequest(version, "json_topic", "fluss.json_topic"), + version); + for (String name : Arrays.asList("json_topic", "fluss.json_topic")) { + assertThat(response.data().topics().find(name).errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(response.data().topics().find(name).partitions()).hasSize(2); + } + if (version >= 10) { + assertThat(response.data().topics().find("json_topic").topicId()) + .isEqualTo(new Uuid(0x466c757373410000L, 140L)); + assertThat(response.data().topics().find("fluss.json_topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 140L)); + } + assertThat(handle(service, allTopicsRequest(version), version).data().topics()) + .extracting(MetadataResponseTopic::name) + .contains("fluss.json_topic") + .doesNotContain("json_topic"); + } + } + + @Test + public void testJsonAdmissionReflectsAuthoritativeSchemaAndProperties() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + TableDescriptor flat = + TableDescriptor.builder(defaultDescriptor()) + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") + .build(); + service.putTable("fluss.json_topic", 140L, flat); + assertThat( + handle(service, namedTopicRequest("json_topic"), (short) 11) + .data() + .topics() + .find("json_topic") + .errorCode()) + .isEqualTo(Errors.NONE.code()); + // Changing the layout without moving business columns invalidates the same table ID. + service.putTable( + "fluss.json_topic", + 140L, + TableDescriptor.builder(flat) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), "nested") + .build()); + assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) + .containsEntry("json_topic", Errors.INVALID_TOPIC_EXCEPTION); + RowType business = + DataTypes.ROW( + DataTypes.FIELD("items", DataTypes.ARRAY(DataTypes.INT())), + DataTypes.FIELD( + "attributes", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW(DataTypes.FIELD("count", DataTypes.INT()))))); + for (boolean nested : new boolean[] {false, true}) { + Schema complexSchema = + nested + ? Schema.newBuilder().column("_kafka_value", business).build() + : Schema.newBuilder().fromRowType(business).build(); + TableDescriptor complex = + TableDescriptor.builder(flat) + .schema(complexSchema) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + nested ? "nested" : "flat") + .build(); + service.putTable("fluss.json_topic", 140L, complex); + assertThat( + handle(service, namedTopicRequest("json_topic"), (short) 11) + .data() + .topics() + .find("json_topic") + .errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat( + handle(service, allTopicsRequest((short) 11), (short) 11) + .data() + .topics() + .find("fluss.json_topic") + .errorCode()) + .isEqualTo(Errors.NONE.code()); + + RowType invalidBusiness = + DataTypes.ROW( + DataTypes.FIELD( + "items", DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()))); + Schema invalidSchema = + nested + ? Schema.newBuilder().column("_kafka_value", invalidBusiness).build() + : Schema.newBuilder().fromRowType(invalidBusiness).build(); + service.putTable( + "fluss.json_topic", + 140L, + TableDescriptor.builder(complex).schema(invalidSchema).build()); + assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) + .containsEntry("json_topic", Errors.INVALID_TOPIC_EXCEPTION); + assertThat(handle(service, allTopicsRequest((short) 11), (short) 11).errors()) + .containsEntry("fluss.json_topic", Errors.INVALID_TOPIC_EXCEPTION); + } + service.putTable( + "fluss.json_topic", + 140L, + TableDescriptor.builder(flat) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + assertThat(handle(service, namedTopicRequest("json_topic"), (short) 11).errors()) + .containsEntry("json_topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + service.putTable("fluss.json_topic", 141L, flat); + MetadataResponse recreated = + handle( + service, + topicsRequest((short) 11, "json_topic", "fluss.json_topic"), + (short) 11); + assertThat(recreated.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(recreated.data().topics().find("json_topic").topicId()) + .isEqualTo(new Uuid(0x466c757373410000L, 141L)); + assertThat(recreated.data().topics().find("fluss.json_topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 141L)); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMetadataColumnContractForEveryVersion(boolean legacyOffset) { + RowType metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + if (legacyOffset) { + metadataType = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT())); + } + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + // Also model an invalid historical descriptor that native DDL would now reject. + service.putTable( + "fluss.metadata_events", + 777L, + TableDescriptor.builder(defaultDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_metadata", metadataType) + .build()) + .build()); + short expected = legacyOffset ? Errors.INVALID_TOPIC_EXCEPTION.code() : Errors.NONE.code(); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle( + service, + topicsRequest( + version, + "metadata_events", + "fluss.metadata_events", + "kafka.topic"), + version); + for (String name : Arrays.asList("metadata_events", "fluss.metadata_events")) { + MetadataResponseTopic topic = named.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(expected); + assertThat(topic.partitions()).hasSize(legacyOffset ? 0 : 2); + } + assertThat(named.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics().find("fluss.metadata_events").errorCode()) + .isEqualTo(expected); + assertThat(all.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .doesNotContain("metadata_events"); + } + } + + @Test + public void testRejectMalformedNamesWithoutHidingValidTopics() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + List invalidNames = Arrays.asList(".topic", "kafka.", "kafka.topic.extra"); + List names = new ArrayList<>(invalidNames); + names.add("kafka.topic"); + MetadataResponse response = + handle( + service, + new MetadataRequest.Builder(names, false).build((short) 11), + (short) 11); + for (String name : invalidNames) { + assertThat(response.data().topics().find(name).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + } + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + + @Test + public void testMetadataV12IsRejectedBeforeCallingGateway() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) + .build((short) 12); + + MetadataResponse response = handle(service, request, (short) 12); + + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + assertThat(service.gatewayCalls).isZero(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMissingDatabaseRetainsBrokersAndAliases(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + service.failListTablesAsFuture = failedFuture; + for (short version = 0; version <= 11; version++) { + for (List names : + Arrays.asList( + Collections.singletonList("missing_db.topic"), + Arrays.asList( + "missing_db.topic", "topic", "fluss.topic", "kafka.topic"))) { + MetadataResponse response = + handle( + service, + topicsRequest(version, names.toArray(new String[0])), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(names.size()); + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + if (names.contains("topic")) { + for (String name : Arrays.asList("topic", "fluss.topic", "kafka.topic")) { + MetadataResponseTopic valid = response.data().topics().find(name); + assertThat(valid.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.partitions()).hasSize(2); + } + assertThat(response.data().topics().find("topic").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, 777L) + : Uuid.ZERO_UUID); + assertThat(response.data().topics().find("fluss.topic").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373000000L, 777L) + : Uuid.ZERO_UUID); + } + } + } + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.listedDatabases) + .containsExactlyInAnyOrder("fluss", "kafka", "missing_db"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testDatabaseDeletedDuringDiscoveryRetainsOtherTopics(boolean failedFuture) { + for (short version = 0; version <= 11; version++) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("deleted_db.topic", 223L); + service.deleteDatabaseBeforeListing = "deleted_db"; + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, allTopicsRequest(version), version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic"); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testUnexpectedListTablesFailureIsNotTreatedAsMissingDatabase(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failNextMetadataAsMissing = true; + service.listTablesFailure = new IllegalStateException("metadata unavailable"); + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_SERVER_ERROR.code()); + } + + @ParameterizedTest + @ValueSource(strings = {"fluss", "custom_db"}) + public void testAliasesPreserveRequestedNamesAndShareIdentity(String defaultDatabase) { + Configuration config = new Configuration(); + if (!"fluss".equals(defaultDatabase)) { + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, defaultDatabase); + } + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + String canonical = defaultDatabase + ".topic"; + service.putTable(canonical, 777L); + service.putTable("sales.topic", 778L); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle( + service, + topicsRequest(version, "topic", canonical, "sales.topic"), + version, + config); + assertThat(named.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("topic", canonical, "sales.topic"); + assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = named.data().topics().find("topic"); + MetadataResponseTopic qualified = named.data().topics().find(canonical); + assertThat(alias.partitions()).isEqualTo(qualified.partitions()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373410000L, 777L) : Uuid.ZERO_UUID); + if (version >= 10) { + assertThat(qualified.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)) + .isNotEqualTo(alias.topicId()) + .isNotEqualTo(named.data().topics().find("sales.topic").topicId()); + } + MetadataResponse all = handle(service, allTopicsRequest(version), version, config); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .contains(canonical, "sales.topic") + .doesNotContain("topic") + .doesNotHaveDuplicates(); + } + } + + @Test + public void testMissingDefaultDatabaseNeverSearchesOtherDatabases() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + MetadataResponse response = + handle( + service, + topicsRequest((short) 11, "topic", "kafka.topic"), + (short) 11, + config); + assertThat(response.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(service.listDatabasesCalls).isZero(); + assertThat(service.requestedTables) + .containsExactlyInAnyOrder("missing_default.topic", "kafka.topic"); + assertThat(service.listedDatabases).containsExactlyInAnyOrder("missing_default", "kafka"); + } + + @Test + public void testUnauthorizedAliasesRemainHiddenWithAndWithoutRetry() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + service.hiddenTopics.add("fluss.topic"); + service.putTable("sales.topic", 778L); + for (short version = 0; version <= 11; version++) { + for (List names : + Arrays.asList( + Arrays.asList("topic", "fluss.topic", "sales.topic"), + Arrays.asList( + "topic", "fluss.topic", "sales.topic", "missing_db.topic"))) { + MetadataResponse response = + handle( + service, + topicsRequest(version, names.toArray(new String[0])), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(names.size()); + for (String hidden : Arrays.asList("topic", "fluss.topic")) { + MetadataResponseTopic topic = response.data().topics().find(hidden); + assertThat(topic.errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + } + MetadataResponseTopic visible = response.data().topics().find("sales.topic"); + assertThat(visible.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(visible.partitions()).hasSize(2); + if (names.contains("missing_db.topic")) { + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + } + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .contains("sales.topic") + .doesNotContain("topic", "fluss.topic"); + } + } + + @Test + public void testPersistedEnablementControlsBothAliasesAndDiscovery() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + TableDescriptor missingFlag = + TableDescriptor.builder() + .schema(defaultDescriptor().getSchema()) + .distributedBy(2) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(); + TableDescriptor disabledInvalid = + TableDescriptor.builder(defaultDescriptor()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "unimplemented") + .build(); + for (TableDescriptor disabled : Arrays.asList(missingFlag, disabledInvalid)) { + service.putTable("fluss.topic", 777L, disabled); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + assertThat(named.errors()) + .containsEntry("topic", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("fluss.topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(named.data().topics()) + .allSatisfy( + topic -> { + assertThat(topic.topicId()).isEqualTo(Uuid.ZERO_UUID); + assertThat(topic.partitions()).isEmpty(); + }); + assertThat(handle(service, allTopicsRequest(version), version).data().topics()) + .extracting(MetadataResponseTopic::name) + .doesNotContain("topic", "fluss.topic"); + } + } + service.putTable("fluss.topic", 777L); + MetadataResponse enabled = + handle(service, topicsRequest((short) 11, "topic", "fluss.topic"), (short) 11); + assertThat(enabled.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(enabled.data().topics().find("topic").topicId()) + .isEqualTo(new Uuid(0x466c757373410000L, 777L)); + assertThat(enabled.data().topics().find("fluss.topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 777L)); + service.putTable("fluss.topic", 777L, disabledInvalid); + assertThat(handle(service, namedTopicRequest("topic"), (short) 11).errors()) + .containsEntry("topic", Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + + @Test + public void testResolvedAliasNameLimitDoesNotHideValidTopic() { + String database = String.join("", Collections.nCopies(125, "d")); + String table = String.join("", Collections.nCopies(125, "t")); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, database); + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + MetadataResponse response = + handle( + service, + topicsRequest((short) 11, table, "kafka.topic"), + (short) 11, + config); + assertThat(response.errors()).containsEntry(table, Errors.INVALID_TOPIC_EXCEPTION); + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(service.requestedTables).containsExactly("kafka.topic"); + + // A historical enabled table with an unrepresentable canonical name must be visible as + // an error during discovery rather than silently classified as disabled. + service.putTable(database + "." + table, 777L); + assertThat(handle(service, allTopicsRequest((short) 11), (short) 11, config).errors()) + .containsEntry(database + "." + table, Errors.INVALID_TOPIC_EXCEPTION); + } + + @ParameterizedTest + @ValueSource(shorts = {10, 11}) + public void testKafkaClientCanCacheCanonicalNameAndAliasTogether(short version) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + Metadata client = + new Metadata(0L, 0L, 60000L, new LogContext(), new ClusterResourceListeners()); + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "fluss.topic"), version), false, 1L); + // A producer discovers the alias after already caching the canonical name. Nonzero + // duplicate IDs would make Kafka's MetadataSnapshot throw while building topicNames. + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "topic"), version), true, 2L); + assertThat(client.fetch().partitionsForTopic("topic")).hasSize(2); + assertThat(client.fetch().partitionsForTopic("fluss.topic")).hasSize(2); + client.updateWithCurrentRequestVersion( + handle(service, topicsRequest(version, "topic", "fluss.topic"), version), + false, + 3L); + assertThat(client.fetch().partitionsForTopic("topic")).hasSize(2); + assertThat(client.fetch().partitionsForTopic("fluss.topic")).hasSize(2); + assertThat(client.topicIds()).containsOnlyKeys("topic", "fluss.topic"); + assertThat(client.topicNames()) + .containsEntry(new Uuid(0x466c757373000000L, 777L), "fluss.topic") + .containsEntry(new Uuid(0x466c757373410000L, 777L), "topic"); + client.close(); + } + + @ParameterizedTest + @ValueSource(shorts = {10, 11}) + public void testKafkaClientRefreshesRecreatedAliasWithLowerLeaderEpoch(short version) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("fluss.topic", 777L); + Metadata client = + new Metadata(0L, 0L, 60000L, new LogContext(), new ClusterResourceListeners()); + MetadataResponse original = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + original.data() + .topics() + .forEach( + topic -> + topic.partitions() + .forEach(p -> p.setLeaderId(1).setLeaderEpoch(5))); + client.updateWithCurrentRequestVersion(original, false, 1L); + for (String name : Arrays.asList("topic", "fluss.topic")) { + assertThat(client.fetch().leaderFor(new TopicPartition(name, 0)).id()).isEqualTo(1); + assertThat(client.lastSeenLeaderEpoch(new TopicPartition(name, 0))).contains(5); + } + + // The client never observes the missing table between deletion and recreation. + service.removeTable("fluss.topic"); + service.putTable("fluss.topic", 778L); + MetadataResponse recreated = + handle(service, topicsRequest(version, "topic", "fluss.topic"), version); + recreated + .data() + .topics() + .forEach( + topic -> + topic.partitions() + .forEach(p -> p.setLeaderId(2).setLeaderEpoch(0))); + client.updateWithCurrentRequestVersion(recreated, false, 2L); + for (String name : Arrays.asList("fluss.topic", "topic")) { + assertThat(client.fetch().leaderFor(new TopicPartition(name, 0)).id()) + .as("Leader after recreating %s", name) + .isEqualTo(2); + assertThat(client.lastSeenLeaderEpoch(new TopicPartition(name, 0))).contains(0); + } + assertThat(client.topicNames()) + .containsEntry(new Uuid(0x466c757373000000L, 778L), "fluss.topic") + .containsEntry(new Uuid(0x466c757373410000L, 778L), "topic") + .doesNotContainKeys( + new Uuid(0x466c757373000000L, 777L), new Uuid(0x466c757373410000L, 777L)); + client.close(); + } + + private static MetadataRequest topicsRequest(short version, String... names) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList(names))), + version); + } + + private static TableDescriptor defaultDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(); + } + + private static MetadataRequest namedTopicRequest(String topicName) { + return topicsRequest((short) 11, topicName); + } + + private static MetadataRequest allTopicsRequest(short version) { + MetadataRequestData data = new MetadataRequestData(); + data.setTopics(version == 0 ? Collections.emptyList() : null); + return new MetadataRequest(data, version); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { + return handle(service, requestBody, version, new Configuration()); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, + MetadataRequest requestBody, + short version, + Configuration config) { + KafkaRequestHandler handler = new KafkaRequestHandler(config, service); + ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); + KafkaRequest request; + try { + request = + new KafkaRequest( + ApiKeys.METADATA, + version, + new RequestHeader(ApiKeys.METADATA, version, "client-id", 1), + requestBody, + "KAFKA", + requestBuffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + } finally { + // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest. + requestBuffer.release(); + } + handler.processRequest(request); + ByteBuf responseBuffer = request.responseBuffer(); + try { + return (MetadataResponse) + AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + assertThat(requestBuffer.refCnt()).isZero(); + } + } + + private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + + private final Set databases = new LinkedHashSet<>(); + private final Set hiddenTopics = new LinkedHashSet<>(); + private final Map tables = new LinkedHashMap<>(); + private final Map descriptors = new LinkedHashMap<>(); + private int gatewayCalls; + private int listDatabasesCalls; + private final Set requestedTables = new LinkedHashSet<>(); + private final Set listedDatabases = new LinkedHashSet<>(); + private String lastListenerName; + private boolean topicLeaderAvailable = true; + private int[] topicIsr = new int[] {1, 2}; + private Integer topicBucketEpoch = 7; + private boolean failMetadata; + private boolean failNextMetadataAsMissing; + private boolean failListTablesAsFuture; + private String deleteDatabaseBeforeListing; + private RuntimeException listTablesFailure; + + private TestingMetadataGatewayService() { + putTable("kafka.topic", 123L); + putTable("kafka.other", 124L); + } + + @Override + public CompletableFuture listDatabases( + ListDatabasesRequest request) { + gatewayCalls++; + listDatabasesCalls++; + assertThat(currentListenerName()).isEqualTo("KAFKA"); + return CompletableFuture.completedFuture( + new ListDatabasesResponse().addAllDatabaseNames(databases)); + } + + @Override + public CompletableFuture listTables(ListTablesRequest request) { + gatewayCalls++; + assertThat(currentListenerName()).isEqualTo("KAFKA"); + String database = request.getDatabaseName(); + listedDatabases.add(database); + if (database.equals(deleteDatabaseBeforeListing)) { + databases.remove(database); + tables.keySet().removeIf(name -> name.startsWith(database + ".")); + } + RuntimeException failure = listTablesFailure; + if (failure == null && !databases.contains(database)) { + failure = new DatabaseNotExistException("Database does not exist."); + } + if (failure != null) { + if (failListTablesAsFuture) { + return FutureUtils.completedExceptionally(new CompletionException(failure)); + } + throw failure; + } + String prefix = request.getDatabaseName() + "."; + List names = + tables.keySet().stream() + .filter(name -> !hiddenTopics.contains(name)) + .filter(name -> name.startsWith(prefix)) + .map(name -> name.substring(prefix.length())) + .collect(Collectors.toList()); + return CompletableFuture.completedFuture( + new ListTablesResponse().addAllTableNames(names)); + } + + @Override + public CompletableFuture metadata( + org.apache.fluss.rpc.messages.MetadataRequest request) { + gatewayCalls++; + lastListenerName = currentListenerName(); + if (failMetadata) { + CompletableFuture failure = + new CompletableFuture<>(); + failure.completeExceptionally(new IllegalStateException("metadata unavailable")); + return failure; + } + if (failNextMetadataAsMissing) { + failNextMetadataAsMissing = false; + throw new TableNotExistException("table was deleted"); + } + List topics = new ArrayList<>(); + for (PbTablePath tablePath : request.getTablePathsList()) { + String topicName = tablePath.getDatabaseName() + "." + tablePath.getTableName(); + requestedTables.add(topicName); + // Native Metadata and listTables omit tables without DESCRIBE permission. + if (hiddenTopics.contains(topicName)) { + continue; + } + Long tableId = tables.get(topicName); + if (tableId == null) { + throw new TableNotExistException("Table does not exist: " + topicName); + } + topics.add( + tableMetadata( + topicName, + tableId, + !"kafka.topic".equals(topicName) || topicLeaderAvailable, + "kafka.topic".equals(topicName) + ? topicIsr + : new int[] {1, 2}, + "kafka.topic".equals(topicName) + ? topicBucketEpoch + : Integer.valueOf(7)) + .setTableJson(descriptors.get(topicName).toJsonBytes())); + } + return CompletableFuture.completedFuture( + new org.apache.fluss.rpc.messages.MetadataResponse() + .addAllTabletServers( + Arrays.asList( + new PbServerNode() + .setNodeId(1) + .setHost("broker-1") + .setPort(9092) + .setRack("rack-a"), + new PbServerNode() + .setNodeId(2) + .setHost("broker-2") + .setPort(9093))) + .addAllTableMetadatas(topics)); + } + + private void putTable(String topic, long tableId) { + putTable(topic, tableId, defaultDescriptor()); + } + + private void putTable(String topic, long tableId, TableDescriptor descriptor) { + databases.add(topic.substring(0, topic.indexOf('.'))); + tables.put(topic, tableId); + descriptors.put(topic, descriptor); + } + + private void removeTable(String topic) { + tables.remove(topic); + descriptors.remove(topic); + } + + private static PbTableMetadata tableMetadata( + String topic, + long tableId, + boolean leaderAvailable, + int[] isr, + Integer bucketEpoch) { + PbTableMetadata table = + new PbTableMetadata() + .setTablePath( + new PbTablePath() + .setDatabaseName(topic.substring(0, topic.indexOf('.'))) + .setTableName(topic.substring(topic.indexOf('.') + 1))) + .setTableId(tableId) + .addAllBucketMetadatas( + Arrays.asList( + new PbBucketMetadata() + .setBucketId(0) + .setLeaderId(leaderAvailable ? 1 : 3) + .setLeaderEpoch(5) + .setReplicaIds( + leaderAvailable + ? new int[] {1, 2} + : new int[] {1, 2, 3}) + .setIsrs( + bucketEpoch == null ? new int[0] : isr), + new PbBucketMetadata() + .setBucketId(1) + .setLeaderId(2) + .setLeaderEpoch(6) + .setReplicaIds(new int[] {1, 2}) + .setIsrs( + bucketEpoch == null + ? new int[0] + : new int[] {1, 2}))); + if (bucketEpoch != null) { + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + bucket.setBucketEpoch(bucketEpoch); + } + } + return table; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java new file mode 100644 index 00000000000..b281349a68f --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.dropDatabase; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newDatabaseExistsRequest; +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies qualified Metadata discovery over a real Kafka listener and TabletServer. */ +public class KafkaMetadataITCase { + + @RegisterExtension + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(2) + .setClusterConf(clusterConfiguration()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exception { + TableDescriptor descriptor = topicDescriptor(); + long existing = createTable(CLUSTER, TablePath.of("existing_db", "events"), descriptor); + long deleted = createTable(CLUSTER, TablePath.of("deleted_db", "events"), descriptor); + CLUSTER.waitUntilTableReady(existing); + CLUSTER.waitUntilTableReady(deleted); + dropDatabase(CLUSTER, "deleted_db"); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + + for (short version = 0; version <= 11; version++) { + MetadataResponse missing = query(node, named(version, "deleted_db.events")); + assertThat(missing.brokers()).hasSize(2); + assertThat(missing.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + + MetadataResponse mixed = + query(node, named(version, "deleted_db.events", "existing_db.events")); + assertThat(mixed.brokers()).hasSize(2); + assertThat(mixed.brokers().iterator().next().port()).isEqualTo(node.port()); + assertThat(mixed.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(mixed.data().topics().find("existing_db.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(mixed.data().topics().find("existing_db.events").partitions()).hasSize(2); + + MetadataResponse all = query(node, allTopics(version)); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("existing_db.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @Test + public void testAliasesAndOptionalMetadataAcrossDatabases() throws Exception { + long other = createTable(CLUSTER, TablePath.of("sales", "events"), topicDescriptor()); + CLUSTER.waitUntilTableReady(other); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + for (short version = 0; version <= 11; version++) { + MetadataResponse before = query(node, named(version, "events", "sales.events")); + assertThat(before.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(before.data().topics().find("sales.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + // Exercise PR03's optional metadata column through the same alias and discovery paths. + TableDescriptor withMetadata = + TableDescriptor.builder(topicDescriptor()) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()))) + .build()) + .build(); + long tableId = createTable(CLUSTER, TablePath.of("fluss", "events"), withMetadata); + CLUSTER.waitUntilTableReady(tableId); + for (short version = 0; version <= 11; version++) { + for (ServerNode broker : CLUSTER.getTabletServerNodes("KAFKA")) { + // The missing database forces the same discovery retry that used to discard + // aliases. + MetadataResponse response = + query( + broker, + named( + version, + "events", + "fluss.events", + "sales.events", + "fluss.missing", + "missing_db.events")); + assertThat(response.brokers()).hasSize(2); + assertThat(response.brokers()).extracting(Node::port).contains(broker.port()); + for (String name : Arrays.asList("fluss.events", "sales.events")) { + MetadataResponseTopic topic = response.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + } + assertThat(response.data().topics().find("sales.events").topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373000000L, other) + : Uuid.ZERO_UUID); + MetadataResponseTopic alias = response.data().topics().find("events"); + MetadataResponseTopic canonical = response.data().topics().find("fluss.events"); + assertThat(alias.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(alias.partitions()).isEqualTo(canonical.partitions()).hasSize(2); + assertThat(alias.partitions()) + .allSatisfy( + partition -> { + assertThat(partition.leaderId()).isIn(0, 1); + assertThat(partition.replicaNodes()) + .containsExactlyInAnyOrder(0, 1); + assertThat(partition.isrNodes()) + .containsExactlyInAnyOrder(0, 1); + }); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, tableId) + : Uuid.ZERO_UUID); + if (version >= 10) { + assertThat(canonical.topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, tableId)) + .isNotEqualTo(alias.topicId()) + .isNotEqualTo(response.data().topics().find("sales.events").topicId()); + } + for (String missing : Arrays.asList("fluss.missing", "missing_db.events")) { + assertThat(response.data().topics().find(missing).errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + MetadataResponse all = query(broker, allTopics(version)); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events", "sales.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + } + + @Test + public void testPersistedEnablementChangesAreVisibleThroughBothNames() throws Exception { + TablePath path = TablePath.of("fluss", "events"); + TableDescriptor disabled = + TableDescriptor.builder(topicDescriptor()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build(); + TableDescriptor unset = + TableDescriptor.builder() + .schema(topicDescriptor().getSchema()) + .distributedBy(2) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(); + long tableId = createTable(CLUSTER, path, disabled); + long unsetId = createTable(CLUSTER, TablePath.of("sales", "unset"), unset); + CLUSTER.waitUntilTableReady(tableId); + CLUSTER.waitUntilTableReady(unsetId); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + assertExposure(node, false, tableId); + setEnabled(CLUSTER, path, true); + assertExposure(node, true, tableId); + setEnabled(CLUSTER, path, false); + assertExposure(node, false, tableId); + setEnabled(CLUSTER, path, true); + assertExposure(node, true, tableId); + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList(ConfigOptions.TABLE_KAFKA_ENABLED.key()), + Collections.emptyList(), + false)) + .get(); + assertExposure(node, false, tableId); + } + + @Test + public void testInvalidEnabledContractRemainsVisibleInNamedAndAllTopics() throws Exception { + TablePath path = TablePath.of("fluss", "events"); + long tableId = createTable(CLUSTER, path, topicDescriptor()); + long validId = createTable(CLUSTER, TablePath.of("sales", "valid"), topicDescriptor()); + CLUSTER.waitUntilTableReady(tableId); + CLUSTER.waitUntilTableReady(validId); + TableRegistration registration = CLUSTER.getZooKeeperClient().getTable(path).get(); + Map invalidProperties = new HashMap<>(registration.properties); + invalidProperties.put(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "string"); + // Bypass current DDL validation to model historical persisted metadata with a damaged + // contract. Metadata must expose the enabled table's error, not hide it as disabled. + CLUSTER.getZooKeeperClient() + .updateTable( + path, + registration.newProperties( + invalidProperties, registration.customProperties)); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + query(node, named((short) 11, "events")) + .data() + .topics() + .find("events") + .errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code())); + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + query(node, named(version, "events", "fluss.events", "sales.valid")); + for (String topic : Arrays.asList("events", "fluss.events")) { + assertThat(named.data().topics().find(topic).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(named.data().topics().find(topic).partitions()).isEmpty(); + } + MetadataResponse all = query(node, allTopics(version)); + assertThat(all.brokers()).hasSize(2); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events", "sales.valid"); + assertThat(all.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(all.data().topics().find("sales.valid").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + } + + @Test + public void testCustomDefaultDatabaseAliasAndNoRecreationByMetadata() throws Exception { + Configuration config = clusterConfiguration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "custom_default"); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + FlussClusterExtension cluster = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(config) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + cluster.start(); + try { + long nativeDefaultId = + createTable(cluster, TablePath.of("fluss", "events"), topicDescriptor()); + cluster.waitUntilTableReady(nativeDefaultId); + ServerNode node = cluster.getTabletServerInfos().get(0).node("KAFKA"); + // PR03 creates the configured database at startup, before any Metadata request. + assertThat( + cluster.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("custom_default")) + .get() + .isExists()) + .isTrue(); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = query(node, named(version, "events", "fluss.events")); + assertThat(response.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(response.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + long customId = + createTable( + cluster, TablePath.of("custom_default", "events"), topicDescriptor()); + cluster.waitUntilTableReady(customId); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + query( + node, + named(version, "events", "custom_default.events", "fluss.events")); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + MetadataResponseTopic alias = response.data().topics().find("events"); + assertThat(alias.partitions()) + .isEqualTo( + response.data() + .topics() + .find("custom_default.events") + .partitions()); + assertThat(alias.topicId()) + .isEqualTo( + version >= 10 + ? new Uuid(0x466c757373410000L, customId) + : Uuid.ZERO_UUID); + if (version >= 10) { + assertThat(response.data().topics().find("custom_default.events").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, customId)) + .isNotEqualTo(alias.topicId()) + .isNotEqualTo(response.data().topics().find("fluss.events").topicId()); + } + assertThat(query(node, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("custom_default.events", "fluss.events"); + } + dropDatabase(cluster, "custom_default"); + MetadataResponse deleted = query(node, named((short) 11, "events", "fluss.events")); + assertThat(deleted.data().topics().find("events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(deleted.data().topics().find("fluss.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat( + cluster.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("custom_default")) + .get() + .isExists()) + .isFalse(); + } finally { + cluster.close(); + } + } + + private static void assertExposure(ServerNode node, boolean enabled, long tableId) + throws Exception { + short expected = enabled ? Errors.NONE.code() : Errors.UNKNOWN_TOPIC_OR_PARTITION.code(); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + query(node, named((short) 11, "events")) + .data() + .topics() + .find("events") + .errorCode()) + .isEqualTo(expected)); + for (short version = 0; version <= 11; version++) { + MetadataResponse response = + query(node, named(version, "events", "fluss.events", "sales.unset")); + for (String name : Arrays.asList("events", "fluss.events")) { + MetadataResponseTopic topic = response.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(expected); + assertThat(topic.topicId()) + .isEqualTo( + enabled && version >= 10 + ? new Uuid( + name.indexOf('.') >= 0 + ? 0x466c757373000000L + : 0x466c757373410000L, + tableId) + : Uuid.ZERO_UUID); + } + assertThat(response.data().topics().find("sales.unset").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + if (enabled) { + assertThat(query(node, allTopics(version)).data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("fluss.events"); + } else { + assertThat(query(node, allTopics(version)).data().topics()).isEmpty(); + } + } + } + + private static void setEnabled(FlussClusterExtension cluster, TablePath path, boolean enabled) + throws Exception { + cluster.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled)), + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private static MetadataRequest named(short version, String... names) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList(names))), + version); + } + + private static MetadataRequest allTopics(short version) { + return new MetadataRequest( + new MetadataRequestData().setTopics(version == 0 ? Collections.emptyList() : null), + version); + } + + private static TableDescriptor topicDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(); + } + + private static Configuration clusterConfiguration() { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.KAFKA_ENABLED, true); + conf.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 2); + conf.set(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS, 3); + return conf; + } + + private static MetadataResponse query(ServerNode node, MetadataRequest request) + throws Exception { + RequestHeader header = + new RequestHeader(ApiKeys.METADATA, request.version(), "metadata-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] requestBytes = new byte[encoded.remaining()]; + encoded.get(requestBytes); + try (Socket socket = new Socket(node.host(), node.port())) { + socket.setSoTimeout(10000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + byte[] responseBytes = new byte[input.readInt()]; + input.readFully(responseBytes); + return (MetadataResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(responseBytes), header); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java new file mode 100644 index 00000000000..88f62125eb0 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -0,0 +1,724 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Exercises native DDL, Kafka discovery and Produce, replication, and native Fluss readback. */ +class KafkaProduceAppendITCase { + private static final String DATABASE = "produce_default"; + private static final byte[] KEY = "key".getBytes(StandardCharsets.UTF_8); + private static final byte[] VALUE = "value".getBytes(StandardCharsets.UTF_8); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception { + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, "produce_" + format + "_" + acks); + String topic = path.toString(); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + // Node discovery does not wait for table replicas. With acks=0 the producer + // cannot observe or retry a write rejected before the leader is ready. + CLUSTER.waitUntilAllReplicaReady( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); + try (KafkaProducer producer = producer(acks)) { + assertThat(producer.partitionsFor(topic)).hasSize(1); + ProducerRecord record = + new ProducerRecord<>( + topic, + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + RecordMetadata first = producer.send(record).get(30, TimeUnit.SECONDS); + RecordMetadata second = + producer.send( + new ProducerRecord<>( + path.getTableName(), + 0, + 123L, + KEY, + VALUE, + record.headers())) + .get(30, TimeUnit.SECONDS); + assertThat(first.topic()).isEqualTo(topic); + assertThat(second.topic()).isEqualTo(path.getTableName()); + if (!acks.equals("0")) { + assertThat(first.offset()).isZero(); + assertThat(second.offset()).isEqualTo(1L); + } + producer.flush(); + assertReadback(connection, path, format, VALUE); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullValuesArePersistedForAllAckModes(String format) throws Exception { + List values = Arrays.asList(null, new byte[0], VALUE, null); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + TablePath path = TablePath.of(DATABASE, "produce_null_" + format + "_" + acks); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + CLUSTER.waitUntilAllReplicaReady( + new TableBucket(admin.getTableInfo(path).get().getTableId(), 0)); + try (KafkaProducer producer = producer(acks, 0)) { + for (int i = 0; i < values.size(); i++) { + ProducerRecord record = + new ProducerRecord<>( + i % 2 == 0 ? path.toString() : path.getTableName(), + 0, + 123L, + KEY, + values.get(i), + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + RecordMetadata metadata = producer.send(record).get(30, TimeUnit.SECONDS); + if (!acks.equals("0")) { + assertThat(metadata.offset()).isEqualTo(i); + } + } + producer.flush(); + assertReadback(connection, path, format, values); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testAllNullAndMixedBatchesPreserveOffsets(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_null_batches_" + format); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.toString()}; + assertProduceOffsets(query(socket, recordBatch(null, null), names), names, 0L); + assertProduceOffsets( + query(socket, recordBatch(VALUE, null, new byte[0], null), names), + names, + 2L); + assertProduceOffsets(query(socket, names), names, 6L); + assertReadback( + connection, + path, + format, + Arrays.asList(null, null, VALUE, null, new byte[0], null, VALUE)); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNotNullValueSchemaIsRejectedWhenCreatingKafkaTable(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_not_null_" + format); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + assertThatThrownBy( + () -> admin.createTable(path, descriptor(format, false), false).get()) + .hasCauseInstanceOf(InvalidTableException.class) + .rootCause() + .hasMessageContaining("Kafka requires nullable fields"); + } + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullAndEmptyValuesPreserveKeysAndOffsets(String format) throws Exception { + TablePath path = TablePath.of(DATABASE, "null_values_" + format); + List values = Arrays.asList(null, new byte[0], VALUE); + List keys = Arrays.asList(new byte[] {(byte) 0xff}, null, null); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + for (int i = 0; i < values.size(); i++) { + RecordMetadata metadata = + producer.send( + new ProducerRecord<>( + path.toString(), + 0, + 123L, + keys.get(i), + values.get(i))) + .get(30, TimeUnit.SECONDS); + assertThat(metadata.offset()).isEqualTo(i); + } + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < values.size() && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + InternalRow row = record.getRow(); + byte[] expected = values.get(count); + if (expected == null) { + assertThat(row.isNullAt(0)).isTrue(); + } else if (format.equals("binary")) { + assertThat(row.getBytes(0)).isEqualTo(expected); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo(new String(expected, StandardCharsets.UTF_8)); + } + if (keys.get(count) == null) { + assertThat(row.isNullAt(3)).isTrue(); + } else { + assertThat(row.getBytes(3)).isEqualTo(keys.get(count)); + } + assertThat(row.getMap(1).size()).isZero(); + assertThat(row.isNullAt(2)).isTrue(); + assertThat(record.logOffset()).isEqualTo(count); + count++; + } + } + assertThat(count).isEqualTo(values.size()); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testSameNamedTablesAcrossDatabases() throws Exception { + Map tables = new LinkedHashMap<>(); + tables.put(TablePath.of("produce_first", "events"), VALUE); + tables.put( + TablePath.of("produce_second", "events"), + "second database".getBytes(StandardCharsets.UTF_8)); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + try { + for (TablePath path : tables.keySet()) { + admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true) + .get(); + admin.createTable(path, descriptor("binary"), false).get(); + } + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all")) { + for (Map.Entry entry : tables.entrySet()) { + String topic = entry.getKey().toString(); + assertThat(producer.partitionsFor(topic)).hasSize(1); + ProducerRecord record = + new ProducerRecord<>( + topic, + 0, + 123L, + KEY, + entry.getValue(), + Arrays.asList( + new RecordHeader("source", entry.getValue()), + new RecordHeader("source", null))); + assertThat(producer.send(record).get(30, TimeUnit.SECONDS).offset()) + .isZero(); + assertThat(producer.send(record).get(30, TimeUnit.SECONDS).offset()) + .isEqualTo(1L); + } + for (Map.Entry entry : tables.entrySet()) { + assertReadback(connection, entry.getKey(), "binary", entry.getValue()); + } + } + } finally { + for (TablePath path : tables.keySet()) { + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testPipelinedProducePreservesAppendOrder() throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_pipelined"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("binary"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + assertThat(producer.partitionsFor(path.toString())).hasSize(1); + List values = new ArrayList<>(); + List> writes = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + byte[] value = new byte[i % 2 == 0 ? 16384 : 1]; + Arrays.fill(value, (byte) i); + values.add(value); + writes.add( + producer.send( + new ProducerRecord<>( + path.toString(), 0, (long) i, KEY, value))); + } + for (int i = 0; i < writes.size(); i++) { + assertThat(writes.get(i).get(30, TimeUnit.SECONDS).offset()).isEqualTo(i); + } + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < values.size() && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.getRow().getBytes(0)).isEqualTo(values.get(count)); + count++; + } + } + assertThat(count).isEqualTo(values.size()); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testAliasesShareOffsetsAndObservePersistedEnablementOnOneConnection() throws Exception { + TablePath path = TablePath.of(DATABASE, "produce_enablement"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable(path, descriptor("binary"), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer("all", 0)) { + Node leader = producer.partitionsFor(path.toString()).get(0).leader(); + try (Socket socket = new Socket(leader.host(), leader.port())) { + String[] names = {path.getTableName(), path.toString()}; + assertProduceOffsets(query(socket, names), names, 0L, 1L); + setEnabled(path, false); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setEnabled(path, true); + assertProduceOffsets(query(socket, names), names, 2L, 3L); + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList( + ConfigOptions.TABLE_KAFKA_ENABLED.key()), + Collections.emptyList(), + false)) + .get(); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertReadback(connection, path, "binary", VALUE, 4); + admin.dropTable(path, true).get(); + assertProduceErrors( + query(socket, names), names, Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testDeletedDefaultDatabaseDoesNotFallbackOrRecreateDatabase() throws Exception { + Configuration config = clusterConfig(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "produce_missing_db"); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 1); + FlussClusterExtension cluster = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(config) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + cluster.start(); + try (Connection connection = ConnectionFactory.createConnection(cluster.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + // Startup creates the configured default; Produce must not recreate a deleted one. + assertThat(admin.databaseExists("produce_missing_db").get()).isTrue(); + admin.dropDatabase("produce_missing_db", false, false).get(); + assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); + TablePath existing = TablePath.of("fluss", "events"); + long tableId = createTable(cluster, existing, descriptor("binary")); + cluster.waitUntilTableReady(tableId); + ServerNode node = cluster.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket(node.host(), node.port())) { + String[] missing = { + "events", + "produce_missing_db.events", + "another_missing_db.events", + "fluss.missing" + }; + assertProduceErrors( + query(socket, missing), missing, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertProduceOffsets( + query(socket, existing.toString()), new String[] {existing.toString()}, 0L); + assertProduceOffsets( + query(socket, existing.toString()), new String[] {existing.toString()}, 1L); + try (KafkaProducer producer = producer(node, "all", 0)) { + assertThat( + producer.send( + new ProducerRecord<>( + existing.toString(), + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader( + "source", VALUE), + new RecordHeader( + "source", null)))) + .get(30, TimeUnit.SECONDS) + .offset()) + .isEqualTo(2L); + } + assertReadback(connection, existing, "binary", VALUE, 3); + assertThat(admin.databaseExists("produce_missing_db").get()).isFalse(); + assertThat(admin.databaseExists("another_missing_db").get()).isFalse(); + } + } finally { + cluster.close(); + } + } + + private static void setEnabled(TablePath path, boolean enabled) throws Exception { + CLUSTER.newCoordinatorClient() + .alterTable( + newAlterTableRequest( + path, + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled)), + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private static MemoryRecords recordBatch(byte[]... values) { + SimpleRecord[] records = new SimpleRecord[values.length]; + for (int i = 0; i < values.length; i++) { + records[i] = + new SimpleRecord( + 123L, + KEY, + values[i], + new Header[] { + new RecordHeader("source", VALUE), new RecordHeader("source", null) + }); + } + return MemoryRecords.withRecords(Compression.NONE, records); + } + + private static ProduceResponse query(Socket socket, String... names) throws Exception { + return query(socket, recordBatch(VALUE), names); + } + + private static ProduceResponse query(Socket socket, MemoryRecords records, String... names) + throws Exception { + List topics = new ArrayList<>(); + for (String name : names) { + topics.add( + new TopicProduceData() + .setName(name) + .setPartitionData( + Collections.singletonList( + new PartitionProduceData() + .setIndex(0) + .setRecords(records)))); + } + ProduceRequest request = + new ProduceRequest( + new ProduceRequestData() + .setAcks((short) -1) + .setTimeoutMs(10000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + topics.iterator())), + (short) 9); + RequestHeader header = + new RequestHeader(ApiKeys.PRODUCE, request.version(), "produce-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] bytes = new byte[encoded.remaining()]; + encoded.get(bytes); + socket.setSoTimeout(15000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(bytes.length); + output.write(bytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + byte[] response = new byte[input.readInt()]; + input.readFully(response); + return (ProduceResponse) AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static void assertProduceOffsets( + ProduceResponse response, String[] names, long... offsets) { + assertProduceErrors(response, names, Errors.NONE); + for (int i = 0; i < names.length; i++) { + assertThat( + response.data() + .responses() + .find(names[i]) + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(offsets[i]); + } + } + + private static void assertProduceErrors( + ProduceResponse response, String[] names, Errors error) { + assertThat(response.data().responses()) + .extracting(topic -> topic.name()) + .containsExactly(names); + for (String name : names) { + List partitions = + response.data().responses().find(name).partitionResponses(); + assertThat(partitions).hasSize(1); + assertThat(partitions.get(0).index()).isZero(); + assertThat(partitions.get(0).errorCode()).isEqualTo(error.code()); + if (error != Errors.NONE) { + assertThat(partitions.get(0).baseOffset()).isEqualTo(-1L); + } + } + } + + private static void assertReadback( + Connection connection, TablePath path, String format, byte[] expectedValue) + throws Exception { + assertReadback(connection, path, format, expectedValue, 2); + } + + private static void assertReadback( + Connection connection, + TablePath path, + String format, + byte[] expectedValue, + int expectedCount) + throws Exception { + assertReadback(connection, path, format, Collections.nCopies(expectedCount, expectedValue)); + } + + private static void assertReadback( + Connection connection, TablePath path, String format, List expectedValues) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < expectedValues.size() && System.nanoTime() < deadline) { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + assertThat(record.logOffset()).isEqualTo(count); + byte[] expectedValue = expectedValues.get(count); + InternalRow row = record.getRow(); + if (expectedValue == null) { + assertThat(row.isNullAt(0)).isTrue(); + } else if (format.equals("binary")) { + assertThat(row.getBytes(0)).isEqualTo(expectedValue); + } else { + assertThat(row.getString(0).toString()) + .isEqualTo(new String(expectedValue, StandardCharsets.UTF_8)); + } + assertThat(row.getBytes(3)).isEqualTo(KEY); + assertThat(row.getMap(1).size()).isEqualTo(1); + assertThat(row.getMap(1).keyArray().getString(0).toString()) + .isEqualTo("source"); + assertThat(row.getMap(1).valueArray().isNullAt(0)).isTrue(); + assertThat(row.isNullAt(2)).isTrue(); + InternalRow metadata = row.getRow(4, 3); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()) + .isEqualTo(123L); + assertThat(metadata.getInt(1)).isZero(); + assertThat(metadata.getInt(2)).isZero(); + count++; + } + } + assertThat(count) + .as("Native readback for %s (%s)", path, format) + .isEqualTo(expectedValues.size()); + } + } + + private static TableDescriptor descriptor(String format) { + return descriptor(format, true); + } + + private static TableDescriptor descriptor(String format, boolean nullableValue) { + boolean binary = format.equals("binary"); + return TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + (binary ? DataTypes.BYTES() : DataTypes.STRING()) + .copy(nullableValue)) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + format.equals("binary") + ? KafkaValueConversionType.BINARY + : KafkaValueConversionType.STRING) + .build(); + } + + private static KafkaProducer producer(String acks) { + return producer(acks, 16384); + } + + private static KafkaProducer producer(String acks, int batchSize) { + return producer(CLUSTER.getTabletServerNodes("KAFKA").get(0), acks, batchSize); + } + + private static KafkaProducer producer( + ServerNode node, String acks, int batchSize) { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); + config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java new file mode 100644 index 00000000000..1a801cd55ed --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -0,0 +1,829 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.kafka.api.produce.ProduceHandler; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; + +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.message.ProduceResponseData.TopicProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.DefaultRecordBatch; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.MemoryRecordsBuilder; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; +import org.apache.kafka.common.utils.Crc32C; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +/** Protocol tests independent of the Fluss append and record conversion backends. */ +class KafkaProduceProtocolTest { + + @Test + void testSupportedVersionsCopyRecordsAndForwardContext() { + for (short version = 3; version <= 11; version++) { + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request(version, (short) -1, partition(0, records())), + command -> { + copied.set(command); + return successful(command); + }); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(42L); + KafkaProduceCommand command = copied.get(); + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(4321); + assertThat(command.listenerName()).isEqualTo("KAFKA"); + KafkaProduceCommand.Record record = + command.topics().get(0).partitions().get(0).records().get(0); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(record.hasTimestamp()).isTrue(); + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()).hasSize(1); + assertThat(record.headers().get(0).value()).isNull(); + byte[] key = record.key(); + key[0] = 9; + assertThat(record.key()).containsExactly((byte) 1); + } + } + + @ParameterizedTest + @ValueSource(shorts = {3, 4, 5, 6, 7, 8, 9, 10, 11}) + void testBareAndQualifiedTopicNamesRemainUnchanged(short version) { + MemoryRecords nullableValues = + MemoryRecords.withRecords( + Compression.NONE, + new SimpleRecord(123L, new byte[] {1}, null), + new SimpleRecord(456L, null, new byte[0])); + ProduceRequest request = + requestForTopics( + version, + (short) 1, + topic("orders", partition(0, nullableValues)), + topic("fluss.orders", partition(0, records())), + topic("sales.orders", partition(0, records()))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()) + .extracting(KafkaProduceCommand.TopicWrite::topicName) + .containsExactly("orders", "fluss.orders", "sales.orders"); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + assertThat(topic.partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(0); + } + List copied = + command.topics().get(0).partitions().get(0).records(); + assertThat(copied).hasSize(2); + // The protocol command preserves null and empty values for the adapter. + assertThat(copied.get(0).value()).isNull(); + assertThat(copied.get(1).value()).isEmpty(); + List results = + new ArrayList<>(successful(command).join().topics()); + Collections.reverse(results); + return CompletableFuture.completedFuture( + new KafkaProduceResult(results)); + }); + + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactly("orders", "fluss.orders", "sales.orders"); + for (TopicProduceResponse topic : response.data().responses()) { + assertThat(topic.partitionResponses()) + .extracting( + PartitionProduceResponse::index, + PartitionProduceResponse::errorCode, + PartitionProduceResponse::baseOffset) + .containsExactly(tuple(0, Errors.NONE.code(), 42L)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"orders", "fluss.orders"}) + void testLocalAndBackendErrorsAreIsolatedByRequestTopicName(String invalidTopic) { + String validTopic = invalidTopic.equals("orders") ? "fluss.orders" : "orders"; + ProduceResponse response = + dispatch( + requestForTopics( + (short) 11, + (short) 1, + topic(invalidTopic, partition(0, MemoryRecords.EMPTY)), + topic(validTopic, partition(0, records())), + topic("sales.orders", partition(0, records()))), + command -> { + assertThat(command.topics()) + .extracting(KafkaProduceCommand.TopicWrite::topicName) + .containsExactly(validTopic, "sales.orders"); + return CompletableFuture.completedFuture( + new KafkaProduceResult( + Arrays.asList( + new TopicResult( + "sales.orders", + Collections.singletonList( + new PartitionResult( + 0, + Errors + .TOPIC_AUTHORIZATION_FAILED, + -1L, + null))), + new TopicResult( + validTopic, + Collections.singletonList( + new PartitionResult( + 0, + Errors.NONE, + 42L, + null)))))); + }); + + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactly(invalidTopic, validTopic, "sales.orders"); + assertTopicError(response, invalidTopic, Errors.INVALID_RECORD); + assertTopicError(response, validTopic, Errors.NONE); + assertTopicError(response, "sales.orders", Errors.TOPIC_AUTHORIZATION_FAILED); + } + + @ParameterizedTest + @ValueSource(shorts = {3, 4, 5, 6, 7, 8, 9, 10, 11}) + void testRequestAndBackendFailuresPreserveTopicNames(short version) { + for (boolean invalidAcks : Arrays.asList(false, true)) { + ProduceResponse response = + dispatch( + requestForTopics( + version, + invalidAcks ? (short) 2 : (short) 1, + topic("orders", partition(0, records())), + topic("fluss.orders", partition(0, records())), + topic("sales.orders", partition(0, records()))), + command -> { + assertThat(invalidAcks).isFalse(); + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally(new TimeoutException("append failed")); + return failed; + }); + assertThat(response.data().responses()) + .extracting(TopicProduceResponse::name) + .containsExactlyInAnyOrder("orders", "fluss.orders", "sales.orders"); + Errors error = invalidAcks ? Errors.INVALID_REQUIRED_ACKS : Errors.REQUEST_TIMED_OUT; + for (String name : Arrays.asList("orders", "fluss.orders", "sales.orders")) { + assertTopicError(response, name, error); + } + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testTimestampTypesRemainLocalToTheirBatches(CompressionType compressionType) { + MemoryRecords createTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + 123L, + 456L); + MemoryRecords appendTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + 123L, + 789L, + 987L); + MemoryRecords missingCreateTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP); + MemoryRecords missingAppendTime = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_TIMESTAMP); + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, createTime), + partition(1, appendTime), + partition(2, missingCreateTime), + partition(3, missingAppendTime)), + command -> { + copied.set(command); + return successful(command); + }); + assertErrors(response, Errors.NONE, Errors.NONE, Errors.NONE, Errors.NONE); + for (MemoryRecords input : + Arrays.asList(createTime, appendTime, missingCreateTime, missingAppendTime)) { + ByteBuffer buffer = input.buffer().duplicate(); + while (buffer.hasRemaining()) { + buffer.put((byte) 0); + } + } + + List partitions = + copied.get().topics().get(0).partitions(); + assertThat(partitions).hasSize(4); + assertThat(partitions.get(0).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(123L, 456L); + // The same numeric timestamp can have a different batch type. + assertThat(partitions.get(1).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(123L, 123L); + for (int i = 0; i < partitions.size(); i++) { + TimestampType expectedType = + i % 2 == 0 ? TimestampType.CREATE_TIME : TimestampType.LOG_APPEND_TIME; + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::timestampType) + .containsExactly(expectedType, expectedType); + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::hasTimestamp) + .containsExactly(i < 2, i < 2); + if (i >= 2) { + assertThat(partitions.get(i).records()) + .extracting(KafkaProduceCommand.Record::timestamp) + .containsExactly(RecordBatch.NO_TIMESTAMP, RecordBatch.NO_TIMESTAMP); + } + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testMultipleBatchesPerPartitionRemainInvalid(CompressionType compressionType) { + MemoryRecords first = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + RecordBatch.NO_TIMESTAMP, + 123L); + MemoryRecords second = + timestampedRecords( + compressionType, + org.apache.kafka.common.record.TimestampType.LOG_APPEND_TIME, + 456L, + 123L); + ByteBuffer combined = ByteBuffer.allocate(first.sizeInBytes() + second.sizeInBytes()); + combined.put(first.buffer().duplicate()).put(second.buffer().duplicate()).flip(); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.readableRecords(combined)), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType) { + byte[] value = new byte[32 * 1024]; + Arrays.fill(value, (byte) 7); + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord( + 123L, + new byte[] {1}, + value, + new Header[] { + new RecordHeader("header", new byte[] {2, 3}), + new RecordHeader("header", null) + }), + new SimpleRecord(456L, null, new byte[0])); + AtomicReference copied = new AtomicReference<>(); + + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + copied.set(command); + return successful(command); + }); + assertErrors(response, Errors.NONE); + ByteBuffer input = records.buffer().duplicate(); + while (input.hasRemaining()) { + input.put((byte) 0); + } + + List result = + copied.get().topics().get(0).partitions().get(0).records(); + assertThat(result).hasSize(2); + assertThat(result.get(0).timestamp()).isEqualTo(123L); + assertThat(result.get(0).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(result.get(0).key()).containsExactly((byte) 1); + assertThat(result.get(0).value()).containsExactly(value); + assertThat(result.get(0).headers()) + .extracting(KafkaProduceCommand.RecordHeader::name) + .containsExactly("header", "header"); + assertThat(result.get(0).headers().get(0).value()).containsExactly((byte) 2, (byte) 3); + assertThat(result.get(0).headers().get(1).value()).isNull(); + assertThat(result.get(1).timestamp()).isEqualTo(456L); + assertThat(result.get(1).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(result.get(1).key()).isNull(); + assertThat(result.get(1).value()).isEmpty(); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testIncorrectRecordCountDoesNotSuppressValidPartition(CompressionType compressionType) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + ByteBuffer bytes = malformed.buffer(); + bytes.putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, 32); + bytes.putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, 31); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testInvalidBatchOffsetRangeDoesNotSuppressValidPartition(CompressionType compressionType) { + for (int[] invalidHeader : + new int[][] {{1, -1}, {1, 9}, {1, Integer.MAX_VALUE}, {0, 0}, {-1, 0}}) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + int recordCount = invalidHeader[0]; + int lastOffsetDelta = invalidHeader[1]; + malformed.buffer().putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, recordCount); + malformed.buffer().putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, lastOffsetDelta); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testValidBatchUsesOffsetAssignedByBackend(CompressionType compressionType) { + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1}), + new SimpleRecord(new byte[] {2}), + new SimpleRecord(new byte[] {3})); + long backendBaseOffset = 1L << 40; + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + assertThat(command.topics().get(0).partitions().get(0).records()) + .hasSize(3); + return CompletableFuture.completedFuture( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, + Errors.NONE, + backendBaseOffset, + null)))))); + }); + assertErrors(response, Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(backendBaseOffset); + } + + @Test + void testCorruptPartitionDoesNotSuppressValidPartition() { + MemoryRecords corrupt = records(); + corrupt.buffer().put(corrupt.sizeInBytes() - 1, (byte) 99); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, corrupt), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + assertThat(command.topics().get(0).partitions().get(0).partitionId()) + .isEqualTo(1); + return successful(command); + }); + assertErrors(response, Errors.CORRUPT_MESSAGE, Errors.NONE); + } + + @Test + void testInvalidTopicIsIsolated() { + ProduceRequest request = request((short) 11, (short) 1, partition(0, records())); + request.data() + .topicData() + .add( + new TopicProduceData() + .setName("bad/name") + .setPartitionData( + Collections.singletonList(partition(0, records())))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()).hasSize(1); + return successful(command); + }); + assertThat(response.errorCounts()) + .containsEntry(Errors.NONE, 1) + .containsEntry(Errors.INVALID_TOPIC_EXCEPTION, 1); + } + + @Test + void testEmptyNegativeAndIdempotentPartitionsAreRejectedLocally() { + MemoryRecords idempotent = + MemoryRecords.withIdempotentRecords( + Compression.NONE, 10L, (short) 0, 0, new SimpleRecord(new byte[] {1})); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(-1, records()), + partition(2, idempotent), + partition(3, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + return successful(command); + }); + assertErrors( + response, + Errors.INVALID_RECORD, + Errors.INVALID_REQUEST, + Errors.INVALID_REQUEST, + Errors.NONE); + } + + @Test + void testRequestValidationNeverCallsBackend() { + KafkaProduceBackend unused = + command -> { + throw new AssertionError("Backend must not be called"); + }; + assertErrors( + dispatch(request((short) 11, (short) 2, partition(0, records())), unused), + Errors.INVALID_REQUIRED_ACKS); + ProduceRequest transactional = request((short) 11, (short) 1, partition(0, records())); + transactional.data().setTransactionalId("transaction"); + assertErrors( + dispatch(new ProduceRequest(transactional.data(), transactional.version()), unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch( + request( + (short) 11, + (short) 1, + partition(0, records()), + partition(0, records())), + unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch(request((short) 2, (short) 1, partition(0, records())), unused), + Errors.UNSUPPORTED_VERSION); + } + + @Test + void testBackendFailuresAndMissingResultsPreserveValidationErrors() { + for (boolean synchronous : Arrays.asList(true, false)) { + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(1, records())), + command -> { + if (synchronous) { + throw new TimeoutException("append timed out"); + } + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally( + new TimeoutException("append timed out")); + return failed; + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.REQUEST_TIMED_OUT); + } + assertErrors( + dispatch( + request((short) 11, (short) 1, partition(0, records())), + command -> + CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList()))), + Errors.UNKNOWN_SERVER_ERROR); + } + + @Test + void testAcksZeroSuccessAndFailureReleaseBufferWithoutSendingResponse() { + for (boolean failure : Arrays.asList(false, true)) { + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + ProduceRequest body = request((short) 11, (short) 0, partition(0, records())); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, (short) 11, "producer", 1); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + CompletableFuture pending = new CompletableFuture<>(); + try { + channel.writeInbound(buffer); + KafkaRequest parsed = (KafkaRequest) requests.pollRequest(1000); + dispatcher( + command -> { + assertThat( + command.topics() + .get(0) + .partitions() + .get(0) + .records() + .get(0) + .timestampType()) + .isEqualTo(TimestampType.CREATE_TIME); + return pending; + }) + .dispatch(parsed) + .whenComplete( + (response, error) -> { + if (error == null) { + parsed.complete(response); + } else { + parsed.fail(error); + } + }); + assertThat(parsed.future()).isNotDone(); + if (failure) { + pending.completeExceptionally(new TimeoutException("failure")); + } else { + pending.complete( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, Errors.NONE, 42L, null)))))); + } + channel.runPendingTasks(); + assertThat(parsed.future()).isDone(); + assertThat((Object) channel.readOutbound()).isNull(); + assertThat(buffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + } + + private static ProduceResponse dispatch(ProduceRequest body, KafkaProduceBackend backend) { + ByteBuf buffer = Unpooled.buffer(1); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, body.version(), "producer", 1); + KafkaRequest request = + new KafkaRequest( + ApiKeys.PRODUCE, + body.version(), + header, + body, + "KAFKA", + buffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + buffer.release(); + request.complete(dispatcher(backend).dispatch(request).join()); + ByteBuf response = request.responseBuffer(); + try { + ByteBuffer bytes = response.nioBuffer(); + ResponseHeader.parse(bytes, header.toResponseHeader().headerVersion()); + return (ProduceResponse) + AbstractResponse.parseResponse(ApiKeys.PRODUCE, bytes, body.version()); + } finally { + response.release(); + } + } + + private static KafkaRequestDispatcher dispatcher(KafkaProduceBackend backend) { + KafkaApiRegistry registry = + new KafkaApiRegistry(Collections.singletonList(new ProduceHandler(backend))); + return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); + } + + private static CompletableFuture successful(KafkaProduceCommand command) { + List topics = new ArrayList<>(); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + List partitions = new ArrayList<>(); + for (KafkaProduceCommand.PartitionWrite partition : topic.partitions()) { + partitions.add( + new PartitionResult(partition.partitionId(), Errors.NONE, 42L, null)); + } + topics.add(new TopicResult(topic.topicName(), partitions)); + } + return CompletableFuture.completedFuture(new KafkaProduceResult(topics)); + } + + private static ProduceRequest request( + short version, short acks, PartitionProduceData... partitions) { + return requestForTopics(version, acks, topic("topic", partitions)); + } + + private static ProduceRequest requestForTopics( + short version, short acks, TopicProduceData... topics) { + return new ProduceRequest( + new ProduceRequestData() + .setAcks(acks) + .setTimeoutMs(4321) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Arrays.asList(topics).iterator())), + version); + } + + private static TopicProduceData topic(String name, PartitionProduceData... partitions) { + return new TopicProduceData().setName(name).setPartitionData(Arrays.asList(partitions)); + } + + private static PartitionProduceData partition(int index, MemoryRecords records) { + return new PartitionProduceData().setIndex(index).setRecords(records); + } + + private static MemoryRecords records() { + return MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.NONE, + new SimpleRecord( + 123L, + new byte[] {1}, + new byte[] {2}, + new Header[] {new RecordHeader("header", null)})); + } + + private static MemoryRecords timestampedRecords( + CompressionType compressionType, + org.apache.kafka.common.record.TimestampType timestampType, + long logAppendTime, + long... timestamps) { + try (MemoryRecordsBuilder builder = + MemoryRecords.builder( + ByteBuffer.allocate(1024), + RecordBatch.MAGIC_VALUE_V2, + Compression.of(compressionType).build(), + timestampType, + 0L, + logAppendTime)) { + for (long timestamp : timestamps) { + builder.append(new SimpleRecord(timestamp, new byte[] {1}, new byte[] {2})); + } + return builder.build(); + } + } + + private static void assertErrors(ProduceResponse response, Errors... errors) { + assertThat(response.data().responses().find("topic").partitionResponses()) + .extracting(PartitionProduceResponse::errorCode) + .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); + } + + private static void assertTopicError(ProduceResponse response, String topic, Errors error) { + assertThat(response.data().responses().find(topic).partitionResponses()) + .extracting(PartitionProduceResponse::index, PartitionProduceResponse::errorCode) + .containsExactly(tuple(0, error.code())); + } + + private static void updateBatchChecksum(MemoryRecords records) { + ByteBuffer bytes = records.buffer(); + int checksumStart = DefaultRecordBatch.CRC_OFFSET + Integer.BYTES; + bytes.putInt( + DefaultRecordBatch.CRC_OFFSET, + (int) Crc32C.compute(bytes, checksumStart, bytes.limit() - checksumStart)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java new file mode 100644 index 00000000000..8601ae69661 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -0,0 +1,989 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.cluster.rebalance.RebalancePlanForBucket; +import org.apache.fluss.cluster.rebalance.RebalanceProgress; +import org.apache.fluss.cluster.rebalance.RebalanceStatus; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.config.cluster.AlterConfig; +import org.apache.fluss.config.cluster.AlterConfigOpType; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableChange; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.coordinator.event.RecoverRebalanceEvent; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.data.RebalanceTask; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.errors.NotEnoughReplicasException; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.assertj.core.groups.Tuple; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.tuple; + +/** Regression coverage for native DDL lifecycle, metadata refresh, and Produce minimum ISR. */ +class KafkaProduceRegressionITCase { + private static final String DATABASE = "fluss"; + private static final short METADATA_VERSION = 11; + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { + TablePath path = TablePath.of(DATABASE, "metadata_failover"); + String topic = path.toString(); + int stoppedLeader = -1; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + MetadataResponsePartition initial = + waitForTopic(topic).data().topics().find(topic).partitions().get(0); + stoppedLeader = initial.leaderId(); + try (KafkaProducer producer = producer("1", true)) { + assertThat(send(producer, topic, "before_failover").offset()).isNotNegative(); + CLUSTER.stopTabletServer(stoppedLeader); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponseTopic metadata = + fetchMetadata(topic).data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = metadata.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(partition.leaderId()) + .isNotEqualTo(-1) + .isNotEqualTo(initial.leaderId()); + assertThat(partition.leaderEpoch()) + .isGreaterThan(initial.leaderEpoch()); + }); + // Reuse the same producer to verify routing recovers after leader disconnect. + // acks=1 permits data loss and offset reuse after failover; this verifies + // continued routing, not durability of the first acknowledged record. + assertThat(send(producer, topic, "after_failover").offset()).isNotNegative(); + } + } finally { + try { + if (stoppedLeader >= 0 && CLUSTER.getTabletServerById(stoppedLeader) == null) { + CLUSTER.startTabletServer(stoppedLeader); + CLUSTER.assertHasTabletServerNumber(3); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testProducerRefreshesMetadataAfterOnlineLeaderChange() throws Exception { + TablePath path = TablePath.of(DATABASE, "metadata_online_leader_change"); + String topic = path.toString(); + String rebalanceId = "kafka-online-leader-change"; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3, metadataSchema(), 1); + TableInfo tableInfo = admin.getTableInfo(path).get(); + TableBucket bucket = new TableBucket(tableInfo.getTableId(), 0); + MetadataResponsePartition initial = + waitForTopic(topic).data().topics().find(topic).partitions().get(0); + int oldLeader = initial.leaderId(); + int newLeader = + initial.replicaNodes().stream() + .filter(id -> id != oldLeader) + .findFirst() + .get(); + List nodes = CLUSTER.getTabletServerNodes("KAFKA"); + ServerNode oldNode = + nodes.stream().filter(node -> node.id() == oldLeader).findFirst().get(); + + // Keep this socket open across the handoff to verify the old leader stays + // reachable. + try (Socket oldLeaderSocket = connect(oldNode); + KafkaProducer producer = producer("all", true)) { + MetadataResponse before = + (MetadataResponse) sendRequest(oldLeaderSocket, metadataRequest(topic)); + assertThat(before.data().topics().find(topic).partitions().get(0).leaderId()) + .isEqualTo(oldLeader); + assertThat(send(producer, topic, 0, 123L, "before_handoff").offset()).isZero(); + assertRecordsWithMetadata( + connection, + path, + 0, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before_handoff", 123L, 0, 0)); + CLUSTER.waitUntilReplicaExpandToIsr(bucket, newLeader); + double retriesBefore = producerMetric(producer, "record-retry-total"); + double disconnectsBefore = producerMetric(producer, "connection-close-total"); + + rebalanceLeader(bucket, initial, newLeader, rebalanceId); + waitForLeaderChange( + admin, topic, 0, newLeader, initial.leaderEpoch(), rebalanceId); + CLUSTER.waitAndGetFollowerReplica(bucket, oldLeader); + assertThat(CLUSTER.getTabletServerNodes("KAFKA")) + .containsExactlyInAnyOrderElementsOf(nodes); + + ProduceResponse rejected = + (ProduceResponse) + sendRequest(oldLeaderSocket, produceRequest(topic, "rejected")); + assertThat(rejected.data().responses().find(topic).partitionResponses()) + .singleElement() + .satisfies( + partition -> { + assertThat(partition.index()).isZero(); + assertThat(partition.errorCode()) + .isEqualTo(Errors.NOT_LEADER_OR_FOLLOWER.code()); + assertThat(partition.baseOffset()).isEqualTo(-1L); + }); + + // partitionsFor uses the already populated producer cache. Separate raw + // Metadata requests above must not refresh this route before the next send. + assertThat(producer.partitionsFor(topic).get(0).leader().id()) + .isEqualTo(oldLeader); + assertThat(send(producer, topic, 0, 456L, "after_handoff").offset()) + .isEqualTo(1L); + assertThat(producerMetric(producer, "record-retry-total")) + .isGreaterThan(retriesBefore); + assertThat(producerMetric(producer, "connection-close-total")) + .isEqualTo(disconnectsBefore); + assertThat(producer.partitionsFor(topic).get(0).leader().id()) + .isEqualTo(newLeader); + assertRecordsWithMetadata( + connection, + path, + 0, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before_handoff", 123L, 0, 0), + tuple(1L, "after_handoff", 456L, 0, 0)); + } + } finally { + try { + admin.cancelRebalance(rebalanceId).get(); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + @Test + void testClusterMinIsrPolicyAndDynamicReload() throws Exception { + TablePath replicated = TablePath.of(DATABASE, "acks_all_replicated"); + TablePath underReplicated = TablePath.of(DATABASE, "acks_all_under_replicated"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, replicated, 3); + createTable(admin, underReplicated, 1); + try (KafkaProducer producer = producer("all", false)) { + assertThat(send(producer, replicated.toString(), "replicated").offset()) + .isZero(); + assertThatThrownBy(() -> send(producer, underReplicated.toString(), "rejected")) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(NotEnoughReplicasException.class); + + setMinIsr(admin, 1); + assertThat(send(producer, underReplicated.toString(), "accepted").offset()) + .isZero(); + // Offset zero and native readback also prove the rejected write was not stored. + assertValues(connection, underReplicated, "accepted"); + } + } finally { + try { + setMinIsr(admin, 2); + } finally { + admin.dropTable(replicated, true).get(); + admin.dropTable(underReplicated, true).get(); + } + } + } + } + + @Test + void testSameProducerAliasesFollowTableLifecycle() throws Exception { + TablePath path = TablePath.of(DATABASE, "cached_lifecycle"); + String full = path.toString(); + String alias = path.getTableName(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + MetadataResponse initial = waitForTopic(full); + assertKafkaListenerEndpoints(initial); + Uuid fullId = initial.data().topics().find(full).topicId(); + assertThat(fullId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(full)).isEqualTo(fullId); + Uuid aliasId = topicId(alias); + assertThat(aliasId).isNotEqualTo(Uuid.ZERO_UUID).isNotEqualTo(fullId); + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, full, "full").offset()).isZero(); + assertThat(send(producer, alias, "alias").offset()).isEqualTo(1L); + setTableEnabled(admin, path, false); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setTableEnabled(admin, path, true); + assertThat(topicId(full)).isEqualTo(fullId); + assertThat(topicId(alias)).isEqualTo(aliasId); + assertThat(send(producer, alias, "enabled").offset()).isEqualTo(2L); + Map properties = + admin.getTableInfo(path).get().getProperties().toMap(); + assertThatThrownBy( + () -> + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE + .key(), + "string")), + false) + .get()) + .isInstanceOf(ExecutionException.class); + assertThat(admin.getTableInfo(path).get().getProperties().toMap()) + .isEqualTo(properties); + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE + .key(), + "BINARY")), + false) + .get(); + assertThat(admin.getTableInfo(path).get().getProperties().toMap()) + .containsEntry( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "BINARY"); + assertThat(send(producer, full, "property").offset()).isEqualTo(3L); + assertValues(connection, path, "full", "alias", "enabled", "property"); + admin.dropTable(path, false).get(); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + createTable(admin, path, 3); + assertThat(topicId(full)).isNotEqualTo(fullId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(alias)).isNotEqualTo(aliasId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(topicId(full)).isNotEqualTo(topicId(alias)); + assertThat(send(producer, full, "recreated_full").offset()).isZero(); + assertThat(send(producer, alias, "recreated_alias").offset()).isEqualTo(1L); + assertValues(connection, path, "recreated_full", "recreated_alias"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testSameProducerAliasesUseAddedMetadataColumn() throws Exception { + TablePath path = TablePath.of(DATABASE, "cached_schema_alter"); + String full = path.toString(); + String alias = path.getTableName(); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3); + TableInfo before = admin.getTableInfo(path).get(); + Uuid fullId = topicId(full); + Uuid aliasId = topicId(alias); + try (KafkaProducer producer = producer("all", true)) { + // Populate both name mappings and the write plan before changing the schema. + assertThat(send(producer, full, "old_full").offset()).isZero(); + assertThat(send(producer, alias, "old_alias").offset()).isEqualTo(1L); + assertValues(connection, path, "old_full", "old_alias"); + + admin.alterTable( + path, + Collections.singletonList( + TableChange.addColumn( + "_kafka_metadata", + metadataType(), + null, + TableChange.ColumnPosition.last())), + false) + .get(); + TableInfo after = admin.getTableInfo(path).get(); + assertThat(after.getTableId()).isEqualTo(before.getTableId()); + assertThat(after.getSchemaId()).isGreaterThan(before.getSchemaId()); + assertThat(after.getProperties().toMap()) + .isEqualTo(before.getProperties().toMap()); + assertThat(after.getSchema().getColumns()).hasSize(2); + assertThat(after.getSchema().getColumn("_kafka_metadata").getDataType()) + .isEqualTo(metadataType()); + + // Neither producer recreation nor a Kafka Metadata refresh should be needed. + assertThat(send(producer, full, 0, 789L, "new_full").offset()).isEqualTo(2L); + assertThat(send(producer, alias, 0, 987L, "new_alias").offset()).isEqualTo(3L); + assertThat(topicId(full)).isEqualTo(fullId); + assertThat(topicId(alias)).isEqualTo(aliasId); + // Read the new batches using the new native schema, independently of the + // native client's earlier cached table handle. + try (Connection reader = + ConnectionFactory.createConnection(CLUSTER.getClientConfig())) { + assertRecordsWithMetadata( + reader, + path, + 0, + 2L, + after.getSchemaId(), + tuple(2L, "new_full", 789L, 0, 0), + tuple(3L, "new_alias", 987L, 0, 0)); + } + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testNeverEnabledTableRejectsBothNamesWithoutAppending() throws Exception { + TablePath path = TablePath.of(DATABASE, "never_enabled"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + assertRejectedNames(path, Errors.UNKNOWN_TOPIC_OR_PARTITION); + setTableEnabled(admin, path, true); + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, path.getTableName(), "accepted").offset()).isZero(); + assertValues(connection, path, "accepted"); + } + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testNotNullKeySchemaIsRejectedWhenCreatingKafkaTable() throws Exception { + TablePath path = TablePath.of(DATABASE, "atomic_conversion"); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES().copy(false)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(); + assertThatThrownBy(() -> admin.createTable(path, descriptor, false).get()) + .hasCauseInstanceOf(InvalidTableException.class) + .rootCause() + .hasMessageContaining("Kafka requires nullable fields"); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testReplicatedOffsetsAndMetadataSurviveLeaderLossAndRestart() throws Exception { + TablePath path = TablePath.of(DATABASE, "durable_restart"); + String topic = path.toString(); + String rebalanceId = "kafka-recovered-leader"; + int partition = 1; + int stopped = -1; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, path, 3, metadataSchema(), 2); + TableInfo tableInfo = admin.getTableInfo(path).get(); + TableBucket bucket = new TableBucket(tableInfo.getTableId(), partition); + MetadataResponsePartition initial = + waitForTopic(topic, 2) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + for (int replica : initial.replicaNodes()) { + CLUSTER.waitUntilReplicaExpandToIsr(bucket, replica); + } + try (KafkaProducer producer = producer("all", true)) { + assertThat(send(producer, topic, partition, 123L, "before").offset()).isZero(); + assertRecordsWithMetadata( + connection, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition)); + stopped = initial.leaderId(); + CLUSTER.stopTabletServer(stopped); + retry( + Duration.ofMinutes(1), + () -> + assertThat( + waitForTopic(topic, 2) + .data() + .topics() + .find(topic) + .partitions() + .get(partition) + .leaderId()) + .isNotEqualTo(initial.leaderId())); + assertThat(send(producer, topic, partition, 456L, "after_loss").offset()) + .isEqualTo(1L); + assertRecordsWithMetadata( + connection, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition), + tuple(1L, "after_loss", 456L, 0, partition)); + CLUSTER.startTabletServer(stopped); + CLUSTER.assertHasTabletServerNumber(3); + CLUSTER.waitUntilReplicaExpandToIsr(bucket, stopped); + // Make the recovered replica serve reads and writes, including the record + // produced while it was offline. ISR membership alone does not prove readback. + MetadataResponsePartition current = + waitForTopic(topic, 2) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + rebalanceLeader(bucket, current, stopped, rebalanceId); + waitForLeaderChange( + admin, topic, partition, stopped, current.leaderEpoch(), rebalanceId); + assertThat( + send( + producer, + path.getTableName(), + partition, + 789L, + "after_restart") + .offset()) + .isEqualTo(2L); + try (Connection reader = + ConnectionFactory.createConnection(CLUSTER.getClientConfig())) { + assertRecordsWithMetadata( + reader, + path, + partition, + 0L, + tableInfo.getSchemaId(), + tuple(0L, "before", 123L, 0, partition), + tuple(1L, "after_loss", 456L, 0, partition), + tuple(2L, "after_restart", 789L, 0, partition)); + } + } + } finally { + if (stopped >= 0 && CLUSTER.getTabletServerById(stopped) == null) { + CLUSTER.startTabletServer(stopped); + CLUSTER.assertHasTabletServerNumber(3); + } + try { + admin.cancelRebalance(rebalanceId).get(); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + private static void setTableEnabled(Admin admin, TablePath path, boolean enabled) + throws Exception { + admin.alterTable( + path, + Collections.singletonList( + TableChange.set( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + Boolean.toString(enabled))), + false) + .get(); + } + + private static Uuid topicId(String topic) { + return waitForTopic(topic).data().topics().find(topic).topicId(); + } + + private static void assertRejectedNames(TablePath path, Errors error) throws Exception { + retry( + Duration.ofMinutes(1), + () -> { + for (String topic : new String[] {path.toString(), path.getTableName()}) { + for (ServerNode node : CLUSTER.getTabletServerNodes("KAFKA")) { + try (Socket socket = connect(node)) { + MetadataResponse response = + (MetadataResponse) + sendRequest(socket, metadataRequest(topic)); + assertThat(response.errors()).containsEntry(topic, error); + } + } + } + }); + for (String topic : new String[] {path.toString(), path.getTableName()}) { + for (ServerNode node : CLUSTER.getTabletServerNodes("KAFKA")) { + try (Socket socket = connect(node)) { + ProduceResponse response = + (ProduceResponse) + sendRequest(socket, produceRequest(topic, "rejected")); + assertThat(response.errorCounts()).containsEntry(error, 1); + } + } + } + } + + private static void createTable(Admin admin, TablePath path, int replicas) throws Exception { + createTable( + admin, + path, + replicas, + Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build(), + 1); + } + + private static void createTable( + Admin admin, TablePath path, int replicas, Schema schema, int buckets) + throws Exception { + admin.createDatabase(path.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + path, + TableDescriptor.builder() + .schema(schema) + .distributedBy(buckets) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + } + + private static Schema metadataSchema() { + return Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_metadata", metadataType()) + .build(); + } + + private static RowType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + + private static void setMinIsr(Admin admin, int minIsr) throws Exception { + admin.alterClusterConfigs( + Collections.singletonList( + new AlterConfig( + ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER.key(), + Integer.toString(minIsr), + AlterConfigOpType.SET))) + .get(30, TimeUnit.SECONDS); + retry( + Duration.ofMinutes(1), + () -> + assertThat(CLUSTER.getTabletServers()) + .allSatisfy( + server -> + assertThat( + server.getReplicaManager() + .getMinInSyncReplicas()) + .isEqualTo(minIsr))); + } + + private static void assertValues(Connection connection, TablePath path, String... expected) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + List values = new ArrayList<>(); + retry( + Duration.ofSeconds(30), + () -> { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + assertThat(record.logOffset()).isEqualTo(values.size()); + values.add( + new String( + record.getRow().getBytes(0), StandardCharsets.UTF_8)); + } + assertThat(values).containsExactly(expected); + }); + } + } + + private static void assertRecordsWithMetadata( + Connection connection, + TablePath path, + int partition, + long startOffset, + int schemaId, + Tuple... expected) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribe(partition, startOffset); + List actual = new ArrayList<>(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (actual.size() < expected.length && System.nanoTime() < deadline) { + for (ScanRecord record : scanner.poll(Duration.ofSeconds(1))) { + assertThat(record.getSchemaId()) + .as("Schema at offset %s", record.logOffset()) + .isEqualTo(schemaId); + InternalRow row = record.getRow(); + assertThat(row.getFieldCount()).isEqualTo(2); + assertThat(row.isNullAt(1)).isFalse(); + InternalRow metadata = row.getRow(1, 3); + assertThat(metadata.getFieldCount()).isEqualTo(3); + assertThat(metadata.isNullAt(0)).isFalse(); + assertThat(metadata.isNullAt(1)).isFalse(); + assertThat(metadata.isNullAt(2)).isFalse(); + actual.add( + tuple( + record.logOffset(), + new String(row.getBytes(0), StandardCharsets.UTF_8), + metadata.getTimestampLtz(0, 3).getEpochMillisecond(), + metadata.getInt(1), + metadata.getInt(2))); + } + } + assertThat(actual) + .as("Native metadata readback for %s bucket %s", path, partition) + .containsExactly(expected); + } + } + + private static MetadataResponse waitForTopic(String topic) { + return waitForTopic(topic, 1); + } + + private static MetadataResponse waitForTopic(String topic, int partitions) { + AtomicReference result = new AtomicReference<>(); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponse response = fetchMetadata(topic); + MetadataResponseTopic metadata = response.data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(metadata.partitions()) + .hasSize(partitions) + .allSatisfy( + partition -> assertThat(partition.leaderId()).isNotEqualTo(-1)); + result.set(response); + }); + return result.get(); + } + + private static void assertKafkaListenerEndpoints(MetadataResponse response) { + assertThat(response.brokers()) + .extracting( + org.apache.kafka.common.Node::id, + org.apache.kafka.common.Node::host, + org.apache.kafka.common.Node::port, + org.apache.kafka.common.Node::rack) + .containsExactlyInAnyOrderElementsOf( + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map( + node -> + tuple( + node.id(), + node.host(), + node.port(), + node.rack())) + .collect(Collectors.toList())); + } + + private static MetadataResponse fetchMetadata(String topic) throws Exception { + try (Socket socket = connect(CLUSTER.getTabletServerNodes("KAFKA").get(0))) { + return (MetadataResponse) sendRequest(socket, metadataRequest(topic)); + } + } + + private static MetadataRequest metadataRequest(String topic) { + return new MetadataRequest.Builder(Collections.singletonList(topic), false) + .build(METADATA_VERSION); + } + + private static ProduceRequest produceRequest(String topic, String value) { + return produceRequest(topic, new SimpleRecord(value.getBytes(StandardCharsets.UTF_8))); + } + + private static ProduceRequest produceRequest(String topic, SimpleRecord... values) { + MemoryRecords records = MemoryRecords.withRecords(Compression.NONE, values); + ProduceRequestData.TopicProduceData topicData = + new ProduceRequestData.TopicProduceData() + .setName(topic) + .setPartitionData( + Collections.singletonList( + new ProduceRequestData.PartitionProduceData() + .setIndex(0) + .setRecords(records))); + return new ProduceRequest( + new ProduceRequestData() + .setAcks((short) 1) + .setTimeoutMs(5000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topicData).iterator())), + (short) 11); + } + + private static Socket connect(ServerNode node) throws Exception { + Socket socket = new Socket(); + try { + socket.connect(new InetSocketAddress(node.host(), node.port()), 5000); + socket.setSoTimeout(5000); + return socket; + } catch (Exception failure) { + socket.close(); + throw failure; + } + } + + private static AbstractResponse sendRequest(Socket socket, AbstractRequest request) + throws Exception { + RequestHeader header = + new RequestHeader(request.apiKey(), request.version(), "regression-test", 1); + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + byte[] requestBytes = new byte[serialized.remaining()]; + serialized.get(requestBytes); + // The caller owns the socket and may send another request on the same connection. + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + int size = input.readInt(); + assertThat(size).isBetween(1, 1024 * 1024); + byte[] response = new byte[size]; + input.readFully(response); + return AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + + private static void rebalanceLeader( + TableBucket bucket, + MetadataResponsePartition initial, + int newLeader, + String rebalanceId) + throws Exception { + List replicas = new ArrayList<>(initial.replicaNodes()); + Collections.swap(replicas, 0, replicas.indexOf(newLeader)); + RebalancePlanForBucket plan = + new RebalancePlanForBucket( + bucket, initial.leaderId(), newLeader, initial.replicaNodes(), replicas); + RebalanceTask task = + new RebalanceTask( + rebalanceId, + RebalanceStatus.NOT_STARTED, + Collections.singletonMap(bucket, plan)); + // Submit a deterministic leader-only plan through the coordinator event thread. + // All replicas and their client connections remain online during the real handoff. + CLUSTER.getZooKeeperClient().registerRebalanceTask(task); + CLUSTER.getCoordinatorServer() + .getCoordinatorEventProcessor() + .getCoordinatorEventManager() + .put(new RecoverRebalanceEvent(task)); + } + + private static void waitForLeaderChange( + Admin admin, + String topic, + int partition, + int newLeader, + int oldEpoch, + String rebalanceId) { + retry( + Duration.ofMinutes(1), + () -> { + assertThat(admin.listRebalanceProgress(rebalanceId).get()) + .isPresent() + .get() + .extracting(RebalanceProgress::status) + .isEqualTo(RebalanceStatus.COMPLETED); + MetadataResponsePartition updated = + fetchMetadata(topic) + .data() + .topics() + .find(topic) + .partitions() + .get(partition); + assertThat(updated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(updated.leaderId()).isEqualTo(newLeader); + assertThat(updated.leaderEpoch()).isGreaterThan(oldEpoch); + }); + } + + private static double producerMetric(KafkaProducer producer, String name) { + return producer.metrics().entrySet().stream() + .filter( + entry -> + entry.getKey().group().equals("producer-metrics") + && entry.getKey().name().equals(name)) + .mapToDouble(entry -> ((Number) entry.getValue().metricValue()).doubleValue()) + .findFirst() + .getAsDouble(); + } + + private static RecordMetadata send( + KafkaProducer producer, String topic, String value) throws Exception { + return producer.send( + new ProducerRecord<>( + topic, 0, null, value.getBytes(StandardCharsets.UTF_8))) + .get(60, TimeUnit.SECONDS); + } + + private static RecordMetadata send( + KafkaProducer producer, + String topic, + int partition, + long timestamp, + String value) + throws Exception { + return producer.send( + new ProducerRecord<>( + topic, + partition, + timestamp, + null, + value.getBytes(StandardCharsets.UTF_8))) + .get(60, TimeUnit.SECONDS); + } + + private static KafkaProducer producer(String acks, boolean retryOnFailure) { + Map config = new HashMap<>(); + config.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map(node -> node.host() + ":" + node.port()) + .collect(Collectors.joining(","))); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.RETRIES_CONFIG, retryOnFailure ? 20 : 0); + config.put(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, 200); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, (int) Duration.ofHours(1).toMillis()); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 40fdab04cc4..9642ed9bfb1 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,6 +17,8 @@ package org.apache.fluss.kafka; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; @@ -76,7 +78,10 @@ public void testUnsupportedWireVersionFallsBackAndAllowsRenegotiation() { ApiVersionsResponse unsupportedResponse = parseWireResponse(channel, (short) 0, 41); assertThat(unsupportedResponse.data().errorCode()) .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); - assertBrokerCapabilities(unsupportedResponse); + // The version error only advertises the range needed to retry ApiVersions. + assertThat(unsupportedResponse.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); ApiVersionsRequestData data = new ApiVersionsRequestData() @@ -145,7 +150,10 @@ private static void assertSuccessfulResponseDefaults(ApiVersionsResponse respons private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) - .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); + .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, (short) 11), + tuple(ApiKeys.METADATA.id, (short) 0, (short) 11), + tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } @ParameterizedTest @@ -292,6 +300,7 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + return new KafkaRequestHandler(new Configuration(), service); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java index 0492ee3572f..9d7912b5b42 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.api.versions; import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; @@ -59,6 +60,11 @@ void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { new KafkaApiRegistry( Arrays.asList( handler, + new ProduceHandler( + command -> { + throw new AssertionError( + "Capability advertisement must not invoke the Produce backend."); + }), metadataHandler, new TestingApiHandler( new KafkaApiSpec( @@ -78,6 +84,7 @@ void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, (short) 11), tuple(ApiKeys.METADATA.id, (short) 2, (short) 7), tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java new file mode 100644 index 00000000000..39603e33a01 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -0,0 +1,1328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.KafkaKeyConversionType; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.exception.AuthorizationException; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.kafka.transcode.KafkaTopicWritePlan; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.protocol.Errors; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests append admission, error isolation, session propagation and asynchronous acknowledgements. + */ +class GatewayKafkaProduceBackendTest { + + @Test + void testAllNullValuesUseNativeAppendAndAuthorization() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service) + .write(command((short) 1, topic("kafka.topic", nullValue(0), nullValue(8)))) + .join(); + assertErrors(result, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.writeChecks).isEqualTo(1); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isZero(); + service.denyWrite = true; + assertErrors( + backend(service) + .write(command((short) 1, topic("kafka.topic", nullValue(0)))) + .join(), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.appends).hasSize(1); + assertThat(service.writeChecks).isEqualTo(2); + } + + @ParameterizedTest + @ValueSource(strings = {"non-null", "all-null", "mixed"}) + void testCachedMetadataDoesNotBypassRevokedWritePermission(String recordsKind) + throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); + PartitionWrite partition = + recordsKind.equals("non-null") + ? good(0) + : recordsKind.equals("all-null") + ? nullValue(0) + : new PartitionWrite( + 0, + Arrays.asList( + nullValue(0).records().get(0), + good(0).records().get(0))); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", partition)); + try { + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + int appendedBeforeRevocation = service.appends.size(); + service.denyWrite = true; + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.metadataPaths).hasSize(2); + assertThat(service.writeChecks).isEqualTo(2); + assertThat(service.appends).hasSize(appendedBeforeRevocation); + assertThat(service.metadataThread).isNotSameAs(Thread.currentThread()); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testNullValuesCannotBypassTableAndSchemaValidation() throws Exception { + TestingProduceService service = new TestingProduceService(); + for (String name : new String[] {"invalid", "missing"}) { + assertErrors( + backend(service) + .write(command((short) 1, topic("kafka." + name, nullValue(0)))) + .join(), + name.equals("invalid") + ? Errors.INVALID_TOPIC_EXCEPTION + : Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + assertThat(service.writeChecks).isZero(); + assertThat(service.append).isNull(); + } + + @Test + void testMixedNullValuesPreserveRecordOrderAndNativeOffsets() throws Exception { + TestingProduceService service = new TestingProduceService(); + List seen = new ArrayList<>(); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + mapper(), + (partitionId, records, table) -> { + seen.addAll(records); + return transcoder.transcode(partitionId, records, table); + }); + Record nullRecord = nullValue(0).records().get(0); + Record empty = new Record(1L, null, new byte[0], Collections.emptyList()); + Record valid = good(0).records().get(0); + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic( + "kafka.topic", + new PartitionWrite( + 0, + Arrays.asList(nullRecord, empty, valid))))) + .join(); + assertErrors(result, Errors.NONE); + assertThat(seen).containsExactly(nullRecord, empty, valid); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + } + + private static PartitionWrite nullValue(int partition) { + return new PartitionWrite( + partition, + Collections.singletonList( + new Record(-1L, new byte[] {(byte) 0xff}, null, Collections.emptyList()))); + } + + @Test + void testAcksAllWaitsForNativeAppendResponse() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.append.getAcks()).isEqualTo(-1); + assertThat(service.append.getTimeoutMs()).isEqualTo(4321); + assertThat(service.append.getTableId()).isEqualTo(42L); + assertThat(result).isNotDone(); + assertThat(service.pendingAppend).isNotDone(); + service.pendingAppend.complete(success(0)); + assertThat(result.join().topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + } + + @Test + void testMapsSynchronousAppendFailure() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.throwAppend = true; + assertErrors( + backend(service).write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.UNKNOWN_SERVER_ERROR); + } + + @Test + void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception { + TestingProduceService service = new TestingProduceService(); + PartitionWrite malformed = + new PartitionWrite( + 0, + Arrays.asList( + good(0).records().get(0), + new Record( + 1L, + null, + new byte[] {(byte) 0xff}, + Collections.emptyList()))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("kafka.topic", malformed, good(1), good(8)))) + .join(); + assertErrors(result, Errors.INVALID_RECORD, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + } + + @Test + void testInvalidAndMissingTablesDoNotSuppressOtherTopics() throws Exception { + for (boolean async : new boolean[] {false, true}) { + TestingProduceService service = new TestingProduceService(); + service.asyncMissing = async; + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("kafka.invalid", good(0)), + topic("kafka.missing", good(0)), + topic("kafka.topic", good(0)))) + .join(); + assertThat(result.topics()).hasSize(3); + assertThat(result.topics().get(0).partitions().get(0).error()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION); + assertThat(result.topics().get(1).partitions().get(0).error()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(result.topics().get(2).partitions().get(0).error()).isEqualTo(Errors.NONE); + } + } + + @Test + void testNoAppendWhenAllPartitionsFailConversion() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service).write(command((short) 1, topic("kafka.topic", good(8)))).join(); + assertErrors(result, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append).isNull(); + } + + @Test + void testMapsReplicationFailuresAndMissingResponsesPerPartition() throws Exception { + org.apache.fluss.rpc.protocol.Errors[] flussErrors = { + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.REQUEST_TIME_OUT, + org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER, + org.apache.fluss.rpc.protocol.Errors.CORRUPT_RECORD_EXCEPTION + }; + Errors[] kafkaErrors = { + Errors.NOT_ENOUGH_REPLICAS, + Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, + Errors.REQUEST_TIMED_OUT, + Errors.NOT_LEADER_OR_FOLLOWER, + Errors.CORRUPT_MESSAGE + }; + for (int i = 0; i < flussErrors.length; i++) { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = + CompletableFuture.completedFuture( + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setErrorCode(flussErrors[i].code()), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setErrorCode(0) + .setBaseOffset(17L)))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) -1, + topic("kafka.topic", good(0), good(1), good(2)))) + .join(); + assertErrors(result, kafkaErrors[i], Errors.NONE, Errors.UNKNOWN_SERVER_ERROR); + assertThat(result.topics().get(0).partitions().get(1).baseOffset()).isEqualTo(17L); + } + } + + @Test + void testAsyncMetadataCompletionRestoresSessionAndAcksZero() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) 0, topic("kafka.topic", good(0)))); + CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))).join(); + assertErrors(result.join(), Errors.NONE); + assertThat(service.append.getAcks()).isZero(); + } + + @Test + void testQualifiedTopicsPreserveDatabaseAndTranscoderTablePath() throws Exception { + TestingProduceService service = new TestingProduceService(); + List transcodedPaths = new ArrayList<>(); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + mapper(), + (partitionId, records, tableInfo) -> { + transcodedPaths.add(tableInfo.getTablePath()); + return transcoder.transcode(partitionId, records, tableInfo); + }); + + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("kafka.topic", good(0)), + topic("other.topic", good(0)))) + .join(); + + assertThat(service.metadataPaths) + .containsExactly(TablePath.of("kafka", "topic"), TablePath.of("other", "topic")); + assertThat(transcodedPaths).containsExactlyElementsOf(service.metadataPaths); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 43L); + assertThat(result.topics()) + .extracting(KafkaProduceResult.TopicResult::topicName) + .containsExactly("kafka.topic", "other.topic"); + assertThat(result.topics()) + .allSatisfy( + topic -> + assertThat(topic.partitions().get(0).error()) + .isEqualTo(Errors.NONE)); + } + + @Test + void testInvalidQualifiedNamesDoNotReachGatewayOrSuppressValidTopic() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic(".", good(0)), + topic("kafka.topic.extra", good(0)), + topic("kafka.__internal", good(0)), + topic("kafka.topic", good(0)))) + .join(); + + assertThat(result.topics()) + .extracting(topic -> topic.partitions().get(0).error()) + .containsExactly( + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION, + Errors.NONE); + assertThat(service.metadataPaths).containsExactly(TablePath.of("kafka", "topic")); + assertThat(service.appends).hasSize(1); + } + + @Test + void testReusesPlansButAlwaysRefreshesMetadataAndValidatesChangedContract() throws Exception { + TestingProduceService service = new TestingProduceService(); + ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); + List prepared = new ArrayList<>(); + List converted = new ArrayList<>(); + List convertedPartitions = new ArrayList<>(); + KafkaRecordTranscoder transcoder = + new KafkaRecordTranscoder() { + @Override + public KafkaTopicWritePlan prepare(TableInfo tableInfo) { + KafkaTopicWritePlan plan = arrow.prepare(tableInfo); + prepared.add(plan); + return plan; + } + + @Override + public BytesView transcode( + int partitionId, List records, TableInfo tableInfo) + throws Exception { + return arrow.transcode(partitionId, records, tableInfo); + } + + @Override + public BytesView transcode( + int partitionId, List records, KafkaTopicWritePlan plan) + throws Exception { + converted.add(plan); + convertedPartitions.add(partitionId); + return arrow.transcode(partitionId, records, plan); + } + }; + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + backend.write(command((short) 1, topic("kafka.topic", good(0), good(1)))).join(); + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(); + assertThat(service.metadataPaths).hasSize(2); + assertThat(prepared).hasSize(2); + assertThat(prepared.get(1)).isSameAs(prepared.get(0)); + assertThat(converted).containsExactly(prepared.get(0), prepared.get(0), prepared.get(0)); + assertThat(convertedPartitions).containsExactly(0, 1, 0); + + service.pendingMetadata = CompletableFuture.completedFuture(metadata(true)); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.metadataPaths).hasSize(3); + assertThat(service.appends).hasSize(2); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingMetadata.completeExceptionally( + new org.apache.fluss.exception.AuthorizationException("revoked")); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.TOPIC_AUTHORIZATION_FAILED); + assertThat(service.metadataPaths).hasSize(4); + assertThat(service.appends).hasSize(2); + } + + @Test + void testOffloadedRequestsAppendInOrderWithoutWaitingForAcknowledgement() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(2, 4); + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingAppend = new CompletableFuture<>(); + service.metadataEntered = new CountDownLatch(1); + service.appended = new CountDownLatch(2); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); + try { + CompletableFuture first = + backend.write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.metadataEntered.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture second = + backend.write(command((short) -1, topic("kafka.topic", good(1)))); + assertThat(service.metadataThread).isNotSameAs(Thread.currentThread()); + CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))) + .join(); + assertThat(service.appended.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(service.appends) + .extracting(request -> request.getBucketsReqsList().get(0).getBucketId()) + .containsExactly(0, 1); + assertThat(first).isNotDone(); + assertThat(second).isNotDone(); + service.pendingAppend.complete( + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setBaseOffset(17L), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setBaseOffset(18L)))); + assertErrors(first.get(10, TimeUnit.SECONDS), Errors.NONE); + assertErrors(second.get(10, TimeUnit.SECONDS), Errors.NONE); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testOffloadedMetadataRefreshUsesNewTableIdentityAndSchema() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 2); + ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + List snapshots = new ArrayList<>(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + mapper(), + (partitionId, records, table) -> { + snapshots.add(table); + return transcoder.transcode(partitionId, records, table); + }, + executor); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); + try { + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + TableDescriptor changedSchema = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(3) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableId(84L) + .setSchemaId(2) + .setBucketCountEpoch(7L) + .setTableJson(changedSchema.toJsonBytes())); + assertErrors(backend.write(command).get(10, TimeUnit.SECONDS), Errors.NONE); + assertThat(snapshots).hasSize(3); + assertThat(snapshots.get(1)).isSameAs(snapshots.get(0)); + assertThat(snapshots.get(2)).isNotSameAs(snapshots.get(0)); + assertThat(snapshots.get(2).getTableId()).isEqualTo(84L); + assertThat(snapshots.get(2).getSchemaId()).isEqualTo(2); + assertThat(snapshots.get(2).getRowType().getTypeAt(0)).isEqualTo(DataTypes.BYTES()); + assertThat(snapshots.get(2).getBucketCountEpoch()).isEqualTo(7L); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L, 84L); + service.pendingMetadata = new CompletableFuture<>(); + service.pendingMetadata.completeExceptionally(new TableNotExistException("deleted")); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), + Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.metadataPaths).hasSize(4); + assertThat(service.appends).hasSize(3); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testShutdownPreservesAlreadySubmittedAppendResult() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = new CompletableFuture<>(); + service.appended = new CountDownLatch(1); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); + try { + CompletableFuture result = + backend.write(command((short) -1, topic("kafka.topic", good(0)))); + assertThat(service.appended.await(10, TimeUnit.SECONDS)).isTrue(); + executor.closeAsync().get(10, TimeUnit.SECONDS); + assertThat(result).isNotDone(); + assertThat(service.pendingAppend).isNotCancelled(); + service.pendingAppend.complete(success(0)); + KafkaProduceResult completed = result.get(10, TimeUnit.SECONDS); + assertErrors(completed, Errors.NONE); + assertThat(completed.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.appends).hasSize(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testSaturationAndShutdownReturnRetriableErrorsWithoutAppend() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + service.metadataEntered = new CountDownLatch(1); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder(), executor); + KafkaProduceCommand command = command((short) 1, topic("kafka.topic", good(0))); + try { + CompletableFuture running = backend.write(command); + assertThat(service.metadataEntered.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture queued = backend.write(command); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + executor.closeAsync().get(10, TimeUnit.SECONDS); + assertErrors(running.get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + assertErrors(queued.get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + assertErrors( + backend.write(command).get(10, TimeUnit.SECONDS), Errors.REQUEST_TIMED_OUT); + service.pendingMetadata.complete(metadata(false)); + assertThat(service.appends).isEmpty(); + assertThat(service.metadataPaths).hasSize(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testAliasesSharePlansAndAuthorizationButRetainRequestNames() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 4); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder, executor); + try { + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("topic", good(0)), + topic("kafka.topic", good(0)))) + .get(10, TimeUnit.SECONDS); + assertThat(result.topics()) + .extracting(KafkaProduceResult.TopicResult::topicName) + .containsExactly("topic", "kafka.topic"); + assertThat(result.topics()) + .allSatisfy( + topic -> + assertThat(topic.partitions().get(0).error()) + .isEqualTo(Errors.NONE)); + assertThat(service.metadataPaths) + .containsExactly( + TablePath.of("kafka", "topic"), TablePath.of("kafka", "topic")); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L); + assertThat(service.writeChecks).isEqualTo(2); + assertThat(transcoder.prepared).hasSize(2); + assertThat(transcoder.prepared.get(0)).isSameAs(transcoder.prepared.get(1)); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDisabledAndResetFlagsRejectBothNamesBeforePrepare(boolean allNull) throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + PartitionWrite partition = allNull ? nullValue(0) : good(0); + assertErrors( + backend.write(command((short) 1, topic("topic", partition))).join(), Errors.NONE); + for (boolean reset : new boolean[] {false, true}) { + TableDescriptor enabled = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + java.util.Map properties = + new java.util.HashMap<>(enabled.getProperties()); + if (reset) { + properties.remove(ConfigOptions.TABLE_KAFKA_ENABLED.key()); + } else { + properties.put(ConfigOptions.TABLE_KAFKA_ENABLED.key(), "false"); + } + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + enabled.withProperties(properties).toJsonBytes())); + int prepared = transcoder.prepared.size(); + int appended = service.appends.size(); + int authorized = service.writeChecks; + for (String name : Arrays.asList("topic", "kafka.topic")) { + assertErrors( + backend.write(command((short) 1, topic(name, partition))).join(), + Errors.UNKNOWN_TOPIC_OR_PARTITION); + } + assertThat(transcoder.prepared).hasSize(prepared); + assertThat(service.appends).hasSize(appended); + assertThat(service.writeChecks).isEqualTo(authorized); + service.pendingMetadata = CompletableFuture.completedFuture(metadata(false)); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", partition))).join(), + Errors.NONE); + assertThat(transcoder.prepared.get(prepared)) + .isNotSameAs(transcoder.prepared.get(prepared - 1)); + } + assertThat(service.metadataPaths).hasSize(7); + } + + @Test + void testPropertyOnlyChangesRejectInvalidContractsAndRecompileRecoveredPlan() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + assertErrors( + backend.write(command((short) 1, topic("topic", good(0)))).join(), Errors.NONE); + TableDescriptor original = + TableDescriptor.builder( + TableDescriptor.fromJsonBytes(metadata(false).getTableJson())) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES()) + .build()) + .build(); + for (TableDescriptor invalid : + Arrays.asList( + TableDescriptor.builder(original) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .build(), + TableDescriptor.builder(original) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), "string") + .build())) { + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(invalid.toJsonBytes())); + for (String name : Arrays.asList("topic", "kafka.topic")) { + assertErrors( + backend.write(command((short) 1, topic(name, nullValue(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + } + } + assertThat(service.appends).hasSize(1); + assertThat(transcoder.prepared).hasSize(1); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + KafkaKeyConversionType.BINARY) + .build() + .toJsonBytes())); + assertErrors( + backend.write(command((short) 1, topic("kafka.topic", good(0)))).join(), + Errors.NONE); + assertThat(transcoder.prepared).hasSize(2); + assertThat(transcoder.prepared.get(1)).isNotSameAs(transcoder.prepared.get(0)); + assertThat(transcoder.prepared.get(1).tableInfo().getSchemaId()).isEqualTo(1); + assertThat(service.appends).hasSize(2); + assertThat(service.metadataPaths).hasSize(6); + } + + @Test + void testMissingDefaultAndExplicitDatabasesDoNotFallback() throws Exception { + TestingProduceService service = new TestingProduceService(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_db"); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend( + service, + service, + new KafkaTopicMapper(config), + new ArrowKafkaRecordTranscoder()); + KafkaProduceResult result = + backend.write( + command( + (short) 1, + topic("topic", good(0)), + topic("missing_db.topic", good(0)), + topic("other.topic", good(0)))) + .join(); + assertThat(result.topics()) + .extracting(topic -> topic.partitions().get(0).error()) + .containsExactly( + Errors.UNKNOWN_TOPIC_OR_PARTITION, + Errors.UNKNOWN_TOPIC_OR_PARTITION, + Errors.NONE); + assertThat(service.metadataPaths) + .containsExactly( + TablePath.of("missing_db", "topic"), + TablePath.of("missing_db", "topic"), + TablePath.of("other", "topic")); + assertThat(service.appends).extracting(ProduceLogRequest::getTableId).containsExactly(43L); + } + + @Test + void testNotNullKeyContractRejectsBeforeAppendingAnyRecords() throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.STRING()) + .column( + "_kafka_key", + DataTypes.BYTES() + .copy(false)) + .build()) + .build() + .toJsonBytes())); + Record valid = new Record(1L, new byte[] {1}, new byte[] {65}, Collections.emptyList()); + assertErrors( + backend(service) + .write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 0, + Arrays.asList( + valid, good(0).records().get(0)))))) + .join(), + Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.appends).isEmpty(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testInvalidRecordDoesNotAppendValidPrefixWithMetadata(boolean malformedUtf8) + throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(original) + .schema(schema) + .build() + .toJsonBytes())); + Record valid = new Record(1L, new byte[] {1}, new byte[] {65}, Collections.emptyList()); + Record invalid = + malformedUtf8 + ? new Record( + 2L, + new byte[] {1}, + new byte[] {(byte) 0xc3, 0x28}, + Collections.emptyList()) + : new Record( + 2L, + new byte[] {1}, + new byte[] {(byte) 0xff}, + Collections.emptyList()); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 2, Arrays.asList(valid, invalid))))) + .join(), + Errors.INVALID_RECORD); + assertThat(service.appends).isEmpty(); + assertErrors( + backend.write( + command( + (short) 1, + topic( + "topic", + new PartitionWrite( + 1, Collections.singletonList(valid))))) + .join(), + Errors.NONE); + assertThat(service.appends).hasSize(1); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNullValueViolationRejectsNotNullTableContractBeforeAppending(String format) + throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + TableDescriptor notNull = + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + (format.equals("binary") + ? DataTypes.BYTES() + : DataTypes.STRING()) + .copy(false)) + .build()) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + format.equals("binary") + ? KafkaValueConversionType.BINARY + : KafkaValueConversionType.STRING) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(notNull.toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.appends).isEmpty(); + + Record valid = good(0).records().get(0); + PartitionWrite invalid = + new PartitionWrite(0, Arrays.asList(valid, nullValue(0).records().get(0), valid)); + assertErrors( + backend.write(command((short) 1, topic("topic", invalid, good(1)))).join(), + Errors.INVALID_TOPIC_EXCEPTION, + Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.appends).isEmpty(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testLegacyMetadataOffsetsRejectedAndJsonNullValuesPersisted(boolean nested) + throws Exception { + TestingProduceService service = new TestingProduceService(); + TableDescriptor original = TableDescriptor.fromJsonBytes(metadata(false).getTableJson()); + TableDescriptor unsupportedMetadata = + TableDescriptor.builder(original) + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()), + DataTypes.FIELD( + "offset", DataTypes.BIGINT()))) + .build()) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableJson(unsupportedMetadata.toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + assertErrors( + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + Schema jsonSchema = + nested + ? Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD("business", DataTypes.STRING()))) + .build() + : Schema.newBuilder().column("business", DataTypes.STRING()).build(); + TableDescriptor json = + TableDescriptor.builder(original) + .schema(jsonSchema) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.PARSE_JSON) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + nested + ? KafkaValueConversionLayout.NESTED + : KafkaValueConversionLayout.FLAT) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture(metadata(false).setTableJson(json.toJsonBytes())); + KafkaProduceResult result = + backend.write(command((short) 1, topic("topic", nullValue(0)))).join(); + assertErrors(result, Errors.NONE); + assertThat(result.topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isZero(); + assertThat(service.writeChecks).isEqualTo(1); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testJsonConversionHonorsRescueAndPartitionAtomicity(boolean nested) throws Exception { + List invalidValues = + new ArrayList<>(Arrays.asList("{", "[]", "{\"id\":\"bad\"}", "{} {}")); + List rescuableValues = + new ArrayList<>(Collections.singletonList("{\"id\":1,\"unknown\":2}")); + for (String reserved : + Arrays.asList( + "_kafka_key", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue", + "_kafka_value")) { + rescuableValues.add("{\"id\":1,\"" + reserved + "\":null}"); + } + invalidValues.addAll(rescuableValues); + for (boolean rescue : new boolean[] {false, true}) { + for (String invalid : invalidValues) { + boolean captured = rescue && rescuableValues.contains(invalid); + TestingProduceService service = new TestingProduceService(); + if (captured) { + ProduceLogResponse response = success(0); + response.addAllBucketsResps(success(1).getBucketsRespsList()); + service.pendingAppend = CompletableFuture.completedFuture(response); + } + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + jsonDescriptor(nested, rescue).toJsonBytes())); + PartitionWrite bad = + new PartitionWrite( + 0, + Arrays.asList( + json(0, "{\"id\":0}").records().get(0), + json(0, invalid).records().get(0))); + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic( + "topic", + bad, + json(1, "{\"id\":1}"), + json(8, "{\"id\":8}")))) + .join(); + assertErrors( + result, + captured ? Errors.NONE : Errors.INVALID_RECORD, + Errors.NONE, + Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.appends).hasSize(1); + assertThat(service.append.getBucketsReqsList()).hasSize(captured ? 2 : 1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()) + .isEqualTo(captured ? 0 : 1); + if (captured) { + assertThat(result.topics().get(0).partitions().get(0).baseOffset()) + .isEqualTo(17L); + } + assertThat(result.topics().get(0).partitions().get(1).baseOffset()).isEqualTo(17L); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testInvalidJsonSchemaIsInvalidTopicEvenForNullValues(boolean nested) throws Exception { + TestingProduceService service = new TestingProduceService(); + Schema schema = + nested + ? Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD( + "id", + DataTypes.MAP( + DataTypes.INT(), + DataTypes.STRING())))) + .build() + : Schema.newBuilder() + .column("id", DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())) + .build(); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(jsonDescriptor(nested, false)) + .schema(schema) + .build() + .toJsonBytes())); + GatewayKafkaProduceBackend backend = backend(service); + for (PartitionWrite partition : Arrays.asList(nullValue(0), json(0, "{\"id\":[1]}"))) { + assertErrors( + backend.write(command((short) 1, topic("topic", partition))).join(), + Errors.INVALID_TOPIC_EXCEPTION); + } + assertThat(service.appends).isEmpty(); + assertThat(service.writeChecks).isZero(); + } + + @Test + void testJsonCacheUsesAuthoritativeLayoutSchemaAndTableIdentity() throws Exception { + TestingProduceService service = new TestingProduceService(); + RecordingTranscoder transcoder = new RecordingTranscoder(); + GatewayKafkaProduceBackend backend = + new GatewayKafkaProduceBackend(service, service, mapper(), transcoder); + TableDescriptor flat = jsonDescriptor(false, false); + service.pendingMetadata = + CompletableFuture.completedFuture(metadata(false).setTableJson(flat.toJsonBytes())); + KafkaProduceCommand bare = command((short) 1, topic("topic", json(0, "{\"id\":1}"))); + KafkaProduceCommand full = command((short) 1, topic("kafka.topic", json(0, "{\"id\":1}"))); + assertErrors(backend.write(bare).join(), Errors.NONE); + assertErrors(backend.write(full).join(), Errors.NONE); + assertThat(transcoder.prepared.get(1)).isSameAs(transcoder.prepared.get(0)); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setTableJson( + TableDescriptor.builder(flat) + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_LAYOUT + .key(), + "nested") + .build() + .toJsonBytes())); + assertErrors(backend.write(bare).join(), Errors.INVALID_TOPIC_EXCEPTION); + assertThat(service.appends).hasSize(2); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false) + .setSchemaId(2) + .setTableJson(jsonDescriptor(true, false).toJsonBytes())); + assertErrors(backend.write(full).join(), Errors.NONE); + assertThat(transcoder.prepared.get(2)).isNotSameAs(transcoder.prepared.get(0)); + assertThat(transcoder.prepared.get(2).tableInfo().getSchemaId()).isEqualTo(2); + service.pendingMetadata = + CompletableFuture.completedFuture( + metadata(false).setTableId(84L).setTableJson(flat.toJsonBytes())); + assertErrors(backend.write(bare).join(), Errors.NONE); + assertThat(transcoder.prepared.get(3)).isNotSameAs(transcoder.prepared.get(0)); + assertThat(transcoder.prepared.get(3).tableInfo().getTableId()).isEqualTo(84L); + assertThat(service.appends) + .extracting(ProduceLogRequest::getTableId) + .containsExactly(42L, 42L, 42L, 84L); + assertThat(service.metadataPaths).hasSize(5); + } + + private static TableDescriptor jsonDescriptor(boolean nested, boolean rescue) { + Schema.Builder schema = Schema.newBuilder(); + if (nested) { + schema.column("_kafka_value", DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT()))); + } else { + schema.column("id", DataTypes.INT()); + } + schema.column("_kafka_key", DataTypes.BYTES()); + if (rescue) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + return TableDescriptor.builder() + .schema(schema.build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + nested ? "nested" : "flat") + .build(); + } + + private static PartitionWrite json(int id, String value) { + return new PartitionWrite( + id, + Collections.singletonList( + new Record( + 123L, + null, + value.getBytes(java.nio.charset.StandardCharsets.UTF_8), + Collections.emptyList()))); + } + + private static final class RecordingTranscoder implements KafkaRecordTranscoder { + private final ArrowKafkaRecordTranscoder arrow = new ArrowKafkaRecordTranscoder(); + private final List prepared = new ArrayList<>(); + + @Override + public KafkaTopicWritePlan prepare(TableInfo table) { + KafkaTopicWritePlan plan = arrow.prepare(table); + prepared.add(plan); + return plan; + } + + @Override + public BytesView transcode(int partitionId, List records, TableInfo table) + throws Exception { + return arrow.transcode(partitionId, records, table); + } + } + + private static KafkaTopicMapper mapper() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "kafka"); + return new KafkaTopicMapper(config); + } + + private static GatewayKafkaProduceBackend backend(TestingProduceService service) { + return new GatewayKafkaProduceBackend( + service, service, mapper(), new ArrowKafkaRecordTranscoder()); + } + + private static KafkaProduceCommand command(short acks, TopicWrite... topics) throws Exception { + return new KafkaProduceCommand( + acks, 4321, Arrays.asList(topics), "KAFKA", InetAddress.getLoopbackAddress()); + } + + private static TopicWrite topic(String name, PartitionWrite... partitions) { + return new TopicWrite(name, Arrays.asList(partitions)); + } + + private static PartitionWrite good(int id) { + return new PartitionWrite( + id, + Collections.singletonList( + new Record(123L, null, new byte[] {65}, Collections.emptyList()))); + } + + private static GetTableInfoResponse metadata(boolean invalid) { + TableDescriptor.Builder descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .build()) + .distributedBy(3) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true); + if (!invalid) { + descriptor.property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, KafkaValueConversionType.STRING); + } + return new GetTableInfoResponse() + .setTableId(42L) + .setSchemaId(1) + .setTableJson(descriptor.build().toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L); + } + + private static ProduceLogResponse success(int id) { + return new ProduceLogResponse() + .addAllBucketsResps( + Collections.singletonList( + new PbProduceLogRespForBucket() + .setBucketId(id) + .setBaseOffset(17L))); + } + + private static void assertErrors(KafkaProduceResult result, Errors... errors) { + List partitions = result.topics().get(0).partitions(); + assertThat(partitions).extracting(PartitionResult::error).containsExactly(errors); + for (PartitionResult partition : partitions) { + if (partition.error() != Errors.NONE) { + assertThat(partition.baseOffset()).isEqualTo(-1L); + } + } + } + + private static class TestingProduceService extends TestingTabletGatewayService { + private final List metadataPaths = new ArrayList<>(); + private final List appends = new ArrayList<>(); + private ProduceLogRequest append; + private CompletableFuture pendingAppend; + private CompletableFuture pendingMetadata; + private boolean throwAppend; + private boolean asyncMissing; + private int writeChecks; + private boolean denyWrite; + private CountDownLatch metadataEntered; + private CountDownLatch appended; + private Thread metadataThread; + + @Override + public CompletableFuture getTableInfo(GetTableInfoRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + String database = request.getTablePath().getDatabaseName(); + String name = request.getTablePath().getTableName(); + metadataPaths.add(TablePath.of(database, name)); + metadataThread = Thread.currentThread(); + if (metadataEntered != null) { + metadataEntered.countDown(); + } + if (database.equals("missing_db")) { + throw new DatabaseNotExistException("missing_db"); + } + if (name.equals("missing")) { + if (!asyncMissing) { + throw new TableNotExistException("missing"); + } + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(new TableNotExistException("missing")); + return failed; + } + return pendingMetadata == null + ? CompletableFuture.completedFuture( + GatewayKafkaProduceBackendTest.metadata(name.equals("invalid")) + .setTableId(database.equals("other") ? 43L : 42L)) + : pendingMetadata; + } + + private void checkWriteAuthorization(long tableId) { + assertThat(tableId).isPositive(); + assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(currentSession().getInetAddress()) + .isEqualTo(InetAddress.getLoopbackAddress()); + writeChecks++; + if (denyWrite) { + throw new AuthorizationException("Write denied"); + } + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + checkWriteAuthorization(request.getTableId()); + assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(currentSession().getInetAddress()) + .isEqualTo(InetAddress.getLoopbackAddress()); + append = request; + appends.add(request); + if (throwAppend) { + throw new IllegalStateException("append failed"); + } + if (appended != null) { + appended.countDown(); + } + return pendingAppend == null + ? CompletableFuture.completedFuture( + success(request.getBucketsReqsList().get(0).getBucketId())) + : pendingAppend; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java new file mode 100644 index 00000000000..6243bb8a900 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests timestamp provenance and ownership of protocol-independent Produce commands. */ +class KafkaProduceCommandTest { + + @ParameterizedTest + @ValueSource(longs = {-1L, 0L, 123L}) + void testLegacyConstructorDoesNotInferBatchType(long timestamp) { + Record record = new Record(timestamp, null, null, Collections.emptyList()); + + assertThat(record.timestamp()).isEqualTo(timestamp); + assertThat(record.hasTimestamp()).isEqualTo(timestamp != Record.NO_TIMESTAMP); + assertThat(record.timestampType()).isEqualTo(TimestampType.UNKNOWN); + assertThat(record.key()).isNull(); + assertThat(record.value()).isNull(); + } + + @ParameterizedTest + @EnumSource(TimestampType.class) + void testTimestampPresenceDoesNotChangeExplicitBatchType(TimestampType timestampType) { + for (long timestamp : new long[] {Record.NO_TIMESTAMP, 0L, 123L}) { + Record record = + new Record(timestamp, timestampType, null, null, Collections.emptyList()); + + assertThat(record.timestampType()).isEqualTo(timestampType); + assertThat(record.timestamp()).isEqualTo(timestamp); + assertThat(record.hasTimestamp()).isEqualTo(timestamp != Record.NO_TIMESTAMP); + } + } + + @Test + void testExplicitBatchTypeMustNotBeNull() { + assertThatThrownBy(() -> new Record(123L, null, null, null, Collections.emptyList())) + .isInstanceOf(NullPointerException.class); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testBothConstructorsOwnTheirRecordData(boolean explicitTimestampType) { + byte[] key = {1}; + byte[] value = {2}; + byte[] headerValue = {3}; + List headers = + new ArrayList<>( + Arrays.asList( + new RecordHeader("header", headerValue), + new RecordHeader("header", null))); + Record record = + explicitTimestampType + ? new Record(123L, TimestampType.LOG_APPEND_TIME, key, value, headers) + : new Record(123L, key, value, headers); + key[0] = 4; + value[0] = 5; + headerValue[0] = 6; + headers.clear(); + record.key()[0] = 7; + record.value()[0] = 8; + record.headers().get(0).value()[0] = 9; + + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()) + .extracting(RecordHeader::name) + .containsExactly("header", "header"); + assertThat(record.headers().get(0).value()).containsExactly((byte) 3); + assertThat(record.headers().get(1).value()).isNull(); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.timestampType()) + .isEqualTo( + explicitTimestampType + ? TimestampType.LOG_APPEND_TIME + : TimestampType.UNKNOWN); + assertThatThrownBy(() -> record.headers().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testCommandPreservesMixedTimestampTypesAndOwnsItsLists() { + List records = + new ArrayList<>( + Arrays.asList( + new Record( + 123L, + TimestampType.CREATE_TIME, + null, + null, + Collections.emptyList()), + new Record( + 123L, + TimestampType.LOG_APPEND_TIME, + null, + null, + Collections.emptyList()), + new Record( + Record.NO_TIMESTAMP, null, null, Collections.emptyList()))); + List partitions = + new ArrayList<>(Collections.singletonList(new PartitionWrite(2, records))); + List topics = + new ArrayList<>(Collections.singletonList(new TopicWrite("db.topic", partitions))); + KafkaProduceCommand command = + new KafkaProduceCommand((short) -1, 1000, topics, "KAFKA", null); + records.clear(); + partitions.clear(); + topics.clear(); + + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(1000); + assertThat(command.topics()).extracting(TopicWrite::topicName).containsExactly("db.topic"); + assertThat(command.topics().get(0).partitions()) + .extracting(PartitionWrite::partitionId) + .containsExactly(2); + List copied = command.topics().get(0).partitions().get(0).records(); + assertThat(copied) + .extracting(Record::timestampType) + .containsExactly( + TimestampType.CREATE_TIME, + TimestampType.LOG_APPEND_TIME, + TimestampType.UNKNOWN); + assertThat(copied) + .extracting(Record::timestamp) + .containsExactly(123L, 123L, Record.NO_TIMESTAMP); + assertThatThrownBy(() -> command.topics().clear()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> command.topics().get(0).partitions().clear()) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(copied::clear).isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testNullValuesPreserveRecordIdentityAndTimestampProvenance() { + Record nullRecord = + new Record( + 123L, + TimestampType.CREATE_TIME, + new byte[] {1}, + null, + Collections.emptyList()); + Record retained = + new Record( + 456L, + TimestampType.LOG_APPEND_TIME, + new byte[] {2}, + new byte[] {3}, + Collections.emptyList()); + Record empty = + new Record( + Record.NO_TIMESTAMP, + TimestampType.CREATE_TIME, + null, + new byte[0], + Collections.emptyList()); + List source = new ArrayList<>(Arrays.asList(nullRecord, retained, empty)); + PartitionWrite partition = new PartitionWrite(0, source); + source.clear(); + + assertThat(partition.records()).containsExactly(nullRecord, retained, empty); + assertThat(partition.records().get(0)).isSameAs(nullRecord); + assertThat(partition.records().get(0).value()).isNull(); + assertThat(partition.records().get(0).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.records().get(0).timestamp()).isEqualTo(123L); + assertThat(partition.records().get(1)).isSameAs(retained); + assertThat(partition.records().get(1).timestampType()) + .isEqualTo(TimestampType.LOG_APPEND_TIME); + assertThat(partition.records().get(1).timestamp()).isEqualTo(456L); + assertThat(partition.records().get(2).timestampType()).isEqualTo(TimestampType.CREATE_TIME); + assertThat(partition.records().get(2).hasTimestamp()).isFalse(); + assertThatThrownBy(() -> partition.records().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testBorrowedPayloadsReuseOwnedArraysWhileCopyAccessorsRemainDefensive() { + byte[] key = {1}; + byte[] value = {2}; + byte[] headerValue = {3}; + RecordHeader header = new RecordHeader("header", headerValue); + List headers = new ArrayList<>(); + headers.add(header); + Record record = new Record(1L, key, value, headers); + + assertThat(record.borrowedKey()).isNotSameAs(key); + assertThat(record.borrowedValue()).isNotSameAs(value); + assertThat(header.borrowedValue()).isNotSameAs(headerValue); + key[0] = 10; + value[0] = 20; + headerValue[0] = 30; + headers.clear(); + + assertThat(record.borrowedKey()).containsExactly((byte) 1); + assertThat(record.borrowedValue()).containsExactly((byte) 2); + assertThat(header.borrowedValue()).containsExactly((byte) 3); + assertThat(record.headers()).containsExactly(header); + assertThat(record.borrowedKey()).isSameAs(record.borrowedKey()); + assertThat(record.borrowedValue()).isSameAs(record.borrowedValue()); + assertThat(header.borrowedValue()).isSameAs(header.borrowedValue()); + + record.key()[0] = 40; + record.value()[0] = 50; + header.value()[0] = 60; + assertThat(record.key()).containsExactly((byte) 1).isNotSameAs(record.borrowedKey()); + assertThat(record.value()).containsExactly((byte) 2).isNotSameAs(record.borrowedValue()); + assertThat(header.value()).containsExactly((byte) 3).isNotSameAs(header.borrowedValue()); + } + + @Test + void testBorrowedPayloadsPreserveNullAndEmptyBytes() { + Record nullRecord = new Record(1L, null, null, Collections.emptyList()); + assertThat(nullRecord.borrowedKey()).isNull(); + assertThat(nullRecord.borrowedValue()).isNull(); + assertThat(new RecordHeader("header", null).borrowedValue()).isNull(); + + Record emptyRecord = new Record(1L, new byte[0], new byte[0], Collections.emptyList()); + assertThat(emptyRecord.borrowedKey()).isEmpty(); + assertThat(emptyRecord.borrowedValue()).isEmpty(); + assertThat(new RecordHeader("header", new byte[0]).borrowedValue()).isEmpty(); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java new file mode 100644 index 00000000000..88e95f420b6 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceConversionExecutorTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests bounded admission, ordering, independent lanes and executor shutdown. */ +class KafkaProduceConversionExecutorTest { + @Test + void testBoundedLanePreservesOrderAndDoesNotBlockAnotherLane() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(2, 2); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + List order = new ArrayList<>(); + AtomicReference worker = new AtomicReference<>(); + AtomicBoolean rejectedRan = new AtomicBoolean(); + try { + CompletableFuture first = + executor.submit( + 0, + () -> { + worker.set(Thread.currentThread()); + started.countDown(); + await(release); + order.add(1); + return CompletableFuture.completedFuture(1); + }); + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture second = + executor.submit( + 0, + () -> { + order.add(2); + return CompletableFuture.completedFuture(2); + }); + CompletableFuture rejected = + executor.submit( + 0, + () -> { + rejectedRan.set(true); + return CompletableFuture.completedFuture(3); + }); + assertThatThrownBy(rejected::join).hasCauseInstanceOf(RejectedExecutionException.class); + assertThat(rejectedRan).isFalse(); + assertThat(worker.get()).isNotSameAs(Thread.currentThread()); + assertThat(second).isNotDone(); + assertThat( + executor.submit(1, () -> CompletableFuture.completedFuture(4)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(4); + release.countDown(); + assertThat(first.get(10, TimeUnit.SECONDS)).isEqualTo(1); + assertThat(second.get(10, TimeUnit.SECONDS)).isEqualTo(2); + assertThat(order).containsExactly(1, 2); + } finally { + release.countDown(); + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testPendingAcknowledgementDoesNotOccupyWorker() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + CompletableFuture acknowledgement = new CompletableFuture<>(); + CountDownLatch submitted = new CountDownLatch(1); + try { + CompletableFuture first = + executor.submit( + 0, + () -> { + submitted.countDown(); + return acknowledgement; + }); + assertThat(submitted.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat( + executor.submit(0, () -> CompletableFuture.completedFuture(2)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(2); + assertThat(first).isNotDone(); + acknowledgement.complete(1); + assertThat(first.get(10, TimeUnit.SECONDS)).isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testCloseInterruptsWorkerCompletesQueuedRequestsAndRejectsNewWork() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + CountDownLatch started = new CountDownLatch(1); + AtomicBoolean queuedRan = new AtomicBoolean(); + CompletableFuture running = + executor.submit( + 0, + () -> { + started.countDown(); + await(new CountDownLatch(1)); + return CompletableFuture.completedFuture(null); + }); + try { + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + CompletableFuture queued = + executor.submit( + 0, + () -> { + queuedRan.set(true); + return CompletableFuture.completedFuture(null); + }); + CompletableFuture closed = executor.closeAsync(); + assertThat(executor.closeAsync()).isSameAs(closed); + closed.get(10, TimeUnit.SECONDS); + assertThatThrownBy(running::join).hasRootCauseInstanceOf(InterruptedException.class); + assertThatThrownBy(queued::join).hasCauseInstanceOf(RejectedExecutionException.class); + assertThat(queuedRan).isFalse(); + assertThatThrownBy( + () -> + executor.submit( + 0, + () -> CompletableFuture.completedFuture(null)) + .join()) + .hasCauseInstanceOf(RejectedExecutionException.class); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + @Test + void testFailedOperationDoesNotStopLane() throws Exception { + KafkaProduceConversionExecutor executor = new KafkaProduceConversionExecutor(1, 1); + try { + CompletableFuture failed = + executor.submit( + 0, + () -> { + throw new IllegalStateException("conversion failed"); + }); + assertThatThrownBy(failed::join).hasCauseInstanceOf(IllegalStateException.class); + assertThat( + executor.submit(0, () -> CompletableFuture.completedFuture(1)) + .get(10, TimeUnit.SECONDS)) + .isEqualTo(1); + } finally { + executor.closeAsync().get(10, TimeUnit.SECONDS); + } + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Test worker was not released."); + } + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(failure); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java new file mode 100644 index 00000000000..b6c8fce097a --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaTableInfoCacheTest.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class KafkaTableInfoCacheTest { + + private static final TablePath TABLE_PATH = TablePath.of("kafka", "orders"); + + @Test + void testReusesUnchangedResponseAndReloadsPropertyOnlyChange() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loadCount = new AtomicInteger(); + GetTableInfoResponse firstResponse = response(1L, 1, descriptor("v1")); + + TableInfo first = + cache.getOrLoad( + TABLE_PATH, + firstResponse, + () -> load(TABLE_PATH, firstResponse, loadCount)); + GetTableInfoResponse equivalentResponse = response(1L, 1, descriptor("v1")); + TableInfo reused = + cache.getOrLoad( + TABLE_PATH, + equivalentResponse, + () -> load(TABLE_PATH, equivalentResponse, loadCount)); + + assertThat(reused).isSameAs(first); + assertThat(loadCount).hasValue(1); + + // A Kafka contract property may change without a schema ID or timestamp change. + GetTableInfoResponse changedResponse = response(1L, 1, descriptor("v2")); + TableInfo changed = + cache.getOrLoad( + TABLE_PATH, + changedResponse, + () -> load(TABLE_PATH, changedResponse, loadCount)); + + assertThat(changed).isNotSameAs(first); + assertThat(loadCount).hasValue(2); + } + + @Test + void testReloadsDeleteAndRecreateAndSupportsExplicitInvalidation() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loadCount = new AtomicInteger(); + GetTableInfoResponse original = response(1L, 1, descriptor("v1")); + TableInfo first = + cache.getOrLoad(TABLE_PATH, original, () -> load(TABLE_PATH, original, loadCount)); + + GetTableInfoResponse recreated = response(2L, 1, descriptor("v1")); + TableInfo second = + cache.getOrLoad( + TABLE_PATH, recreated, () -> load(TABLE_PATH, recreated, loadCount)); + assertThat(second).isNotSameAs(first); + + cache.invalidate(TABLE_PATH); + TableInfo third = + cache.getOrLoad( + TABLE_PATH, recreated, () -> load(TABLE_PATH, recreated, loadCount)); + assertThat(third).isNotSameAs(second); + assertThat(loadCount).hasValue(3); + } + + @Test + void testReloadsWhenOnlyBucketCountEpochChanges() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger loads = new AtomicInteger(); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + TableInfo first = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, loads)); + response.setBucketCountEpoch(7L); + TableInfo changed = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, loads)); + assertThat(changed).isNotSameAs(first); + assertThat(changed.getBucketCountEpoch()).isEqualTo(7L); + assertThat(loads).hasValue(2); + } + + @Test + void testLoadFailureIsNotCached() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + AtomicInteger attempts = new AtomicInteger(); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + + assertThatThrownBy( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + attempts.incrementAndGet(); + throw new IllegalStateException("broken descriptor"); + })) + .isInstanceOf(IllegalStateException.class) + .hasMessage("broken descriptor"); + + TableInfo loaded = + cache.getOrLoad(TABLE_PATH, response, () -> load(TABLE_PATH, response, attempts)); + assertThat(loaded.getTableId()).isEqualTo(1L); + assertThat(attempts).hasValue(2); + } + + @Test + void testConcurrentColdLoadIsSingleFlight() throws Exception { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + futures.add( + executor.submit( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + loadCount.incrementAndGet(); + loaderStarted.countDown(); + await(releaseLoader); + return toTableInfo(TABLE_PATH, response); + }))); + } + loaderStarted.await(); + releaseLoader.countDown(); + + TableInfo first = futures.get(0).get(); + for (Future future : futures) { + assertThat(future.get()).isSameAs(first); + } + assertThat(loadCount).hasValue(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testInvalidationDoesNotAllowInFlightLoadToResurrect() throws Exception { + KafkaTableInfoCache cache = new KafkaTableInfoCache(10); + GetTableInfoResponse response = response(1L, 1, descriptor("v1")); + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch loaderStarted = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future loading = + executor.submit( + () -> + cache.getOrLoad( + TABLE_PATH, + response, + () -> { + loadCount.incrementAndGet(); + loaderStarted.countDown(); + await(releaseLoader); + return toTableInfo(TABLE_PATH, response); + })); + loaderStarted.await(); + Future invalidating = executor.submit(() -> cache.invalidate(TABLE_PATH)); + releaseLoader.countDown(); + + TableInfo loadedBeforeInvalidation = loading.get(); + invalidating.get(); + TableInfo loadedAfterInvalidation = + cache.getOrLoad( + TABLE_PATH, response, () -> load(TABLE_PATH, response, loadCount)); + + assertThat(loadedAfterInvalidation).isNotSameAs(loadedBeforeInvalidation); + assertThat(loadCount).hasValue(2); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCacheSizeIsBounded() { + KafkaTableInfoCache cache = new KafkaTableInfoCache(2); + for (int i = 0; i < 10; i++) { + TablePath path = TablePath.of("kafka", "topic_" + i); + GetTableInfoResponse response = response(i + 1L, 1, descriptor("v1")); + cache.getOrLoad(path, response, () -> toTableInfo(path, response)); + } + + assertThat(cache.estimatedSize()).isLessThanOrEqualTo(2); + } + + private static TableInfo load( + TablePath tablePath, GetTableInfoResponse response, AtomicInteger loadCount) { + loadCount.incrementAndGet(); + return toTableInfo(tablePath, response); + } + + private static TableInfo toTableInfo(TablePath tablePath, GetTableInfoResponse response) { + return TableInfo.of( + tablePath, + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + null, + response.getCreatedTime(), + response.getModifiedTime(), + response.hasBucketCountEpoch() ? response.getBucketCountEpoch() : 0L); + } + + private static GetTableInfoResponse response( + long tableId, int schemaId, TableDescriptor descriptor) { + return new GetTableInfoResponse() + .setTableId(tableId) + .setSchemaId(schemaId) + .setTableJson(descriptor.toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L); + } + + private static TableDescriptor descriptor(String version) { + TableDescriptor.Builder builder = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY); + if ("v2".equals(version)) { + builder.property(ConfigOptions.TABLE_KAFKA_ENABLED, false); + } + return builder.build(); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for test loader.", e); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java new file mode 100644 index 00000000000..5d4cfd0f960 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/KafkaFormatFactoryRegistryTest.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.kafka.transcode.KafkaRowAssembler; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests value decoders against the shared fixed-column table contract. */ +class KafkaFormatFactoryRegistryTest { + private final KafkaFormatFactoryRegistry formats = new KafkaFormatFactoryRegistry(); + + @Test + void testBinaryBorrowsOpaqueBytesAndPreservesNullAndEmpty() { + KafkaFieldDecoder decoder = + formats.createDecoder( + KafkaValueConversionType.BINARY, projection(DataTypes.BYTES())); + byte[] opaque = new byte[] {(byte) 0xff, 0}; + assertThat(decoder.decode(opaque)[0]).isSameAs(opaque); + assertThat(opaque).containsExactly((byte) 0xff, (byte) 0); + assertThat(decoder.decode(null)).containsExactly((Object) null); + byte[] empty = new byte[0]; + assertThat(decoder.decode(empty)[0]).isSameAs(empty); + } + + @Test + void testStringRejectsMalformedUtf8AndCanBeReused() { + KafkaFieldDecoder decoder = + formats.createDecoder( + KafkaValueConversionType.STRING, projection(DataTypes.STRING())); + for (byte[] invalid : + new byte[][] { + {(byte) 0xc3, 0x28}, + {(byte) 0xc0, (byte) 0x80}, + {(byte) 0xed, (byte) 0xa0, (byte) 0x80}, + {(byte) 0xe2, (byte) 0x82} + }) { + byte[] original = invalid.clone(); + assertThatThrownBy(() -> decoder.decode(invalid)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("_kafka_value") + .hasMessageContaining("UTF-8"); + assertThat(invalid).isEqualTo(original); + byte[] valid = "键".getBytes(StandardCharsets.UTF_8); + assertThat(decoder.decode(valid)).containsExactly(BinaryString.fromString("键")); + assertThat(valid).isEqualTo("键".getBytes(StandardCharsets.UTF_8)); + } + assertThat(decoder.decode(null)).containsExactly((Object) null); + assertThat(decoder.decode(new byte[0])).containsExactly(BinaryString.fromString("")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesUseThePublicValueProjection(boolean string) { + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + TableDescriptor table = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES()) + .column(KafkaTableSchema.VALUE_COLUMN, type) + .column( + KafkaTableSchema.HEADERS_COLUMN, + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column(KafkaTableSchema.RESCUE_COLUMN, DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + string + ? KafkaValueConversionType.STRING + : KafkaValueConversionType.BINARY) + .build(); + KafkaTableSchema schema = KafkaTableSchema.resolve(table); + assertThat(schema.valueProjection().positions()).containsExactly(1); + KafkaFieldDecoder decoder = formats.createDecoder(schema); + byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + assertThat(decoder.decode(payload)) + .containsExactly(string ? BinaryString.fromString("payload") : payload); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesRejectIncompatibleProjections(boolean string) { + KafkaValueConversionType format = + string ? KafkaValueConversionType.STRING : KafkaValueConversionType.BINARY; + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + assertThatThrownBy( + () -> + formats.createDecoder( + format, + projection( + string ? DataTypes.BYTES() : DataTypes.STRING()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining(string ? "STRING" : "BYTES"); + KafkaFieldProjection multiple = + new KafkaFieldProjection( + DataTypes.ROW( + DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type), + DataTypes.FIELD(KafkaTableSchema.KEY_COLUMN, DataTypes.BYTES())), + Arrays.asList(0, 1)); + KafkaFieldProjection empty = + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type)), + Collections.emptyList()); + for (KafkaFieldProjection invalid : Arrays.asList(multiple, empty)) { + assertThatThrownBy(() -> formats.createDecoder(format, invalid)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("exactly one"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFactoriesRejectKeyAndArbitraryValueColumnNames(boolean string) { + KafkaValueConversionType format = + string ? KafkaValueConversionType.STRING : KafkaValueConversionType.BINARY; + DataType type = string ? DataTypes.STRING() : DataTypes.BYTES(); + for (String name : Arrays.asList(KafkaTableSchema.KEY_COLUMN, "body")) { + KafkaFieldProjection invalid = + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(name, type)), + Collections.singletonList(0)); + assertThatThrownBy(() -> formats.createDecoder(format, invalid)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testJsonLayoutDeterminesPhysicalValueContainer(boolean nested) { + RowType business = DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT())); + KafkaFieldProjection projection = + nested + ? projection(business) + : new KafkaFieldProjection(business, Collections.singletonList(0)); + KafkaFieldDecoder decoder = + formats.createDecoder( + KafkaValueConversionType.PARSE_JSON, + projection, + nested + ? KafkaValueConversionLayout.NESTED + : KafkaValueConversionLayout.FLAT); + Object[] values = decoder.decode("{\"id\":7}".getBytes(StandardCharsets.UTF_8)); + assertThat(values).hasSize(1); + if (nested) { + assertThat(((InternalRow) values[0]).getInt(0)).isEqualTo(7); + } else { + assertThat(values).containsExactly(7); + } + assertThatThrownBy( + () -> + formats.createDecoder( + KafkaValueConversionType.PARSE_JSON, + projection(DataTypes.STRING()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("Reserved name"); + assertThatThrownBy( + () -> + decoder.decode( + "{\"_kafka_value\":{\"id\":7}}" + .getBytes(StandardCharsets.UTF_8))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("reserved field"); + } + + @ParameterizedTest + @CsvSource({"flat,false", "flat,true", "nested,false", "nested,true"}) + void testSchemaFactoryAndAssemblerKeepRescueSeparateFromPhysicalOrder( + String layout, boolean first) { + Schema.Builder schema = Schema.newBuilder(); + if (first) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + schema.column("_kafka_key", DataTypes.BYTES()); + schema.column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()))); + if (!first) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + if (layout.equals("nested")) { + schema.column("_kafka_value", DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT()))); + } else { + schema.column("id", DataTypes.INT()); + } + KafkaTableSchema contract = + KafkaTableSchema.resolve( + TableDescriptor.builder() + .schema(schema.build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + layout) + .build()); + KafkaFieldDecoder decoder = formats.createDecoder(contract); + KafkaRowAssembler assembler = new KafkaRowAssembler(contract); + String rescued = "{\"extra\":2,\"_kafka_metadata\":{\"timestamp\":0,\"partition\":99}}"; + Object[] values = + decoder.decode( + ("{\"id\":7," + rescued.substring(1)).getBytes(StandardCharsets.UTF_8)); + assertThat(values).hasSize(2); + byte[] key = {(byte) 0xff}; + Record record = + new Record(123L, TimestampType.LOG_APPEND_TIME, key, null, Collections.emptyList()); + InternalRow row = assembler.assemble(values, record, 7); + assertThat(row.getString(contract.rescuePosition()).toString()).isEqualTo(rescued); + assertThat(row.getBytes(contract.keyPosition())).isEqualTo(key); + InternalRow metadata = row.getRow(contract.metadataPosition(), 3); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(metadata.getInt(1)).isEqualTo(1); + assertThat(metadata.getInt(2)).isEqualTo(7); + int valuePosition = contract.valueProjection().positionAt(0); + assertThat( + layout.equals("nested") + ? row.getRow(valuePosition, 1).getInt(0) + : row.getInt(valuePosition)) + .isEqualTo(7); + InternalRow nullRow = assembler.assemble(decoder.decode(null), record, 7); + assertThat(nullRow.isNullAt(valuePosition)).isTrue(); + assertThat(nullRow.isNullAt(contract.rescuePosition())).isTrue(); + assertThat(nullRow.getBytes(contract.keyPosition())).isEqualTo(key); + InternalRow nullMetadata = nullRow.getRow(contract.metadataPosition(), 3); + assertThat(nullMetadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(nullMetadata.getInt(1)).isEqualTo(1); + assertThat(nullMetadata.getInt(2)).isEqualTo(7); + assertThat( + assembler + .assemble( + decoder.decode( + "{\"id\":8}".getBytes(StandardCharsets.UTF_8)), + record, + 7) + .isNullAt(contract.rescuePosition())) + .isTrue(); + for (Object[] invalid : new Object[][] {new Object[1], new Object[3]}) { + assertThatThrownBy(() -> assembler.assemble(invalid, record, 7)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("contract requires 2"); + } + } + + private static KafkaFieldProjection projection(DataType type) { + return new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, type)), + Collections.singletonList(0)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java new file mode 100644 index 00000000000..6de4094616f --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaComplexTypesTest.java @@ -0,0 +1,744 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.types.DataField; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests recursive JSON conversion, nested validation and resource bounds. */ +class JsonKafkaComplexTypesTest { + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDecodesRecursiveRowArrayAndMapTypes(boolean nested) { + RowType rowType = complexRowType(); + + Object[] values = + decodeBusinessFields( + rowType, + "{" + + "\"customer\":{\"name\":\"Alice\"," + + "\"address\":{\"city\":\"Hangzhou\"}}," + + "\"items\":[{\"sku\":\"A-100\"," + + "\"quantity\":2}]," + + "\"attributes\":{\"scores\":[7,null,9]}" + + "}", + nested); + + GenericRow customer = (GenericRow) values[0]; + assertThat(customer.getString(0).toString()).isEqualTo("Alice"); + assertThat(customer.getRow(1, 1).getString(0).toString()).isEqualTo("Hangzhou"); + + GenericArray items = (GenericArray) values[1]; + assertThat(items.size()).isEqualTo(1); + assertThat(items.getRow(0, 2).getString(0).toString()).isEqualTo("A-100"); + assertThat(items.getRow(0, 2).getInt(1)).isEqualTo(2); + + GenericMap attributes = (GenericMap) values[2]; + GenericArray scores = (GenericArray) attributes.get(BinaryString.fromString("scores")); + assertThat(scores.toObjectArray()).containsExactly(7, null, 9); + } + + @Test + void testRejectsInvalidNestedValuesWithPrecisePaths() { + assertFailure( + complexRowType(), + "{\"customer\":{\"address\":{\"city\":\"Hangzhou\"}}," + + "\"items\":[],\"attributes\":{}}", + "$[\"customer\"][\"name\"]", + "missing or null"); + assertFailure( + complexRowType(), + "{\"customer\":{\"name\":\"Alice\"," + + "\"address\":{\"city\":\"Hangzhou\"}}," + + "\"items\":[{\"sku\":\"A-100\",\"quantity\":2}," + + "{\"sku\":\"A-200\",\"quantity\":\"bad\"}]," + + "\"attributes\":{}}", + "$[\"items\"][1][\"quantity\"]", + "expected a JSON integer"); + assertFailure( + complexRowType(), + "{\"customer\":{\"name\":\"Alice\"," + + "\"address\":{\"city\":\"Hangzhou\"}}," + + "\"items\":[],\"attributes\":{\"bad\\\"key\":[\"bad\"]}}", + "$[\"attributes\"][\"bad\\\"key\"][0]", + "expected a JSON integer"); + } + + @Test + void testRejectsUnknownRootAndNestedRowFieldsButAllowsMapKeys() { + assertFailure( + DataTypes.ROW(DataTypes.FIELD("id", DataTypes.BIGINT())), + "{\"id\":1,\"extra\":\"unexpected\"}", + "$[\"extra\"]", + "unknown field"); + assertFailure( + complexRowType(), + "{\"customer\":{\"name\":\"Alice\"," + + "\"address\":{\"city\":\"Hangzhou\",\"extra\":1}}," + + "\"items\":[],\"attributes\":{}}", + "$[\"customer\"][\"address\"][\"extra\"]", + "unknown field"); + + Object[] values = + decoder( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.INT())))) + .decode(bytes("{\"attributes\":{\"dynamic-key\":1}}")); + GenericMap attributes = (GenericMap) values[0]; + assertThat(attributes.size()).isEqualTo(1); + assertThat(attributes.get(BinaryString.fromString("dynamic-key"))).isEqualTo(1); + } + + @Test + void testRejectsNonStringMapKeyBeforeDecodingRecords() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD( + "attributes", DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()))); + + assertThatThrownBy(() -> decoder(rowType)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("only supports STRING map keys") + .hasMessageContaining("MAP decoder(DataTypes.ROW(DataTypes.FIELD("nested", tooDeepType)))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("maximum nesting depth"); + + StringBuilder nestedJson = new StringBuilder("{\"value\":"); + for (int i = 0; i < JsonKafkaFieldDecoder.MAX_JSON_NESTING_DEPTH; i++) { + nestedJson.append("{\"nested\":"); + } + nestedJson.append('0'); + for (int i = 0; i < JsonKafkaFieldDecoder.MAX_JSON_NESTING_DEPTH; i++) { + nestedJson.append('}'); + } + nestedJson.append('}'); + JsonKafkaFieldDecoder complexDecoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD( + "value", + DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING())))); + assertThatThrownBy(() -> complexDecoder.decode(bytes(nestedJson.toString()))) + .isInstanceOf(KafkaRecordEncodingException.class); + + StringBuilder oversizedArray = new StringBuilder("{\"values\":["); + for (int i = 0; i <= JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; i++) { + if (i > 0) { + oversizedArray.append(','); + } + oversizedArray.append('0'); + } + oversizedArray.append("]}"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("values", DataTypes.ARRAY(DataTypes.INT()))), + oversizedArray.toString(), + "$[\"values\"]", + "container size exceeds"); + } + + @ParameterizedTest(name = "{index}: {0}") + @MethodSource("oversizedContainers") + void testRejectsOversizedContainersBeforeReadingExcessChild( + String description, DataType dataType, String json, String path) { + // The excess child is deliberately unfinished. Reading it before checking the + // containing ARRAY/MAP/ROW limit would fail with a syntax error instead. + assertFailure( + DataTypes.ROW(DataTypes.FIELD("value", dataType)), + "{\"value\":" + json, + path, + "container size exceeds " + JsonToFlussConverters.MAX_CONTAINER_ELEMENTS); + } + + private static Stream oversizedContainers() { + String array = containerAtLimit(false); + String object = containerAtLimit(true); + String oversizedArray = array.substring(0, array.length() - 1) + ",["; + String oversizedObject = object.substring(0, object.length() - 1) + ",\"excess\":{"; + DataType arrayType = DataTypes.ARRAY(DataTypes.INT()); + DataType mapType = DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()); + RowType rowType = rowTypeAtLimit(); + return Stream.of( + Arguments.of("ARRAY", arrayType, oversizedArray, "$[\"value\"]"), + Arguments.of("MAP", mapType, oversizedObject, "$[\"value\"]"), + Arguments.of("ROW", rowType, oversizedObject, "$[\"value\"]"), + Arguments.of( + "ARRAY inside ROW", + DataTypes.ROW(DataTypes.FIELD("nested", arrayType)), + "{\"nested\":" + oversizedArray, + "$[\"value\"][\"nested\"]"), + Arguments.of( + "MAP inside ARRAY", + DataTypes.ARRAY(mapType), + "[" + oversizedObject, + "$[\"value\"][0]"), + Arguments.of( + "ROW inside MAP", + DataTypes.MAP(DataTypes.STRING(), rowType), + "{\"nested\":" + oversizedObject, + "$[\"value\"][\"nested\"]")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testAcceptsLimitForEachSiblingContainer(boolean nested) { + String array = containerAtLimit(false); + String object = containerAtLimit(true); + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD( + "arrays", DataTypes.ARRAY(DataTypes.ARRAY(DataTypes.INT()))), + DataTypes.FIELD( + "maps", + DataTypes.ARRAY( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))), + DataTypes.FIELD("rows", DataTypes.ARRAY(rowTypeAtLimit()))); + Object[] values = + decodeBusinessFields( + rowType, + "{\"arrays\":[" + + array + + "," + + array + + "],\"maps\":[" + + object + + "," + + object + + "],\"rows\":[" + + object + + "," + + object + + "]}", + nested); + + int limit = JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; + for (int i = 0; i < 2; i++) { + GenericArray arrayValue = (GenericArray) ((GenericArray) values[0]).getArray(i); + assertThat(arrayValue.size()).isEqualTo(limit); + assertThat(arrayValue.getInt(limit - 1)).isZero(); + GenericMap mapValue = (GenericMap) ((GenericArray) values[1]).getMap(i); + assertThat(mapValue.size()).isEqualTo(limit); + assertThat(mapValue.get(BinaryString.fromString("k" + (limit - 1)))).isEqualTo(0); + GenericRow rowValue = (GenericRow) ((GenericArray) values[2]).getRow(i, limit); + assertThat(rowValue.getFieldCount()).isEqualTo(limit); + assertThat(rowValue.getInt(limit - 1)).isZero(); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "{\"value\":[{\"a\":[],\"a\":[]}]}", + "{\"value\":[{\"a\":[1,]}]}", + "{\"value\":[{\"a\":[1]", + "{\"value\":[{\"a\":[1]}]} {}", + }) + void testRetainsStrictParsingForNestedContainers(String json) { + JsonKafkaFieldDecoder nestedDecoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD( + "value", + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ARRAY(DataTypes.INT())))))); + assertThatThrownBy(() -> nestedDecoder.decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("not valid strict UTF-8 JSON"); + } + + private static RowType rowTypeAtLimit() { + DataField[] fields = new DataField[JsonToFlussConverters.MAX_CONTAINER_ELEMENTS]; + for (int i = 0; i < fields.length; i++) { + fields[i] = DataTypes.FIELD("k" + i, DataTypes.INT()); + } + return DataTypes.ROW(fields); + } + + private static String containerAtLimit(boolean object) { + StringBuilder json = new StringBuilder(object ? "{" : "["); + for (int i = 0; i < JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; i++) { + if (i > 0) { + json.append(','); + } + if (object) { + json.append('"').append('k').append(i).append("\":"); + } + json.append('0'); + } + return json.append(object ? '}' : ']').toString(); + } + + @Test + void testBoundsAndEscapesMapKeysInErrorPaths() { + StringBuilder mapKey = new StringBuilder("quote\""); + for (int i = 0; i < 300; i++) { + mapKey.append('x'); + } + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD( + "attributes", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))); + + assertThatThrownBy( + () -> + decoder(rowType) + .decode( + bytes( + "{\"attributes\":{" + + quoteJson(mapKey.toString()) + + ":\"bad\"}}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$[\"attributes\"][\"quote\\\"") + .hasMessageContaining("...") + .satisfies( + error -> assertThat(error.getMessage()).doesNotContain(mapKey.toString())); + + StringBuilder controlCharacters = new StringBuilder(); + for (int i = 0; i < 300; i++) { + controlCharacters.append('\u0001'); + } + String boundedPath = JsonPath.field(JsonPath.ROOT, controlCharacters.toString()); + assertThat(boundedPath) + .hasSizeLessThanOrEqualTo(512) + .endsWith("\"]") + .doesNotContain("\\u0..."); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDecodesTheSharedDepthBoundaryIncludingEmptyRows(boolean nested) { + for (boolean emptyRow : new boolean[] {false, true}) { + DataType type = emptyRow ? new RowType(Collections.emptyList()) : DataTypes.INT(); + String json = emptyRow ? "{}" : "7"; + Object expected = emptyRow ? new GenericRow(0) : 7; + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + switch (depth % 3) { + case 0: + type = DataTypes.ROW(DataTypes.FIELD("child", type)); + json = "{\"child\":" + json + "}"; + expected = GenericRow.of(expected); + break; + case 1: + type = DataTypes.ARRAY(type); + json = "[" + json + "]"; + expected = new GenericArray(new Object[] {expected}); + break; + default: + type = DataTypes.MAP(DataTypes.STRING(), type); + json = "{\"entry\":" + json + "}"; + expected = + new GenericMap( + Collections.singletonMap( + BinaryString.fromString("entry"), expected)); + } + } + KafkaTableSchema.validateJsonType(type); + RowType business = DataTypes.ROW(DataTypes.FIELD("value", type)); + // An empty ROW at business depth 64 is JSON container depth 66, including the root. + assertThat(decodeBusinessFields(business, "{\"value\":" + json + "}", nested)) + .containsExactly(expected); + + DataType tooDeep = DataTypes.ARRAY(type); + assertThatThrownBy(() -> KafkaTableSchema.validateJsonType(tooDeep)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("maximum nesting depth of 64"); + assertThatThrownBy( + () -> decoder(DataTypes.ROW(DataTypes.FIELD("value", tooDeep)), nested)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("maximum nesting depth of 64"); + String invalidJson = + "{\"value\":" + + (emptyRow + ? json.replace("{}", "{\"extra\":{}}") + : json.replace("7", "[7]")) + + "}"; + assertThatThrownBy(() -> decoder(business, nested).decode(bytes(invalidJson))) + .isInstanceOf(KafkaRecordEncodingException.class); + } + } + + @ParameterizedTest + @MethodSource("floatingPointZeroTokens") + void testPreservesFloatingPointZeroInEveryContainer(boolean nested, String token) { + for (Object value : numericContainerValues(DataTypes.FLOAT(), token, nested)) { + assertThat(Float.floatToRawIntBits((Float) value)) + .isEqualTo(Float.floatToRawIntBits(-0.0F)); + } + for (Object value : numericContainerValues(DataTypes.DOUBLE(), token, nested)) { + assertThat(Double.doubleToRawLongBits((Double) value)) + .isEqualTo(Double.doubleToRawLongBits(-0.0D)); + } + } + + private static Stream floatingPointZeroTokens() { + return Stream.of(false, true) + .flatMap( + nested -> + Stream.of("-0", "-0.0", "-0e0") + .map(token -> Arguments.of(nested, token))); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFloatingPointUnderflowPreservesSignAndRoundsOnceInEveryContainer(boolean nested) { + // The last two pairs straddle half of the smallest subnormal for each target. + // Parsing FLOAT through DOUBLE first would round both tokens to the same midpoint. + String[][] tokens = { + {"1e-1000", "1e-1000"}, + {"7.006492321624085e-46", "2.4703282292062327e-324"}, + {"7.006492321624086e-46", "2.4703282292062328e-324"} + }; + for (boolean negative : new boolean[] {false, true}) { + String sign = negative ? "-" : ""; + for (int i = 0; i < tokens.length; i++) { + int magnitude = i == tokens.length - 1 ? 1 : 0; + String floatToken = sign + tokens[i][0]; + for (Object value : numericContainerValues(DataTypes.FLOAT(), floatToken, nested)) { + assertThat(Float.floatToRawIntBits((Float) value)) + .as("FLOAT %s, nested=%s", floatToken, nested) + .isEqualTo((negative ? Integer.MIN_VALUE : 0) | magnitude); + } + String doubleToken = sign + tokens[i][1]; + for (Object value : + numericContainerValues(DataTypes.DOUBLE(), doubleToken, nested)) { + assertThat(Double.doubleToRawLongBits((Double) value)) + .as("DOUBLE %s, nested=%s", doubleToken, nested) + .isEqualTo((negative ? Long.MIN_VALUE : 0L) | magnitude); + } + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testPreservesOriginalNumericRoundingAndExactDecimalsInEveryContainer(boolean nested) { + for (String sign : new String[] {"", "-"}) { + float direction = sign.isEmpty() ? 1.0F : -1.0F; + assertThat( + numericContainerValues( + DataTypes.FLOAT(), sign + "1.0000000596046448", nested)) + .containsOnly(direction * 1.0000001F); + assertThat( + numericContainerValues( + DataTypes.FLOAT(), sign + "1.0000000596046447", nested)) + .containsOnly(direction); + assertThat( + numericContainerValues( + DataTypes.DOUBLE(), + sign + + "1.00000000000000011102230246251565404236316680908203126", + nested)) + .containsOnly(direction * 1.0000000000000002D); + assertThat( + numericContainerValues( + DataTypes.DOUBLE(), + sign + + "1.00000000000000011102230246251565404236316680908203124", + nested)) + .containsOnly((double) direction); + String amount = sign + "12345678901234567890.123456789012345678"; + for (Object value : numericContainerValues(DataTypes.DECIMAL(38, 18), amount, nested)) { + assertThat(((Decimal) value).toBigDecimal()).isEqualTo(new BigDecimal(amount)); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMapKeysRejectUnpairedSurrogatesBeforeEncodingOrColliding(boolean nested) { + RowType business = + DataTypes.ROW( + DataTypes.FIELD( + "value", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()))); + JsonKafkaFieldDecoder decoder = decoder(business, nested); + for (String key : + new String[] { + "\\ud800", "\\udc00", "a\\ud800b", "\\ud800\\ud800", "\\udc00\\ud800" + }) { + for (String object : + new String[] {"{\"?\":1,\"" + key + "\":2}", "{\"" + key + "\":2,\"?\":1}"}) { + assertThatThrownBy(() -> decoder.decode(bytes("{\"value\":" + object + "}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$[\"value\"]") + .hasMessageContaining("surrogate"); + } + } + GenericMap values = + (GenericMap) + decodeBusinessFields( + business, + "{\"value\":{\"?\":1,\"\\ud83d\\ude00\":2,\"\\\\ud800\":3}}", + nested)[0]; + assertThat(values.size()).isEqualTo(3); + assertThat(values.get(BinaryString.fromString("?"))).isEqualTo(1); + assertThat(values.get(BinaryString.fromString("😀"))).isEqualTo(2); + assertThat(values.get(BinaryString.fromString("\\ud800"))).isEqualTo(3); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDeclaredReservedNamesAreOrdinaryFieldsBelowTheJsonRoot(boolean nested) { + String[] names = { + "_kafka_value", "_kafka_key", "_kafka_headers", "_kafka_metadata", "_kafka_rescue" + }; + DataField[] fields = new DataField[names.length]; + StringBuilder object = new StringBuilder("{"); + for (int i = 0; i < names.length; i++) { + fields[i] = DataTypes.FIELD(names[i], DataTypes.INT().copy(false)); + if (i > 0) { + object.append(','); + } + object.append('"').append(names[i]).append("\":").append(i); + } + object.append('}'); + RowType inner = DataTypes.ROW(fields); + RowType business = + DataTypes.ROW( + DataTypes.FIELD("row", inner), + DataTypes.FIELD("array", DataTypes.ARRAY(inner)), + DataTypes.FIELD("map", DataTypes.MAP(DataTypes.STRING(), inner))); + Object[] values = + decodeBusinessFields( + business, + "{\"row\":" + + object + + ",\"array\":[" + + object + + "],\"map\":{\"_kafka_value\":" + + object + + "}}", + nested); + GenericRow expected = GenericRow.of(0, 1, 2, 3, 4); + assertThat(values[0]).isEqualTo(expected); + assertThat(((GenericArray) values[1]).getRow(0, names.length)).isEqualTo(expected); + assertThat(((GenericMap) values[2]).get(BinaryString.fromString("_kafka_value"))) + .isEqualTo(expected); + } + + @Test + void testNestedNullabilityAppliesToRowsArrayElementsAndMapValues() { + RowType business = + DataTypes.ROW( + DataTypes.FIELD( + "row", + DataTypes.ROW( + DataTypes.FIELD("required", DataTypes.INT().copy(false)))), + DataTypes.FIELD("array", DataTypes.ARRAY(DataTypes.INT().copy(false))), + DataTypes.FIELD( + "map", + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT().copy(false)))); + assertFailure(business, "{\"row\":{}}", "$[\"row\"][\"required\"]", "missing or null"); + assertFailure( + business, + "{\"row\":{\"required\":null}}", + "$[\"row\"][\"required\"]", + "missing or null"); + assertFailure(business, "{\"array\":[1,null]}", "$[\"array\"][1]", "missing or null"); + assertFailure( + business, "{\"map\":{\"entry\":null}}", "$[\"map\"][\"entry\"]", "missing or null"); + for (boolean nested : new boolean[] {false, true}) { + assertThat( + decodeBusinessFields( + business, "{\"row\":null,\"array\":null,\"map\":null}", nested)) + .containsExactly(null, null, null); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testNullValueRespectsComplexContainerAndParentNullability(boolean nested) { + DataType[] containers = { + DataTypes.ROW(DataTypes.FIELD("required", DataTypes.INT().copy(false))), + DataTypes.ARRAY(DataTypes.INT().copy(false)), + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT().copy(false)) + }; + for (DataType container : containers) { + RowType business = DataTypes.ROW(DataTypes.FIELD("value", container)); + assertThat(decoder(business, nested).decode(null)).containsExactly((Object) null); + assertThat(decodeBusinessFields(business, "{}", nested)).containsExactly((Object) null); + assertThat(decodeBusinessFields(business, "{\"value\":null}", nested)) + .containsExactly((Object) null); + assertThatThrownBy(() -> decoder(business, nested).decode(bytes("null"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("object root"); + + RowType requiredContainer = + DataTypes.ROW(DataTypes.FIELD("value", container.copy(false))); + for (String json : new String[] {"{}", "{\"value\":null}"}) { + assertThatThrownBy(() -> decoder(requiredContainer, nested).decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$[\"value\"]") + .hasMessageContaining("missing or null"); + } + if (nested) { + // An absent parent does not require values for its non-nullable children. + assertThat(decoder(requiredContainer, true).decode(null)) + .containsExactly((Object) null); + RowType requiredParent = (RowType) business.copy(false); + assertThatThrownBy(() -> decoder(requiredParent, true).decode(null)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL field '_kafka_value'"); + assertThat(decodeBusinessFields(requiredParent, "{}", true)) + .containsExactly((Object) null); + } else { + assertThatThrownBy(() -> decoder(requiredContainer, false).decode(null)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL field 'value'"); + } + } + } + + private static Object[] numericContainerValues(DataType type, String token, boolean nested) { + RowType business = + DataTypes.ROW( + DataTypes.FIELD("row", DataTypes.ROW(DataTypes.FIELD("value", type))), + DataTypes.FIELD("array", DataTypes.ARRAY(type)), + DataTypes.FIELD("map", DataTypes.MAP(DataTypes.STRING(), type))); + Object[] values = + decodeBusinessFields( + business, + "{\"row\":{\"value\":" + + token + + "},\"array\":[" + + token + + "],\"map\":{\"entry\":" + + token + + "}}", + nested); + return new Object[] { + ((GenericRow) values[0]).getField(0), + ((GenericArray) values[1]).toObjectArray()[0], + ((GenericMap) values[2]).get(BinaryString.fromString("entry")) + }; + } + + private static Object[] decodeBusinessFields(RowType rowType, String json, boolean nested) { + Object[] values = decoder(rowType, nested).decode(bytes(json)); + if (!nested) { + return values; + } + GenericRow row = (GenericRow) values[0]; + Object[] fields = new Object[row.getFieldCount()]; + for (int i = 0; i < fields.length; i++) { + fields[i] = row.getField(i); + } + return fields; + } + + private static void assertFailure(RowType rowType, String json, String path, String reason) { + for (boolean nested : new boolean[] {false, true}) { + assertThatThrownBy(() -> decoder(rowType, nested).decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining(path) + .hasMessageContaining(reason); + } + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType) { + return decoder(rowType, false); + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType, boolean nested) { + if (nested) { + return new JsonKafkaFieldDecoder( + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, rowType)), + Collections.singletonList(0)), + KafkaValueConversionLayout.NESTED); + } + List positions = new ArrayList<>(); + for (int i = 0; i < rowType.getFieldCount(); i++) { + positions.add(i); + } + return new JsonKafkaFieldDecoder(new KafkaFieldProjection(rowType, positions)); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static RowType complexRowType() { + return DataTypes.ROW( + DataTypes.FIELD( + "customer", + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING().copy(false)), + DataTypes.FIELD( + "address", + DataTypes.ROW( + DataTypes.FIELD( + "city", DataTypes.STRING().copy(false)))))), + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("sku", DataTypes.STRING().copy(false)), + DataTypes.FIELD("quantity", DataTypes.INT().copy(false))))), + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(DataTypes.INT())))); + } + + private static String quoteJson(String value) { + return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java new file mode 100644 index 00000000000..78083b6932d --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaFieldDecoderTest.java @@ -0,0 +1,509 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaFieldProjection; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.StreamReadConstraints; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.exc.StreamConstraintsException; +import org.apache.fluss.types.DataField; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests strict schema-aware JSON scalar conversion. */ +class JsonKafkaFieldDecoderTest { + + @Test + void testRejectsMalformedAndAmbiguousJson() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.BIGINT().copy(false)), + DataTypes.FIELD("optional", DataTypes.STRING())); + JsonKafkaFieldDecoder decoder = decoder(rowType); + + assertThatThrownBy(() -> decoder.decode(bytes("[]"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("object root"); + assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":1,\"id\":2}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":1} {}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + assertThatThrownBy(() -> decoder.decode(bytes("{"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + } + + @Test + void testRejectsMissingTypeOverflowAndPrecisionLoss() { + assertFailure( + DataTypes.ROW(DataTypes.FIELD("id", DataTypes.BIGINT().copy(false))), + "{}", + "$[\"id\"]", + "missing or null"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT())), + "{\"id\":2147483648}", + "$[\"id\"]", + "integer overflow"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("amount", DataTypes.DECIMAL(5, 2))), + "{\"amount\":1.234}", + "$[\"amount\"]", + "scale exceeds"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3))), + "{\"timestamp\":\"2026-07-27T19:30:12.123\"}", + "$[\"timestamp\"]", + "offset or Z is required"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("fixed", DataTypes.BINARY(2))), + "{\"fixed\":\"AQ==\"}", + "$[\"fixed\"]", + "length must equal 2"); + } + + @Test + void testNullKafkaValueRequiresNullableProjection() { + JsonKafkaFieldDecoder nullableDecoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING()))); + assertThat(nullableDecoder.decode(null)).containsExactly((Object) null); + + JsonKafkaFieldDecoder notNullDecoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING().copy(false)))); + assertThatThrownBy(() -> notNullDecoder.decode(null)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL field 'value'"); + } + + @Test + void testExactDecimalParsingAndUnknownFields() { + JsonKafkaFieldDecoder decoder = + decoder(DataTypes.ROW(DataTypes.FIELD("amount", DataTypes.DECIMAL(30, 10)))); + Decimal value = + (Decimal) decoder.decode(bytes("{\"amount\":12345678901234567890.1234567890}"))[0]; + assertThat(value.toBigDecimal()) + .isEqualByComparingTo(new BigDecimal("12345678901234567890.1234567890")); + assertThatThrownBy(() -> decoder.decode(bytes("{\"extra\":1}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("unknown field"); + assertThat(decoder.decode(bytes("{}"))).containsExactly((Object) null); + } + + @ParameterizedTest + @ValueSource(strings = {"[", "{"}) + void testRejectsContainersBeforeReadingTheirContents(String container) { + // The unfinished value must fail the scalar check before parsing its contents. + assertFailure( + DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING())), + "{\"value\":" + container, + "$[\"value\"]", + "scalar"); + assertFailure( + DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING())), + "{\"extra\":" + container, + "$[\"extra\"]", + "unknown field"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFieldCountLimitBeforeReadingExcessFieldValue(boolean nested) { + int maximumFields = JsonKafkaFieldDecoder.MAX_CONTAINER_ELEMENTS; + List fields = new ArrayList<>(); + StringBuilder json = new StringBuilder("{"); + Object[] expected = new Object[maximumFields + 1]; + for (int i = 0; i <= maximumFields; i++) { + fields.add(DataTypes.FIELD("field" + i, DataTypes.INT())); + if (i < maximumFields) { + if (i > 0) { + json.append(','); + } + json.append("\"field").append(i).append("\":").append(i); + expected[i] = i; + } + } + JsonKafkaFieldDecoder decoder = decoder(new RowType(fields), nested); + Object[] result = decoder.decode(bytes(json + "}")); + assertThat(result) + .containsExactly(nested ? new Object[] {GenericRow.of(expected)} : expected); + + String excessField = json + ",\"field" + maximumFields + "\":"; + for (String value : new String[] {"0}", "["}) { + assertThatThrownBy(() -> decoder.decode(bytes(excessField + value))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("maximum field count of " + maximumFields); + } + } + + @ParameterizedTest + @ValueSource(strings = {"-0", "-0.0", "-0e3", "-0e-3", "0", "0.0", "0e3", "0e-3"}) + void testPreservesFloatingPointZeroSign(String number) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE()))); + Object[] values = + decoder.decode( + bytes("{\"floatValue\":" + number + ",\"doubleValue\":" + number + "}")); + float expectedFloat = number.startsWith("-") ? -0.0F : 0.0F; + double expectedDouble = number.startsWith("-") ? -0.0D : 0.0D; + assertThat(Float.floatToRawIntBits((Float) values[0])) + .isEqualTo(Float.floatToRawIntBits(expectedFloat)); + assertThat(Double.doubleToRawLongBits((Double) values[1])) + .isEqualTo(Double.doubleToRawLongBits(expectedDouble)); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFloatingPointUnderflowPreservesSignAndRoundsOnce(boolean nested) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE())), + nested); + // The final two pairs straddle half of the smallest subnormal for each target. + // Converting the FLOAT token through DOUBLE first rounds both to the same midpoint. + String[][] tokens = { + {"0", "0"}, + {"0.0", "0.0"}, + {"1e-1000", "1e-1000"}, + {"7.006492321624085e-46", "2.4703282292062327e-324"}, + {"7.006492321624086e-46", "2.4703282292062328e-324"} + }; + for (boolean negative : new boolean[] {false, true}) { + String sign = negative ? "-" : ""; + for (int i = 0; i < tokens.length; i++) { + Object[] decoded = + decoder.decode( + bytes( + "{\"floatValue\":" + + sign + + tokens[i][0] + + ",\"doubleValue\":" + + sign + + tokens[i][1] + + "}")); + GenericRow row = nested ? (GenericRow) decoded[0] : GenericRow.of(decoded); + int magnitude = i == tokens.length - 1 ? 1 : 0; + assertThat(Float.floatToRawIntBits(row.getFloat(0))) + .as("FLOAT %s%s, nested=%s", sign, tokens[i][0], nested) + .isEqualTo((negative ? Integer.MIN_VALUE : 0) | magnitude); + assertThat(Double.doubleToRawLongBits(row.getDouble(1))) + .as("DOUBLE %s%s, nested=%s", sign, tokens[i][1], nested) + .isEqualTo((negative ? Long.MIN_VALUE : 0L) | magnitude); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"", "-"}) + void testFloatingPointRoundingAndExactDecimalParsing(String sign) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE()), + DataTypes.FIELD("amount", DataTypes.DECIMAL(38, 18)))); + String amount = sign + "12345678901234567890.123456789012345678"; + Object[] values = + decoder.decode( + bytes( + "{\"floatValue\":" + + sign + + "1.0000000596046448,\"doubleValue\":" + + sign + + "1.00000000000000011102230246251565404236316680908203126," + + "\"amount\":" + + amount + + "}")); + float direction = sign.isEmpty() ? 1.0F : -1.0F; + assertThat(values[0]).isEqualTo(direction * 1.0000001F); + assertThat(values[1]).isEqualTo(direction * 1.0000000000000002D); + assertThat(((Decimal) values[2]).toBigDecimal()).isEqualTo(new BigDecimal(amount)); + + assertThat( + decoder.decode( + bytes( + "{\"floatValue\":" + + sign + + "1.0000000596046447,\"doubleValue\":" + + sign + + "1.00000000000000011102230246251565404236316680908203124}"))) + .containsExactly(direction, (double) direction, null); + } + + @ParameterizedTest + @ValueSource(strings = {"", "-"}) + void testFloatingPointOverflowBoundaries(String sign) { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE())); + JsonKafkaFieldDecoder decoder = decoder(rowType); + float direction = sign.isEmpty() ? 1.0F : -1.0F; + String largestFloat = sign + "340282356779733661637539395458142568447"; + for (String suffix : new String[] {"", ".0"}) { + assertThat(decoder.decode(bytes("{\"floatValue\":" + largestFloat + suffix + "}"))) + .containsExactly(direction * Float.MAX_VALUE, null); + assertFailure( + rowType, + "{\"floatValue\":" + + sign + + "340282356779733661637539395458142568448" + + suffix + + "}", + "$[\"floatValue\"]", + "overflowing number"); + } + assertThat(decoder.decode(bytes("{\"doubleValue\":" + sign + "1.7976931348623157e308}"))) + .containsExactly(null, direction * Double.MAX_VALUE); + assertFailure( + rowType, + "{\"doubleValue\":" + sign + "1e309}", + "$[\"doubleValue\"]", + "overflowing number"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRawFloatingPointTokensStillEnforceNumberLengthLimit(boolean nested) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE())), + nested); + // This token underflows to zero rather than overflowing the target. It must still be + // rejected by the parser before the direct FLOAT/DOUBLE token conversion. + StringBuilder number = new StringBuilder("-0."); + for (int i = 0; i < StreamReadConstraints.DEFAULT_MAX_NUM_LEN; i++) { + number.append('0'); + } + number.append('1'); + for (String field : new String[] {"floatValue", "doubleValue"}) { + assertThatThrownBy(() -> decoder.decode(bytes("{\"" + field + "\":" + number + "}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasRootCauseInstanceOf(StreamConstraintsException.class); + } + } + + @Test + void testMixedScalarTokensPreserveProjectionAndMissingFields() { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("text", DataTypes.STRING()), + DataTypes.FIELD("amount", DataTypes.DECIMAL(5, 2)), + DataTypes.FIELD("floatValue", DataTypes.FLOAT()), + DataTypes.FIELD("doubleValue", DataTypes.DOUBLE()), + DataTypes.FIELD("optional", DataTypes.INT()))); + Object[] values = + decoder.decode( + bytes( + "{\"floatValue\":-0.0,\"text\":\"hello\",\"amount\":12.34," + + "\"doubleValue\":2.5,\"optional\":null}")); + assertThat(values[0].toString()).isEqualTo("hello"); + assertThat(((Decimal) values[1]).toBigDecimal()).isEqualTo(new BigDecimal("12.34")); + assertThat(Float.floatToRawIntBits((Float) values[2])) + .isEqualTo(Float.floatToRawIntBits(-0.0F)); + assertThat(values[3]).isEqualTo(2.5D); + assertThat(values[4]).isNull(); + assertThat(decoder.decode(bytes("{\"doubleValue\":2.5}"))) + .containsExactly(null, null, null, 2.5D, null); + } + + @Test + void testFloatingPointParsingStillRequiresStrictJsonSyntax() { + JsonKafkaFieldDecoder decoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.DOUBLE()))); + for (String json : + new String[] { + "{\"value\":}", + "{\"value\":-0.0 \"value\":1}", + "{\"value\":-0.0,}", + "{\"value\":01}", + "{\"value\":NaN}", + "{\"value\":-Infinity}", + "{\"value\":0x1.0p0}", + "{\"value\":1f}", + "{\"value\":1}null" + }) { + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + } + } + + @Test + void testRejectsInvalidUtf8AfterTrailingWhitespaceAcrossReaderBuffers() { + JsonKafkaFieldDecoder decoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING()))); + byte[] json = bytes("{\"value\":\"ok\"}"); + byte[] input = Arrays.copyOf(json, json.length + 32_768); + Arrays.fill(input, json.length, input.length - 1, (byte) ' '); + input[input.length - 1] = (byte) 0xff; + assertThatThrownBy(() -> decoder.decode(input)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + } + + @Test + void testRejectsInvalidUtf8AndAlternateEncodings() { + JsonKafkaFieldDecoder decoder = + decoder(DataTypes.ROW(DataTypes.FIELD("value", DataTypes.STRING()))); + for (byte[] input : + new byte[][] { + new byte[] {(byte) 0xff}, "{\"value\":\"ok\"}".getBytes(StandardCharsets.UTF_16) + }) { + assertThatThrownBy(() -> decoder.decode(input)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("strict UTF-8 JSON"); + } + } + + private static void assertFailure(RowType rowType, String json, String path, String reason) { + assertThatThrownBy(() -> decoder(rowType).decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining(path) + .hasMessageContaining(reason); + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType) { + return decoder(rowType, false); + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType, boolean nested) { + if (nested) { + return new JsonKafkaFieldDecoder( + new KafkaFieldProjection( + DataTypes.ROW(DataTypes.FIELD(KafkaTableSchema.VALUE_COLUMN, rowType)), + Collections.singletonList(0)), + KafkaValueConversionLayout.NESTED); + } + List positions = new ArrayList<>(); + for (int i = 0; i < rowType.getFieldCount(); i++) { + positions.add(i); + } + return new JsonKafkaFieldDecoder(new KafkaFieldProjection(rowType, positions)); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRootContractAndEarlyRejectionAreIndependentOfLayout(boolean nested) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT().copy(false)), + DataTypes.FIELD("optional", DataTypes.STRING())), + nested); + for (String name : + new String[] { + "_kafka_key", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue", + "_kafka_value" + }) { + assertThatThrownBy(() -> decoder.decode(bytes("{\"" + name + "\":["))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("reserved field"); + assertThatThrownBy( + () -> + decoder( + DataTypes.ROW(DataTypes.FIELD(name, DataTypes.INT())), + nested)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("Reserved name"); + } + assertThatThrownBy(() -> decoder.decode(bytes("{\"unknown\":["))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("unknown field"); + assertThatThrownBy(() -> decoder.decode(bytes("{\"id\":["))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("expected a JSON scalar"); + for (byte[] invalid : + new byte[][] { + bytes("null"), + bytes("[]"), + bytes("{}"), + bytes("{\"id\":1,\"id\":2}"), + bytes("{\"id\":1} {}"), + {(byte) 0xff} + }) { + assertThatThrownBy(() -> decoder.decode(invalid)) + .isInstanceOf(KafkaRecordEncodingException.class); + } + Object[] result = decoder.decode(bytes("{\"optional\":null,\"id\":7}")); + assertThat(result) + .containsExactly( + nested ? new Object[] {GenericRow.of(7, null)} : new Object[] {7, null}); + } + + @Test + void testNestedRootPreservesScalarValuesAndContainerNullability() { + RowType business = + DataTypes.ROW( + DataTypes.FIELD("f", DataTypes.FLOAT()), + DataTypes.FIELD("d", DataTypes.DOUBLE()), + DataTypes.FIELD("amount", DataTypes.DECIMAL(38, 10).copy(false))); + JsonKafkaFieldDecoder decoder = decoder(business, true); + GenericRow row = + (GenericRow) + decoder.decode( + bytes( + "{\"amount\":1234567890123456789012345678.1234567890,\"d\":-0.0,\"f\":-0.0}"))[ + 0]; + assertThat(Float.floatToRawIntBits(row.getFloat(0))) + .isEqualTo(Float.floatToRawIntBits(-0.0F)); + assertThat(Double.doubleToRawLongBits(row.getDouble(1))) + .isEqualTo(Double.doubleToRawLongBits(-0.0D)); + assertThat(row.getDecimal(2, 38, 10).toBigDecimal()) + .isEqualTo(new BigDecimal("1234567890123456789012345678.1234567890")); + assertThat(decoder.decode(null)).containsExactly((Object) null); + assertThatThrownBy(() -> decoder((RowType) business.copy(false), true).decode(null)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL field '_kafka_value'"); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java new file mode 100644 index 00000000000..5163b4cae76 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonKafkaRescueTest.java @@ -0,0 +1,734 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; +import org.apache.fluss.types.DataField; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Tests unknown-field rescue while keeping known-field and resource violations fatal. */ +class JsonKafkaRescueTest { + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testOtherStringColumnDoesNotEnableRescue(boolean nested) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW(DataTypes.FIELD("custom_rescue", DataTypes.STRING())), + nested); + Object[] values = decoder.decode(bytes("{\"custom_rescue\":\"business\"}")); + assertThat(values).hasSize(1); + assertThat(nested ? ((GenericRow) values[0]).getString(0).toString() : values[0].toString()) + .isEqualTo("business"); + for (String json : + new String[] {"{\"extra\":1}", "{\"_kafka_rescue\":1}", "{\"_kafka_key\":1}"}) { + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class); + } + } + + @Test + void testUnexpectedConversionFailureIsNotRescued() { + JsonNode broken = mock(JsonNode.class); + IllegalStateException failure = new IllegalStateException("unexpected conversion failure"); + when(broken.getNodeType()).thenReturn(JsonNodeType.BOOLEAN); + when(broken.booleanValue()).thenThrow(failure); + assertThatThrownBy( + () -> + JsonToFlussConverters.create(DataTypes.BOOLEAN()) + .convert(broken, "$")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasCause(failure); + } + + @ParameterizedTest + @MethodSource("invalidKnownValues") + void testKnownFailuresRemainFatalAcrossNullableContainers( + DataType dataType, String jsonValue, String reason) { + assertRejectedInAllContainers(dataType.copy(true), jsonValue, reason); + } + + private static Stream invalidKnownValues() { + return Stream.of( + Arguments.of(DataTypes.BOOLEAN(), "1", "expected a JSON boolean"), + Arguments.of(DataTypes.TINYINT(), "128", "integer overflow"), + Arguments.of(DataTypes.SMALLINT(), "32768", "integer overflow"), + Arguments.of(DataTypes.INT(), "2147483648", "integer overflow"), + Arguments.of(DataTypes.INT(), "1.5", "expected a JSON integer"), + Arguments.of(DataTypes.INT(), "\"bad\"", "expected a JSON integer"), + Arguments.of(DataTypes.BIGINT(), "9223372036854775808", "integer overflow"), + Arguments.of(DataTypes.DECIMAL(3, 1), "123.4", "decimal precision exceeds target"), + Arguments.of(DataTypes.DECIMAL(3, 1), "1.23", "decimal scale exceeds target"), + Arguments.of(DataTypes.FLOAT(), "1e40", "non-finite or overflowing number"), + Arguments.of(DataTypes.DOUBLE(), "1e400", "non-finite or overflowing number"), + Arguments.of(DataTypes.DATE(), "\"invalid\"", "invalid ISO-8601 date"), + Arguments.of(DataTypes.TIME(3), "\"invalid\"", "invalid ISO-8601 local time"), + Arguments.of( + DataTypes.TIMESTAMP(3), "\"invalid\"", "invalid ISO-8601 local timestamp"), + Arguments.of( + DataTypes.TIMESTAMP_LTZ(3), "\"invalid\"", "invalid ISO-8601 timestamp"), + Arguments.of(DataTypes.CHAR(1), "\"ab\"", "character length exceeds target"), + Arguments.of(DataTypes.BINARY(2), "\"YQ==\"", "decoded binary length must equal 2"), + Arguments.of(DataTypes.BYTES(), "\"%%%\"", "invalid Base64 string"), + Arguments.of(DataTypes.STRING(), "1", "expected a JSON string"), + Arguments.of(DataTypes.STRING(), "{}", "expected a JSON"), + Arguments.of(DataTypes.STRING(), "[]", "expected a JSON"), + Arguments.of( + DataTypes.ROW(DataTypes.FIELD("n", DataTypes.INT())), + "[]", + "expected a JSON object"), + Arguments.of(DataTypes.ARRAY(DataTypes.INT()), "{}", "expected a JSON array"), + Arguments.of( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + "[]", + "expected a JSON object")); + } + + private static void assertRejectedInAllContainers( + DataType dataType, String value, String reason) { + for (boolean nested : new boolean[] {false, true}) { + JsonKafkaFieldDecoder decoder = + decoder( + DataTypes.ROW( + DataTypes.FIELD("value", dataType), + DataTypes.FIELD( + "row", + DataTypes.ROW(DataTypes.FIELD("value", dataType))), + DataTypes.FIELD("items", DataTypes.ARRAY(dataType)), + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.STRING(), dataType)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())), + nested); + String[][] cases = { + {"\"value\":" + value, "$[\"value\"]"}, + {"\"row\":{\"value\":" + value + ",\"extra\":2}", "$[\"row\"][\"value\"]"}, + {"\"items\":[" + value + "]", "$[\"items\"][0]"}, + {"\"attributes\":{\"dynamic\":" + value + "}", "$[\"attributes\"][\"dynamic\"]"} + }; + for (String[] entry : cases) { + String json = "{\"extra\":1," + entry[0] + "}"; + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .as("%s: %s", dataType, json) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining(entry[1]) + .hasMessageContaining(reason); + } + } + } + + @Test + void testNotNullBusinessTypesAreRejectedBeforeRescue() { + RowType required = + DataTypes.ROW( + DataTypes.FIELD("value", DataTypes.INT().copy(false)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + for (boolean nested : new boolean[] {false, true}) { + assertThatThrownBy(() -> decoder(required, nested)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("nullable fields"); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "{", + "[]", + "{\"extra\":1,\"extra\":2}", + "{\"_kafka_rescue\":null,\"_kafka_rescue\":1}", + "{\"extra\":{\"x\":1,\"x\":2}}", + "{} {}", + "{\"_kafka_rescue\":\"\\uD800\"}", + "{\"_kafka_rescue\":1e400}" + }) + void testInvalidJsonCannotBeRescued(String json) { + assertThatThrownBy(() -> fidelityDecoder().decode(bytes(json))) + .isInstanceOf(KafkaRecordEncodingException.class); + } + + @Test + void testRescueNameStillEnforcesResourceLimitsAndUtf8() { + JsonKafkaFieldDecoder decoder = + rescueDecoder(DataTypes.ROW(DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); + String atLimit = containerAtLimit(false); + assertThat(decoder.decode(bytes("{\"_kafka_rescue\":" + atLimit + "}"))[0].toString()) + .isEqualTo("{\"_kafka_rescue\":" + atLimit + "}"); + assertThatThrownBy( + () -> + decoder.decode( + bytes( + "{\"_kafka_rescue\":" + + atLimit.substring(0, atLimit.length() - 1) + + ",0]}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("container size exceeds"); + String nested = "0"; + for (int i = 0; i < JsonKafkaFieldDecoder.MAX_JSON_NESTING_DEPTH; i++) { + nested = "[" + nested + "]"; + } + byte[] tooDeep = bytes("{\"_kafka_rescue\":" + nested + "}"); + assertThatThrownBy(() -> decoder.decode(tooDeep)) + .isInstanceOf(KafkaRecordEncodingException.class); + byte[] invalidUtf8 = bytes("{\"_kafka_rescue\":\"x\"}"); + invalidUtf8[invalidUtf8.length - 3] = (byte) 0xff; + assertThatThrownBy(() -> decoder.decode(invalidUtf8)) + .isInstanceOf(KafkaRecordEncodingException.class); + } + + @Test + void testRescuesUnknownRootAndNestedFieldsAsJsonString() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.BIGINT()), + DataTypes.FIELD( + "customer", + DataTypes.ROW(DataTypes.FIELD("name", DataTypes.STRING()))), + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.ROW(DataTypes.FIELD("sku", DataTypes.STRING())))), + DataTypes.FIELD( + "attributes", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW(DataTypes.FIELD("known", DataTypes.INT())))), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + JsonKafkaFieldDecoder decoder = rescueDecoder(rowType); + + Object[] values = + decoder.decode( + bytes( + "{" + + "\"id\":1," + + "\"customer\":{\"name\":\"Alice\",\"extra_nested\":2}," + + "\"items\":[{\"sku\":\"A-1\",\"extra_item\":3}]," + + "\"attributes\":{\"dynamic\":{\"known\":1," + + "\"extra_map_value\":4}}," + + "\"extra\":\"unexpected\"," + + "\"_kafka_rescue\":null" + + "}")); + + assertThat(values[0]).isEqualTo(1L); + assertThat(((GenericRow) values[1]).getString(0).toString()).isEqualTo("Alice"); + assertThat(((GenericArray) values[2]).getRow(0, 1).getString(0).toString()) + .isEqualTo("A-1"); + GenericMap attributes = (GenericMap) values[3]; + assertThat(((GenericRow) attributes.get(BinaryString.fromString("dynamic"))).getInt(0)) + .isEqualTo(1); + assertThat(((BinaryString) values[4]).toString()) + .isEqualTo( + "{\"extra\":\"unexpected\",\"_kafka_rescue\":null," + + "\"customer\":{\"extra_nested\":2}," + + "\"items\":[{\"extra_item\":3}]," + + "\"attributes\":{\"dynamic\":{\"extra_map_value\":4}}}"); + + assertThat( + decoder.decode( + bytes( + "{\"id\":1,\"customer\":null,\"items\":null," + + "\"attributes\":null}"))[4]) + .isNull(); + } + + @ParameterizedTest + @ValueSource( + strings = { + "null", + "\"payload\"", + "7", + "false", + "[1,{\"x\":2}]", + "{\"_kafka_rescue\":\"nested\",\"x\":1}" + }) + void testPayloadRescueNameIsPreservedAsJsonMember(String value) { + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT()), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); + Object[] decoded = + decoder.decode(bytes("{\"id\":1,\"_kafka_rescue\":" + value + ",\"extra\":2}")); + assertThat(decoded[0]).isEqualTo(1); + assertThat(decoded[1].toString()) + .isEqualTo("{\"_kafka_rescue\":" + value + ",\"extra\":2}"); + assertThat(decoder.decode(bytes("{\"id\":1}"))[1]).isNull(); + assertThat(decoder.decode(bytes("{\"_kafka_rescue\":null}"))[1].toString()) + .isEqualTo("{\"_kafka_rescue\":null}"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFixedReservedNamesAndDeepBusinessNames(boolean nested) { + String[] names = { + "_kafka_key", "_kafka_headers", "_kafka_metadata", "_kafka_rescue", "_kafka_value" + }; + List fields = new ArrayList<>(); + for (String name : names) { + fields.add(DataTypes.FIELD(name, DataTypes.STRING())); + } + RowType schema = + DataTypes.ROW( + DataTypes.FIELD("row", new RowType(fields)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + StringBuilder json = new StringBuilder("{\"row\":{"); + for (int i = 0; i < names.length; i++) { + if (i > 0) { + json.append(','); + } + json.append('"').append(names[i]).append("\":\"business\""); + } + json.append("}"); + for (String name : names) { + json.append(",\"").append(name).append("\":null"); + } + json.append('}'); + Object[] values = decoder(schema, nested).decode(bytes(json.toString())); + GenericRow business = businessRow(values, nested); + GenericRow row = (GenericRow) business.getField(0); + for (int i = 0; i < names.length; i++) { + assertThat(row.getString(i).toString()).isEqualTo("business"); + } + assertThat(values[values.length - 1].toString()) + .isEqualTo( + "{\"_kafka_key\":null,\"_kafka_headers\":null,\"_kafka_metadata\":null,\"_kafka_rescue\":null,\"_kafka_value\":null}"); + RowType noRescue = + DataTypes.ROW( + DataTypes.FIELD("row", new RowType(fields)), + DataTypes.FIELD("rescue", DataTypes.STRING())); + for (String name : names) { + assertThatThrownBy( + () -> + decoder(noRescue, nested) + .decode(bytes("{\"" + name + "\":null}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("reserved field"); + } + assertThatThrownBy(() -> decoder(noRescue, nested).decode(bytes("{\"unknown\":1}"))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("unknown field"); + Object[] plain = decoder(noRescue, nested).decode(bytes("{\"rescue\":\"business\"}")); + assertThat((nested ? ((GenericRow) plain[0]).getField(1) : plain[1]).toString()) + .isEqualTo("business"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRescueKeepsTypedNestedNumbersAndUnknownStructure(boolean nested) { + RowType numbers = + DataTypes.ROW( + DataTypes.FIELD("f", DataTypes.FLOAT()), + DataTypes.FIELD("d", DataTypes.DOUBLE()), + DataTypes.FIELD("n", DataTypes.DECIMAL(38, 18))); + RowType type = + DataTypes.ROW( + DataTypes.FIELD("row", numbers), + DataTypes.FIELD("array", DataTypes.ARRAY(numbers)), + DataTypes.FIELD("map", DataTypes.MAP(DataTypes.STRING(), numbers)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + String decimal = "12345678901234567890.123456789012345678"; + for (String token : + new String[] { + "-0", + "-0.0", + "-0e0", + "1.0000000596046448", + "-1.0000000596046448", + "-1e-100", + "-1e-400", + "7.006492321624085e-46", + "-7.006492321624085e-46" + }) { + String row = + "{\"f\":" + + token + + ",\"d\":" + + token + + ",\"n\":" + + decimal + + ",\"extra\":{\"a\":[1,null,2]}}"; + Object[] decoded = + decoder(type, nested) + .decode( + bytes( + "{\"row\":" + + row + + ",\"array\":[null," + + row + + "],\"map\":{\"k\":" + + row + + "}}")); + GenericRow values = businessRow(decoded, nested); + GenericRow[] rows = { + (GenericRow) values.getField(0), + (GenericRow) ((GenericArray) values.getField(1)).getRow(1, 3), + (GenericRow) ((GenericMap) values.getField(2)).get(BinaryString.fromString("k")) + }; + for (GenericRow number : rows) { + assertThat(Float.floatToRawIntBits(number.getFloat(0))) + .isEqualTo(Float.floatToRawIntBits(Float.parseFloat(token))); + assertThat(Double.doubleToRawLongBits(number.getDouble(1))) + .isEqualTo(Double.doubleToRawLongBits(Double.parseDouble(token))); + assertThat(((Decimal) number.getField(2)).toBigDecimal()) + .isEqualTo(new BigDecimal(decimal)); + } + assertThat(decoded[decoded.length - 1].toString()) + .isEqualTo( + "{\"row\":{\"extra\":{\"a\":[1,null,2]}},\"array\":[null,{\"extra\":{\"a\":[1,null,2]}}],\"map\":{\"k\":{\"extra\":{\"a\":[1,null,2]}}}}"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRescueSharesTheMaximumBusinessDepthIncludingEmptyRow(boolean nested) { + DataType type = new RowType(Collections.emptyList()); + String json = "{}"; + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + type = DataTypes.ARRAY(type); + json = "[" + json + "]"; + } + RowType fields = + DataTypes.ROW( + DataTypes.FIELD("value", type), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + Object[] decoded = + decoder(fields, nested).decode(bytes("{\"value\":" + json + ",\"extra\":1}")); + assertThat(decoded[decoded.length - 1].toString()).isEqualTo("{\"extra\":1}"); + assertThat(decoder(fields, nested).decode(bytes("{}"))[1]).isNull(); + } + + private static GenericRow businessRow(Object[] values, boolean nested) { + return nested + ? (GenericRow) values[0] + : GenericRow.of(Arrays.copyOf(values, values.length - 1)); + } + + @Test + void testRescueArrayPreservesOriginalElementPositions() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT())))), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())); + Object[] values = + rescueDecoder(rowType) + .decode(bytes("{\"items\":[{\"id\":1},{\"id\":2,\"extra\":3}]}")); + assertThat(((BinaryString) values[1]).toString()) + .isEqualTo("{\"items\":[null,{\"extra\":3}]}"); + assertThat(((GenericArray) values[0]).getRow(1, 1).getInt(0)).isEqualTo(2); + } + + @Test + void testRescuesRootObjectAtLimitAndRejectsExcessFieldBeforeReadingItsValue() { + JsonKafkaFieldDecoder decoder = + rescueDecoder(DataTypes.ROW(DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); + String object = containerAtLimit(true); + assertThat(((BinaryString) decoder.decode(bytes(object))[0]).toString()).isEqualTo(object); + + String excessField = object.substring(0, object.length() - 1) + ",\"excess\":{"; + assertThatThrownBy(() -> decoder.decode(bytes(excessField))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining( + "maximum field count of " + JsonToFlussConverters.MAX_CONTAINER_ELEMENTS); + } + + @ParameterizedTest(name = "{index}: {0}") + @MethodSource("rescuedContainers") + void testBoundsUnknownContainersBeforeReadingExcessChild( + String description, String prefix, String suffix, String path, boolean object) { + RowType nestedRow = DataTypes.ROW(DataTypes.FIELD("id", DataTypes.INT())); + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("details", nestedRow), + DataTypes.FIELD("items", DataTypes.ARRAY(nestedRow)), + DataTypes.FIELD( + "attributes", DataTypes.MAP(DataTypes.STRING(), nestedRow)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); + String container = containerAtLimit(object); + String json = prefix + container + suffix; + assertThat(((BinaryString) decoder.decode(bytes(json))[3]).toString()).isEqualTo(json); + + // An unfinished excess child distinguishes the container limit from a syntax error. + String excessChild = + prefix + + container.substring(0, container.length() - 1) + + (object ? ",\"excess\":{" : ",["); + assertThatThrownBy(() -> decoder.decode(bytes(excessChild))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining(path) + .hasMessageContaining( + "container size exceeds " + JsonToFlussConverters.MAX_CONTAINER_ELEMENTS); + } + + private static Stream rescuedContainers() { + return Stream.of(false, true) + .flatMap( + object -> + Stream.of( + Arguments.of( + "unknown root " + (object ? "object" : "array"), + "{\"extra\":", + "}", + "$[\"extra\"]", + object), + Arguments.of( + "unknown subtree " + (object ? "object" : "array"), + "{\"extra\":{\"nested\":", + "}}", + "$[\"extra\"][\"nested\"]", + object), + Arguments.of( + "inside ROW " + (object ? "object" : "array"), + "{\"details\":{\"extra\":", + "}}", + "$[\"details\"][\"extra\"]", + object), + Arguments.of( + "inside ARRAY " + (object ? "object" : "array"), + "{\"items\":[{\"extra\":", + "}]}", + "$[\"items\"][0][\"extra\"]", + object), + Arguments.of( + "inside MAP " + (object ? "object" : "array"), + "{\"attributes\":{\"dynamic\":{\"extra\":", + "}}}", + "$[\"attributes\"][\"dynamic\"][\"extra\"]", + object))); + } + + @ParameterizedTest + @ValueSource( + strings = {"-0", "-0.0", "-0e3", "-0e-3", "1.0000000596046448", "-1.0000000596046448"}) + void testRescuePreservesRootFloatToken(String number) { + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("value", DataTypes.FLOAT()), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); + Object[] values = decoder.decode(bytes("{\"extra\":1,\"value\":" + number + "}")); + assertThat(Float.floatToRawIntBits((Float) values[0])) + .isEqualTo(Float.floatToRawIntBits(Float.parseFloat(number))); + assertThat(((BinaryString) values[1]).toString()).isEqualTo("{\"extra\":1}"); + } + + @ParameterizedTest + @ValueSource(strings = {"[", "{"}) + void testRescueRejectsContainersForKnownScalarsBeforeReadingTheirContents(String container) { + JsonKafkaFieldDecoder decoder = + rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("value", DataTypes.STRING()), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); + assertThatThrownBy(() -> decoder.decode(bytes("{\"extra\":1,\"value\":" + container))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$[\"value\"]") + .hasMessageContaining("expected a JSON scalar"); + } + + @ParameterizedTest + @ValueSource(strings = {"\\uD800", "\\uDC00", "a\\uD800b", "\\uD800\\uD800", "\\uDC00\\uD800"}) + void testRejectsUnpairedSurrogatesBeforeRescueEncoding(String escaped) { + JsonKafkaFieldDecoder decoder = fidelityDecoder(); + for (String json : + new String[] { + "{\"" + escaped + "\":1,\"?\":2}", + "{\"extra\":\"" + escaped + "\"}", + "{\"extra\":{\"" + escaped + "\":1,\"?\":2}}", + "{\"extra\":[{\"key\":\"" + escaped + "\"}]}", + "{\"details\":{\"" + escaped + "\":1,\"?\":2}}", + "{\"items\":[{\"extra\":\"" + escaped + "\"}]}", + "{\"attributes\":{\"" + escaped + "\":{\"known\":1},\"?\":{\"known\":2}}}", + "{\"attributes\":{\"dynamic\":{\"extra\":\"" + escaped + "\"}}}" + }) { + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .as(json) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("unpaired UTF-16 surrogate"); + } + } + + @ParameterizedTest + @ValueSource(strings = {"1e400", "-1e400", "1e10000", "-1e10000", "1.7976931348623159e308"}) + void testRejectsOverflowingNumbersBeforeRescueEncoding(String number) { + JsonKafkaFieldDecoder decoder = fidelityDecoder(); + for (String json : + new String[] { + "{\"extra\":" + number + "}", + "{\"extra\":{\"nested\":" + number + "}}", + "{\"extra\":[" + number + "]}", + "{\"details\":{\"extra\":" + number + "}}", + "{\"items\":[{\"extra\":" + number + "}]}", + "{\"attributes\":{\"dynamic\":{\"extra\":" + number + "}}}" + }) { + assertThatThrownBy(() -> decoder.decode(bytes(json))) + .as(json) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("non-finite or overflowing number"); + } + } + + @Test + void testRescuePreservesValidUnicodeAndExactNumbers() throws Exception { + String largeInteger = BigInteger.TEN.pow(400).subtract(BigInteger.ONE).toString(); + String decimal = "12345678901234567890.1234567890123456789"; + Object[] values = + fidelityDecoder() + .decode( + bytes( + "{\"\\uD83D\\uDE00\":\"\\uD83D\\uDE00\"," + + "\"extra\":{\"integer\":" + + largeInteger + + ",\"decimal\":" + + decimal + + ",\"small\":1e-400,\"large\":1e308," + + "\"text\":\"Infinity\",\"literal\":\"\\\\uD800\"}," + + "\"details\":{\"known\":7,\"\\uD83D\\uDE00\":\"ok\"}," + + "\"items\":[{\"known\":8,\"extra\":\"\\uD83D\\uDE00\"}]," + + "\"attributes\":{\"\\uD83D\\uDE00\":{\"known\":9,\"extra\":1}}}")); + ObjectMapper mapper = + new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + JsonNode rescued = mapper.readTree(((BinaryString) values[3]).toString()); + assertThat(rescued.get("😀").textValue()).isEqualTo("😀"); + JsonNode extra = rescued.get("extra"); + assertThat(extra.get("integer").bigIntegerValue()).isEqualTo(new BigInteger(largeInteger)); + assertThat(extra.get("decimal").decimalValue()) + .isEqualByComparingTo(new BigDecimal(decimal)); + assertThat(extra.get("small").decimalValue()) + .isEqualByComparingTo(new BigDecimal("1e-400")); + assertThat(extra.get("large").decimalValue()).isEqualByComparingTo(new BigDecimal("1e308")); + assertThat(extra.get("text").textValue()).isEqualTo("Infinity"); + assertThat(extra.get("literal").textValue()).isEqualTo("\\uD800"); + assertThat(rescued.get("details").get("😀").textValue()).isEqualTo("ok"); + assertThat(rescued.get("items").get(0).get("extra").textValue()).isEqualTo("😀"); + assertThat(rescued.get("attributes").get("😀").get("extra").intValue()).isEqualTo(1); + assertThat(((GenericRow) values[0]).getInt(0)).isEqualTo(7); + assertThat(((GenericArray) values[1]).getRow(0, 1).getInt(0)).isEqualTo(8); + assertThat( + ((GenericRow) ((GenericMap) values[2]).get(BinaryString.fromString("😀"))) + .getInt(0)) + .isEqualTo(9); + } + + private static JsonKafkaFieldDecoder fidelityDecoder() { + RowType nestedRow = DataTypes.ROW(DataTypes.FIELD("known", DataTypes.INT())); + return rescueDecoder( + DataTypes.ROW( + DataTypes.FIELD("details", nestedRow), + DataTypes.FIELD("items", DataTypes.ARRAY(nestedRow)), + DataTypes.FIELD("attributes", DataTypes.MAP(DataTypes.STRING(), nestedRow)), + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING()))); + } + + private static String containerAtLimit(boolean object) { + StringBuilder json = new StringBuilder(object ? "{" : "["); + for (int i = 0; i < JsonToFlussConverters.MAX_CONTAINER_ELEMENTS; i++) { + if (i > 0) { + json.append(','); + } + if (object) { + json.append('"').append('k').append(i).append("\":"); + } + json.append('0'); + } + return json.append(object ? '}' : ']').toString(); + } + + private static JsonKafkaFieldDecoder rescueDecoder(RowType rowType) { + return decoder(rowType, false); + } + + private static JsonKafkaFieldDecoder decoder(RowType rowType, boolean nested) { + Schema.Builder schema = Schema.newBuilder(); + List business = new ArrayList<>(); + for (DataField field : rowType.getFields()) { + if (!KafkaTableSchema.RESCUE_COLUMN.equals(field.getName())) { + business.add(field); + } + } + if (nested) { + schema.column(KafkaTableSchema.VALUE_COLUMN, new RowType(business)); + } else { + for (DataField field : business) { + schema.column(field.getName(), field.getType()); + } + } + int rescue = rowType.getFieldIndex(KafkaTableSchema.RESCUE_COLUMN); + if (rescue >= 0) { + schema.column(KafkaTableSchema.RESCUE_COLUMN, rowType.getTypeAt(rescue)); + } + return new JsonKafkaFieldDecoder( + KafkaTableSchema.resolve( + TableDescriptor.builder() + .schema(schema.build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + nested ? "nested" : "flat") + .build())); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java new file mode 100644 index 00000000000..90b67058b41 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/format/json/JsonToFlussConvertersTest.java @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format.json; + +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DecimalNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.DoubleNode; +import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.databind.node.FloatNode; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Collections; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies strict schema-aware conversion without transport or Arrow dependencies. */ +class JsonToFlussConvertersTest { + private static final ObjectMapper MAPPER = + new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + + @Test + void testNumericRepresentationsAndExactDecimals() throws Exception { + assertThat(convert(DataTypes.BOOLEAN(), "true")).isEqualTo(true); + assertThat(convert(DataTypes.TINYINT(), "-128")).isEqualTo((byte) -128); + assertThat(convert(DataTypes.SMALLINT(), "32767")).isEqualTo((short) 32767); + assertThat(convert(DataTypes.INT(), "2147483647")).isEqualTo(Integer.MAX_VALUE); + assertThat(convert(DataTypes.BIGINT(), "9223372036854775807")).isEqualTo(Long.MAX_VALUE); + assertThat(convert(DataTypes.FLOAT(), "1.25")).isEqualTo(1.25F); + assertThat(convert(DataTypes.DOUBLE(), "2.5")).isEqualTo(2.5D); + Decimal decimal = + (Decimal) convert(DataTypes.DECIMAL(30, 10), "12345678901234567890.1234567890"); + assertThat(decimal.toBigDecimal()) + .isEqualByComparingTo(new BigDecimal("12345678901234567890.1234567890")); + } + + @ParameterizedTest + @ValueSource(strings = {"", "-"}) + void testFloatRoundsDirectlyWithoutDoubleRounding(String sign) throws Exception { + float direction = sign.isEmpty() ? 1.0F : -1.0F; + assertThat(convert(DataTypes.FLOAT(), sign + "1.0000000596046448")) + .isEqualTo(direction * 1.0000001F); + assertThat(convert(DataTypes.FLOAT(), sign + "1.0000000596046447")).isEqualTo(direction); + // The integer and decimal nodes must both round to the largest finite float. + String belowOverflow = "340282356779733661637539395458142568447"; + assertThat(convert(DataTypes.FLOAT(), sign + belowOverflow)) + .isEqualTo(direction * Float.MAX_VALUE); + assertThat(convert(DataTypes.FLOAT(), sign + belowOverflow + ".0")) + .isEqualTo(direction * Float.MAX_VALUE); + assertThatThrownBy( + () -> + convert( + DataTypes.FLOAT(), + sign + "340282356779733661637539395458142568448")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("overflowing number"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFloatingPointPreservesZeroSignIncludingUnderflow(boolean negative) { + double zero = negative ? -0.0D : 0.0D; + for (JsonNode node : + new JsonNode[] { + DoubleNode.valueOf(zero), + FloatNode.valueOf((float) zero), + DecimalNode.valueOf(new BigDecimal(negative ? "-1e-1000" : "1e-1000")) + }) { + float floatValue = + (float) JsonToFlussConverters.create(DataTypes.FLOAT()).convert(node, "$.f"); + double doubleValue = + (double) JsonToFlussConverters.create(DataTypes.DOUBLE()).convert(node, "$.d"); + assertThat(Float.floatToRawIntBits(floatValue)) + .isEqualTo(Float.floatToRawIntBits((float) zero)); + assertThat(Double.doubleToRawLongBits(doubleValue)) + .isEqualTo(Double.doubleToRawLongBits(zero)); + } + } + + @ParameterizedTest + @MethodSource("nonFiniteNumbers") + void testRejectsNonFiniteNumericNodes(JsonNode node) { + for (DataType type : + new DataType[] {DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.DECIMAL(38, 18)}) { + assertThatThrownBy(() -> JsonToFlussConverters.create(type).convert(node, "$.number")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.number") + .hasMessageContaining(type.asSummaryString()); + } + } + + private static Stream nonFiniteNumbers() { + return Stream.of( + DoubleNode.valueOf(Double.NaN), + DoubleNode.valueOf(Double.NEGATIVE_INFINITY), + DoubleNode.valueOf(Double.POSITIVE_INFINITY), + FloatNode.valueOf(Float.NaN), + FloatNode.valueOf(Float.NEGATIVE_INFINITY), + FloatNode.valueOf(Float.POSITIVE_INFINITY)); + } + + @ParameterizedTest + @ValueSource(strings = {"A", "AQ===", "AQ==x", "A Q==", "AQ==\n", "_w=="}) + void testBase64RejectsInvalidPaddingAlphabetAndWhitespace(String value) { + for (DataType type : new DataType[] {DataTypes.BYTES(), DataTypes.BINARY(1)}) { + assertThatThrownBy( + () -> + JsonToFlussConverters.create(type) + .convert( + MAPPER.getNodeFactory().textNode(value), "$.b")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.b") + .hasMessageContaining("invalid Base64"); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "1969-12-31T23:59:59.999999999Z", + "1970-01-01T05:44:59.999999999+05:45", + "1969-12-31T20:29:59.999999999-03:30" + }) + void testTimestampLtzNormalizesOffsetsBeforeEpoch(String value) throws Exception { + TimestampLtz timestamp = + (TimestampLtz) convert(DataTypes.TIMESTAMP_LTZ(9), "\"" + value + "\""); + assertThat(timestamp.toInstant()).isEqualTo(Instant.EPOCH.minusNanos(1)); + } + + @Test + void testStringsBinaryAndTemporalRepresentations() throws Exception { + assertThat(convert(DataTypes.CHAR(2), "\"😀a\"")).isEqualTo(BinaryString.fromString("😀a")); + assertThat(convert(DataTypes.STRING(), "\"文本\"")).isEqualTo(BinaryString.fromString("文本")); + assertThat((byte[]) convert(DataTypes.BINARY(2), "\"AQI=\"")).containsExactly(1, 2); + assertThat((byte[]) convert(DataTypes.BYTES(), "\"\"")).isEmpty(); + assertThat(convert(DataTypes.DATE(), "\"1970-01-02\"")) + .isEqualTo((int) LocalDate.of(1970, 1, 2).toEpochDay()); + assertThat(convert(DataTypes.TIME(3), "\"00:00:01.123\"")).isEqualTo(1123); + assertThat(convert(DataTypes.TIMESTAMP(3), "\"2026-09-11T12:00:00.123\"")) + .isEqualTo( + TimestampNtz.fromLocalDateTime( + LocalDateTime.of(2026, 9, 11, 12, 0, 0, 123000000))); + assertThat(convert(DataTypes.TIMESTAMP_LTZ(3), "\"2026-09-11T12:00:00.123+08:00\"")) + .isEqualTo(TimestampLtz.fromInstant(Instant.parse("2026-09-11T04:00:00.123Z"))); + } + + @Test + void testMissingAndNullRespectNullability() throws Exception { + assertThat(JsonToFlussConverters.create(DataTypes.INT()).convert(null, "$.field")).isNull(); + assertThat(convert(DataTypes.INT(), "null")).isNull(); + assertThatThrownBy( + () -> + JsonToFlussConverters.create(DataTypes.INT().copy(false)) + .convert(null, "$.field")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("missing or null"); + assertThatThrownBy(() -> convert(DataTypes.INT().copy(false), "null")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("missing or null"); + } + + @Test + void testDecimalBoundsBeforeRescaling() throws Exception { + for (int scale : new int[] {100_000_000, Integer.MAX_VALUE, Integer.MIN_VALUE}) { + assertThatThrownBy( + () -> + JsonToFlussConverters.create(DataTypes.DECIMAL(5, 2)) + .convert( + DecimalNode.valueOf( + new BigDecimal(BigInteger.ONE, scale)), + "$.amount")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.amount") + .hasMessageContaining(scale > 0 ? "scale" : "precision"); + Decimal zero = + (Decimal) + JsonToFlussConverters.create(DataTypes.DECIMAL(5, 2)) + .convert( + DecimalNode.valueOf( + new BigDecimal(BigInteger.ZERO, scale)), + "$.amount"); + assertThat(zero.toBigDecimal()).isEqualTo(new BigDecimal("0.00")); + } + for (String json : new String[] {"12.3400", "1.234e1", "-12.3400", "1e2", "1e-2"}) { + Decimal decimal = (Decimal) convert(DataTypes.DECIMAL(5, 2), json); + assertThat(decimal.toBigDecimal()).isEqualByComparingTo(new BigDecimal(json)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"\\uD800", "\\uDC00", "a\\uD800b", "\\uD800\\uD800", "\\uDC00\\uD800"}) + void testRejectsUnpairedSurrogates(String escaped) { + for (DataType type : new DataType[] {DataTypes.STRING(), DataTypes.CHAR(8)}) { + assertThatThrownBy(() -> convert(type, "\"" + escaped + "\"")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("surrogate"); + } + assertThatThrownBy( + () -> + convert( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + "{\"?\":1,\"" + escaped + "\":2}")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("surrogate"); + } + + @Test + void testAcceptsPairedSurrogatesAndLiteralEscape() throws Exception { + String escapedEmoji = "\"\\uD83D\\uDE00\""; + assertThat(convert(DataTypes.STRING(), escapedEmoji)) + .isEqualTo(BinaryString.fromString("😀")); + assertThat(convert(DataTypes.CHAR(1), escapedEmoji)) + .isEqualTo(BinaryString.fromString("😀")); + assertThat(convert(DataTypes.STRING(), "\"\\\\uD800\"")) + .isEqualTo(BinaryString.fromString("\\uD800")); + } + + @ParameterizedTest + @ValueSource(ints = {3, 6, 9}) + void testTimestampStorageBoundaries(int precision) throws Exception { + long unitsPerSecond = precision == 3 ? 1_000 : precision == 6 ? 1_000_000 : 1_000_000_000; + long nanosPerUnit = 1_000_000_000 / unitsPerSecond; + for (long units : new long[] {Long.MIN_VALUE, -1, 0, 1, Long.MAX_VALUE}) { + Instant instant = + Instant.ofEpochSecond( + Math.floorDiv(units, unitsPerSecond), + Math.floorMod(units, unitsPerSecond) * nanosPerUnit); + for (boolean withZone : new boolean[] {false, true}) { + Object actual = + convert( + timestampType(precision, withZone), + timestampJson(instant, withZone)); + Object expected = + withZone + ? TimestampLtz.fromEpochMillis( + instant.toEpochMilli(), instant.getNano() % 1_000_000) + : TimestampNtz.fromMillis( + instant.toEpochMilli(), instant.getNano() % 1_000_000); + assertThat(actual).isEqualTo(expected); + if (withZone) { + assertThat(((TimestampLtz) actual).toInstant()).isEqualTo(instant); + } else { + assertThat(((TimestampNtz) actual).toLocalDateTime()) + .isEqualTo(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); + } + } + } + for (long limit : new long[] {Long.MIN_VALUE, Long.MAX_VALUE}) { + Instant outside = + Instant.ofEpochSecond( + Math.floorDiv(limit, unitsPerSecond), + Math.floorMod(limit, unitsPerSecond) * nanosPerUnit) + .plusNanos(limit < 0 ? -nanosPerUnit : nanosPerUnit); + for (boolean withZone : new boolean[] {false, true}) { + assertThatThrownBy( + () -> + convert( + timestampType(precision, withZone), + timestampJson(outside, withZone))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field") + .hasMessageContaining("timestamp exceeds storage range"); + } + } + } + + @Test + void testRejectsTimestampOverflowAndKeepsRepresentableDates() throws Exception { + for (boolean withZone : new boolean[] {false, true}) { + String suffix = withZone ? "Z" : ""; + for (String date : new String[] {"3000-01-01T00:00:00", "1600-01-01T00:00:00"}) { + assertThatThrownBy( + () -> + convert( + timestampType(9, withZone), + "\"" + date + suffix + "\"")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("timestamp exceeds storage range"); + assertThat(convert(timestampType(6, withZone), "\"" + date + suffix + "\"")) + .isNotNull(); + } + assertThatThrownBy( + () -> + convert( + timestampType(3, withZone), + "\"+999999999-01-01T00:00:00" + suffix + "\"")) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("timestamp exceeds storage range"); + } + } + + private static DataType timestampType(int precision, boolean withZone) { + return withZone ? DataTypes.TIMESTAMP_LTZ(precision) : DataTypes.TIMESTAMP(precision); + } + + private static String timestampJson(Instant instant, boolean withZone) { + return "\"" + + (withZone + ? instant.toString() + : LocalDateTime.ofInstant(instant, ZoneOffset.UTC).toString()) + + "\""; + } + + @ParameterizedTest + @MethodSource("invalidScalars") + void testRejectsCoercionOverflowAndPrecisionLoss(DataType type, String json) { + assertThatThrownBy(() -> convert(type, json)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("$.field"); + } + + private static Stream invalidScalars() { + return Stream.of( + Arguments.of(DataTypes.BOOLEAN(), "1"), + Arguments.of(DataTypes.INT(), "\"1\""), + Arguments.of(DataTypes.INT(), "1.0"), + Arguments.of(DataTypes.INT(), "2147483648"), + Arguments.of(DataTypes.TINYINT(), "128"), + Arguments.of(DataTypes.SMALLINT(), "-32769"), + Arguments.of(DataTypes.BIGINT(), "9223372036854775808"), + Arguments.of(DataTypes.FLOAT(), "1e39"), + Arguments.of(DataTypes.DOUBLE(), "1e309"), + Arguments.of(DataTypes.DECIMAL(5, 2), "1.234"), + Arguments.of(DataTypes.DECIMAL(5, 2), "1000"), + Arguments.of(DataTypes.CHAR(1), "\"ab\""), + Arguments.of(DataTypes.STRING(), "false"), + Arguments.of(DataTypes.BINARY(2), "\"AQ==\""), + Arguments.of(DataTypes.BYTES(), "\"!invalid!\""), + Arguments.of(DataTypes.DATE(), "\"2026-02-30\""), + Arguments.of(DataTypes.TIME(0), "\"00:00:00.001\""), + Arguments.of(DataTypes.TIME(3), "\"00:00:00.0001\""), + Arguments.of(DataTypes.TIMESTAMP(3), "\"2026-09-11T00:00:00.0001\""), + Arguments.of(DataTypes.TIMESTAMP_LTZ(3), "\"2026-09-11T00:00:00\"")); + } + + @Test + void testComplexTypesUseThePublicSchemaContract() throws Exception { + DataType array = DataTypes.ARRAY(DataTypes.INT()); + DataType map = DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()); + DataType row = DataTypes.ROW(DataTypes.FIELD("value", DataTypes.INT())); + for (DataType type : new DataType[] {array, map, row}) { + KafkaTableSchema.validateJsonType(type); + assertThat(JsonToFlussConverters.create(type)).isNotNull(); + } + assertThat(convert(array, "[1,null]")).isEqualTo(new GenericArray(new Object[] {1, null})); + assertThat(convert(map, "{\"key\":1}")) + .isEqualTo( + new GenericMap( + Collections.singletonMap(BinaryString.fromString("key"), 1))); + assertThat(convert(row, "{\"value\":1}")).isEqualTo(GenericRow.of(1)); + } + + @ParameterizedTest + @MethodSource("invalidMapKeyTypes") + void testInvalidMapKeysUseThePublicSchemaError(DataType type) { + assertThatThrownBy(() -> KafkaTableSchema.validateJsonType(type)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("only supports STRING map keys"); + assertThatThrownBy(() -> JsonToFlussConverters.create(type)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("only supports STRING map keys"); + } + + private static Stream invalidMapKeyTypes() { + return Stream.of( + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()), + DataTypes.MAP(DataTypes.CHAR(8), DataTypes.STRING()), + DataTypes.ROW(DataTypes.ARRAY(DataTypes.MAP(DataTypes.BYTES(), DataTypes.INT()))), + DataTypes.MAP( + DataTypes.STRING(), DataTypes.MAP(DataTypes.BOOLEAN(), DataTypes.INT()))); + } + + @Test + void testSchemaDepthUsesTheSharedBusinessFieldLimit() { + DataType type = DataTypes.INT(); + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + switch (depth % 3) { + case 0: + type = DataTypes.ROW(type); + break; + case 1: + type = DataTypes.ARRAY(type); + break; + default: + type = DataTypes.MAP(DataTypes.STRING(), type); + } + } + // The top-level business field starts at zero; each child consumes one level. + DataType atLimit = type; + KafkaTableSchema.validateJsonType(atLimit); + assertThat(JsonToFlussConverters.create(atLimit)).isNotNull(); + + DataType tooDeep = DataTypes.ROW(atLimit); + String message = + "Kafka JSON schema exceeds the maximum nesting depth of " + + KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH + + "."; + assertThatThrownBy(() -> KafkaTableSchema.validateJsonType(tooDeep)) + .isInstanceOf(InvalidTableException.class) + .hasMessage(message); + assertThatThrownBy(() -> JsonToFlussConverters.create(tooDeep)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessage(message); + } + + private static Object convert(DataType type, String json) throws Exception { + return JsonToFlussConverters.create(type).convert(MAPPER.readTree(json), "$.field"); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index c2aa5206640..5638c9a0666 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -162,6 +162,22 @@ public void testTopicIdMapping() { assertThat(mapper.toTableId(firstTableTopicId)).isZero(); } + @Test + public void testAliasTopicIdsKeepTableIdentityAndDetectRecreation() { + for (long tableId : new long[] {0L, 123L, Long.MAX_VALUE}) { + Uuid canonical = mapper.toTopicId(tableId); + Uuid alias = mapper.toAliasTopicId(tableId); + assertThat(alias) + .isNotIn(canonical, Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID) + .isEqualTo(new KafkaTopicMapper().toAliasTopicId(tableId)); + assertThat(mapper.isMappedTopicId(alias)).isTrue(); + assertThat(mapper.toTableId(alias)).isEqualTo(mapper.toTableId(canonical)); + } + assertThat(mapper.toAliasTopicId(124L)).isNotEqualTo(mapper.toAliasTopicId(123L)); + assertThatThrownBy(() -> mapper.toAliasTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + } + @Test public void testRejectsInvalidTopicIds() { assertThatThrownBy(() -> mapper.toTopicId(-1L)) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java new file mode 100644 index 00000000000..7657a7ff7d3 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaJsonSchemaTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests JSON table admission shared by Metadata and Produce. */ +class KafkaJsonSchemaTest { + private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); + + @Test + void testFlatJsonProjectsOnlyBusinessColumnsAroundOptionalSystemColumns() { + TableDescriptor descriptor = + descriptor( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column("amount", DataTypes.DECIMAL(8, 2)) + .column("_kafka_rescue", DataTypes.STRING()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), "binary") + .build(); + KafkaTableSchema schema = resolver.resolve(descriptor); + assertThat(schema).isEqualTo(KafkaTableSchema.resolve(descriptor)); + assertThat(schema.valueConversionType()).isEqualTo(KafkaValueConversionType.PARSE_JSON); + assertThat(schema.valueConversionLayout()).isEqualTo(KafkaValueConversionLayout.FLAT); + assertThat(schema.valueProjection().positions()).containsExactly(0, 3); + assertThat(schema.keyPosition()).isEqualTo(1); + assertThat(schema.headersPosition()).isEqualTo(2); + assertThat(schema.rescuePosition()).isEqualTo(4); + assertThat(schema.rowType()).isEqualTo(descriptor.getSchema().getRowType()); + } + + @Test + void testNestedJsonProjectsItsPhysicalValueContainer() { + RowType value = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.INT()), + DataTypes.FIELD("name", DataTypes.STRING())); + KafkaTableSchema schema = + resolver.resolve( + descriptor( + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES()) + .column("_kafka_value", value) + .column("_kafka_rescue", DataTypes.STRING()) + .build()) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + "nested") + .build()); + assertThat(schema.valueConversionLayout()).isEqualTo(KafkaValueConversionLayout.NESTED); + assertThat(schema.valueProjection().positions()).containsExactly(1); + assertThat(schema.valueProjection().dataTypeAt(0)).isEqualTo(value); + assertThat(schema.keyPosition()).isZero(); + assertThat(schema.rescuePosition()).isEqualTo(2); + } + + @Test + void testJsonContractRequiresExplicitEnablementAndIgnoresCustomProperties() { + TableDescriptor.Builder table = + descriptor(Schema.newBuilder().column("id", DataTypes.INT()).build()); + TableDescriptor disabled = table.property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + assertThat(resolver.isEnabled(disabled)).isFalse(); + assertThatThrownBy(() -> resolver.resolve(disabled)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); + for (String option : + Arrays.asList( + "kafka.value.format", + "kafka.key.fields", + "table.kafka.convert.value.type")) { + TableDescriptor withCustomProperty = + descriptor(disabled.getSchema()).customProperty(option, "json").build(); + assertThat(withCustomProperty.getCustomProperties()).containsEntry(option, "json"); + assertThat(resolver.resolve(withCustomProperty).valueConversionType()) + .isEqualTo(KafkaValueConversionType.PARSE_JSON); + } + assertThatThrownBy( + () -> + resolver.resolve( + descriptor(disabled.getSchema()) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE + .key(), + "string") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("Could not parse value 'string'"); + } + + @Test + void testNestedJsonRejectsUnmappedTopLevelBusinessColumns() { + assertThatThrownBy( + () -> + resolver.resolve( + descriptor( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD( + "id", + DataTypes + .INT()))) + .column("other", DataTypes.STRING()) + .build()) + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_LAYOUT + .key(), + "nested") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("no value mapping"); + } + + @ParameterizedTest + @ValueSource( + strings = { + "_kafka_key", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue", + "_kafka_value" + }) + void testNestedJsonRejectsReservedRootBusinessNames(String name) { + assertThatThrownBy( + () -> + resolver.resolve( + descriptor( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD( + name, + DataTypes + .STRING()))) + .build()) + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_LAYOUT + .key(), + "nested") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("Reserved name"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testComplexBusinessFieldsUseTheSharedContractInBothLayouts(boolean nested) { + for (DataType type : + Arrays.asList( + DataTypes.ARRAY(DataTypes.INT()), + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + DataTypes.ROW(DataTypes.FIELD("child", DataTypes.INT())))) { + Schema schema = + nested + ? Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("items", type))) + .build() + : Schema.newBuilder().column("items", type).build(); + TableDescriptor table = + descriptor(schema) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + nested ? "nested" : "flat") + .build(); + assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); + assertThat(resolver.resolve(table).valueConversionType()) + .isEqualTo(KafkaValueConversionType.PARSE_JSON); + } + } + + private static TableDescriptor.Builder descriptor(Schema schema) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "parse_json"); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 43eb7c9a40e..f2f5f847347 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -327,7 +327,7 @@ void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { .schema(schema) .distributedBy(1) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .customProperty("table.kafka.convert.value.type", "custom-value") + .customProperty("custom.kafka.convert.value.type", "custom-value") .customProperty("kafka.value.format", "custom-format") .customProperty("fluss.owner", "team") .build(); @@ -337,7 +337,8 @@ void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { assertThat(stored.getCustomProperties()).isEqualTo(descriptor.getCustomProperties()); assertThat(kafkaProperties(stored)) .isEqualTo(Collections.singletonMap("table.kafka.enabled", "true")); - assertThat(new KafkaTopicSchemaResolver().resolve(stored).valueProjection().positions()) + // This checks persisted DDL mapping; recursive gateway decoding is a later stage. + assertThat(KafkaTableSchema.resolve(stored).valueProjection().positions()) .containsExactly(1); set(path, Collections.singletonMap("table.kafka.convert.key.type", "binary")); assertThat(KafkaTableSchema.resolve(stored(path))) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java new file mode 100644 index 00000000000..7dd03050925 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -0,0 +1,1070 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TimestampType; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalMap; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.CloseableIterator; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; + +/** Verifies fixed-column Arrow records after transcoding resources have closed. */ +class ArrowKafkaRecordTranscoderTest { + private final ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFixedColumnsAndAllOptionalSystemColumnCombinations(boolean string) throws Exception { + // The value looks like JSON but binary/string must not interpret or rescue its members. + byte[] value = bytes("{\"_kafka_rescue\":\"消息\",\"_kafka_key\":123}"); + byte[] key = {(byte) 0xff, 0, (byte) 0xc3}; + Record record = + new Record( + 123L, + TimestampType.LOG_APPEND_TIME, + key, + value, + Arrays.asList( + new RecordHeader("duplicate", bytes("first")), + new RecordHeader("nullable", bytes("discarded")), + new RecordHeader("duplicate", bytes("last")), + new RecordHeader("nullable", null))); + for (int mask = 0; mask < 16; mask++) { + TableInfo table = envelope(string, mask); + TableDescriptor before = table.toTableDescriptor(); + KafkaTableSchema contract = KafkaTableSchema.resolve(before); + read( + transcoder.transcode(7, Collections.singletonList(record), table), + table, + 1, + row -> { + assertThat(row.getFieldCount()) + .isEqualTo(table.getRowType().getFieldCount()); + assertValue(row, contract.valueProjection().positionAt(0), string, value); + if (contract.keyPosition() >= 0) { + assertThat(row.getBytes(contract.keyPosition())).isEqualTo(key); + } + if (contract.headersPosition() >= 0) { + Map headers = + headers(row.getMap(contract.headersPosition())); + assertThat(headers).hasSize(2).containsKey("nullable"); + assertThat(headers.get("duplicate")).isEqualTo(bytes("last")); + assertThat(headers.get("nullable")).isNull(); + } + if (contract.metadataPosition() >= 0) { + assertMetadata(row, contract.metadataPosition(), 123L, 1, 7); + } + if (contract.rescuePosition() >= 0) { + assertThat(row.isNullAt(contract.rescuePosition())).isTrue(); + } + }); + assertThat(table.toTableDescriptor()).isEqualTo(before); + assertThat(before.getProperties()).doesNotContainKey("table.kafka.convert.key.type"); + assertThat(before.getCustomProperties()).isEmpty(); + } + assertThat(record.timestampType()).isEqualTo(TimestampType.LOG_APPEND_TIME); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.borrowedKey()).isEqualTo(key); + assertThat(record.borrowedValue()).isEqualTo(value); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testNullAndEmptyPayloadsRemainDistinct(boolean string) throws Exception { + TableInfo table = envelope(string, 7); + List records = + Arrays.asList( + new Record(Record.NO_TIMESTAMP, null, null, Collections.emptyList()), + new Record(0L, new byte[0], new byte[0], Collections.emptyList())); + AtomicInteger next = new AtomicInteger(); + read( + transcoder.transcode(0, records, table), + table, + 2, + row -> { + int value = table.getRowType().getFieldIndex("_kafka_value"); + int key = table.getRowType().getFieldIndex("_kafka_key"); + if (next.getAndIncrement() == 0) { + assertThat(row.isNullAt(value)).isTrue(); + assertThat(row.isNullAt(key)).isTrue(); + } else { + assertValue(row, value, string, new byte[0]); + assertThat(row.getBytes(key)).isEmpty(); + } + assertThat( + row.getMap(table.getRowType().getFieldIndex("_kafka_headers")) + .size()) + .isZero(); + assertThat(row.isNullAt(table.getRowType().getFieldIndex("_kafka_rescue"))) + .isTrue(); + }); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testKeyNullabilityAndRawKeyBytes(boolean string) throws Exception { + TableInfo nonNullableKeyTable = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES().copy(false)) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, + null, + bytes("valid"), + Collections.emptyList())), + nonNullableKeyTable)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_key' is NOT NULL"); + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES()) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record(1L, null, bytes("valid"), Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.isNullAt(0)).isTrue()); + byte[] invalidUtf8Key = {(byte) 0xff}; + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, + invalidUtf8Key, + bytes("valid"), + Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.getBytes(0)).isEqualTo(invalidUtf8Key)); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testHeaderNullabilityIsCheckedAfterLastValueWins(boolean nullableValue) throws Exception { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING() + .copy(false), + DataTypes.BYTES() + .copy( + nullableValue)) + .copy(true)) + .column("_kafka_value", DataTypes.STRING()) + .build(), + true) + .build()); + List overwrittenNull = + Arrays.asList( + new RecordHeader("same", null), + new RecordHeader("", new byte[0]), + new RecordHeader("same", bytes("last"))); + if (!nullableValue) { + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, null, bytes("v"), overwrittenNull)), + table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_headers"); + return; + } + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record(1L, null, bytes("v"), overwrittenNull)), + table), + table, + 1, + row -> { + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(2); + assertThat(map.get("same")).isEqualTo(bytes("last")); + assertThat(map.get("")).isEmpty(); + }); + Record finalNull = + new Record( + 1L, + null, + bytes("v"), + Arrays.asList( + new RecordHeader("same", bytes("first")), + new RecordHeader("same", null))); + read( + transcoder.transcode(0, Collections.singletonList(finalNull), table), + table, + 1, + row -> + assertThat(headers(row.getMap(0))) + .containsOnlyKeys("same") + .containsEntry("same", null)); + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record(1L, null, bytes("v"), Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.getMap(0).size()).isZero()); + } + + @Test + void testOmittedSystemColumnsIgnoreTimestampKeyAndHeaders() throws Exception { + TableInfo table = valueTable(false); + List records = new ArrayList<>(); + for (TimestampType type : TimestampType.values()) { + for (long timestamp : new long[] {Record.NO_TIMESTAMP, 0L, Long.MAX_VALUE}) { + records.add( + new Record( + timestamp, + type, + new byte[] {(byte) 0xff}, + bytes("v"), + Collections.singletonList(new RecordHeader("ignored", null)))); + } + } + read( + transcoder.transcode(0, records, table), + table, + records.size(), + row -> { + assertThat(row.getFieldCount()).isEqualTo(1); + assertThat(row.getString(0).toString()).isEqualTo("v"); + }); + } + + @Test + void testBinaryValuePreservesArbitraryBytesAcrossOutputPages() throws Exception { + byte[] value = new byte[32768]; + new Random(42L).nextBytes(value); + value[0] = (byte) 0xff; + TableInfo table = envelope(false, 4); + BytesView encoded = + transcoder.transcode( + 0, + Collections.singletonList( + new Record(1L, null, value, Collections.emptyList())), + table); + assertThat(encoded.getBytesLength()).isGreaterThan(4096); + read( + encoded, + table, + 1, + row -> { + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.getBytes(1)).isEqualTo(value); + }); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testValueNotNullConstraint(boolean string) { + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_value", + valueType(string).copy(false)) + .build(), + string) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, null, null, Collections.emptyList())), + table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value' is NOT NULL"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMetadataPreservesInputTimestampBatchTypeAndRequestedPartition(boolean string) + throws Exception { + long[] timestamps = {Record.NO_TIMESTAMP, 0L, 123L, 1728912345678L, Long.MAX_VALUE}; + TimestampType[] types = { + TimestampType.UNKNOWN, TimestampType.CREATE_TIME, TimestampType.LOG_APPEND_TIME + }; + int[] typeIds = {-1, 0, 1}; + List records = new ArrayList<>(); + for (TimestampType type : types) { + for (long timestamp : timestamps) { + records.add(new Record(timestamp, type, null, bytes("v"), Collections.emptyList())); + } + } + TableInfo table = metadataTable(string, true, true); + TableDescriptor before = table.toTableDescriptor(); + // Reuse the same converter and records across requests without retaining partition state. + for (int partition : new int[] {7, 0, 3}) { + AtomicInteger next = new AtomicInteger(); + read( + transcoder.transcode(partition, records, table), + table, + records.size(), + row -> { + int i = next.getAndIncrement(); + assertMetadata( + row, + 0, + timestamps[i % timestamps.length], + typeIds[i / timestamps.length], + partition); + assertValue(row, 1, string, bytes("v")); + }); + } + assertThat(table.toTableDescriptor()).isEqualTo(before); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMetadataTimestampNotNullIsRejectedAtSchemaResolution(boolean string) { + TableInfo table = metadataTable(string, true, false); + assertThatThrownBy( + () -> + transcoder.transcode( + 7, + Collections.singletonList( + new Record( + Record.NO_TIMESTAMP, + TimestampType.UNKNOWN, + null, + bytes("v"), + Collections.emptyList())), + table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_metadata.timestamp' is NOT NULL"); + } + + @Test + void testLegacyRecordDoesNotInferBatchTimestampType() throws Exception { + TableInfo table = metadataTable(true, true, true); + read( + transcoder.transcode( + 7, + Collections.singletonList( + new Record(123L, null, bytes("v"), Collections.emptyList())), + table), + table, + 1, + row -> assertMetadata(row, 0, 123L, -1, 7)); + } + + @Test + void testRejectsNegativePartitionId() { + assertThatThrownBy( + () -> + transcoder.transcode( + -1, + Collections.singletonList( + new Record( + Record.NO_TIMESTAMP, + null, + bytes("v"), + Collections.emptyList())), + envelope(false, 8))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Kafka partition ID must be non-negative"); + } + + @Test + void testRejectsLegacyMetadataOffsetField() { + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))) + .build(); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 123L, + TimestampType.CREATE_TIME, + null, + bytes("v"), + Collections.emptyList())), + table(descriptor(schema, true).build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining( + "_kafka_metadata must be ROW"); + } + + @Test + void testRejectsDisabledAndLegacyContractsAndEmptyPartitions() throws Exception { + List records = + Collections.singletonList( + new Record(1L, null, bytes("v"), Collections.emptyList())); + Schema fixed = Schema.newBuilder().column("_kafka_value", DataTypes.STRING()).build(); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + records, + table( + descriptor(fixed, true) + .property( + ConfigOptions.TABLE_KAFKA_ENABLED, + false) + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + records, + table( + descriptor( + Schema.newBuilder() + .column( + "body", + DataTypes.STRING()) + .build(), + true) + .build()))) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_value"); + TableInfo unrelatedCustomProperty = + table( + descriptor(fixed, true) + .customProperty("kafka.value.format", "string") + .build()); + read( + transcoder.transcode(0, records, unrelatedCustomProperty), + unrelatedCustomProperty, + records.size(), + row -> assertThat(row.getString(0).toString()).isEqualTo("v")); + assertThatThrownBy( + () -> transcoder.transcode(0, Collections.emptyList(), valueTable(false))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("empty"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testComplexJsonFailureDiscardsOutputAndCanBeReused(boolean nested) throws Exception { + RowType business = + DataTypes.ROW( + DataTypes.FIELD( + "customer", + DataTypes.ROW(DataTypes.FIELD("name", DataTypes.STRING()))), + DataTypes.FIELD("items", DataTypes.ARRAY(DataTypes.INT())), + DataTypes.FIELD( + "attributes", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW(DataTypes.FIELD("count", DataTypes.INT()))))); + Schema.Builder schema = + nested + ? Schema.newBuilder().column("_kafka_value", business) + : Schema.newBuilder().fromRowType(business); + schema.column("_kafka_metadata", metadataType(true)); + TableInfo table = + table( + descriptor(schema.build(), false) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + nested ? "nested" : "flat") + .build()); + KafkaTopicWritePlan plan = transcoder.prepare(table); + int partitionId = 3; + Record valid = + new Record( + 1L, + TimestampType.CREATE_TIME, + null, + bytes( + "{\"customer\":{\"name\":\"Alice\"},\"items\":[1,2]," + + "\"attributes\":{\"?\":{\"count\":3}}}"), + Collections.emptyList()); + for (String invalid : + Arrays.asList( + "{\"customer\":{\"name\":1}}", + "{\"customer\":{\"name\":\"Alice\",\"unknown\":1}}", + "{\"items\":[1,\"bad\"]}", + "{\"attributes\":[]}", + "{\"attributes\":{\"?\":{\"count\":1},\"\\ud800\":{\"count\":2}}}")) { + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + partitionId, + Arrays.asList( + valid, + new Record( + 2L, + null, + bytes(invalid), + Collections.emptyList())), + plan))); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + read( + transcoder.transcode(partitionId, Collections.singletonList(valid), plan), + table, + 1, + physical -> { + InternalRow row = nested ? physical.getRow(0, 3) : physical; + assertThat(row.getRow(0, 1).getString(0).toString()).isEqualTo("Alice"); + assertThat(row.getArray(1).size()).isEqualTo(2); + assertThat(row.getArray(1).getInt(1)).isEqualTo(2); + assertThat(row.getMap(2).keyArray().getString(0).toString()).isEqualTo("?"); + assertThat(row.getMap(2).valueArray().getRow(0, 1).getInt(0)).isEqualTo(3); + assertMetadata(physical, nested ? 1 : 3, 1L, 0, partitionId); + }); + } + } + + @Test + void testInvalidJsonMapKeysFailDuringPreparation() { + TableInfo table = + table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column( + "body", + DataTypes.MAP( + DataTypes.INT(), + DataTypes.STRING())) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.PARSE_JSON) + .build()); + assertThatThrownBy(() -> transcoder.prepare(table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("only supports STRING map keys"); + for (byte[] value : new byte[][] {null, bytes("{}")}) { + assertThatThrownBy( + () -> + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 1L, + null, + value, + Collections.emptyList())), + table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("only supports STRING map keys"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean string) + throws Exception { + TableInfo table = envelope(string, 15); + List records = new ArrayList<>(); + for (int i = 0; i < 2050; i++) { + records.add( + new Record( + i % 3 == 0 ? Record.NO_TIMESTAMP : i, + i % 3 == 0 + ? TimestampType.UNKNOWN + : (i % 3 == 1 + ? TimestampType.CREATE_TIME + : TimestampType.LOG_APPEND_TIME), + bytes("key-" + i), + bytes("消息-" + i), + Arrays.asList( + new RecordHeader("duplicate", null), + new RecordHeader("duplicate", bytes("header-" + i))))); + } + AtomicInteger next = new AtomicInteger(); + read( + transcoder.transcode(7, records, table), + table, + records.size(), + row -> { + int i = next.getAndIncrement(); + assertValue(row, 2, string, bytes("消息-" + i)); + assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(1); + assertThat(map.get("duplicate")).isEqualTo(bytes("header-" + i)); + assertThat(row.isNullAt(1)).isTrue(); + assertMetadata(row, 4, i % 3 == 0 ? Record.NO_TIMESTAMP : i, i % 3 - 1, 7); + }); + } + + @Test + void testOneRecordWithManyDistinctHeadersGrowsArrowMapVectors() throws Exception { + TableInfo table = envelope(false, 2); + List input = new ArrayList<>(); + for (int i = 0; i < 2050; i++) { + input.add(new RecordHeader("header-" + i, bytes("value-" + i))); + } + read( + transcoder.transcode( + 0, + Collections.singletonList(new Record(1L, null, bytes("v"), input)), + table), + table, + 1, + row -> { + Map map = headers(row.getMap(0)); + assertThat(map).hasSize(2050); + for (int i = 0; i < 2050; i++) { + assertThat(map.get("header-" + i)).isEqualTo(bytes("value-" + i)); + } + }); + } + + @Test + void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exception { + TableInfo table = envelope(false, 7); + byte[] key = new byte[1]; + byte[] value = new byte[1]; + byte[] headerValue = new byte[1]; + GenericRow row = + GenericRow.of( + GenericMap.of(BinaryString.fromString("header"), headerValue), + null, + value, + key); + BytesView encoded = + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + for (int i = 0; i < 3; i++) { + key[0] = (byte) i; + value[0] = (byte) (i + 10); + headerValue[0] = (byte) (i + 20); + consumer.append(row); + } + Arrays.fill(key, (byte) -1); + Arrays.fill(value, (byte) -1); + Arrays.fill(headerValue, (byte) -1); + row.setField(2, null); + }, + table); + AtomicInteger next = new AtomicInteger(); + read( + encoded, + table, + 3, + actual -> { + int i = next.getAndIncrement(); + assertThat(actual.getBytes(3)).containsExactly((byte) i); + assertThat(actual.getBytes(2)).containsExactly((byte) (i + 10)); + assertThat(headers(actual.getMap(0)).get("header")) + .containsExactly((byte) (i + 20)); + }); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDecodeFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable(boolean metadata) + throws Exception { + Schema.Builder schema = Schema.newBuilder().column("_kafka_value", DataTypes.STRING()); + if (metadata) { + schema.column("_kafka_metadata", metadataType(true)); + } + TableInfo table = table(descriptor(schema.build(), true).build()); + KafkaTopicWritePlan plan = transcoder.prepare(table); + Record valid = + new Record( + Record.NO_TIMESTAMP, + TimestampType.LOG_APPEND_TIME, + null, + bytes("valid"), + Collections.emptyList()); + for (byte[] invalid : new byte[][] {{(byte) 0xc3, 0x28}, {(byte) 0xff}}) { + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + 7, + Arrays.asList( + valid, + new Record( + 2L, + TimestampType.LOG_APPEND_TIME, + null, + invalid, + Collections.emptyList())), + plan))); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + } + read( + transcoder.transcode(3, Collections.singletonList(valid), plan), + table, + 1, + row -> { + assertThat(row.getString(0).toString()).isEqualTo("valid"); + if (metadata) { + assertMetadata(row, 1, Record.NO_TIMESTAMP, 1, 3); + } + }); + read( + transcoder.transcode( + 0, + Collections.singletonList( + new Record( + 2L, + TimestampType.LOG_APPEND_TIME, + null, + null, + Collections.emptyList())), + plan), + table, + 1, + row -> { + assertThat(row.isNullAt(0)).isTrue(); + if (metadata) { + assertMetadata(row, 1, 2L, 1, 0); + } + }); + } + + @Test + void testHeaderFailureAfterValidRecordHasNoPartialOutputAndRemainsUsable() throws Exception { + TableInfo invalidTable = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.BYTES().copy(false))) + .build(), + true) + .build()); + Record valid = + new Record( + 1L, + null, + bytes("v"), + Collections.singletonList(new RecordHeader("h", bytes("v")))); + Record invalid = + new Record( + 1L, + null, + bytes("v"), + Collections.singletonList(new RecordHeader("h", null))); + AtomicReference output = new AtomicReference<>(); + Throwable failure = + catchThrowable( + () -> + output.set( + transcoder.transcode( + 0, Arrays.asList(valid, invalid), invalidTable))); + assertThat(failure) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("_kafka_headers"); + assertThat(failure.getSuppressed()).isEmpty(); + assertThat(output).hasNullValue(); + TableInfo table = + table( + descriptor( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.BYTES())) + .build(), + true) + .build()); + read( + transcoder.transcode(0, Collections.singletonList(valid), table), + table, + 1, + row -> assertThat(headers(row.getMap(1)).get("h")).isEqualTo(bytes("v"))); + } + + @Test + void testArrowWriterFailureClosesAllocatorWithoutLeaking() { + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encode( + Collections.singletonList(GenericRow.of(123)), + valueTable(false))); + assertThat(failure).isInstanceOf(ClassCastException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + + @Test + void testStreamingSourceFailureClosesAllocator() { + IOException expected = new IOException("source failed after the first row"); + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + consumer.append( + GenericRow.of( + BinaryString.fromString( + "valid"))); + throw expected; + }, + valueTable(false))); + assertThat(failure).isSameAs(expected); + assertThat(failure.getSuppressed()).isEmpty(); + } + + private static TableInfo valueTable(boolean notNull) { + return table( + descriptor( + Schema.newBuilder() + .column("_kafka_value", DataTypes.STRING().copy(!notNull)) + .build(), + true) + .build()); + } + + private static TableInfo envelope(boolean string, int mask) { + Schema.Builder schema = Schema.newBuilder(); + if ((mask & 2) != 0) { + schema.column("_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + } + if ((mask & 4) != 0) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + schema.column("_kafka_value", valueType(string)); + if ((mask & 1) != 0) { + schema.column("_kafka_key", DataTypes.BYTES()); + } + if ((mask & 8) != 0) { + schema.column("_kafka_metadata", metadataType(true)); + } + return table(descriptor(schema.build(), string).build()); + } + + private static DataType metadataType(boolean nullableTimestamp) { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3).copy(nullableTimestamp)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + + private static TableInfo metadataTable( + boolean string, boolean nullableRow, boolean nullableTimestamp) { + return table( + descriptor( + Schema.newBuilder() + .column( + "_kafka_metadata", + metadataType(nullableTimestamp).copy(nullableRow)) + .column("_kafka_value", valueType(string)) + .build(), + string) + .build()); + } + + private static DataType valueType(boolean string) { + return string ? DataTypes.STRING() : DataTypes.BYTES(); + } + + private static TableDescriptor.Builder descriptor(Schema schema, boolean string) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(8) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + string ? KafkaValueConversionType.STRING : KafkaValueConversionType.BINARY); + } + + private static TableInfo table(TableDescriptor descriptor) { + return TableInfo.of(TablePath.of("fluss", "topic"), 42L, 3, descriptor, null, 1L, 1L); + } + + private static Map headers(InternalMap map) { + Map result = new LinkedHashMap<>(); + for (int i = 0; i < map.size(); i++) { + assertThat(map.keyArray().isNullAt(i)).isFalse(); + String key = map.keyArray().getString(i).toString(); + assertThat(result).doesNotContainKey(key); + result.put(key, map.valueArray().isNullAt(i) ? null : map.valueArray().getBytes(i)); + } + return result; + } + + private static void assertValue(InternalRow row, int position, boolean string, byte[] value) { + if (string) { + assertThat(row.getString(position).toString()) + .isEqualTo(new String(value, StandardCharsets.UTF_8)); + } else { + assertThat(row.getBytes(position)).isEqualTo(value); + } + } + + private static void assertMetadata( + InternalRow row, int position, long timestamp, int timestampType, int partition) { + assertThat(row.isNullAt(position)).isFalse(); + InternalRow metadata = row.getRow(position, 3); + assertThat(metadata.getFieldCount()).isEqualTo(3); + if (timestamp == Record.NO_TIMESTAMP) { + assertThat(metadata.isNullAt(0)).isTrue(); + } else { + assertThat(metadata.isNullAt(0)).isFalse(); + assertThat(metadata.getTimestampLtz(0, 3).getEpochMillisecond()).isEqualTo(timestamp); + } + assertThat(metadata.isNullAt(1)).isFalse(); + assertThat(metadata.getInt(1)).isEqualTo(timestampType); + assertThat(metadata.isNullAt(2)).isFalse(); + assertThat(metadata.getInt(2)).isEqualTo(partition); + } + + private static void read( + BytesView encoded, TableInfo table, int count, Consumer verify) + throws Exception { + ByteBuf buffer = encoded.getByteBuf(); + try { + LogRecordBatch batch = + MemoryLogRecords.pointToByteBuffer(buffer.nioBuffer()) + .batches() + .iterator() + .next(); + batch.ensureValid(); + assertThat(batch.schemaId()).isEqualTo((short) table.getSchemaId()); + assertThat(batch.getRecordCount()).isEqualTo(count); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + table.getRowType(), + table.getSchemaId(), + new TestingSchemaGetter( + new SchemaInfo( + table.getSchema(), table.getSchemaId()))); + CloseableIterator records = batch.records(context)) { + for (int i = 0; i < count; i++) { + assertThat(records.hasNext()).isTrue(); + verify.accept(records.next().getRow()); + } + assertThat(records.hasNext()).isFalse(); + } + } finally { + buffer.release(); + } + } + + private static byte[] bytes(String text) { + return text.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java new file mode 100644 index 00000000000..f5beaae2b61 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/KafkaCompiledWritePlanCacheTest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class KafkaCompiledWritePlanCacheTest { + + @Test + void testReusesSameIdentityAndSeparatesEquivalentAndChangedInstances() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo original = tableInfo("orders", 1L, 1, "v1"); + + KafkaTopicWritePlan first = + cache.getOrCompile(original, () -> compile(original, compileCount)); + KafkaTopicWritePlan reused = + cache.getOrCompile(original, () -> compile(original, compileCount)); + assertThat(reused).isSameAs(first); + assertThat(cache.getIfPresent(original)).isSameAs(first); + + // The authoritative TableInfo cache owns canonical instances. A separately constructed, + // structurally equivalent object is conservatively treated as a new metadata snapshot. + TableInfo equivalent = tableInfo("orders", 1L, 1, "v1"); + assertThat(cache.getIfPresent(equivalent)).isNull(); + KafkaTopicWritePlan equivalentPlan = + cache.getOrCompile(equivalent, () -> compile(equivalent, compileCount)); + assertThat(equivalentPlan).isNotSameAs(first); + + TableInfo propertyChange = tableInfo("orders", 1L, 1, "v2"); + KafkaTopicWritePlan propertyPlan = + cache.getOrCompile(propertyChange, () -> compile(propertyChange, compileCount)); + TableInfo schemaChange = tableInfo("orders", 1L, 2, "v2"); + KafkaTopicWritePlan schemaPlan = + cache.getOrCompile(schemaChange, () -> compile(schemaChange, compileCount)); + TableInfo recreated = tableInfo("orders", 2L, 2, "v2"); + KafkaTopicWritePlan recreatedPlan = + cache.getOrCompile(recreated, () -> compile(recreated, compileCount)); + + assertThat(propertyPlan).isNotSameAs(first); + assertThat(schemaPlan).isNotSameAs(propertyPlan); + assertThat(recreatedPlan).isNotSameAs(schemaPlan); + assertThat(compileCount).hasValue(5); + } + + @Test + void testEquivalentInstanceCannotInvalidateCurrentIdentity() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo current = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan currentPlan = + cache.getOrCompile(current, () -> compile(current, compileCount)); + + cache.invalidate(tableInfo("orders", 1L, 1, "v1")); + + assertThat(cache.getOrCompile(current, () -> compile(current, compileCount))) + .isSameAs(currentPlan); + assertThat(compileCount).hasValue(1); + } + + @Test + void testInvalidationAndCompileFailureRecovery() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + AtomicInteger compileCount = new AtomicInteger(); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan first = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + + cache.invalidate(tableInfo); + KafkaTopicWritePlan second = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + assertThat(second).isNotSameAs(first); + + TableInfo failingTable = tableInfo("payments", 2L, 1, "v1"); + assertThatThrownBy( + () -> + cache.getOrCompile( + failingTable, + () -> { + compileCount.incrementAndGet(); + throw new IllegalStateException("compile failed"); + })) + .isInstanceOf(IllegalStateException.class) + .hasMessage("compile failed"); + assertThat(cache.getOrCompile(failingTable, () -> compile(failingTable, compileCount))) + .isNotNull(); + assertThat(compileCount).hasValue(4); + } + + @Test + void testConcurrentColdCompileIsSingleFlight() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + AtomicInteger compileCount = new AtomicInteger(); + CountDownLatch compilerStarted = new CountDownLatch(1); + CountDownLatch releaseCompiler = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + futures.add( + executor.submit( + () -> + cache.getOrCompile( + tableInfo, + () -> { + compileCount.incrementAndGet(); + compilerStarted.countDown(); + await(releaseCompiler); + return compile(tableInfo); + }))); + } + compilerStarted.await(); + releaseCompiler.countDown(); + + KafkaTopicWritePlan first = futures.get(0).get(); + for (Future future : futures) { + assertThat(future.get()).isSameAs(first); + } + assertThat(compileCount).hasValue(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testInvalidationDoesNotAllowInFlightCompileToResurrect() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + AtomicInteger compileCount = new AtomicInteger(); + CountDownLatch compilerStarted = new CountDownLatch(1); + CountDownLatch releaseCompiler = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future compiling = + executor.submit( + () -> + cache.getOrCompile( + tableInfo, + () -> { + compileCount.incrementAndGet(); + compilerStarted.countDown(); + await(releaseCompiler); + return compile(tableInfo); + })); + compilerStarted.await(); + Future invalidating = executor.submit(() -> cache.invalidate(tableInfo)); + releaseCompiler.countDown(); + + KafkaTopicWritePlan planBeforeInvalidation = compiling.get(); + invalidating.get(); + KafkaTopicWritePlan planAfterInvalidation = + cache.getOrCompile(tableInfo, () -> compile(tableInfo, compileCount)); + + assertThat(planAfterInvalidation).isNotSameAs(planBeforeInvalidation); + assertThat(compileCount).hasValue(2); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCompiledAssemblerIsSafeToShare() throws Exception { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(10); + TableInfo tableInfo = tableInfo("orders", 1L, 1, "v1"); + KafkaTopicWritePlan plan = cache.getOrCompile(tableInfo, () -> compile(tableInfo)); + ExecutorService executor = Executors.newFixedThreadPool(8); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + final int id = i; + futures.add( + executor.submit( + () -> + plan.rowAssembler() + .assemble( + new Object[] { + Integer.toString(id) + .getBytes( + StandardCharsets.UTF_8) + }, + new Record( + id, null, null, new ArrayList<>()), + 0) + .getField(0))); + } + for (int i = 0; i < futures.size(); i++) { + assertThat((byte[]) futures.get(i).get()) + .isEqualTo(Integer.toString(i).getBytes(StandardCharsets.UTF_8)); + } + } finally { + executor.shutdownNow(); + } + } + + @Test + void testCacheSizeIsBounded() { + KafkaCompiledWritePlanCache cache = new KafkaCompiledWritePlanCache(2); + for (int i = 0; i < 10; i++) { + TableInfo tableInfo = tableInfo("topic_" + i, i + 1L, 1, "v1"); + cache.getOrCompile(tableInfo, () -> compile(tableInfo)); + } + + assertThat(cache.estimatedSize()).isLessThanOrEqualTo(2); + } + + private static KafkaTopicWritePlan compile(TableInfo tableInfo, AtomicInteger compileCount) { + compileCount.incrementAndGet(); + return compile(tableInfo); + } + + private static KafkaTopicWritePlan compile(TableInfo tableInfo) { + KafkaTableSchema schema = + new KafkaTopicSchemaResolver().resolve(tableInfo.toTableDescriptor()); + return new KafkaTopicWritePlan(tableInfo, schema); + } + + private static TableInfo tableInfo( + String tableName, long tableId, int schemaId, String propertyValue) { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), + propertyValue.equals("v1") ? "binary" : "BINARY") + .build(); + return TableInfo.of( + TablePath.of("kafka", tableName), tableId, schemaId, descriptor, null, 1L, 1L); + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for test compiler.", e); + } + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java index fa0174cb492..7b2af609728 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java @@ -80,6 +80,7 @@ public final class NettyServer implements RpcServer { private EventLoopGroup selectorGroup; private volatile boolean isRunning; + private CompletableFuture closeFuture; public NettyServer( Configuration conf, @@ -104,7 +105,8 @@ public NettyServer( } @Override - public void start() throws IOException { + public synchronized void start() throws IOException { + checkState(closeFuture == null, "Netty server has already been closed."); checkState(bindChannels.isEmpty(), "Netty server has already been initialized."); int numNetworkThreads = conf.getInt(ConfigOptions.NETTY_SERVER_NUM_NETWORK_THREADS); int numWorkerThreads = conf.getInt(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS); @@ -298,34 +300,50 @@ public ScheduledExecutorService getScheduledExecutor() { @Override public CompletableFuture closeAsync() { - if (!isRunning) { - return CompletableFuture.completedFuture(null); + synchronized (this) { + if (closeFuture != null) { + return closeFuture; + } + isRunning = false; + + CompletableFuture channelShutdownFuture = + FutureUtils.completeAll( + bindChannels.stream() + .map(NettyUtils::shutdownChannel) + .collect(Collectors.toList())); + CompletableFuture workerShutdownFuture = + workerPool == null + ? CompletableFuture.completedFuture(null) + : workerPool.closeAsync(); + // Event loops must remain active while accepted requests drain: their responses and + // backpressure updates still use those loops. Close the loops only after both the + // listening channels and request workers have stopped. + CompletableFuture transportShutdownFuture = + FutureUtils.composeAfterwards( + CompletableFuture.allOf(channelShutdownFuture, workerShutdownFuture), + () -> + CompletableFuture.allOf( + shutdownGroup(acceptorGroup), + shutdownGroup(selectorGroup))); + closeFuture = closeProtocolsAfter(transportShutdownFuture, protocols); + return closeFuture; } + } - isRunning = false; - - // Stop accepting new connections. - CompletableFuture channelShutdownFuture = - FutureUtils.completeAll( - bindChannels.stream() - .map(NettyUtils::shutdownChannel) - .collect(Collectors.toList())); - // Drain the requests that were already accepted. - CompletableFuture workerShutdownFuture; - if (workerPool != null) { - workerShutdownFuture = workerPool.closeAsync(); - } else { - workerShutdownFuture = CompletableFuture.completedFuture(null); - } - // Shut down the event loops only after both of the above completed. The event loops - // terminate without a quiet period, so they must outlive the tasks still submitted to - // them: the bind channel close listeners, and the responses (and backpressure updates) - // of the requests drained by the worker pool. Shutting the event loops down closes the - // remaining client connections. + static CompletableFuture closeProtocolsAfter( + CompletableFuture shutdownFuture, List protocols) { return FutureUtils.composeAfterwards( - CompletableFuture.allOf(channelShutdownFuture, workerShutdownFuture), - () -> - CompletableFuture.allOf( - shutdownGroup(acceptorGroup), shutdownGroup(selectorGroup))); + shutdownFuture, + () -> { + List> closeFutures = new ArrayList<>(); + for (NetworkProtocolPlugin protocol : protocols) { + try { + closeFutures.add(protocol.closeAsync()); + } catch (Throwable failure) { + closeFutures.add(FutureUtils.completedExceptionally(failure)); + } + } + return FutureUtils.completeAll(closeFutures); + }); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java index f60202228ef..c8921473fad 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/NetworkProtocolPlugin.java @@ -25,6 +25,7 @@ import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import java.util.List; +import java.util.concurrent.CompletableFuture; /** A network protocol plugin that provides the server side implementation of a network protocol. */ public interface NetworkProtocolPlugin extends Plugin { @@ -52,4 +53,9 @@ public interface NetworkProtocolPlugin extends Plugin { * protocol. */ RequestHandler createRequestHandler(RpcGatewayService service); + + /** Closes protocol-owned resources after transport and request processors have stopped. */ + default CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } } diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java index 6e442187019..528f9fbcef8 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/NettyServerTest.java @@ -25,6 +25,7 @@ import org.apache.fluss.config.MemorySize; import org.apache.fluss.metrics.groups.MetricGroup; import org.apache.fluss.metrics.util.NOPMetricsGroup; +import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.RpcServer; import org.apache.fluss.rpc.TestingGatewayService; import org.apache.fluss.rpc.messages.ApiMessage; @@ -33,6 +34,8 @@ import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; import org.apache.fluss.rpc.netty.client.NettyClient; import org.apache.fluss.rpc.protocol.ApiKeys; +import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; +import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; import org.apache.fluss.utils.NetUtils; import org.junit.jupiter.api.Test; @@ -40,11 +43,14 @@ import java.net.ConnectException; import java.net.Socket; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.apache.fluss.utils.NetUtils.getAvailablePort; @@ -54,6 +60,24 @@ /** Test for {@link NettyServer}. */ public class NettyServerTest { + @Test + void testCannotStartAfterClose() { + MetricGroup metricGroup = NOPMetricsGroup.newInstance(); + NettyServer server = + new NettyServer( + new Configuration(), + Endpoint.fromListenersString("FLUSS://localhost:0"), + new TestingGatewayService(), + metricGroup, + RequestsMetrics.createCoordinatorServerRequestMetrics(metricGroup)); + + server.closeAsync().join(); + + assertThatThrownBy(server::start) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already been closed"); + } + @Test void testPortAsZero() throws Exception { List endpoints = Endpoint.fromListenersString("FLUSS://localhost:0"); @@ -210,4 +234,94 @@ public CompletableFuture apiVersions( } } } + + @Test + void testProtocolsCloseOnlyAfterTransportShutdown() { + CompletableFuture transportShutdown = new CompletableFuture<>(); + AtomicInteger closeCalls = new AtomicInteger(); + NetworkProtocolPlugin protocol = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + return CompletableFuture.completedFuture(null); + }); + + CompletableFuture result = + NettyServer.closeProtocolsAfter( + transportShutdown, Collections.singletonList(protocol)); + assertThat(closeCalls).hasValue(0); + assertThat(result).isNotDone(); + + transportShutdown.complete(null); + result.join(); + assertThat(closeCalls).hasValue(1); + } + + @Test + void testAllProtocolsCloseAfterExceptionalTransportShutdown() { + CompletableFuture transportShutdown = new CompletableFuture<>(); + AtomicInteger closeCalls = new AtomicInteger(); + RuntimeException asyncFailure = new RuntimeException("async protocol close failure"); + RuntimeException synchronousFailure = + new RuntimeException("synchronous protocol close failure"); + NetworkProtocolPlugin asyncFailing = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(asyncFailure); + return failed; + }); + NetworkProtocolPlugin synchronouslyFailing = + testingProtocol( + () -> { + closeCalls.incrementAndGet(); + throw synchronousFailure; + }); + CompletableFuture result = + NettyServer.closeProtocolsAfter( + transportShutdown, Arrays.asList(asyncFailing, synchronouslyFailing)); + + transportShutdown.completeExceptionally(new RuntimeException("transport failure")); + + assertThatThrownBy(result::join) + .hasStackTraceContaining("transport failure") + .hasStackTraceContaining("async protocol close failure") + .hasStackTraceContaining("synchronous protocol close failure"); + assertThat(closeCalls).hasValue(2); + } + + private static NetworkProtocolPlugin testingProtocol( + Supplier> closeAction) { + return new NetworkProtocolPlugin() { + @Override + public String name() { + return "testing"; + } + + @Override + public void setup(Configuration conf) {} + + @Override + public List listenerNames() { + return Collections.emptyList(); + } + + @Override + public ChannelHandler createChannelHandler( + RequestChannel[] requestChannels, String listenerName) { + return null; + } + + @Override + public RequestHandler createRequestHandler(RpcGatewayService service) { + return null; + } + + @Override + public CompletableFuture closeAsync() { + return closeAction.get(); + } + }; + } }