diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/FlussSparkSessionExtensions.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/FlussSparkSessionExtensions.scala index dc12ed50342..5c3faf8638c 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/FlussSparkSessionExtensions.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/FlussSparkSessionExtensions.scala @@ -17,7 +17,8 @@ package org.apache.fluss.spark -import org.apache.fluss.spark.catalyst.analysis.FlussProcedureResolver +import org.apache.fluss.spark.catalyst.analysis.{FlussProcedureResolver, FlussTableValuedFunctionResolver} +import org.apache.fluss.spark.catalyst.plans.logical.FlussTableValuedFunctions import org.apache.fluss.spark.execution.FlussStrategy import org.apache.spark.sql.SparkSessionExtensions @@ -32,6 +33,14 @@ class FlussSparkSessionExtensions extends (SparkSessionExtensions => Unit) { // analyzer extensions extensions.injectResolutionRule(spark => FlussProcedureResolver(spark)) + extensions.injectResolutionRule(spark => FlussTableValuedFunctionResolver(spark)) + + // table function extensions + FlussTableValuedFunctions.supportedFnNames.foreach { + fnName => + extensions.injectTableFunction( + FlussTableValuedFunctions.getTableValueFunctionInjection(fnName)) + } // planner extensions extensions.injectPlannerStrategy(spark => FlussStrategy(spark)) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala index aac6a698daf..24fbfb20483 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala @@ -44,6 +44,30 @@ object SparkFlussConf { .defaultValue(StartUpMode.FULL.toString) .withDescription("The start up mode when read Fluss table.") + val SCAN_INCREMENTAL_START_TIMESTAMP: ConfigOption[String] = + ConfigBuilder + .key("scan.incremental.start.timestamp") + .stringType() + .noDefaultValue() + .withDescription( + "Enables an incremental (time-range) batch read and sets the inclusive lower bound of " + + "the window. Accepts either epoch milliseconds (e.g. '1678883047356') or a " + + "'yyyy-MM-dd HH:mm:ss' datetime string (e.g. '2023-12-09 23:09:12') interpreted in " + + "the Spark session time zone. Batch read only; it has no effect on streaming reads.") + + val SCAN_INCREMENTAL_END_TIMESTAMP: ConfigOption[String] = + ConfigBuilder + .key("scan.incremental.end.timestamp") + .stringType() + .noDefaultValue() + .withDescription( + "The exclusive upper bound of an incremental (time-range) batch read, yielding a " + + "left-closed right-open '[start, end)' window. Accepts epoch milliseconds or a " + + "'yyyy-MM-dd HH:mm:ss' datetime string interpreted in the Spark session time zone; " + + "when unset the read runs up to the latest committed data. Setting it without " + + "'scan.incremental.start.timestamp' fails fast, as does a window whose start is not " + + "strictly before its end.") + val SCAN_POLL_TIMEOUT: ConfigOption[Duration] = ConfigBuilder .key("scan.poll.timeout") diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkTable.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkTable.scala index 144db03aeaa..b2bd2fdf508 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkTable.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkTable.scala @@ -41,30 +41,36 @@ class SparkTable( with SupportsWrite with SQLConfHelper { - private def populateSparkConf(flussConfig: FlussConfiguration): Unit = { + /** + * Merges the current `spark.sql.fluss.*` session configuration onto a copy of the catalog + * configuration, which is shared by every table of this catalog and must not be mutated. + */ + private def configWithSessionConfs(): FlussConfiguration = { + val merged = new FlussConfiguration(flussConfig) conf.getAllConfs .filter(_._1.startsWith(SparkFlussConf.SPARK_FLUSS_CONF_PREFIX)) .foreach { case (k, v) => - flussConfig.setString(k.substring(SparkFlussConf.SPARK_FLUSS_CONF_PREFIX.length), v) + merged.setString(k.substring(SparkFlussConf.SPARK_FLUSS_CONF_PREFIX.length), v) } + merged } override def newWriteBuilder(logicalWriteInfo: LogicalWriteInfo): WriteBuilder = { - populateSparkConf(flussConfig) + val config = configWithSessionConfs() if (tableInfo.getPrimaryKeys.isEmpty) { - new FlussAppendWriteBuilder(tablePath, logicalWriteInfo.schema(), flussConfig) + new FlussAppendWriteBuilder(tablePath, logicalWriteInfo.schema(), config) } else { - new FlussUpsertWriteBuilder(tablePath, logicalWriteInfo.schema(), flussConfig) + new FlussUpsertWriteBuilder(tablePath, logicalWriteInfo.schema(), config) } } override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { - populateSparkConf(flussConfig) + val config = configWithSessionConfs() if (tableInfo.getPrimaryKeys.isEmpty) { - new FlussAppendScanBuilder(tablePath, tableInfo, options, flussConfig) + new FlussAppendScanBuilder(tablePath, tableInfo, options, config) } else { - new FlussUpsertScanBuilder(tablePath, tableInfo, options, flussConfig) + new FlussUpsertScanBuilder(tablePath, tableInfo, options, config) } } } diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/analysis/FlussTableValuedFunctionResolver.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/analysis/FlussTableValuedFunctionResolver.scala new file mode 100644 index 00000000000..62f0faab691 --- /dev/null +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/analysis/FlussTableValuedFunctionResolver.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark.catalyst.analysis + +import org.apache.fluss.spark.catalyst.plans.logical.{FlussTableValuedFunctions, FlussTableValueFunction} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule + +/** + * Resolution rule for Fluss table-valued functions. The injected table function builder produces an + * unresolved [[FlussTableValueFunction]]; once its arguments are resolved this rule rewrites it + * into a plain DataSourceV2 relation carrying the derived scan options. + */ +case class FlussTableValuedFunctionResolver(sparkSession: SparkSession) extends Rule[LogicalPlan] { + + override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsDown { + case func: FlussTableValueFunction if func.args.forall(_.resolved) => + FlussTableValuedFunctions.resolveFlussTableValuedFunction(sparkSession, func) + } +} diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala new file mode 100644 index 00000000000..c8001ef6a26 --- /dev/null +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark.catalyst.plans.logical + +import org.apache.fluss.spark.{SparkFlussConf, SparkTable} +import org.apache.fluss.spark.catalyst.plans.logical.FlussTableValuedFunctions._ +import org.apache.fluss.spark.read.FlussOffsetInitializers + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.FunctionIdentifier +import org.apache.spark.sql.catalyst.analysis.FunctionRegistryBase +import org.apache.spark.sql.catalyst.analysis.TableFunctionRegistry.TableFunctionBuilder +import org.apache.spark.sql.catalyst.expressions.{Attribute, CurrentTimestamp, Expression, ExpressionInfo, RuntimeReplaceable} +import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} +import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{DateType, IntegerType, LongType, ShortType, StringType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +import java.time.{LocalDate, LocalDateTime, ZoneId, ZoneOffset} + +import scala.collection.JavaConverters._ +import scala.util.control.NonFatal + +/** + * Fluss table-valued functions (TVFs), usable from pure SQL. A call is translated into the `scan.*` + * options the DataFrame API accepts and resolved into a plain [[DataSourceV2Relation]], so it is + * scoped to the single query and keeps projection, filter push down and metrics working. + */ +object FlussTableValuedFunctions { + + val INCREMENTAL_BETWEEN_TIMESTAMP = "fluss_incremental_between_timestamp" + + val supportedFnNames: Seq[String] = Seq(INCREMENTAL_BETWEEN_TIMESTAMP) + + private type TableFunctionDescription = + (FunctionIdentifier, ExpressionInfo, TableFunctionBuilder) + + def getTableValueFunctionInjection(fnName: String): TableFunctionDescription = { + val (info, builder) = fnName match { + case INCREMENTAL_BETWEEN_TIMESTAMP => + FunctionRegistryBase.build[IncrementalBetweenTimestamp](fnName, since = None) + case _ => + throw new IllegalArgumentException( + s"Function $fnName isn't a supported Fluss table valued function.") + } + (FunctionIdentifier(fnName), info, builder) + } + + /** Resolves a TVF call into a relation over the referenced Fluss table. */ + def resolveFlussTableValuedFunction( + spark: SparkSession, + tvf: FlussTableValueFunction): LogicalPlan = { + val args = tvf.args + val sessionState = spark.sessionState + val catalogManager = sessionState.catalogManager + + if (args.isEmpty) { + throw new IllegalArgumentException( + s"${tvf.fnName} requires a table identifier as its first argument.") + } + + // Parse the arguments first, so an argument error does not depend on the table being resolvable. + val options = tvf.parseArgs(args.tail) + + val tableArg = args.head.eval() + if (tableArg == null) { + throw new IllegalArgumentException( + s"The first argument of ${tvf.fnName} must be a non-null table identifier.") + } + val tableIdentifier = tableArg.toString + + val (catalogName, namespace, tableName) = + sessionState.sqlParser.parseMultipartIdentifier(tableIdentifier) match { + case Seq(table) => + (catalogManager.currentCatalog.name(), catalogManager.currentNamespace, table) + case Seq(db, table) => (catalogManager.currentCatalog.name(), Array(db), table) + case Seq(catalog, db, table) => (catalog, Array(db), table) + case _ => + throw new IllegalArgumentException( + s"Invalid table identifier '$tableIdentifier' for ${tvf.fnName}. Expected " + + "'table', 'database.table' or 'catalog.database.table'.") + } + val fullTableIdentifier = (catalogName +: namespace :+ tableName).mkString(".") + + val catalogPlugin = catalogManager.catalog(catalogName) + if (!catalogPlugin.isInstanceOf[TableCatalog]) { + throw new IllegalArgumentException( + s"${tvf.fnName} requires a table catalog, but catalog '$catalogName' is " + + s"${catalogPlugin.getClass.getName}.") + } + val tableCatalog = catalogPlugin.asInstanceOf[TableCatalog] + val ident = Identifier.of(namespace, tableName) + val table = tableCatalog.loadTable(ident) + if (!table.isInstanceOf[SparkTable]) { + throw new IllegalArgumentException( + s"${tvf.fnName} only supports Fluss tables, but '$fullTableIdentifier' is " + + s"backed by ${table.getClass.getName}.") + } + + DataSourceV2Relation.create( + table, + Some(tableCatalog), + Some(ident), + new CaseInsensitiveStringMap(options.asJava)) + } + + /** + * Normalizes a timestamp argument, which may be any constant expression, to the string form the + * `scan.incremental.*` options accept: a STRING is passed through (the option layer reads both + * epoch millis and `yyyy-MM-dd HH:mm:ss`), an integral value is epoch millis, a DATE becomes the + * start of that day, and TIMESTAMP / TIMESTAMP_NTZ are converted from Spark's internal + * microseconds. + */ + private[logical] def toTimestampOptionValue(fnName: String, expr: Expression): String = { + // RuntimeReplaceable expressions (e.g. the `-` in `now() - INTERVAL 1 HOUR`) only become + // evaluable once the optimizer rewrites them, which has not happened yet during analysis. + val evaluable = expr.transformUp { case r: RuntimeReplaceable => r.replacement } + + val value = + try { + evaluable.eval() + } catch { + case NonFatal(e) => + throw new IllegalArgumentException( + s"Failed to evaluate the timestamp argument '${expr.sql}' of $fnName. It must be a " + + "constant expression; literals and datetime functions such as now() or " + + "unix_timestamp() are supported, references to table columns are not.", + e + ) + } + if (value == null) { + throw new IllegalArgumentException(s"Timestamp arguments of $fnName must not be null.") + } + val normalized = evaluable.dataType match { + case StringType => value.toString + case ShortType | IntegerType | LongType => value.toString + case DateType => dateDaysToEpochMillis(value.asInstanceOf[Int]).toString + case TimestampType => (value.asInstanceOf[Long] / 1000L).toString + case TimestampNTZType => ntzMicrosToEpochMillis(value.asInstanceOf[Long]).toString + case other => + throw new IllegalArgumentException( + s"Unsupported timestamp argument type $other for $fnName. Use a STRING (epoch " + + "milliseconds or 'yyyy-MM-dd HH:mm:ss'), an integral epoch milliseconds value, a " + + "DATE or a TIMESTAMP.") + } + if (normalized.trim.isEmpty) { + throw new IllegalArgumentException( + s"Timestamp arguments of $fnName must not be blank. Provide epoch milliseconds or a " + + "'yyyy-MM-dd HH:mm:ss' timestamp.") + } + normalized + } + + private val MICROS_PER_SECOND = 1000000L + + /** + * Converts a `DATE` argument to epoch milliseconds, at the start of that day in the Spark session + * time zone. + */ + private def dateDaysToEpochMillis(days: Int): Long = + LocalDate + .ofEpochDay(days.toLong) + .atStartOfDay(ZoneId.of(SQLConf.get.sessionLocalTimeZone)) + .toInstant + .toEpochMilli + + /** + * Converts a `TIMESTAMP_NTZ` argument to epoch milliseconds. Its wall-clock microseconds are + * re-interpreted in the Spark session time zone, the same convention the `yyyy-MM-dd HH:mm:ss` + * string form uses, so both forms resolve to the same instant. + */ + private def ntzMicrosToEpochMillis(micros: Long): Long = { + val seconds = Math.floorDiv(micros, MICROS_PER_SECOND) + val nanos = Math.floorMod(micros, MICROS_PER_SECOND) * 1000L + LocalDateTime + .ofEpochSecond(seconds, nanos.toInt, ZoneOffset.UTC) + .atZone(ZoneId.of(SQLConf.get.sessionLocalTimeZone)) + .toInstant + .toEpochMilli + } +} + +/** + * An unresolved Fluss table-valued function. + * + * @param fnName + * one of [[FlussTableValuedFunctions.supportedFnNames]]. + */ +abstract class FlussTableValueFunction(val fnName: String) extends LeafNode { + + override def output: Seq[Attribute] = Nil + + override lazy val resolved = false + + val args: Seq[Expression] + + /** Translates the arguments following the table identifier into Fluss scan options. */ + def parseArgs(argsWithoutTable: Seq[Expression]): Map[String, String] +} + +/** + * Plan for `fluss_incremental_between_timestamp(table, startTimestamp[, endTimestamp])`. + * + * The window is left-closed and right-open, `[start, end)`, on the record commit timestamp, and + * covers the data Fluss still retains inside it. An omitted end timestamp means "up to now". + */ +case class IncrementalBetweenTimestamp(override val args: Seq[Expression]) + extends FlussTableValueFunction(INCREMENTAL_BETWEEN_TIMESTAMP) { + + override def parseArgs(argsWithoutTable: Seq[Expression]): Map[String, String] = { + if (argsWithoutTable.size != 1 && argsWithoutTable.size != 2) { + throw new IllegalArgumentException( + s"$INCREMENTAL_BETWEEN_TIMESTAMP needs a table identifier followed by a startTimestamp " + + s"and an optional endTimestamp, e.g. " + + s"$INCREMENTAL_BETWEEN_TIMESTAMP('db.t', '2026-01-01 00:00:00', '2026-01-01 01:00:00'). " + + s"Got ${argsWithoutTable.size + 1} arguments.") + } + + val start = toTimestampOptionValue(INCREMENTAL_BETWEEN_TIMESTAMP, argsWithoutTable.head) + // Resolving the end here pins the window at analysis time, so rows committed while the query is + // planned stay out of it and re-executing the same relation reads the same window. + val endArg = if (argsWithoutTable.size == 2) argsWithoutTable.last else CurrentTimestamp() + val end = toTimestampOptionValue(INCREMENTAL_BETWEEN_TIMESTAMP, endArg) + requireValidWindow(start, end) + + Map( + SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() -> start, + SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() -> end) + } + + /** A reversed or degenerate window fails fast instead of silently returning no rows. */ + private def requireValidWindow(start: String, end: String): Unit = { + val startMs = FlussOffsetInitializers.parseTimestamp( + start, + SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) + val endMs = FlussOffsetInitializers.parseTimestamp( + end, + SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key()) + FlussOffsetInitializers.requireValidWindow(start, end, startMs, endMs) + } +} diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala index 354c9b8f2a4..ae147e9fd08 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala @@ -47,6 +47,12 @@ class FlussAppendPartitionReader( // The latest offset of fluss is -2 private var currentOffset: Long = flussPartition.startOffset.max(0L) + private val timeRange: Option[FlussTimeRange] = flussPartition.timeRange + + // Set once a record at or after the end of the requested window is seen; commit timestamps are + // non-decreasing within a bucket, so no later record can belong to the window either. + private var reachedWindowEnd = false + // initialize log scanner initialize() @@ -60,26 +66,28 @@ class FlussAppendPartitionReader( } override def next0(): Boolean = { - if (closed || currentOffset >= flussPartition.stopOffset) { - return false - } - - if (!currentRecords.hasNext) { - pollMoreRecords() - } + while (!closed && !reachedWindowEnd && currentOffset < flussPartition.stopOffset) { + if (!currentRecords.hasNext) { + pollMoreRecords() + } + if (!currentRecords.hasNext) { + throw new IllegalStateException(s"No more data from fluss server," + + s" but current offset $currentOffset not reach the stop offset ${flussPartition.stopOffset}") + } - // If we have records in current batch, return next one - if (currentRecords.hasNext) { val scanRecord = currentRecords.next() - currentRow = convertToSparkRow(scanRecord) currentOffset = scanRecord.logOffset() + 1 - true - } else if (currentOffset < flussPartition.stopOffset) { - throw new IllegalStateException(s"No more data from fluss server," + - s" but current offset $currentOffset not reach the stop offset ${flussPartition.stopOffset}") - } else { - false + timeRange match { + case Some(range) if range.isAfter(scanRecord.timestamp()) => reachedWindowEnd = true + // The record precedes the requested window: the start offset resolved from the start + // timestamp is only time-index accurate on tiered segments, so it can undershoot. + case Some(range) if !range.contains(scanRecord.timestamp()) => // skip + case _ => + currentRow = convertToSparkRow(scanRecord) + return true + } } + false } override def close0(): Unit = { diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussInputPartition.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussInputPartition.scala index b4ea3b9581a..b126cefc14e 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussInputPartition.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussInputPartition.scala @@ -30,18 +30,39 @@ trait FlussInputPartition extends InputPartition { } +/** + * The `[startMs, endMs)` window a time-range batch read was asked for, on the record commit + * timestamp. The reader applies it because offsets resolved from timestamps are only as accurate as + * the server-side time index. Timestamps are non-decreasing within a bucket, so the first record at + * or after `endMs` also ends the partition. + */ +case class FlussTimeRange(startMs: Long, endMs: Long) { + + def contains(timestampMs: Long): Boolean = timestampMs >= startMs && timestampMs < endMs + + def isAfter(timestampMs: Long): Boolean = timestampMs >= endMs + + override def toString: String = s"TimeRange[$startMs - $endMs]" +} + /** * Represents an input partition for reading data from a Fluss table bucket. * * @param tableBucket * the table bucket to read from + * @param timeRange + * the requested time window, set for a time-range batch read only */ -case class FlussAppendInputPartition(tableBucket: TableBucket, startOffset: Long, stopOffset: Long) +case class FlussAppendInputPartition( + tableBucket: TableBucket, + startOffset: Long, + stopOffset: Long, + timeRange: Option[FlussTimeRange] = None) extends FlussInputPartition { override def toString: String = { s"FlussAppendInputPartition{tableId=${tableBucket.getTableId}, bucketId=${tableBucket.getBucket}," + s" partitionId=${tableBucket.getPartitionId}" + - s" logStartOffset=$startOffset, logStopOffset=$stopOffset" + s" logStartOffset=$startOffset, logStopOffset=$stopOffset, timeRange=$timeRange" } } @@ -58,16 +79,20 @@ case class FlussAppendInputPartition(tableBucket: TableBucket, startOffset: Long * the log offset where incremental reading should start * @param logStoppingOffset * the log offset where incremental reading should end + * @param timeRange + * the requested time window, set for a time-range batch read only */ case class FlussUpsertInputPartition( tableBucket: TableBucket, snapshotId: Long, logStartingOffset: Long, - logStoppingOffset: Long) + logStoppingOffset: Long, + timeRange: Option[FlussTimeRange] = None) extends FlussInputPartition { override def toString: String = { s"FlussUpsertInputPartition{tableId=${tableBucket.getTableId}, bucketId=${tableBucket.getBucket}," + s" partitionId=${tableBucket.getPartitionId}, snapshotId=$snapshotId," + - s" logStartOffset=$logStartingOffset, logStopOffset=$logStoppingOffset}" + s" logStartOffset=$logStartingOffset, logStopOffset=$logStoppingOffset," + + s" timeRange=$timeRange}" } } diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussMicroBatchStream.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussMicroBatchStream.scala index 6351dba48f4..0a40f5bdd22 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussMicroBatchStream.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussMicroBatchStream.scala @@ -68,7 +68,7 @@ abstract class FlussMicroBatchStream( FlussOffsetInitializers.startOffsetsInitializer(options, flussConfig) val stoppingOffsetsInitializer: OffsetsInitializer = - FlussOffsetInitializers.stoppingOffsetsInitializer(false, options, flussConfig) + FlussOffsetInitializers.stoppingOffsetsInitializer(false, options) protected def projection: Array[Int] = FlussScanBuilder.projectionOf(tableInfo, Some(readSchema)) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala index 1f0a8806ae3..22ae841b531 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala @@ -18,20 +18,63 @@ package org.apache.fluss.spark.read import org.apache.fluss.client.initializer.{NoStoppingOffsetsInitializer, OffsetsInitializer} -import org.apache.fluss.config.Configuration +import org.apache.fluss.config.{ConfigOption, Configuration} +import org.apache.fluss.metadata.TablePath import org.apache.fluss.spark.SparkFlussConf +import org.apache.spark.internal.Logging +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.util.CaseInsensitiveStringMap -object FlussOffsetInitializers { +import java.time.{Duration, LocalDateTime, ZoneId} +import java.time.format.DateTimeFormatter + +object FlussOffsetInitializers extends Logging { + + private val DATE_TIME_FORMATTER: DateTimeFormatter = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") + + /** + * Whether an incremental (time-range) batch read is requested, i.e. + * `scan.incremental.start.timestamp` is set on the relation being scanned. + * + * The `scan.incremental.*` options are read from the per-query scan options only — set by the + * `fluss_incremental_between_timestamp` table-valued function or `DataFrameReader.option` — and + * deliberately not from session configuration, so a window can never leak into another query. + * Streaming reads ignore them. + * + * An explicitly set but blank start timestamp fails fast instead of silently falling back to a + * full-table batch read. + */ + def isIncrementalRead(options: CaseInsensitiveStringMap): Boolean = { + val startOption = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP + val rawValue = Option(options.get(startOption.key())) + if (rawValue.exists(_.trim.isEmpty)) { + throw new IllegalArgumentException( + s"'${startOption.key()}' must not be blank. Provide epoch milliseconds or a " + + s"'yyyy-MM-dd HH:mm:ss' timestamp, or omit the option for a full-table batch read.") + } + rawValue.isDefined + } + + /** + * Start offsets of an incremental batch read, resolved from `scan.incremental.start.timestamp`. + * Requires that option to be set. + */ + def incrementalStartOffsetsInitializer(options: CaseInsensitiveStringMap): OffsetsInitializer = { + val start = requiredTimestamp(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP) + OffsetsInitializer.timestamp( + parseTimestamp(start, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key())) + } + + /** + * Start offsets of a streaming read, driven by `scan.startup.mode`. Batch reads ignore this + * option (see [[incrementalStartOffsetsInitializer]]). + */ def startOffsetsInitializer( options: CaseInsensitiveStringMap, flussConfig: Configuration): OffsetsInitializer = { - val startupMode = options - .getOrDefault( - SparkFlussConf.SCAN_START_UP_MODE.key(), - flussConfig.get(SparkFlussConf.SCAN_START_UP_MODE)) - .toUpperCase + val startupMode = resolveStartupMode(options, flussConfig).toUpperCase SparkFlussConf.StartUpMode.withName(startupMode) match { case SparkFlussConf.StartUpMode.EARLIEST => OffsetsInitializer.earliest() @@ -39,18 +82,140 @@ object FlussOffsetInitializers { case SparkFlussConf.StartUpMode.LATEST => OffsetsInitializer.latest() case _ => throw new IllegalArgumentException( - s"Unsupported scan start up mode: ${options.get(SparkFlussConf.SCAN_START_UP_MODE.key())}") + s"Unsupported scan start up mode: " + + s"${resolveStartupMode(options, flussConfig)}. Supported values are 'full', " + + s"'earliest' and 'latest'. For a time-range batch read set " + + s"'${SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()}' instead.") } } def stoppingOffsetsInitializer( isBatch: Boolean, - options: CaseInsensitiveStringMap, - flussConfig: Configuration): OffsetsInitializer = { - if (isBatch) { + options: CaseInsensitiveStringMap): OffsetsInitializer = { + if (!isBatch) { + new NoStoppingOffsetsInitializer() + } else { + if (!isIncrementalRead(options)) { + val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() + if (Option(options.get(endKey)).isDefined) { + throw new IllegalArgumentException( + s"'$endKey' is set but " + + s"'${SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()}' is missing. An end " + + s"timestamp alone cannot truncate a batch read; set a start timestamp for an " + + s"incremental read, or remove the end option.") + } + } + // A batch read stops at the latest committed data, an incremental one included: its end bound + // is applied by the reader on the record commit timestamp (see [[incrementalTimeRange]]). OffsetsInitializer.latest() + } + } + + /** The `[start, end)` window of an incremental read, empty for a plain batch read. */ + def incrementalTimeRange(options: CaseInsensitiveStringMap): Option[FlussTimeRange] = { + if (!isIncrementalRead(options)) { + return None + } + val start = requiredTimestamp(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP) + val startMs = parseTimestamp(start, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) + val end = incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP) + .map(_.trim) + val endMs = end + .map(parseTimestamp(_, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key())) + .getOrElse(Long.MaxValue) + end.foreach(requireValidWindow(start, _, startMs, endMs)) + Some(FlussTimeRange(startMs, endMs)) + } + + /** Rejects a reversed or degenerate window instead of silently reading nothing. */ + private[spark] def requireValidWindow( + start: String, + end: String, + startMs: Long, + endMs: Long): Unit = { + if (startMs >= endMs) { + throw new IllegalArgumentException( + s"Invalid incremental time range: the start timestamp '$start' must be strictly before " + + s"the end timestamp '$end'. The window is left-closed right-open '[start, end)'.") + } + } + + /** + * Warns when the requested window starts before what the table is still guaranteed to retain: the + * log below `table.log.ttl` has been dropped, so the read can only return a subset of the window. + * + * Stays a warning rather than an error: TTL is a lower bound — expired segments are deleted + * lazily — so the data may in fact still be there, and a partial window is the documented + * behavior of this read mode. + */ + def warnIfWindowPredatesRetention( + tablePath: TablePath, + range: FlussTimeRange, + logTtlMs: Long): Unit = { + if (predatesRetention(range.startMs, logTtlMs, System.currentTimeMillis())) { + logWarning( + s"Incremental read of $tablePath starts at ${range.startMs}, which is earlier than the " + + s"table is guaranteed to retain (table.log.ttl = ${Duration.ofMillis(logTtlMs)}). " + + s"Records that already expired cannot be returned, so the result may be a subset of " + + s"the requested window. Increase table.log.ttl or move the window forward.") + } + } + + private[spark] def predatesRetention(startMs: Long, logTtlMs: Long, nowMs: Long): Boolean = + startMs < nowMs - logTtlMs + + /** + * Reads a `scan.incremental.*` option from the scan options, falling back to its default. A blank + * value counts as unset; a blank start timestamp is rejected by [[isIncrementalRead]] before it + * can reach here. + */ + private def incrementalOption( + options: CaseInsensitiveStringMap, + option: ConfigOption[String]): Option[String] = + Option(options.getOrDefault(option.key(), option.defaultValue())).filter(_.trim.nonEmpty) + + private def resolveStartupMode( + options: CaseInsensitiveStringMap, + flussConfig: Configuration): String = + options.getOrDefault( + SparkFlussConf.SCAN_START_UP_MODE.key(), + flussConfig.get(SparkFlussConf.SCAN_START_UP_MODE)) + + private def requiredTimestamp( + options: CaseInsensitiveStringMap, + option: ConfigOption[String]): String = { + val value = incrementalOption(options, option) + if (value.getOrElse("").isEmpty) { + throw new IllegalArgumentException( + s"'${option.key()}' must not be empty. Provide epoch milliseconds or a " + + s"'yyyy-MM-dd HH:mm:ss' timestamp.") + } + value.get.trim + } + + /** + * Parses a timestamp option value to epoch milliseconds: a purely numeric string is epoch + * milliseconds, otherwise it is parsed as 'yyyy-MM-dd HH:mm:ss' in the Spark session time zone. + */ + private[spark] def parseTimestamp(timestampStr: String, optionKey: String): Long = { + if (timestampStr.matches("\\d+")) { + timestampStr.toLong } else { - new NoStoppingOffsetsInitializer() + try { + LocalDateTime + .parse(timestampStr, DATE_TIME_FORMATTER) + .atZone(ZoneId.of(SQLConf.get.sessionLocalTimeZone)) + .toInstant + .toEpochMilli + } catch { + case e: Exception => + throw new IllegalArgumentException( + s"Invalid value for '$optionKey': '$timestampStr'. It should be epoch milliseconds or " + + s"follow the format 'yyyy-MM-dd HH:mm:ss', e.g. '2023-12-09 23:09:12' or " + + s"'1678883047356'.", + e + ) + } } } } diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussScan.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussScan.scala index 0b242e0ce2d..96def8ee814 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussScan.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussScan.scala @@ -44,6 +44,8 @@ trait FlussScan extends Scan { def limit: Option[Int] = None + def timeRange: Option[FlussTimeRange] = None + protected def scanType: String override def readSchema(): StructType = { @@ -59,10 +61,16 @@ trait FlussScan extends Scan { case Some(p) => s"$withPushed [PartitionFilter: $p]" case None => withPushed } - limit match { - case Some(l) => s"$withPartition [Limit: $l]" + val withTimeRange = timeRange match { + case Some(r) if r.endMs == Long.MaxValue => + s"$withPartition [TimeRange: [${r.startMs}, latest)]" + case Some(r) => s"$withPartition [TimeRange: [${r.startMs}, ${r.endMs})]" case None => withPartition } + limit match { + case Some(l) => s"$withTimeRange [Limit: $l]" + case None => withTimeRange + } } override def supportedCustomMetrics(): Array[CustomMetric] = @@ -90,6 +98,8 @@ case class FlussAppendScan( override protected lazy val scanType: String = if (planner.hasLakeSnapshot) "LakeAppend" else "Append" + override def timeRange: Option[FlussTimeRange] = planner.timeRange + override def toBatch: Batch = { new FlussAppendBatch( tablePath, @@ -134,6 +144,8 @@ case class FlussUpsertScan( override protected lazy val scanType: String = if (planner.hasLakeSnapshot) "LakeUpsert" else "Upsert" + override def timeRange: Option[FlussTimeRange] = planner.timeRange + override def toBatch: Batch = { new FlussUpsertBatch( tablePath, diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala index 6fc0cd26ddf..af55cca3e26 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala @@ -59,6 +59,7 @@ class FlussUpsertPartitionReader( private val snapshotId: Long = flussPartition.snapshotId private val logStartingOffset: Long = flussPartition.logStartingOffset private val logStoppingOffset: Long = flussPartition.logStoppingOffset + private val timeRange: Option[FlussTimeRange] = flussPartition.timeRange private val logScanFinished = logStartingOffset >= logStoppingOffset || logStoppingOffset <= 0 private val (projectionWithPks, pkProjection) = { @@ -122,7 +123,7 @@ class FlussUpsertPartitionReader( } } - def createLogChangesIterator(): LogChangesIterator = { + def createLogChangesIterator(): CloseableIterator[KeyValueRow] = { // Initialize the log scanner logScanner = table.newScan().project(projectionWithPks).createLogScanner() if (tableBucket.getPartitionId == null) { @@ -140,20 +141,33 @@ class FlussUpsertPartitionReader( if (!records.isEmpty) { val flatRecords = records.asScala for (scanRecord <- flatRecords) { - // Maybe data with logStoppingOffset doesn't exist. - if (scanRecord.logOffset() < logStoppingOffset - 1) { - allLogRecords += scanRecord - } else if (scanRecord.logOffset() == logStoppingOffset - 1) { - allLogRecords += scanRecord + if (timeRange.exists(_.isAfter(scanRecord.timestamp()))) { + // Past the end of the requested window, and commit timestamps only grow from here. continue = false } else { - continue = false // Stop if we reach the stopping offset + // Maybe data with logStoppingOffset doesn't exist. + if ( + scanRecord.logOffset() <= logStoppingOffset - 1 && + timeRange.forall(_.contains(scanRecord.timestamp())) + ) { + allLogRecords += scanRecord + } + if (scanRecord.logOffset() >= logStoppingOffset - 1) { + continue = false // Stop if we reach the stopping offset + } } } } } - LogChangesIterator(allLogRecords.toArray, pkProjection, comparator) + // An incremental read scans up to the latest offset and applies its end bound above, so a + // non-empty offset range can still leave nothing behind. LogChangesIterator seeds its cursor + // from the first record, so it must not be handed an empty batch. + if (allLogRecords.isEmpty) { + CloseableIterator.emptyIterator[KeyValueRow]() + } else { + LogChangesIterator(allLogRecords.toArray, pkProjection, comparator) + } } def createSnapshotIterator(): CloseableIterator[LogRecord] = { diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala index 40a633135f9..e8386c94ffa 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala @@ -83,6 +83,12 @@ sealed trait SplitPlanner extends AutoCloseable { * snapshots. */ def logTailPredicate: Option[FlussPredicate] + + /** + * The requested window of an incremental read, empty otherwise. The scan positions itself with + * offsets, the reader cuts the window by record commit timestamp. + */ + def timeRange: Option[FlussTimeRange] } /** Marker: planner yields partitions consumable by an append (log-table) reader factory. */ @@ -215,10 +221,10 @@ abstract class AbstractSplitPlanner( } /** - * Single append (log-table) planner. Probes a readable lake snapshot at construction; if present, - * the plan is a union of lake splits and the Fluss log-tail (from each bucket's snapshotLogOffset - * to committed). If absent, the plan is a pure Fluss log scan from earliest to committed - * (SCAN_START_UP_MODE deliberately ignored — see class scaladoc note below). + * Single append (log-table) planner. Probes a readable lake snapshot at construction; if present + * (and not an incremental read), the plan is a union of lake splits and the Fluss log-tail (from + * each bucket's snapshotLogOffset to committed). If absent, the plan is a pure Fluss log scan from + * earliest to committed (SCAN_START_UP_MODE deliberately ignored — see class scaladoc note below). * * Batch semantics note: start offset is hardcoded to [[OffsetsInitializer.full]] instead of * consuming the user-facing SCAN_START_UP_MODE. Rationale — batch reads semantically mean "the full @@ -227,8 +233,10 @@ abstract class AbstractSplitPlanner( * snapshot has no partial-read semantics), which is confusing; (b) with mode=latest and no writes * since planning time, start==stop==tail — an empty range that trips the reader-side * `Invalid offset range` guard. Symmetric "batch = earliest → committed" closes both concerns and - * keeps append/upsert planners aligned. Time-range batch reads should be expressed via predicate - * pushdown on the timestamp column, not startup mode. + * keeps append/upsert planners aligned. A bounded time range is instead requested with the + * batch-only `scan.incremental.start.timestamp` / `scan.incremental.end.timestamp` options, which + * resolve to log offsets identically on append and upsert tables; such an incremental read always + * takes the log-only branch and never unions a lake snapshot. * * `OffsetsInitializer.full()` is chosen over `OffsetsInitializer.earliest()` intentionally: for a * log table the two are semantically equivalent (see OffsetsInitializer.full javadoc), but full() @@ -247,12 +255,25 @@ class AppendPlanner( extends AbstractSplitPlanner(tablePath, tableInfo, flussConfig) with AppendSplitPlanner { - override def hasLakeSnapshot: Boolean = readableLakeSnapshot.isDefined + override def hasLakeSnapshot: Boolean = readableLakeSnapshot.isDefined && !incrementalMode + + private val incrementalMode: Boolean = + FlussOffsetInitializers.isIncrementalRead(options) - private val startOffsetsInitializer: OffsetsInitializer = OffsetsInitializer.full() + override val timeRange: Option[FlussTimeRange] = + FlussOffsetInitializers.incrementalTimeRange(options) + + // An incremental read starts at scan.incremental.start.timestamp, a plain batch read at the + // beginning of the table (see class scaladoc). + private val startOffsetsInitializer: OffsetsInitializer = + if (incrementalMode) { + FlussOffsetInitializers.incrementalStartOffsetsInitializer(options) + } else { + OffsetsInitializer.full() + } override protected val stoppingOffsetsInitializer: OffsetsInitializer = - FlussOffsetInitializers.stoppingOffsetsInitializer(true, options, flussConfig) + FlussOffsetInitializers.stoppingOffsetsInitializer(true, options) // Server-side log filter requires ARROW format. Pushdown already gates this on the log-only // path (never sets pushedPredicate for non-ARROW), but re-checking here keeps the planner @@ -263,16 +284,20 @@ class AppendPlanner( override def plan(): Array[InputPartition] = try { + timeRange.foreach( + FlussOffsetInitializers + .warnIfWindowPredatesRetention(tablePath, _, tableInfo.getTableConfig.getLogTTLMs)) readableLakeSnapshot match { - case Some(snap) => planLakeUnion(snap) - case None => planLogOnly() + // An incremental read never unions a lake snapshot; it reads only Fluss. + case Some(snap) if !incrementalMode => planLakeUnion(snap) + case _ => planLogOnly() } } finally { close() } // --------------------------------------------------------------------------------------------- - // Log-only branch: pure Fluss log scan from earliest → committed with optional range splitting. + // Log-only branch: pure Fluss log scan over [start, stop) with optional range splitting. // --------------------------------------------------------------------------------------------- private def planLogOnly(): Array[InputPartition] = { @@ -281,10 +306,10 @@ class AppendPlanner( if (value > 0) Some(value) else None } - val bucketOffsetsRetrieverImpl = maxRecordsPerPartition match { - case Some(_) => new BucketOffsetsRetrieverImpl(admin, tablePath, true) - case _ => new BucketOffsetsRetrieverImpl(admin, tablePath) - } + // Only the max-records splitter needs concrete earliest offsets; otherwise the earliest + // sentinel (-2) is enough. + val bucketOffsetsRetrieverImpl = + new BucketOffsetsRetrieverImpl(admin, tablePath, maxRecordsPerPartition.isDefined) val buckets = (0 until tableInfo.getNumBuckets).toSeq def splitOffsetRange( @@ -295,7 +320,7 @@ class AppendPlanner( if ( startOffset < 0 || stopOffset <= startOffset || stopOffset <= (startOffset + maxRecords) ) { - return Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset)) + return Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset, timeRange)) } val rangeSize = stopOffset - startOffset val numSplits = ((rangeSize + maxRecords - 1) / maxRecords).toInt @@ -306,7 +331,12 @@ class AppendPlanner( .take(numSplits) .map(i => startOffset + i * step) .map { - from => FlussAppendInputPartition(tableBucket, from, math.min(from + step, stopOffset)) + from => + FlussAppendInputPartition( + tableBucket, + from, + math.min(from + step, stopOffset), + timeRange) } .toSeq } @@ -319,13 +349,20 @@ class AppendPlanner( bucketId => val (startOffset, stopOffset) = (startBucketOffsets(bucketId), stoppingBucketOffsets(bucketId)) - val tableBucket = partitionId match { - case Some(pid) => new TableBucket(tableInfo.getTableId, pid, bucketId) - case None => new TableBucket(tableInfo.getTableId, bucketId) - } - maxRecordsPerPartition match { - case Some(maxRecs) => splitOffsetRange(tableBucket, startOffset, stopOffset, maxRecs) - case _ => Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset)) + if (startOffset >= stopOffset) { + // Empty range (e.g. a time-range window with no data, or an empty bucket): emit no + // partition so the append reader is not handed an invalid [start, start) range. + Seq.empty[InputPartition] + } else { + val tableBucket = partitionId match { + case Some(pid) => new TableBucket(tableInfo.getTableId, pid, bucketId) + case None => new TableBucket(tableInfo.getTableId, bucketId) + } + maxRecordsPerPartition match { + case Some(maxRecs) => splitOffsetRange(tableBucket, startOffset, stopOffset, maxRecs) + case _ => + Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset, timeRange)) + } } }.toArray } @@ -338,12 +375,13 @@ class AppendPlanner( matching .map { partitionInfo => + val partitionName = partitionInfo.getPartitionName val startBucketOffsets = startOffsetsInitializer.getBucketOffsets( - partitionInfo.getPartitionName, + partitionName, buckets.map(Integer.valueOf).asJava, bucketOffsetsRetrieverImpl) val stoppingBucketOffsets = stoppingOffsetsInitializer.getBucketOffsets( - partitionInfo.getPartitionName, + partitionName, buckets.map(Integer.valueOf).asJava, bucketOffsetsRetrieverImpl) ( @@ -567,7 +605,9 @@ class AppendPlanner( * partitions. If absent, the plan is a pure Fluss upsert scan derived from kv snapshots + log tail. * * Startup-mode gating has been removed: a batch upsert scan is always full-table regardless of the - * user-facing SCAN_START_UP_MODE setting — same rationale as [[AppendPlanner]]. + * user-facing SCAN_START_UP_MODE setting — same rationale as [[AppendPlanner]]. Setting + * `scan.incremental.start.timestamp` instead yields an incremental read that folds only the + * changelog within the requested window. */ class UpsertPlanner( override val tablePath: TablePath, @@ -580,10 +620,22 @@ class UpsertPlanner( extends AbstractSplitPlanner(tablePath, tableInfo, flussConfig) with UpsertSplitPlanner { - override def hasLakeSnapshot: Boolean = readableLakeSnapshot.isDefined + override def hasLakeSnapshot: Boolean = readableLakeSnapshot.isDefined && !incrementalMode + + private val incrementalMode: Boolean = + FlussOffsetInitializers.isIncrementalRead(options) + + override val timeRange: Option[FlussTimeRange] = + FlussOffsetInitializers.incrementalTimeRange(options) + + // Start offset of an incremental read, resolved from scan.incremental.start.timestamp. Lazy on + // purpose: resolving it requires that option, while a plain batch scan derives its start from kv + // snapshots instead. + private lazy val incrementalStartOffsetsInitializer: OffsetsInitializer = + FlussOffsetInitializers.incrementalStartOffsetsInitializer(options) override protected val stoppingOffsetsInitializer: OffsetsInitializer = - FlussOffsetInitializers.stoppingOffsetsInitializer(true, options, flussConfig) + FlussOffsetInitializers.stoppingOffsetsInitializer(true, options) // Upsert never pushes a server-side log filter (kv+log union semantics require full log tail // to be reconciled with kv snapshots — see FlussUpsertPartitionReader). @@ -591,7 +643,13 @@ class UpsertPlanner( override def plan(): Array[InputPartition] = try { + timeRange.foreach( + FlussOffsetInitializers + .warnIfWindowPredatesRetention(tablePath, _, tableInfo.getTableConfig.getLogTTLMs)) readableLakeSnapshot match { + // An incremental read reads neither the lake nor the kv snapshot; it folds only the Fluss + // changelog within [start, end). + case _ if incrementalMode => planIncrementalLogOnly() case Some(snap) => planLakeUnion(snap) case None => planLogOnly() } @@ -661,6 +719,83 @@ class UpsertPlanner( .toArray } + // --------------------------------------------------------------------------------------------- + // Incremental branch: fold the Fluss changelog within [start, end) per bucket, with no kv + // snapshot and no lake. Emitting NO_SNAPSHOT_ID makes FlussUpsertPartitionReader skip the + // snapshot and fold only the log range; SortMergeReader drops delete rows, so the output is the + // surviving +I/+U rows (keys inserted or updated in the window; deleted keys excluded). + // --------------------------------------------------------------------------------------------- + + private def planIncrementalLogOnly(): Array[InputPartition] = { + if (flussConfig.get(SparkFlussConf.READ_OPTIMIZED_OPTION)) { + throw new IllegalArgumentException( + s"'${SparkFlussConf.READ_OPTIMIZED_OPTION.key()}' must not be enabled for an " + + s"incremental (time-range) read: it skips log changes and reads only snapshots, while " + + s"an incremental read folds only the changelog, so this combination would silently " + + s"return no rows.") + } + val bucketOffsetsRetriever = new BucketOffsetsRetrieverImpl(admin, tablePath) + val buckets = (0 until tableInfo.getNumBuckets).toSeq + + if (tableInfo.isPartitioned) { + val matching = SparkPartitionPredicate.filterPartitions( + tableInfo, + partitionInfos.asScala.toSeq, + partitionPredicate) + matching.flatMap { + partitionInfo => + createIncrementalUpsertPartitions( + partitionInfo.getPartitionName, + Some(partitionInfo.getPartitionId), + buckets, + bucketOffsetsRetriever) + }.toArray + } else { + createIncrementalUpsertPartitions(null, None, buckets, bucketOffsetsRetriever) + } + } + + private def createIncrementalUpsertPartitions( + partitionName: String, + partitionId: Option[Long], + buckets: Seq[Int], + bucketOffsetsRetriever: BucketOffsetsRetrieverImpl): Array[InputPartition] = { + val jBuckets = buckets.map(Integer.valueOf).asJava + val startBucketOffsets = + incrementalStartOffsetsInitializer.getBucketOffsets( + partitionName, + jBuckets, + bucketOffsetsRetriever) + val stoppingBucketOffsets = + stoppingOffsetsInitializer.getBucketOffsets(partitionName, jBuckets, bucketOffsetsRetriever) + + val tableId = tableInfo.getTableId + buckets.flatMap { + bucketId => + val tableBucket = partitionId match { + case Some(pid) => new TableBucket(tableId, pid, bucketId) + case None => new TableBucket(tableId, bucketId) + } + val startOffset = Long2long(startBucketOffsets.get(Integer.valueOf(bucketId))) + val stopOffset = Long2long(stoppingBucketOffsets.get(Integer.valueOf(bucketId))) + if (startOffset >= stopOffset) { + // The start timestamp resolved past the end of the log, so there is nothing to read. + // A window that is empty only in time still yields start < stop, because the stop offset + // is the latest offset and the end bound is applied by the reader. + None + } else { + Some( + FlussUpsertInputPartition( + tableBucket, + TableBucketSnapshot.NO_SNAPSHOT_ID, + startOffset, + stopOffset, + timeRange) + .asInstanceOf[InputPartition]) + } + }.toArray + } + // --------------------------------------------------------------------------------------------- // Lake-union branch: lake splits (upsert view) + Fluss log tail after snapshotLogOffset. // --------------------------------------------------------------------------------------------- diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala new file mode 100644 index 00000000000..9011434832e --- /dev/null +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala @@ -0,0 +1,520 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark + +import org.apache.fluss.config.Configuration +import org.apache.fluss.row.{BinaryString, GenericRow} +import org.apache.fluss.spark.read.{FlussOffsetInitializers, FlussTimeRange} + +import org.apache.spark.sql.Row +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.assertj.core.api.Assertions.assertThat + +/** + * Verifies the `fluss_incremental_between_timestamp` table-valued function. The window is + * left-closed, right-open `[start, end)` on the record commit timestamp, and the function's options + * are scoped to the single query. Tables are partitioned unless a case says otherwise. + */ +class SparkTimeRangeTvfTest extends FlussSparkTestBase { + + /** Snapshots are triggered explicitly, so no window can straddle an automatic one. */ + override protected def flussConf: Configuration = new Configuration() + + private val TVF = "fluss_incremental_between_timestamp" + + private val P1 = "2026-01-01" + private val P2 = "2026-01-02" + + test("TVF: log table window") { + withTable("t_log") { + createLogTable("t_log") + + val t0 = boundary() + insert("t_log", s"""(1L, 11L, 101, "a1", "$P1"), (2L, 12L, 102, "a2", "$P2")""") + val t1 = boundary() + insert("t_log", s"""(3L, 13L, 103, "a3", "$P1"), (4L, 14L, 104, "a4", "$P2")""") + val t2 = boundary() + insert("t_log", s"""(5L, 15L, 105, "a5", "$P1")""") + val t3 = boundary() + // nothing is committed in [t3, t4), but rows follow it + val t4 = boundary() + insert("t_log", s"""(6L, 16L, 106, "a6", "$P2"), (7L, 17L, 107, "a7", "$P1")""") + val t5 = boundary() + val t6 = boundary() + + val batch1 = Row(1L, 11L, 101, "a1", P1) :: Row(2L, 12L, 102, "a2", P2) :: Nil + val batch2 = Row(3L, 13L, 103, "a3", P1) :: Row(4L, 14L, 104, "a4", P2) :: Nil + val batch3 = Row(5L, 15L, 105, "a5", P1) :: Nil + val batch4 = Row(6L, 16L, 106, "a6", P2) :: Row(7L, 17L, 107, "a7", P1) :: Nil + + // consecutive windows partition the table exactly, each one spanning every partition + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t0, t1)} ORDER BY orderId"), batch1) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t1, t2)} ORDER BY orderId"), batch2) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t2, t3)} ORDER BY orderId"), batch3) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t4, t5)} ORDER BY orderId"), batch4) + + // a window may span several writes, and one covering everything matches a plain batch read + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t1, t4)} ORDER BY orderId"), batch2 ::: batch3) + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_log", t0, t6)} ORDER BY orderId"), + batch1 ::: batch2 ::: batch3 ::: batch4) + + // projection, filter and partition pruning still work on top of the TVF relation + checkAnswer( + sql(s"SELECT address FROM ${tvf("t_log", t1, t2)} WHERE amount = 104"), + Row("a4") :: Nil) + checkAnswer( + sql(s"SELECT orderId FROM ${tvf("t_log", t1, t2)} WHERE dt = '$P1'"), + Row(3L) :: Nil) + + // without an end timestamp the window runs up to the latest data + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t2)} ORDER BY orderId"), batch3 ::: batch4) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t4)} ORDER BY orderId"), batch4) + + // a window without writes yields nothing, whether it precedes, splits or trails the data + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t3, t4)}"), Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t5, t6)}"), Nil) + + // the table argument may be unqualified or fully qualified + checkAnswer(sql(s"SELECT * FROM $TVF('t_log', '$t4')"), batch4) + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_CATALOG.$DEFAULT_DATABASE.t_log', '$t4')"), + batch4) + + // the omitted end bound is pinned when the statement is analyzed, so a row committed before + // the scan is planned stays outside the window + val pinned = sql(s"SELECT * FROM ${tvf("t_log", t4)} ORDER BY orderId") + insert("t_log", s"""(8L, 18L, 108, "a8", "$P1")""") + Thread.sleep(200) + checkAnswer(pinned, batch4) + } + } + + test("TVF: primary key table folds the window changelog") { + withTable("t_fold") { + createPkTable("t_fold") + + val writer = loadFlussTable(createTablePath("t_fold")).newUpsert().createWriter() + writer.upsert(row(1L, 11L, 101, "a1", P1)).get() + writer.upsert(row(2L, 12L, 102, "a2", P2)).get() + writer.upsert(row(3L, 13L, 103, "a3", P1)).get() + writer.upsert(row(4L, 14L, 104, "a4", P2)).get() + writer.flush() + val t1 = boundary() + + // key 1 updated twice, key 2 updated once, key 3 deleted then re-inserted, key 4 deleted, + // key 5 inserted then deleted again + writer.upsert(row(1L, 110L, 1001, "a1_v2", P1)).get() + writer.upsert(row(1L, 111L, 1002, "a1_v3", P1)).get() + writer.upsert(row(2L, 120L, 1002, "a2_upd", P2)).get() + writer.delete(deleteKey(3L, P1)).get() + writer.upsert(row(3L, 130L, 1003, "a3_new", P1)).get() + writer.delete(deleteKey(4L, P2)).get() + writer.upsert(row(5L, 15L, 105, "a5", P2)).get() + writer.delete(deleteKey(5L, P2)).get() + writer.flush() + val t2 = boundary() + + writer.upsert(row(1L, 112L, 1004, "a1_v4", P1)).get() + writer.flush() + val t3 = boundary() + // nothing is committed in [t3, t4), but changes follow it: the start offset then resolves to + // a record past the end bound instead of to the end of the log + val t4 = boundary() + + writer.delete(deleteKey(2L, P2)).get() + writer.flush() + val t5 = boundary() + + writer.upsert(row(6L, 16L, 106, "a6", P1)).get() + writer.flush() + val t6 = boundary() + val t7 = boundary() + + // each changed key appears once with its last in-window value; deleted keys and an insert + // cancelled by a delete are excluded + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_fold", t1, t2)} ORDER BY orderId"), + Row(1L, 111L, 1002, "a1_v3", P1) :: + Row(2L, 120L, 1002, "a2_upd", P2) :: + Row(3L, 130L, 1003, "a3_new", P1) :: Nil + ) + + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_fold", t2, t3)} ORDER BY orderId"), + Row(1L, 112L, 1004, "a1_v4", P1) :: Nil) + + // a window whose only change is a delete folds to nothing, even though it does read records + checkAnswer(sql(s"SELECT * FROM ${tvf("t_fold", t4, t5)}"), Nil) + + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_fold", t5, t6)} ORDER BY orderId"), + Row(6L, 16L, 106, "a6", P1) :: Nil) + + // spanning several batches: key 1 keeps its last value, key 2 is dropped by the later delete + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_fold", t2, t6)} ORDER BY orderId"), + Row(1L, 112L, 1004, "a1_v4", P1) :: Row(6L, 16L, 106, "a6", P1) :: Nil) + + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_fold", t1)} ORDER BY orderId"), + Row(1L, 112L, 1004, "a1_v4", P1) :: + Row(3L, 130L, 1003, "a3_new", P1) :: + Row(6L, 16L, 106, "a6", P1) :: Nil + ) + + checkAnswer( + sql(s"SELECT orderId FROM ${tvf("t_fold", t1, t2)} WHERE dt = '$P1' ORDER BY orderId"), + Row(1L) :: Row(3L) :: Nil) + + // windows without any change yield nothing, whether they split or trail the changelog + checkAnswer(sql(s"SELECT * FROM ${tvf("t_fold", t3, t4)}"), Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_fold", t6, t7)}"), Nil) + } + } + + test("TVF: primary key table with several buckets") { + withTable("t_multi") { + createPkTable("t_multi", buckets = 3) + + val writer = loadFlussTable(createTablePath("t_multi")).newUpsert().createWriter() + (1 to 6).foreach { + k => + val dt = if (k % 2 == 0) P2 else P1 + writer.upsert(row(k.toLong, (10 + k).toLong, 100 + k, s"a$k", dt)).get() + } + writer.flush() + val t1 = boundary() + + // the window touches only some of the six bucket/partition pairs, so the rest are planned + // and read down to nothing + writer.upsert(row(2L, 120L, 1002, "a2_upd", P2)).get() + writer.delete(deleteKey(5L, P1)).get() + writer.upsert(row(7L, 17L, 107, "a7", P1)).get() + writer.flush() + val t2 = boundary() + + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_multi", t1, t2)} ORDER BY orderId"), + Row(2L, 120L, 1002, "a2_upd", P2) :: Row(7L, 17L, 107, "a7", P1) :: Nil) + } + } + + test("TVF: primary key table never serves the kv snapshot") { + withTable("t_snap") { + createPkTable("t_snap") + val tablePath = createTablePath("t_snap") + val writer = loadFlussTable(tablePath).newUpsert().createWriter() + + val t0 = boundary() + writer.upsert(row(1L, 11L, 101, "a1", P1)).get() + writer.upsert(row(2L, 12L, 102, "a2", P2)).get() + writer.flush() + // key 1 and key 2 now live in a kv snapshot, so any read that consults it sees them + flussServer.triggerAndWaitSnapshot(tablePath) + val t1 = boundary() + + writer.upsert(row(1L, 110L, 1001, "a1_upd", P1)).get() + writer.upsert(row(3L, 13L, 103, "a3", P1)).get() + writer.flush() + val t2 = boundary() + // the second batch is snapshotted as well, so the window's own rows are in the snapshot too + flussServer.triggerAndWaitSnapshot(tablePath) + val t3 = boundary() + + // a read-optimized full read serves the snapshot alone, which is how we know it is there + withSQLConf(sessionKey(SparkFlussConf.READ_OPTIMIZED_OPTION.key()) -> "true") { + checkAnswer( + sql(s"SELECT * FROM $DEFAULT_DATABASE.t_snap ORDER BY orderId"), + Row(1L, 110L, 1001, "a1_upd", P1) :: + Row(2L, 12L, 102, "a2", P2) :: + Row(3L, 13L, 103, "a3", P1) :: Nil + ) + } + + // key 2 changed before the window and only survives in the snapshot, so it must not appear + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_snap", t1, t2)} ORDER BY orderId"), + Row(1L, 110L, 1001, "a1_upd", P1) :: Row(3L, 13L, 103, "a3", P1) :: Nil) + + // an earlier window still folds from the changelog, at the values it held back then + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_snap", t0, t1)} ORDER BY orderId"), + Row(1L, 11L, 101, "a1", P1) :: Row(2L, 12L, 102, "a2", P2) :: Nil) + + // a window with no changes stays empty instead of falling back to the whole snapshot + checkAnswer(sql(s"SELECT * FROM ${tvf("t_snap", t2, t3)}"), Nil) + } + } + + test("TVF: non-partitioned tables") { + withTable("t_np", "t_np_pk") { + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t_np + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) + |""".stripMargin) + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t_np_pk + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) + |TBLPROPERTIES("primary.key" = "orderId", "bucket.num" = 1) + |""".stripMargin) + + val writer = loadFlussTable(createTablePath("t_np_pk")).newUpsert().createWriter() + val t0 = boundary() + insert("t_np", """(1L, 11L, 101, "a1")""") + writer.upsert(unpartitionedRow(1L, 11L, 101, "a1")).get() + writer.flush() + val t1 = boundary() + + insert("t_np", """(2L, 12L, 102, "a2")""") + writer.upsert(unpartitionedRow(1L, 110L, 1001, "a1_upd")).get() + writer.upsert(unpartitionedRow(2L, 12L, 102, "a2")).get() + writer.flush() + val t2 = boundary() + + // nothing is committed in [t2, t3), but writes follow it + val t3 = boundary() + + insert("t_np", """(3L, 13L, 103, "a3")""") + writer.delete(unpartitionedDeleteKey(2L)).get() + writer.flush() + val t4 = boundary() + + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t0, t1)}"), Row(1L, 11L, 101, "a1") :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np_pk", t0, t1)}"), Row(1L, 11L, 101, "a1") :: Nil) + + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t1, t2)}"), Row(2L, 12L, 102, "a2") :: Nil) + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_np_pk", t1, t2)} ORDER BY orderId"), + Row(1L, 110L, 1001, "a1_upd") :: Row(2L, 12L, 102, "a2") :: Nil) + + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t2, t3)}"), Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np_pk", t2, t3)}"), Nil) + + // the pk window only deletes, so it folds to nothing while the log table keeps its append + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t3, t4)}"), Row(3L, 13L, 103, "a3") :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np_pk", t3, t4)}"), Nil) + + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_np", t1)} ORDER BY orderId"), + Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np_pk", t1)}"), Row(1L, 110L, 1001, "a1_upd") :: Nil) + } + } + + test("TVF: timestamp argument forms resolve to the same window") { + withTable("t_ts") { + createLogTable("t_ts") + + // every accepted form of the same instant must resolve to the same window; asserted on the + // options the reader consumes, so no data has to be written + def assertAllFormsAgree(): Unit = { + val startTs = "2026-01-01 00:00:00" + val endTs = "2026-01-02 00:00:00" + val expected = FlussTimeRange(parseTs(startTs), parseTs(endTs)) + Seq( + s"'${expected.startMs}', '${expected.endMs}'", + s"${expected.startMs}L, ${expected.endMs}L", + s"'$startTs', '$endTs'", + s"TIMESTAMP '$startTs', TIMESTAMP '$endTs'", + s"TIMESTAMP_NTZ '$startTs', TIMESTAMP_NTZ '$endTs'", + "DATE '2026-01-01', DATE '2026-01-02'" + ).foreach(args => assertThat(resolvedWindow("t_ts", args)).isEqualTo(expected)) + } + + assertAllFormsAgree() + // the session time zone applies to the datetime, TIMESTAMP_NTZ and DATE forms alike + withSQLConf("spark.sql.session.timeZone" -> "Asia/Shanghai")(assertAllFormsAgree()) + + // an omitted end bound is pinned to the analysis time + val before = System.currentTimeMillis() + val openEnded = resolvedWindow("t_ts", "'2026-01-01 00:00:00'") + assertThat(openEnded.endMs).isBetween(before, System.currentTimeMillis()) + + // constant expressions are evaluated during analysis + val lastHour = resolvedWindow( + "t_ts", + "date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss'), " + + "CAST(unix_timestamp() * 1000 AS STRING)") + assertThat(lastHour.endMs - lastHour.startMs).isBetween(3590000L, 3610000L) + + val aroundToday = + resolvedWindow("t_ts", "current_date() - INTERVAL 1 DAY, current_date() + INTERVAL 1 DAY") + assertThat(aroundToday.startMs).isLessThan(before) + assertThat(aroundToday.endMs).isGreaterThan(before) + } + } + + test("TVF: invalid usage fails fast") { + withTable("t_bad", "t_bad_pk") { + createLogTable("t_bad") + createPkTable("t_bad_pk") + + val writer = loadFlussTable(createTablePath("t_bad_pk")).newUpsert().createWriter() + insert("t_bad", s"""(1L, 11L, 101, "a1", "$P1")""") + writer.upsert(row(1L, 11L, 101, "a1", P1)).get() + writer.flush() + Thread.sleep(300) + val t1 = System.currentTimeMillis() + + // a blank start must not silently read the full table + assertThat(failureOf(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_bad', ' ')")) + .contains(TVF) + .contains("must not be blank") + + assertThat(failureOf(s"SELECT * FROM ${tvf("t_bad", t1 + 1000, t1)}")) + .contains("strictly before") + assertThat(failureOf(s"SELECT * FROM ${tvf("t_bad", t1, t1)}")) + .contains("strictly before") + + assertThat(failureOf(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_bad')")) + .contains("endTimestamp") + assertThat(failureOf(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_bad', '1', '2', '3')")) + .contains("endTimestamp") + + assertThat(failureOf(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.not_exist', '1', '2')")) + .contains("not_exist") + + // an incremental read reconciles the changelog and cannot serve a read-optimized scan + withSQLConf(sessionKey(SparkFlussConf.READ_OPTIMIZED_OPTION.key()) -> "true") { + assertThat(failureOf(s"SELECT * FROM ${tvf("t_bad_pk", t1)}")) + .contains(SparkFlussConf.READ_OPTIMIZED_OPTION.key()) + } + } + } + + test("TVF: window bounds are never read from session configuration") { + withTable("t_conf", "t_conf_pk") { + createLogTable("t_conf") + createPkTable("t_conf_pk") + + val writer = loadFlussTable(createTablePath("t_conf_pk")).newUpsert().createWriter() + insert("t_conf", s"""(1L, 11L, 101, "a1", "$P1")""") + writer.upsert(row(1L, 11L, 101, "a1", P1)).get() + writer.flush() + val t1 = boundary() + + insert("t_conf", s"""(2L, 12L, 102, "a2", "$P2")""") + writer.upsert(row(2L, 12L, 102, "a2", P2)).get() + writer.flush() + Thread.sleep(200) + + val window = Row(2L, 12L, 102, "a2", P2) :: Nil + + withSQLConf( + sessionKey(SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) -> "2000-01-01 00:00:00", + sessionKey(SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key()) -> "2000-01-02 00:00:00" + ) { + checkAnswer( + sql(s"SELECT * FROM $DEFAULT_DATABASE.t_conf ORDER BY orderId"), + Row(1L, 11L, 101, "a1", P1) :: Row(2L, 12L, 102, "a2", P2) :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_conf", t1)}"), window) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_conf_pk", t1)}"), window) + } + } + } + + private def createLogTable(name: String): Unit = + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.$name + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING, dt STRING) + |PARTITIONED BY (dt) + |""".stripMargin) + + private def createPkTable(name: String, buckets: Int = 1): Unit = + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.$name + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING, dt STRING) + |PARTITIONED BY (dt) + |TBLPROPERTIES("primary.key" = "orderId,dt", "bucket.num" = $buckets) + |""".stripMargin) + + private def insert(table: String, values: String): Unit = + sql(s"INSERT INTO $DEFAULT_DATABASE.$table VALUES $values") + + private def tvf(table: String, timestamps: Long*): String = + s"$TVF('$DEFAULT_DATABASE.$table'${timestamps.map(ts => s", '$ts'").mkString})" + + private def sessionKey(option: String): String = + s"${SparkFlussConf.SPARK_FLUSS_CONF_PREFIX}$option" + + /** A timestamp after everything written so far and before anything written next. */ + private def boundary(): Long = { + Thread.sleep(500) + val ms = System.currentTimeMillis() + Thread.sleep(50) + ms + } + + /** + * The window the reader would apply for a TVF call, taken from the scan options of the analyzed + * relation. Only the table metadata is touched; no data is read. + */ + private def resolvedWindow(table: String, args: String): FlussTimeRange = { + val plan = + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.$table', $args)").queryExecution.analyzed + val options = plan + .collectFirst { case relation: DataSourceV2Relation => relation.options } + .getOrElse(fail(s"no Fluss relation resolved for $TVF($args)")) + FlussOffsetInitializers.incrementalTimeRange(options).get + } + + /** Parses a `yyyy-MM-dd HH:mm:ss` string the way the scan options do. */ + private def parseTs(datetime: String): Long = + FlussOffsetInitializers.parseTimestamp( + datetime, + SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) + + /** The full stack trace of the failure raised by `query`, which must fail. */ + private def failureOf(query: String): String = { + val ex = intercept[Exception](sql(query).collect()) + val sw = new java.io.StringWriter() + ex.printStackTrace(new java.io.PrintWriter(sw)) + sw.toString + } + + private def row( + orderId: Long, + itemId: Long, + amount: Int, + address: String, + dt: String): GenericRow = + GenericRow.of( + Long.box(orderId), + Long.box(itemId), + Int.box(amount), + BinaryString.fromString(address), + BinaryString.fromString(dt)) + + private def deleteKey(orderId: Long, dt: String): GenericRow = + GenericRow.of(Long.box(orderId), null, null, null, BinaryString.fromString(dt)) + + private def unpartitionedRow( + orderId: Long, + itemId: Long, + amount: Int, + address: String): GenericRow = + GenericRow.of( + Long.box(orderId), + Long.box(itemId), + Int.box(amount), + BinaryString.fromString(address)) + + private def unpartitionedDeleteKey(orderId: Long): GenericRow = + GenericRow.of(Long.box(orderId), null, null, null) +} diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala new file mode 100644 index 00000000000..78dbf58a097 --- /dev/null +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark.lake + +import org.apache.fluss.config.{ConfigOptions, Configuration} +import org.apache.fluss.metadata.{DataLakeFormat, TableBucketSnapshot} +import org.apache.fluss.spark.SparkConnectorOptions.{BUCKET_NUMBER, PRIMARY_KEY} +import org.apache.fluss.spark.read.{FlussAppendInputPartition, FlussUpsertInputPartition} + +import org.apache.spark.sql.Row + +import java.nio.file.Files + +/** + * Verifies that an incremental (time-range) batch read on a lake-enabled table is forced to the + * log-only branch: even when a readable lake snapshot exists, the plan never unions lake splits and + * never reads the kv/lake snapshot, so only the data still retained in Fluss is returned. The + * result is the `[t1, t2)` window folded per the underlying table type. + */ +abstract class SparkLakeTimeRangeReadTest extends SparkLakeTableReadTestBase { + + private val TVF = "fluss_incremental_between_timestamp" + + test("Spark Lake Read: log table time-range forces log-only (skips lake snapshot)") { + withTable("t_lake_tr_log") { + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t_lake_tr_log (id INT, name STRING) + | TBLPROPERTIES ( + | '${ConfigOptions.TABLE_DATALAKE_ENABLED.key()}' = true, + | '${ConfigOptions.TABLE_DATALAKE_FRESHNESS.key()}' = '1s', + | '${BUCKET_NUMBER.key()}' = 1) + |""".stripMargin) + + // group 1 (before the window) -> tiered to lake, but also still retained in Fluss + sql(s"""INSERT INTO $DEFAULT_DATABASE.t_lake_tr_log VALUES (1, "hello"), (2, "world")""") + tierToLake("t_lake_tr_log") + + val t1 = System.currentTimeMillis() + Thread.sleep(50) + // group 2 (inside the window) + sql(s"""INSERT INTO $DEFAULT_DATABASE.t_lake_tr_log VALUES (3, "fluss"), (4, "spark")""") + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + // group 3 (after the window) + sql(s"""INSERT INTO $DEFAULT_DATABASE.t_lake_tr_log VALUES (5, "lake")""") + Thread.sleep(200) + + val df = + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_lake_tr_log', '$t1', '$t2') ORDER BY id") + val partitions = lakeInputPartitions(df) + assert(partitions.nonEmpty, "expected at least one Fluss log partition") + assert( + partitions.forall(_.isInstanceOf[FlussAppendInputPartition]), + s"time-range read must be log-only (no lake splits), got: ${partitions.mkString(", ")}" + ) + checkAnswer(df, Row(3, "fluss") :: Row(4, "spark") :: Nil) + } + } + + test("Spark Lake Read: pk table time-range forces log-only (skips lake + kv snapshot)") { + withTable("t_lake_tr_pk") { + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t_lake_tr_pk (id INT, name STRING, score INT) + | TBLPROPERTIES ( + | '${ConfigOptions.TABLE_DATALAKE_ENABLED.key()}' = true, + | '${ConfigOptions.TABLE_DATALAKE_FRESHNESS.key()}' = '1s', + | '${PRIMARY_KEY.key()}' = 'id', + | '${BUCKET_NUMBER.key()}' = 1) + |""".stripMargin) + + // group 1 (before the window) -> tiered to lake, still retained in Fluss changelog + sql(s""" + |INSERT INTO $DEFAULT_DATABASE.t_lake_tr_pk VALUES + |(1, "alice", 90), (2, "bob", 85), (3, "charlie", 95) + |""".stripMargin) + tierToLake("t_lake_tr_pk") + + val t1 = System.currentTimeMillis() + Thread.sleep(50) + // group 2 (inside the window): update id=2, insert id=4 + sql(s""" + |INSERT INTO $DEFAULT_DATABASE.t_lake_tr_pk VALUES + |(2, "bob_updated", 100), (4, "david", 88) + |""".stripMargin) + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + // group 3 (after the window): update id=1, insert id=5 + sql(s""" + |INSERT INTO $DEFAULT_DATABASE.t_lake_tr_pk VALUES + |(1, "alice_updated", 91), (5, "eve", 92) + |""".stripMargin) + Thread.sleep(200) + + val df = + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_lake_tr_pk', '$t1', '$t2') ORDER BY id") + val partitions = lakeInputPartitions(df) + assert(partitions.nonEmpty, "expected at least one Fluss changelog partition") + assert( + partitions.forall { + case p: FlussUpsertInputPartition => + p.snapshotId == TableBucketSnapshot.NO_SNAPSHOT_ID + case _ => false + }, + s"time-range read must be log-only with no kv/lake snapshot " + + s"(snapshotId == NO_SNAPSHOT_ID), got: ${partitions.mkString(", ")}" + ) + // Only keys inserted/updated within [t1, t2): id=2 (updated), id=4 (inserted). + checkAnswer(df, Row(2, "bob_updated", 100) :: Row(4, "david", 88) :: Nil) + } + } +} + +@SparkLakeTest +class SparkLakePaimonTimeRangeReadTest extends SparkLakeTimeRangeReadTest { + + override protected def dataLakeFormat: DataLakeFormat = DataLakeFormat.PAIMON + + override protected def flussConf: Configuration = { + val conf = super.flussConf + conf.setString("datalake.format", DataLakeFormat.PAIMON.toString) + conf.setString("datalake.paimon.metastore", "filesystem") + conf.setString("datalake.paimon.cache-enabled", "false") + warehousePath = + Files.createTempDirectory("fluss-testing-paimon-timerange-lake").resolve("warehouse").toString + conf.setString("datalake.paimon.warehouse", warehousePath) + conf + } + + override protected def lakeCatalogConf: Configuration = { + val conf = new Configuration() + conf.setString("metastore", "filesystem") + conf.setString("warehouse", warehousePath) + conf + } +} diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala new file mode 100644 index 00000000000..f5c53261eec --- /dev/null +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark.read + +import org.apache.fluss.config.Configuration +import org.apache.fluss.spark.SparkFlussConf + +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.assertj.core.api.Assertions.assertThat +import org.scalatest.funsuite.AnyFunSuite + +/** + * Unit tests for how scan options are resolved into offset initializers, and for the retention + * guard of an incremental (time-range) read. The end-to-end behavior is covered by + * [[org.apache.fluss.spark.SparkTimeRangeTvfTest]]. + */ +class FlussOffsetInitializersTest extends AnyFunSuite { + + private def scanOptions(entries: (String, String)*): CaseInsensitiveStringMap = { + val map = new java.util.HashMap[String, String]() + entries.foreach { case (k, v) => map.put(k, v) } + new CaseInsensitiveStringMap(map) + } + + test("incremental read is enabled by the presence of a start timestamp") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + assertThat(FlussOffsetInitializers.isIncrementalRead(scanOptions())).isFalse + assertThat( + FlussOffsetInitializers.isIncrementalRead(scanOptions(startKey -> "1767225600000"))).isTrue + } + + test("a blank start timestamp fails fast instead of silently reading the full table") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.isIncrementalRead(scanOptions(startKey -> " ")) + } + assertThat(ex.getMessage).contains(startKey) + assertThat(ex.getMessage).contains("must not be blank") + } + + test("an end timestamp without a start timestamp fails fast") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.stoppingOffsetsInitializer( + true, + scanOptions(endKey -> "1767312000000")) + } + assertThat(ex.getMessage).contains(endKey) + assertThat(ex.getMessage).contains(startKey) + } + + test("the time range carried to the reader mirrors the requested window") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() + // a plain batch read has no window to apply + assertThat(FlussOffsetInitializers.incrementalTimeRange(scanOptions()).isDefined).isFalse + assertThat( + FlussOffsetInitializers + .incrementalTimeRange(scanOptions(startKey -> "1767225600000", endKey -> "1767312000000")) + .get) + .isEqualTo(FlussTimeRange(1767225600000L, 1767312000000L)) + // an unset end leaves the upper bound open + assertThat( + FlussOffsetInitializers.incrementalTimeRange(scanOptions(startKey -> "1767225600000")).get) + .isEqualTo(FlussTimeRange(1767225600000L, Long.MaxValue)) + } + + test("a window whose start is not strictly before its end fails fast") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() + for (end <- Seq("1767225600000", "1767139200000")) { + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.incrementalTimeRange( + scanOptions(startKey -> "1767225600000", endKey -> end)) + } + assertThat(ex.getMessage).contains("must be strictly before") + assertThat(ex.getMessage).contains("1767225600000") + } + } + + test("predatesRetention compares the window start against now minus table.log.ttl") { + val nowMs = 1767225600000L + val ttlMs = 7L * 24 * 60 * 60 * 1000 + // a start well before the retention horizon predates it + assertThat(FlussOffsetInitializers.predatesRetention(nowMs - ttlMs - 1, ttlMs, nowMs)).isTrue + // a start inside the retention horizon does not + assertThat(FlussOffsetInitializers.predatesRetention(nowMs - ttlMs + 1, ttlMs, nowMs)).isFalse + // a start exactly at the horizon is not warned about + assertThat(FlussOffsetInitializers.predatesRetention(nowMs - ttlMs, ttlMs, nowMs)).isFalse + // a ttl larger than the elapsed time retains everything since epoch 0 + assertThat(FlussOffsetInitializers.predatesRetention(0L, nowMs + 1, nowMs)).isFalse + } + + test("invalid start timestamp format fails with the option name") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.incrementalStartOffsetsInitializer( + scanOptions(startKey -> "not-a-timestamp")) + } + assertThat(ex.getMessage).contains(startKey) + } + + test("scan.startup.mode=timestamp is not a batch option") { + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.startOffsetsInitializer( + scanOptions(SparkFlussConf.SCAN_START_UP_MODE.key() -> "timestamp"), + new Configuration()) + } + assertThat(ex.getMessage).contains("Unsupported scan start up mode") + assertThat(ex.getMessage).contains(SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) + } +} diff --git a/website/docs/engine-spark/options.md b/website/docs/engine-spark/options.md index b1f74cbc5ef..c9d9a4614a5 100644 --- a/website/docs/engine-spark/options.md +++ b/website/docs/engine-spark/options.md @@ -14,6 +14,17 @@ The following Spark configurations can be used to control read behavior for both | Option | Default | Description | |--------|---------|-------------| -| `spark.sql.fluss.scan.startup.mode` | `full` | The startup mode when reading a Fluss table. Supported values: **Note:** For Structured Streaming read, only `latest` mode is currently supported. | +| `spark.sql.fluss.scan.startup.mode` | `full` | The startup mode when reading a Fluss table. Supported values: **Note:** This option only affects Structured Streaming reads, and only `latest` mode is currently supported there. Batch reads ignore it: a plain batch read is always the full table, and a time-range batch read is requested per query (see below). | | `spark.sql.fluss.read.optimized` | `false` | If `true`, Spark will only read data from the data lake snapshot or KV snapshot, without merging log changes. This can improve read performance but may return stale data for primary key tables. | | `spark.sql.fluss.scan.poll.timeout` | `10000ms` | The timeout for the log scanner to poll records. | + +## Per-Query Read Options + +The following options configure a single read and are **not** read from session configuration, so a time window can never leak into later reads. In SQL they are set by the `fluss_incremental_between_timestamp(...)` table-valued function; in the DataFrame API by `spark.read.option(...)` (without the `spark.sql.fluss.` prefix). See [Reads](reads.md#time-range-batch-read) for the full semantics. + +| Option | Default | Description | +|--------|---------|-------------| +| `scan.incremental.start.timestamp` | (none) | Enables an incremental (time-range) batch read and sets the **inclusive** lower bound of the window. Accepts epoch milliseconds (e.g. `1678883047356`) or a `yyyy-MM-dd HH:mm:ss` datetime (e.g. `2023-12-09 23:09:12`) interpreted in the Spark session time zone (`spark.sql.session.timeZone`). A blank or unparseable value fails fast instead of falling back to a full-table read. Batch read only; it has no effect on streaming reads. Only the data Fluss still retains (bounded by `table.log.ttl`) is returned, so a timestamp predating it simply yields fewer rows. | +| `scan.incremental.end.timestamp` | (none) | The **exclusive** upper bound of an incremental batch read, producing a left-closed right-open `[start, end)` window. Same value format as `scan.incremental.start.timestamp`; when unset the read runs up to the latest committed data. The table-valued function always writes a concrete timestamp, pinning the bound when the statement is analyzed. Setting it without `scan.incremental.start.timestamp` fails fast, as does a window whose start is not strictly before its end. | + +Both bounds are compared against the record commit timestamp while reading, so the window is exact even for data already tiered to remote storage, where resolving a timestamp to a log offset is only as accurate as the server-side time index. diff --git a/website/docs/engine-spark/reads.md b/website/docs/engine-spark/reads.md index ca48bdb2017..9ee5dbf852b 100644 --- a/website/docs/engine-spark/reads.md +++ b/website/docs/engine-spark/reads.md @@ -254,6 +254,94 @@ INSERT INTO fluss_order_with_lake VALUES SELECT SUM(total_price) AS total_revenue FROM fluss_order_with_lake; ``` +## Time-Range Batch Read + +A time-range batch read returns the data written to Fluss within a `[start, end)` time window (left-closed, right-open on the commit timestamp). This is the building block for incremental pipelines, e.g. an hourly job that reads only the rows written in the past hour. + +The timestamp value is either epoch milliseconds or a `yyyy-MM-dd HH:mm:ss` datetime interpreted in the Spark session time zone (`spark.sql.session.timeZone`). + +### Output semantics + +| Table type | Output of a `[t1, t2)` read | +|------------|-----------------------------| +| **Log table** | Every record appended within the window. | +| **Primary key table** | The rows whose keys were **inserted or updated** within the window, folded to their latest value as of `t2`. Keys that were only deleted in the window are excluded (i.e. the result keeps `+I`/`+U` after-images and drops `-U`/`-D`). | +| **Lake-enabled table** | Same as the underlying log or primary key table, but read **only from Fluss**. A time-range read never unions the lake snapshot, so it is limited to the data still retained in Fluss (see below). | + +### Using the table-valued function (recommended for SQL) + +`fluss_incremental_between_timestamp(table, start[, end])` reads a time window in a single statement. Omit `end` to read up to the moment the statement is analyzed. + +```sql title="Spark SQL" +-- Read the past hour on a log table (epoch-millis form) +SELECT * FROM fluss_incremental_between_timestamp('log_table', '1767225600000', '1767312000000') +ORDER BY order_id; +``` + +```sql title="Spark SQL" +-- Incremental changes on a primary key table (datetime form) +-- Returns the latest value of every key changed in the window; deleted keys are excluded +SELECT * FROM fluss_incremental_between_timestamp( + 'pk_table', '2026-01-01 00:00:00', '2026-01-02 00:00:00') +ORDER BY order_id; +``` + +```sql title="Spark SQL" +-- Omit the end to read from a start timestamp up to the analysis time +SELECT * FROM fluss_incremental_between_timestamp('log_table', '2026-01-01 00:00:00'); +``` + +The start/end arguments may be any constant expression, so a rolling window does not need to be computed outside SQL: + +```sql title="Spark SQL" +-- The past hour, as epoch milliseconds (unix_timestamp() returns seconds) +SELECT * FROM fluss_incremental_between_timestamp( + 'log_table', + CAST((unix_timestamp() - 3600) * 1000 AS STRING), + CAST(unix_timestamp() * 1000 AS STRING)); + +-- The past hour, as datetime strings +SELECT * FROM fluss_incremental_between_timestamp( + 'log_table', + date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss'), + date_format(now(), 'yyyy-MM-dd HH:mm:ss')); +``` + +The table argument is a string and accepts `table`, `database.table` or `catalog.database.table`; unqualified names resolve against the current catalog and database. The start/end arguments accept a string (epoch milliseconds or `yyyy-MM-dd HH:mm:ss`), an integral epoch-milliseconds value, a `DATE` (the start of that day), or a `TIMESTAMP`/`TIMESTAMP_NTZ` literal — all interpreted in the Spark session time zone — and may be produced by constant expressions such as the datetime functions above (column references are not allowed). The result is an ordinary relation, so projection, filters and joins work as usual. + +:::note +The function is provided by the Fluss Spark session extension, so `spark.sql.extensions=org.apache.fluss.spark.FlussSparkSessionExtensions` must be configured (see [Getting Started](getting-started.md)). Its options apply to that single query only. + +Fluss uses `[start, end)` (start inclusive, end exclusive). This differs from Paimon's similarly named `paimon_incremental_between_timestamp`, which is start-exclusive and end-inclusive. +::: + +### Using the DataFrame API + +The same window can be expressed with scan options, which is how it is configured from the DataFrame API. Setting `scan.incremental.start.timestamp` is what turns a batch read into an incremental one; `scan.incremental.end.timestamp` is optional and the read runs up to the latest committed data when it is left unset: + +```scala title="Spark Scala" +spark.read + .option("scan.incremental.start.timestamp", "2026-01-01 00:00:00") + .option("scan.incremental.end.timestamp", "2026-01-02 00:00:00") + .table("fluss_catalog.fluss.log_table") +``` + +:::note +The `scan.incremental.*` options are per-query read options only. Unlike the options in [Options](options.md), they are **not** picked up from session configuration (`SET spark.sql.fluss.scan.incremental...` has no effect), which keeps a time window from silently applying to later reads in the same session. +::: + +:::warning Retention boundary +A time-range read returns the data Fluss still retains, which is bounded by `table.log.ttl` (default 7 days). A window reaching further back is **not** an error: the part that has already been dropped simply yields fewer rows, or none at all, so the result is always a subset of the requested window. Increase `table.log.ttl` if you need to read further back. Data available only in tiered lake storage is not read by this mode. + +The start timestamp positions the scan through the server's timestamp-to-offset lookup, and both bounds are then applied on each record's commit timestamp, so the window stays exact even for data already tiered to remote storage. + +When the window starts before the table is guaranteed to retain, planning logs a `WARN` on the driver. This is a conservative hint derived from the table's current `table.log.ttl`, not an exact statement about what was deleted: it may over-report (expired segments are deleted lazily, so the data may still be readable) and may under-report (it cannot know that a window predates the table's creation, or that whole partitions were dropped by `table.auto-partition.num-retention`), and it reflects only the current value of `table.log.ttl`, not past changes to it. +::: + +:::note Invalid windows fail fast +Malformed window specifications are rejected instead of silently changing semantics: a blank or unparseable start timestamp, an end timestamp set without a start timestamp (it cannot truncate a plain batch read on its own), and a window whose start is not strictly before its end — whether given through the table-valued function (rejected while the statement is analyzed) or through DataFrame options. A bucket that has no data inside a valid window yields an empty result. +::: + ## All Data Types Fluss Spark connector supports reading all Fluss data types including nested types: diff --git a/website/docs/engine-spark/structured-streaming.md b/website/docs/engine-spark/structured-streaming.md index 4fcf2fafd71..0f6e3ba9ac7 100644 --- a/website/docs/engine-spark/structured-streaming.md +++ b/website/docs/engine-spark/structured-streaming.md @@ -51,7 +51,7 @@ Fluss supports exactly-once semantics for streaming writes through Spark's check Fluss supports reading data from Fluss tables using Spark Structured Streaming. The streaming source continuously reads new data as it arrives. :::caution Limitations -- Streaming read currently only supports the `latest` startup mode. Other modes (`full`, `earliest`, `timestamp`) are not yet supported and will be available in a future release. +- Streaming read currently only supports the `latest` startup mode. Other modes (`full`, `earliest`) are not yet supported and will be available in a future release. To read a bounded time range, use an incremental batch read instead (see [Reads](reads.md#time-range-batch-read)). ::: ### Read from Log Table