From e9d7cc065e5950bb771a97adfef05e93765ad989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Thu, 6 Aug 2026 19:34:08 +0800 Subject: [PATCH 01/10] [spark] Support time-range incremental batch reads (#3842) Add timestamp-bounded batch reads to the Spark connector so downstream pipelines can incrementally read rows written within a [t1, t2) window: - scan.startup.mode=timestamp + scan.startup.timestamp (inclusive start) - scan.bounded.mode=timestamp + scan.bounded.timestamp (exclusive end, defaults to latest committed data at planning time) - Log tables return raw records in the window; primary key tables return keys inserted/updated in the window folded to their latest value - Out-of-range start fails fast by default; scan.startup.timestamp.out-of-range=adjust clamps to earliest retained data - Default behavior unchanged (scan.startup.mode=full) --- .../spark/FlussSparkSessionExtensions.scala | 11 +- .../apache/fluss/spark/SparkFlussConf.scala | 41 ++ .../FlussTableValuedFunctionResolver.scala | 37 ++ .../logical/FlussTableValuedFunctions.scala | 225 +++++++++++ .../spark/read/FlussMicroBatchStream.scala | 2 +- .../spark/read/FlussOffsetInitializers.scala | 159 +++++++- .../fluss/spark/read/SplitPlanner.scala | 207 ++++++++-- .../fluss/spark/SparkTimeRangeTvfTest.scala | 367 ++++++++++++++++++ .../lake/SparkLakeTimeRangeReadTest.scala | 151 +++++++ .../read/FlussOffsetInitializersTest.scala | 103 +++++ website/docs/engine-spark/options.md | 12 +- website/docs/engine-spark/reads.md | 80 ++++ .../docs/engine-spark/structured-streaming.md | 2 +- 13 files changed, 1354 insertions(+), 43 deletions(-) create mode 100644 fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/analysis/FlussTableValuedFunctionResolver.scala create mode 100644 fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala create mode 100644 fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala create mode 100644 fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala create mode 100644 fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/FlussSparkSessionExtensions.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/FlussSparkSessionExtensions.scala index dc12ed50342..5c3faf8638c 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/FlussSparkSessionExtensions.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/FlussSparkSessionExtensions.scala @@ -17,7 +17,8 @@ package org.apache.fluss.spark -import org.apache.fluss.spark.catalyst.analysis.FlussProcedureResolver +import org.apache.fluss.spark.catalyst.analysis.{FlussProcedureResolver, FlussTableValuedFunctionResolver} +import org.apache.fluss.spark.catalyst.plans.logical.FlussTableValuedFunctions import org.apache.fluss.spark.execution.FlussStrategy import org.apache.spark.sql.SparkSessionExtensions @@ -32,6 +33,14 @@ class FlussSparkSessionExtensions extends (SparkSessionExtensions => Unit) { // analyzer extensions extensions.injectResolutionRule(spark => FlussProcedureResolver(spark)) + extensions.injectResolutionRule(spark => FlussTableValuedFunctionResolver(spark)) + + // table function extensions + FlussTableValuedFunctions.supportedFnNames.foreach { + fnName => + extensions.injectTableFunction( + FlussTableValuedFunctions.getTableValueFunctionInjection(fnName)) + } // planner extensions extensions.injectPlannerStrategy(spark => FlussStrategy(spark)) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala index aac6a698daf..aaa8ca1d75f 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala @@ -37,6 +37,13 @@ object SparkFlussConf { val FULL, EARLIEST, LATEST, TIMESTAMP = Value } + object TimestampOutOfRangeMode extends Enumeration { + val ERROR, ADJUST = Value + } + + /** Reserved value of [[SCAN_INCREMENTAL_END_TIMESTAMP]] meaning "the latest committed data". */ + val END_TIMESTAMP_LATEST = "latest" + val SCAN_START_UP_MODE: ConfigOption[String] = ConfigBuilder .key("scan.startup.mode") @@ -44,6 +51,40 @@ object SparkFlussConf { .defaultValue(StartUpMode.FULL.toString) .withDescription("The start up mode when read Fluss table.") + val SCAN_INCREMENTAL_START_TIMESTAMP: ConfigOption[String] = + ConfigBuilder + .key("scan.incremental.start.timestamp") + .stringType() + .noDefaultValue() + .withDescription( + "Enables an incremental (time-range) batch read and sets the inclusive lower bound of " + + "the window. Accepts either epoch milliseconds (e.g. '1678883047356') or a " + + "'yyyy-MM-dd HH:mm:ss' datetime string (e.g. '2023-12-09 23:09:12') interpreted in " + + "the Spark session time zone. Batch read only; it has no effect on streaming reads.") + + val SCAN_INCREMENTAL_END_TIMESTAMP: ConfigOption[String] = + ConfigBuilder + .key("scan.incremental.end.timestamp") + .stringType() + .defaultValue(END_TIMESTAMP_LATEST) + .withDescription( + "The exclusive upper bound of an incremental (time-range) batch read, yielding a " + + "left-closed right-open '[start, end)' window. 'latest' (default) stops at the " + + "latest committed data captured at planning time; otherwise accepts epoch " + + "milliseconds or a 'yyyy-MM-dd HH:mm:ss' datetime string interpreted in the Spark " + + "session time zone. Only honored when 'scan.incremental.start.timestamp' is set.") + + val SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE: ConfigOption[String] = + ConfigBuilder + .key("scan.incremental.timestamp.out-of-range") + .stringType() + .defaultValue(TimestampOutOfRangeMode.ERROR.toString) + .withDescription( + "Behavior when 'scan.incremental.start.timestamp' precedes the earliest data still " + + "retained by Fluss (bounded by 'table.log.ttl'). 'error' (default): fail fast so a " + + "truncated window is never returned silently. 'adjust': clamp the start to the " + + "earliest retained offset and read from there.") + val SCAN_POLL_TIMEOUT: ConfigOption[Duration] = ConfigBuilder .key("scan.poll.timeout") diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/analysis/FlussTableValuedFunctionResolver.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/analysis/FlussTableValuedFunctionResolver.scala new file mode 100644 index 00000000000..62f0faab691 --- /dev/null +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/analysis/FlussTableValuedFunctionResolver.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark.catalyst.analysis + +import org.apache.fluss.spark.catalyst.plans.logical.{FlussTableValuedFunctions, FlussTableValueFunction} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule + +/** + * Resolution rule for Fluss table-valued functions. The injected table function builder produces an + * unresolved [[FlussTableValueFunction]]; once its arguments are resolved this rule rewrites it + * into a plain DataSourceV2 relation carrying the derived scan options. + */ +case class FlussTableValuedFunctionResolver(sparkSession: SparkSession) extends Rule[LogicalPlan] { + + override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsDown { + case func: FlussTableValueFunction if func.args.forall(_.resolved) => + FlussTableValuedFunctions.resolveFlussTableValuedFunction(sparkSession, func) + } +} diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala new file mode 100644 index 00000000000..0adf8b9248a --- /dev/null +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark.catalyst.plans.logical + +import org.apache.fluss.spark.{SparkFlussConf, SparkTable} +import org.apache.fluss.spark.catalyst.plans.logical.FlussTableValuedFunctions._ + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.FunctionIdentifier +import org.apache.spark.sql.catalyst.analysis.FunctionRegistryBase +import org.apache.spark.sql.catalyst.analysis.TableFunctionRegistry.TableFunctionBuilder +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExpressionInfo, RuntimeReplaceable} +import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} +import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.types.{IntegerType, LongType, ShortType, StringType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +import scala.collection.JavaConverters._ +import scala.util.control.NonFatal + +/** + * Fluss table-valued functions (TVFs), usable from pure SQL. + * + * A TVF is only sugar over per-relation scan options: the function arguments are translated into + * the same `scan.*` options the DataFrame API accepts, and the call is then resolved into a plain + * [[DataSourceV2Relation]]. Consequently projection, filter push down and metrics keep working, and + * the options are scoped to the single query instead of leaking through session configuration. + */ +object FlussTableValuedFunctions { + + val INCREMENTAL_BETWEEN_TIMESTAMP = "fluss_incremental_between_timestamp" + + val supportedFnNames: Seq[String] = Seq(INCREMENTAL_BETWEEN_TIMESTAMP) + + private type TableFunctionDescription = + (FunctionIdentifier, ExpressionInfo, TableFunctionBuilder) + + def getTableValueFunctionInjection(fnName: String): TableFunctionDescription = { + val (info, builder) = fnName match { + case INCREMENTAL_BETWEEN_TIMESTAMP => + FunctionRegistryBase.build[IncrementalBetweenTimestamp](fnName, since = None) + case _ => + throw new IllegalArgumentException( + s"Function $fnName isn't a supported Fluss table valued function.") + } + (FunctionIdentifier(fnName), info, builder) + } + + /** + * Resolves a Fluss TVF call into a [[DataSourceV2Relation]] over the referenced Fluss table, with + * the function arguments translated into scan options. + */ + def resolveFlussTableValuedFunction( + spark: SparkSession, + tvf: FlussTableValueFunction): LogicalPlan = { + val args = tvf.args + val sessionState = spark.sessionState + val catalogManager = sessionState.catalogManager + + if (args.isEmpty) { + throw new IllegalArgumentException( + s"${tvf.fnName} requires a table identifier as its first argument.") + } + + // Parse the remaining arguments first so that an argument error is reported without depending + // on the referenced table being resolvable. + val options = tvf.parseArgs(args.tail) + + val tableArg = args.head.eval() + if (tableArg == null) { + throw new IllegalArgumentException( + s"The first argument of ${tvf.fnName} must be a non-null table identifier.") + } + val tableIdentifier = tableArg.toString + + val (catalogName, namespace, tableName) = + sessionState.sqlParser.parseMultipartIdentifier(tableIdentifier) match { + case Seq(table) => + (catalogManager.currentCatalog.name(), catalogManager.currentNamespace.head, table) + case Seq(db, table) => (catalogManager.currentCatalog.name(), db, table) + case Seq(catalog, db, table) => (catalog, db, table) + case _ => + throw new IllegalArgumentException( + s"Invalid table identifier '$tableIdentifier' for ${tvf.fnName}. Expected " + + "'table', 'database.table' or 'catalog.database.table'.") + } + + val catalogPlugin = catalogManager.catalog(catalogName) + if (!catalogPlugin.isInstanceOf[TableCatalog]) { + throw new IllegalArgumentException( + s"${tvf.fnName} requires a table catalog, but catalog '$catalogName' is " + + s"${catalogPlugin.getClass.getName}.") + } + val tableCatalog = catalogPlugin.asInstanceOf[TableCatalog] + val ident = Identifier.of(Array(namespace), tableName) + val table = tableCatalog.loadTable(ident) + if (!table.isInstanceOf[SparkTable]) { + throw new IllegalArgumentException( + s"${tvf.fnName} only supports Fluss tables, but '$catalogName.$namespace.$tableName' is " + + s"backed by ${table.getClass.getName}.") + } + + DataSourceV2Relation.create( + table, + Some(tableCatalog), + Some(ident), + new CaseInsensitiveStringMap(options.asJava)) + } + + /** + * Normalizes a timestamp argument to the string form accepted by the `scan.incremental.*` + * timestamp options. + * + * A STRING argument is passed through untouched, so both epoch milliseconds and + * `yyyy-MM-dd HH:mm:ss` keep being interpreted by the option layer. Integral arguments are epoch + * milliseconds. TIMESTAMP arguments are converted from Spark's internal microseconds, otherwise a + * `TIMESTAMP '...'` literal would silently be read as epoch milliseconds. + * + * Any constant expression is accepted, e.g. `CAST(unix_timestamp() * 1000 AS STRING)` or + * `date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss')`. + */ + private[logical] def toTimestampOptionValue(fnName: String, expr: Expression): String = { + // `RuntimeReplaceable` expressions (such as the `-` in `now() - INTERVAL 1 HOUR`) only become + // evaluable once the optimizer's ReplaceExpressions rule rewrites them, which has not happened + // yet while the analyzer resolves this function. Apply the same rewrite bottom-up here. + val evaluable = expr.transformUp { case r: RuntimeReplaceable => r.replacement } + + val value = + try { + evaluable.eval() + } catch { + case NonFatal(e) => + throw new IllegalArgumentException( + s"Failed to evaluate the timestamp argument '${expr.sql}' of $fnName. It must be a " + + "constant expression; literals and datetime functions such as now() or " + + "unix_timestamp() are supported, references to table columns are not.", + e + ) + } + if (value == null) { + throw new IllegalArgumentException(s"Timestamp arguments of $fnName must not be null.") + } + evaluable.dataType match { + case StringType => value.toString + case ShortType | IntegerType | LongType => value.toString + case TimestampType | TimestampNTZType => (value.asInstanceOf[Long] / 1000L).toString + case other => + throw new IllegalArgumentException( + s"Unsupported timestamp argument type $other for $fnName. Use a STRING (epoch " + + "milliseconds or 'yyyy-MM-dd HH:mm:ss'), an integral epoch milliseconds value, or a " + + "TIMESTAMP.") + } + } +} + +/** + * An unresolved Fluss table-valued function. + * + * @param fnName + * one of [[FlussTableValuedFunctions.supportedFnNames]]. + */ +abstract class FlussTableValueFunction(val fnName: String) extends LeafNode { + + override def output: Seq[Attribute] = Nil + + override lazy val resolved = false + + val args: Seq[Expression] + + /** Translates the arguments following the table identifier into Fluss scan options. */ + def parseArgs(argsWithoutTable: Seq[Expression]): Map[String, String] +} + +/** + * Plan for [[FlussTableValuedFunctions.INCREMENTAL_BETWEEN_TIMESTAMP]]. + * + * Usage: + * - `fluss_incremental_between_timestamp(table, startTimestamp, endTimestamp)` + * - `fluss_incremental_between_timestamp(table, startTimestamp)` reads up to the latest data + * + * The window is left-closed and right-open, `[start, end)`, on the record commit timestamp. + */ +case class IncrementalBetweenTimestamp(override val args: Seq[Expression]) + extends FlussTableValueFunction(INCREMENTAL_BETWEEN_TIMESTAMP) { + + override def parseArgs(argsWithoutTable: Seq[Expression]): Map[String, String] = { + if (argsWithoutTable.size != 1 && argsWithoutTable.size != 2) { + throw new IllegalArgumentException( + s"$INCREMENTAL_BETWEEN_TIMESTAMP needs a table identifier followed by a startTimestamp " + + s"and an optional endTimestamp, e.g. " + + s"$INCREMENTAL_BETWEEN_TIMESTAMP('db.t', '2026-01-01 00:00:00', '2026-01-01 01:00:00'). " + + s"Got ${argsWithoutTable.size + 1} arguments.") + } + + val start = toTimestampOptionValue(INCREMENTAL_BETWEEN_TIMESTAMP, argsWithoutTable.head) + val startOptions = + Map(SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() -> start) + + // The end bound is always written explicitly so the call stays self-contained: options take + // precedence over session configuration, which may still hold a stale end timestamp. + if (argsWithoutTable.size == 2) { + val end = toTimestampOptionValue(INCREMENTAL_BETWEEN_TIMESTAMP, argsWithoutTable.last) + startOptions + (SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() -> end) + } else { + startOptions + + (SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() -> SparkFlussConf.END_TIMESTAMP_LATEST) + } + } +} diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussMicroBatchStream.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussMicroBatchStream.scala index 6351dba48f4..0a40f5bdd22 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussMicroBatchStream.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussMicroBatchStream.scala @@ -68,7 +68,7 @@ abstract class FlussMicroBatchStream( FlussOffsetInitializers.startOffsetsInitializer(options, flussConfig) val stoppingOffsetsInitializer: OffsetsInitializer = - FlussOffsetInitializers.stoppingOffsetsInitializer(false, options, flussConfig) + FlussOffsetInitializers.stoppingOffsetsInitializer(false, options) protected def projection: Array[Int] = FlussScanBuilder.projectionOf(tableInfo, Some(readSchema)) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala index 1f0a8806ae3..fc111b08a2c 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala @@ -18,20 +18,90 @@ package org.apache.fluss.spark.read import org.apache.fluss.client.initializer.{NoStoppingOffsetsInitializer, OffsetsInitializer} -import org.apache.fluss.config.Configuration +import org.apache.fluss.config.{ConfigOption, Configuration} import org.apache.fluss.spark.SparkFlussConf +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.util.CaseInsensitiveStringMap +import java.time.{LocalDateTime, ZoneId} +import java.time.format.DateTimeFormatter + object FlussOffsetInitializers { + + private val DATE_TIME_FORMATTER: DateTimeFormatter = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") + + /** + * Whether an incremental (time-range) batch read is requested, i.e. + * `scan.incremental.start.timestamp` is set on the relation being scanned. + * + * The `scan.incremental.*` options are read from the per-query scan options only — set by the + * `fluss_incremental_between_timestamp` table-valued function or `DataFrameReader.option` — and + * deliberately not from session configuration, so a window can never leak into another query. + * Streaming reads ignore them. + */ + def isIncrementalRead(options: CaseInsensitiveStringMap): Boolean = { + incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP).isDefined + } + + /** + * Whether a resolved start offset predates the data Fluss still retains for a bucket. A bucket + * whose earliest offset is still 0 has dropped nothing and is never flagged. + */ + def isBeforeRetention(startOffset: Long, earliestOffset: Long): Boolean = + earliestOffset > 0 && startOffset <= earliestOffset + + /** + * Rejects a start offset that predates the earliest retained data (see [[isBeforeRetention]]), so + * a truncated window is never returned silently. Callers must pass a concrete earliest offset, + * i.e. from a retriever built with `fetchEarliestOffset = true`. + */ + def requireStartWithinRetention( + tableDescription: String, + partitionName: String, + bucketId: Int, + startOffset: Long, + earliestOffset: Long): Unit = { + if (isBeforeRetention(startOffset, earliestOffset)) { + val partitionDesc = if (partitionName != null) s" partition '$partitionName'" else "" + throw new IllegalArgumentException( + s"The requested start timestamp resolves to log offset $startOffset for bucket " + + s"$bucketId$partitionDesc of table $tableDescription, which is at or before the " + + s"earliest retained offset $earliestOffset. The requested time range exceeds Fluss " + + s"retention (table.log.ttl); narrow the time range or increase table.log.ttl.") + } + } + + /** + * Whether a start timestamp preceding the earliest retained data fails fast (default) instead of + * being clamped to that offset. Controlled by `scan.incremental.timestamp.out-of-range`. + */ + def failOnTimestampOutOfRange(options: CaseInsensitiveStringMap): Boolean = { + val mode = + incrementalOption( + options, + SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE).get.toUpperCase + SparkFlussConf.TimestampOutOfRangeMode.withName(mode) == + SparkFlussConf.TimestampOutOfRangeMode.ERROR + } + + /** + * Start offsets of an incremental batch read, resolved from `scan.incremental.start.timestamp`. + * Requires that option to be set. + */ + def incrementalStartOffsetsInitializer(options: CaseInsensitiveStringMap): OffsetsInitializer = + OffsetsInitializer.timestamp( + requiredTimestamp(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP)) + + /** + * Start offsets of a streaming read, driven by `scan.startup.mode`. Batch reads ignore this + * option (see [[incrementalStartOffsetsInitializer]]). + */ def startOffsetsInitializer( options: CaseInsensitiveStringMap, flussConfig: Configuration): OffsetsInitializer = { - val startupMode = options - .getOrDefault( - SparkFlussConf.SCAN_START_UP_MODE.key(), - flussConfig.get(SparkFlussConf.SCAN_START_UP_MODE)) - .toUpperCase + val startupMode = resolveStartupMode(options, flussConfig).toUpperCase SparkFlussConf.StartUpMode.withName(startupMode) match { case SparkFlussConf.StartUpMode.EARLIEST => OffsetsInitializer.earliest() @@ -39,18 +109,85 @@ object FlussOffsetInitializers { case SparkFlussConf.StartUpMode.LATEST => OffsetsInitializer.latest() case _ => throw new IllegalArgumentException( - s"Unsupported scan start up mode: ${options.get(SparkFlussConf.SCAN_START_UP_MODE.key())}") + s"Unsupported scan start up mode: " + + s"${resolveStartupMode(options, flussConfig)}. Supported values are 'full', " + + s"'earliest' and 'latest'. For a time-range batch read set " + + s"'${SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()}' instead.") } } def stoppingOffsetsInitializer( isBatch: Boolean, - options: CaseInsensitiveStringMap, - flussConfig: Configuration): OffsetsInitializer = { - if (isBatch) { + options: CaseInsensitiveStringMap): OffsetsInitializer = { + if (!isBatch) { + new NoStoppingOffsetsInitializer() + } else if (!isIncrementalRead(options)) { + // A plain batch read stops at the latest committed data; an end timestamp alone must not + // truncate it. OffsetsInitializer.latest() } else { - new NoStoppingOffsetsInitializer() + val end = + incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP).getOrElse("").trim + if ( + end.isEmpty || + end.equalsIgnoreCase(SparkFlussConf.END_TIMESTAMP_LATEST) + ) { + OffsetsInitializer.latest() + } else { + OffsetsInitializer.timestamp( + parseTimestamp(end.trim, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key())) + } + } + } + + /** Reads a `scan.incremental.*` option from the scan options, falling back to its default. */ + private def incrementalOption( + options: CaseInsensitiveStringMap, + option: ConfigOption[String]): Option[String] = + Option(options.getOrDefault(option.key(), option.defaultValue())) + + private def resolveStartupMode( + options: CaseInsensitiveStringMap, + flussConfig: Configuration): String = + options.getOrDefault( + SparkFlussConf.SCAN_START_UP_MODE.key(), + flussConfig.get(SparkFlussConf.SCAN_START_UP_MODE)) + + private def requiredTimestamp( + options: CaseInsensitiveStringMap, + option: ConfigOption[String]): Long = { + val value = incrementalOption(options, option) + if (value.getOrElse("").isEmpty) { + throw new IllegalArgumentException( + s"'${option.key()}' must not be empty. Provide epoch milliseconds or a " + + s"'yyyy-MM-dd HH:mm:ss' timestamp.") + } + parseTimestamp(value.get.trim, option.key()) + } + + /** + * Parses a timestamp option value to epoch milliseconds: a purely numeric string is epoch + * milliseconds, otherwise it is parsed as 'yyyy-MM-dd HH:mm:ss' in the Spark session time zone. + */ + private def parseTimestamp(timestampStr: String, optionKey: String): Long = { + if (timestampStr.matches("\\d+")) { + timestampStr.toLong + } else { + try { + LocalDateTime + .parse(timestampStr, DATE_TIME_FORMATTER) + .atZone(ZoneId.of(SQLConf.get.sessionLocalTimeZone)) + .toInstant + .toEpochMilli + } catch { + case e: Exception => + throw new IllegalArgumentException( + s"Invalid value for '$optionKey': '$timestampStr'. It should be epoch milliseconds or " + + s"follow the format 'yyyy-MM-dd HH:mm:ss', e.g. '2023-12-09 23:09:12' or " + + s"'1678883047356'.", + e + ) + } } } } diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala index 40a633135f9..9414d868091 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala @@ -198,6 +198,32 @@ abstract class AbstractSplitPlanner( .toMap } + /** + * Fail-fast guard for an incremental batch read: rejects a start offset that predates the data + * Fluss still retains (bounded by `table.log.ttl`) instead of silently returning a truncated + * window. Requires a retriever created with `fetchEarliestOffset = true`; otherwise + * `earliestOffsets` returns the EARLIEST_OFFSET sentinel (-2) and the guard is a no-op. + */ + protected def checkTimeRangeWithinRetention( + partitionName: String, + buckets: Seq[Int], + startOffsets: scala.collection.Map[Integer, java.lang.Long], + bucketOffsetsRetriever: BucketOffsetsRetrieverImpl): Unit = { + val earliestOffsets = bucketOffsetsRetriever + .earliestOffsets(partitionName, buckets.map(Integer.valueOf).asJava) + .asScala + buckets.foreach { + bucketId => + val bucket = Integer.valueOf(bucketId) + FlussOffsetInitializers.requireStartWithinRetention( + tablePath.toString, + partitionName, + bucketId, + Long2long(startOffsets(bucket)), + Long2long(earliestOffsets(bucket))) + } + } + /** * Releases the Fluss client connection. Idempotent and null-safe; it never forces the lazily * opened connection into existence, so it is a no-op when no metadata access ever occurred. @@ -215,10 +241,10 @@ abstract class AbstractSplitPlanner( } /** - * Single append (log-table) planner. Probes a readable lake snapshot at construction; if present, - * the plan is a union of lake splits and the Fluss log-tail (from each bucket's snapshotLogOffset - * to committed). If absent, the plan is a pure Fluss log scan from earliest to committed - * (SCAN_START_UP_MODE deliberately ignored — see class scaladoc note below). + * Single append (log-table) planner. Probes a readable lake snapshot at construction; if present + * (and not an incremental read), the plan is a union of lake splits and the Fluss log-tail (from + * each bucket's snapshotLogOffset to committed). If absent, the plan is a pure Fluss log scan from + * earliest to committed (SCAN_START_UP_MODE deliberately ignored — see class scaladoc note below). * * Batch semantics note: start offset is hardcoded to [[OffsetsInitializer.full]] instead of * consuming the user-facing SCAN_START_UP_MODE. Rationale — batch reads semantically mean "the full @@ -227,8 +253,10 @@ abstract class AbstractSplitPlanner( * snapshot has no partial-read semantics), which is confusing; (b) with mode=latest and no writes * since planning time, start==stop==tail — an empty range that trips the reader-side * `Invalid offset range` guard. Symmetric "batch = earliest → committed" closes both concerns and - * keeps append/upsert planners aligned. Time-range batch reads should be expressed via predicate - * pushdown on the timestamp column, not startup mode. + * keeps append/upsert planners aligned. A bounded time range is instead requested with the + * batch-only `scan.incremental.start.timestamp` / `scan.incremental.end.timestamp` options, which + * resolve to log offsets identically on append and upsert tables; such an incremental read always + * takes the log-only branch and never unions a lake snapshot. * * `OffsetsInitializer.full()` is chosen over `OffsetsInitializer.earliest()` intentionally: for a * log table the two are semantically equivalent (see OffsetsInitializer.full javadoc), but full() @@ -247,12 +275,26 @@ class AppendPlanner( extends AbstractSplitPlanner(tablePath, tableInfo, flussConfig) with AppendSplitPlanner { - override def hasLakeSnapshot: Boolean = readableLakeSnapshot.isDefined + override def hasLakeSnapshot: Boolean = readableLakeSnapshot.isDefined && !incrementalMode + + private val incrementalMode: Boolean = + FlussOffsetInitializers.isIncrementalRead(options) + + // Whether a start timestamp preceding retained data fails (default) or is clamped to earliest. + private val failOnOutOfRange: Boolean = + FlussOffsetInitializers.failOnTimestampOutOfRange(options) - private val startOffsetsInitializer: OffsetsInitializer = OffsetsInitializer.full() + // An incremental read starts at scan.incremental.start.timestamp, a plain batch read at the + // beginning of the table (see class scaladoc). + private val startOffsetsInitializer: OffsetsInitializer = + if (incrementalMode) { + FlussOffsetInitializers.incrementalStartOffsetsInitializer(options) + } else { + OffsetsInitializer.full() + } override protected val stoppingOffsetsInitializer: OffsetsInitializer = - FlussOffsetInitializers.stoppingOffsetsInitializer(true, options, flussConfig) + FlussOffsetInitializers.stoppingOffsetsInitializer(true, options) // Server-side log filter requires ARROW format. Pushdown already gates this on the log-only // path (never sets pushedPredicate for non-ARROW), but re-checking here keeps the planner @@ -264,15 +306,16 @@ class AppendPlanner( override def plan(): Array[InputPartition] = try { readableLakeSnapshot match { - case Some(snap) => planLakeUnion(snap) - case None => planLogOnly() + // An incremental read never unions a lake snapshot; it reads only Fluss. + case Some(snap) if !incrementalMode => planLakeUnion(snap) + case _ => planLogOnly() } } finally { close() } // --------------------------------------------------------------------------------------------- - // Log-only branch: pure Fluss log scan from earliest → committed with optional range splitting. + // Log-only branch: pure Fluss log scan over [start, stop) with optional range splitting. // --------------------------------------------------------------------------------------------- private def planLogOnly(): Array[InputPartition] = { @@ -281,10 +324,13 @@ class AppendPlanner( if (value > 0) Some(value) else None } - val bucketOffsetsRetrieverImpl = maxRecordsPerPartition match { - case Some(_) => new BucketOffsetsRetrieverImpl(admin, tablePath, true) - case _ => new BucketOffsetsRetrieverImpl(admin, tablePath) - } + // Both the retention guard and the max-records splitter need concrete earliest offsets; + // otherwise the earliest sentinel (-2) is enough. + val bucketOffsetsRetrieverImpl = + new BucketOffsetsRetrieverImpl( + admin, + tablePath, + maxRecordsPerPartition.isDefined || incrementalMode) val buckets = (0 until tableInfo.getNumBuckets).toSeq def splitOffsetRange( @@ -319,13 +365,19 @@ class AppendPlanner( bucketId => val (startOffset, stopOffset) = (startBucketOffsets(bucketId), stoppingBucketOffsets(bucketId)) - val tableBucket = partitionId match { - case Some(pid) => new TableBucket(tableInfo.getTableId, pid, bucketId) - case None => new TableBucket(tableInfo.getTableId, bucketId) - } - maxRecordsPerPartition match { - case Some(maxRecs) => splitOffsetRange(tableBucket, startOffset, stopOffset, maxRecs) - case _ => Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset)) + if (startOffset >= stopOffset) { + // Empty range (e.g. a time-range window with no data, or an empty bucket): emit no + // partition so the append reader is not handed an invalid [start, start) range. + Seq.empty[InputPartition] + } else { + val tableBucket = partitionId match { + case Some(pid) => new TableBucket(tableInfo.getTableId, pid, bucketId) + case None => new TableBucket(tableInfo.getTableId, bucketId) + } + maxRecordsPerPartition match { + case Some(maxRecs) => splitOffsetRange(tableBucket, startOffset, stopOffset, maxRecs) + case _ => Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset)) + } } }.toArray } @@ -338,14 +390,22 @@ class AppendPlanner( matching .map { partitionInfo => + val partitionName = partitionInfo.getPartitionName val startBucketOffsets = startOffsetsInitializer.getBucketOffsets( - partitionInfo.getPartitionName, + partitionName, buckets.map(Integer.valueOf).asJava, bucketOffsetsRetrieverImpl) val stoppingBucketOffsets = stoppingOffsetsInitializer.getBucketOffsets( - partitionInfo.getPartitionName, + partitionName, buckets.map(Integer.valueOf).asJava, bucketOffsetsRetrieverImpl) + if (incrementalMode && failOnOutOfRange) { + checkTimeRangeWithinRetention( + partitionName, + buckets, + startBucketOffsets.asScala, + bucketOffsetsRetrieverImpl) + } ( partitionInfo.getPartitionId, startBucketOffsets.asScala.map(e => (e._1, Long2long(e._2))), @@ -368,6 +428,13 @@ class AppendPlanner( null, buckets.map(Integer.valueOf).asJava, bucketOffsetsRetrieverImpl) + if (incrementalMode && failOnOutOfRange) { + checkTimeRangeWithinRetention( + null, + buckets, + startBucketOffsets.asScala, + bucketOffsetsRetrieverImpl) + } createPartitions( None, startBucketOffsets.asScala.map(e => (e._1, Long2long(e._2))).toMap, @@ -567,7 +634,9 @@ class AppendPlanner( * partitions. If absent, the plan is a pure Fluss upsert scan derived from kv snapshots + log tail. * * Startup-mode gating has been removed: a batch upsert scan is always full-table regardless of the - * user-facing SCAN_START_UP_MODE setting — same rationale as [[AppendPlanner]]. + * user-facing SCAN_START_UP_MODE setting — same rationale as [[AppendPlanner]]. Setting + * `scan.incremental.start.timestamp` instead yields an incremental read that folds only the + * changelog within the requested window. */ class UpsertPlanner( override val tablePath: TablePath, @@ -580,10 +649,23 @@ class UpsertPlanner( extends AbstractSplitPlanner(tablePath, tableInfo, flussConfig) with UpsertSplitPlanner { - override def hasLakeSnapshot: Boolean = readableLakeSnapshot.isDefined + override def hasLakeSnapshot: Boolean = readableLakeSnapshot.isDefined && !incrementalMode + + private val incrementalMode: Boolean = + FlussOffsetInitializers.isIncrementalRead(options) + + // Whether a start timestamp preceding retained data fails (default) or is clamped to earliest. + private val failOnOutOfRange: Boolean = + FlussOffsetInitializers.failOnTimestampOutOfRange(options) + + // Start offset of an incremental read, resolved from scan.incremental.start.timestamp. Lazy on + // purpose: resolving it requires that option, while a plain batch scan derives its start from kv + // snapshots instead. + private lazy val incrementalStartOffsetsInitializer: OffsetsInitializer = + FlussOffsetInitializers.incrementalStartOffsetsInitializer(options) override protected val stoppingOffsetsInitializer: OffsetsInitializer = - FlussOffsetInitializers.stoppingOffsetsInitializer(true, options, flussConfig) + FlussOffsetInitializers.stoppingOffsetsInitializer(true, options) // Upsert never pushes a server-side log filter (kv+log union semantics require full log tail // to be reconciled with kv snapshots — see FlussUpsertPartitionReader). @@ -592,6 +674,9 @@ class UpsertPlanner( override def plan(): Array[InputPartition] = try { readableLakeSnapshot match { + // An incremental read reads neither the lake nor the kv snapshot; it folds only the Fluss + // changelog within [start, end). + case _ if incrementalMode => planIncrementalLogOnly() case Some(snap) => planLakeUnion(snap) case None => planLogOnly() } @@ -661,6 +746,72 @@ class UpsertPlanner( .toArray } + // --------------------------------------------------------------------------------------------- + // Incremental branch: fold the Fluss changelog within [start, end) per bucket, with no kv + // snapshot and no lake. Emitting snapshotId = -1 makes FlussUpsertPartitionReader skip the + // snapshot and fold only the log range; SortMergeReader drops delete rows, so the output is the + // surviving +I/+U rows (keys inserted or updated in the window; deleted keys excluded). + // --------------------------------------------------------------------------------------------- + + private def planIncrementalLogOnly(): Array[InputPartition] = { + val bucketOffsetsRetriever = new BucketOffsetsRetrieverImpl(admin, tablePath, true) + val buckets = (0 until tableInfo.getNumBuckets).toSeq + + if (tableInfo.isPartitioned) { + val matching = SparkPartitionPredicate.filterPartitions( + tableInfo, + partitionInfos.asScala.toSeq, + partitionPredicate) + matching.flatMap { + partitionInfo => + createIncrementalUpsertPartitions( + partitionInfo.getPartitionName, + Some(partitionInfo.getPartitionId), + buckets, + bucketOffsetsRetriever) + }.toArray + } else { + createIncrementalUpsertPartitions(null, None, buckets, bucketOffsetsRetriever) + } + } + + private def createIncrementalUpsertPartitions( + partitionName: String, + partitionId: Option[Long], + buckets: Seq[Int], + bucketOffsetsRetriever: BucketOffsetsRetrieverImpl): Array[InputPartition] = { + val jBuckets = buckets.map(Integer.valueOf).asJava + val startBucketOffsets = + incrementalStartOffsetsInitializer.getBucketOffsets( + partitionName, + jBuckets, + bucketOffsetsRetriever) + val stoppingBucketOffsets = + stoppingOffsetsInitializer.getBucketOffsets(partitionName, jBuckets, bucketOffsetsRetriever) + if (failOnOutOfRange) { + checkTimeRangeWithinRetention( + partitionName, + buckets, + startBucketOffsets.asScala, + bucketOffsetsRetriever) + } + + val tableId = tableInfo.getTableId + buckets.map { + bucketId => + val tableBucket = partitionId match { + case Some(pid) => new TableBucket(tableId, pid, bucketId) + case None => new TableBucket(tableId, bucketId) + } + FlussUpsertInputPartition( + tableBucket, + -1L, + Long2long(startBucketOffsets.get(Integer.valueOf(bucketId))), + Long2long(stoppingBucketOffsets.get(Integer.valueOf(bucketId)))) + .asInstanceOf[InputPartition] + }.toArray + } + // --------------------------------------------------------------------------------------------- // Lake-union branch: lake splits (upsert view) + Fluss log tail after snapshotLogOffset. // --------------------------------------------------------------------------------------------- diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala new file mode 100644 index 00000000000..b28c721d78a --- /dev/null +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark + +import org.apache.fluss.row.{BinaryString, GenericRow} + +import org.apache.spark.sql.Row +import org.assertj.core.api.Assertions.assertThat + +import java.time.{Instant, ZoneId} +import java.time.format.DateTimeFormatter + +/** + * Verifies the `fluss_incremental_between_timestamp` table-valued function. The window is + * left-closed, right-open `[start, end)` on the record commit timestamp, and the function's options + * are scoped to the single query. + */ +class SparkTimeRangeTvfTest extends FlussSparkTestBase { + + private val TVF = "fluss_incremental_between_timestamp" + + private def createLogTable(name: String): Unit = + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.$name + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) + |""".stripMargin) + + /** Truncates to a whole second so a millis value, a datetime string and a TIMESTAMP agree. */ + private def secondAligned(ms: Long): Long = (ms / 1000L) * 1000L + + private def waitPast(ms: Long): Unit = { + while (System.currentTimeMillis() <= ms) { + Thread.sleep(20) + } + } + + private def formatTs(ms: Long): String = + Instant + .ofEpochMilli(ms) + .atZone(ZoneId.of(spark.sessionState.conf.sessionLocalTimeZone)) + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + + private def fullMessage(t: Throwable): String = { + val sw = new java.io.StringWriter() + t.printStackTrace(new java.io.PrintWriter(sw)) + sw.toString + } + + test("TVF: log table window [t1, t2)") { + withTable("t") { + createLogTable("t") + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES + |(1L, 11L, 101, "a1"), (2L, 12L, 102, "a2")""".stripMargin) + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES + |(3L, 13L, 103, "a3"), (4L, 14L, 104, "a4")""".stripMargin) + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (5L, 15L, 105, "a5")""") + Thread.sleep(200) + + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), + Row(3L, 13L, 103, "a3") :: Row(4L, 14L, 104, "a4") :: Nil) + + // projection and filter still work on top of the TVF relation + checkAnswer( + sql(s"""SELECT address FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') + |WHERE amount = 104""".stripMargin), + Row("a4") :: Nil) + } + } + + test("TVF: two-argument form reads up to the latest data") { + withTable("t") { + createLogTable("t") + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") + Thread.sleep(300) + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (3L, 13L, 103, "a3")""") + Thread.sleep(200) + + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1') ORDER BY orderId"), + Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil) + } + } + + test("TVF: primary key table folds to +I/+U and excludes deletes") { + withTable("t") { + val tablePath = createTablePath("t") + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) + |TBLPROPERTIES("primary.key" = "orderId", "bucket.num" = 1) + |""".stripMargin) + + val writer = loadFlussTable(tablePath).newUpsert().createWriter() + writer.upsert(row(1L, 11L, 101, "a1")).get() + writer.upsert(row(2L, 12L, 102, "a2")).get() + writer.upsert(row(3L, 13L, 103, "a3")).get() + writer.flush() + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + + // in-window: update key 2, insert key 4, delete key 1 + writer.upsert(row(2L, 120L, 1002, "a2_upd")).get() + writer.upsert(row(4L, 14L, 104, "a4")).get() + writer.delete(deleteKey(1L)).get() + writer.flush() + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + + // after the window + writer.upsert(row(5L, 15L, 105, "a5")).get() + writer.flush() + Thread.sleep(200) + + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), + Row(2L, 120L, 1002, "a2_upd") :: Row(4L, 14L, 104, "a4") :: Nil) + } + } + + test("TVF: options are scoped to the query and do not leak into later reads") { + withTable("t") { + createLogTable("t") + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (3L, 13L, 103, "a3")""") + Thread.sleep(200) + + // No SET is required for the TVF to work. + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), + Row(2L, 12L, 102, "a2") :: Nil) + + // The following plain read must still see the whole table. + checkAnswer( + sql(s"SELECT * FROM $DEFAULT_DATABASE.t ORDER BY orderId"), + Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil) + } + } + + test("TVF: empty window returns no rows") { + withTable("t") { + createLogTable("t") + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") + Thread.sleep(400) + val ta = System.currentTimeMillis() + Thread.sleep(300) + val tb = System.currentTimeMillis() + Thread.sleep(300) + // written after the [ta, tb) gap, so the window contains no data + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") + Thread.sleep(200) + + checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$ta', '$tb')"), Nil) + } + } + + test("TVF: future end timestamp is rejected") { + withTable("t") { + createLogTable("t") + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") + Thread.sleep(200) + + val start = System.currentTimeMillis() - 60000 + val future = System.currentTimeMillis() + 3600000 + val ex = intercept[Exception] { + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$start', '$future')").collect() + } + assertThat(fullMessage(ex)).contains("current timestamp") + } + } + + test("TVF: epoch millis, datetime string and TIMESTAMP literal yield the same window") { + withTable("t") { + createLogTable("t") + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") + Thread.sleep(1500) + val t1 = secondAligned(System.currentTimeMillis()) + waitPast(t1) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") + Thread.sleep(1500) + val t2 = secondAligned(System.currentTimeMillis()) + waitPast(t2) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (3L, 13L, 103, "a3")""") + Thread.sleep(200) + + val expected = Row(2L, 12L, 102, "a2") :: Nil + + // epoch milliseconds as a string + checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2')"), expected) + // epoch milliseconds as an integral literal + checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', ${t1}L, ${t2}L)"), expected) + // 'yyyy-MM-dd HH:mm:ss' in the session time zone + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '${formatTs(t1)}', '${formatTs(t2)}')"), + expected) + // TIMESTAMP literals + checkAnswer( + sql(s"""SELECT * FROM $TVF('$DEFAULT_DATABASE.t', + |TIMESTAMP '${formatTs(t1)}', TIMESTAMP '${formatTs(t2)}')""".stripMargin), + expected + ) + } + } + + test("TVF: Spark expressions as epoch-millis arguments") { + withTable("t") { + createLogTable("t") + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES + |(1L, 11L, 101, "a1"), (2L, 12L, 102, "a2"), (3L, 13L, 103, "a3")""".stripMargin) + // unix_timestamp() has second granularity, so make every row strictly older than the + // truncated "now" to keep the window boundaries deterministic. + Thread.sleep(1300) + + // [now - 1h, now) covers every row written above + checkAnswer( + sql(s"""SELECT * FROM $TVF( + | '$DEFAULT_DATABASE.t', + | CAST((unix_timestamp() - 3600) * 1000 AS STRING), + | CAST(unix_timestamp() * 1000 AS STRING)) ORDER BY orderId""".stripMargin), + Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil + ) + + // [now, latest) excludes them, proving the expression is really evaluated and applied + checkAnswer( + sql(s"""SELECT * FROM $TVF( + | '$DEFAULT_DATABASE.t', + | CAST(unix_timestamp() * 1000 AS STRING))""".stripMargin), + Nil + ) + } + } + + test("TVF: Spark expressions as datetime-string arguments") { + withTable("t") { + createLogTable("t") + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES + |(1L, 11L, 101, "a1"), (2L, 12L, 102, "a2")""".stripMargin) + Thread.sleep(1300) + + checkAnswer( + sql(s"""SELECT * FROM $TVF( + | '$DEFAULT_DATABASE.t', + | date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss'), + | date_format(now(), 'yyyy-MM-dd HH:mm:ss')) ORDER BY orderId""".stripMargin), + Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Nil + ) + } + } + + test("TVF: partitioned log table window read") { + withTable("t_part") { + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t_part + |(orderId BIGINT, itemId BIGINT, amount INT, dt STRING) + |PARTITIONED BY (dt) + |""".stripMargin) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t_part VALUES + |(1L, 11L, 101, "2026-01-01"), (2L, 12L, 102, "2026-01-02")""".stripMargin) + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t_part VALUES + |(3L, 13L, 103, "2026-01-01"), (4L, 14L, 104, "2026-01-02")""".stripMargin) + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t_part VALUES (5L, 15L, 105, "2026-01-01")""") + Thread.sleep(200) + + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_part', '$t1', '$t2') ORDER BY orderId"), + Row(3L, 13L, 103, "2026-01-01") :: Row(4L, 14L, 104, "2026-01-02") :: Nil + ) + + // partition filter on top of the TVF relation + checkAnswer( + sql(s"""SELECT orderId FROM $TVF('$DEFAULT_DATABASE.t_part', '$t1', '$t2') + |WHERE dt = '2026-01-01'""".stripMargin), + Row(3L) :: Nil) + } + } + + test("TVF: wrong argument count fails with a usage hint") { + withTable("t") { + createLogTable("t") + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") + + // only the table identifier + val tooFew = intercept[Exception] { + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t')").collect() + } + assertThat(fullMessage(tooFew)).contains("endTimestamp") + + // one argument too many + val tooMany = intercept[Exception] { + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '1', '2', '3')").collect() + } + assertThat(fullMessage(tooMany)).contains("endTimestamp") + } + } + + test("TVF: unknown table fails") { + val ex = intercept[Exception] { + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.not_exist_tvf_table', '1', '2')").collect() + } + assertThat(fullMessage(ex)).contains("not_exist_tvf_table") + } + + private def row(orderId: Long, itemId: Long, amount: Int, address: String): GenericRow = + GenericRow.of( + Long.box(orderId), + Long.box(itemId), + Int.box(amount), + BinaryString.fromString(address)) + + private def deleteKey(orderId: Long): GenericRow = + GenericRow.of(Long.box(orderId), null, null, null) +} diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala new file mode 100644 index 00000000000..84d9ba9e9eb --- /dev/null +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark.lake + +import org.apache.fluss.config.{ConfigOptions, Configuration} +import org.apache.fluss.metadata.DataLakeFormat +import org.apache.fluss.spark.SparkConnectorOptions.{BUCKET_NUMBER, PRIMARY_KEY} +import org.apache.fluss.spark.read.{FlussAppendInputPartition, FlussUpsertInputPartition} + +import org.apache.spark.sql.Row + +import java.nio.file.Files + +/** + * Verifies that an incremental (time-range) batch read on a lake-enabled table is forced to the + * log-only branch: even when a readable lake snapshot exists, the plan never unions lake splits and + * never reads the kv/lake snapshot, so only the data still retained in Fluss is returned. The + * result is the `[t1, t2)` window folded per the underlying table type. + */ +abstract class SparkLakeTimeRangeReadTest extends SparkLakeTableReadTestBase { + + private val TVF = "fluss_incremental_between_timestamp" + + test("Spark Lake Read: log table time-range forces log-only (skips lake snapshot)") { + withTable("t_lake_tr_log") { + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t_lake_tr_log (id INT, name STRING) + | TBLPROPERTIES ( + | '${ConfigOptions.TABLE_DATALAKE_ENABLED.key()}' = true, + | '${ConfigOptions.TABLE_DATALAKE_FRESHNESS.key()}' = '1s', + | '${BUCKET_NUMBER.key()}' = 1) + |""".stripMargin) + + // group 1 (before the window) -> tiered to lake, but also still retained in Fluss + sql(s"""INSERT INTO $DEFAULT_DATABASE.t_lake_tr_log VALUES (1, "hello"), (2, "world")""") + tierToLake("t_lake_tr_log") + + val t1 = System.currentTimeMillis() + Thread.sleep(50) + // group 2 (inside the window) + sql(s"""INSERT INTO $DEFAULT_DATABASE.t_lake_tr_log VALUES (3, "fluss"), (4, "spark")""") + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + // group 3 (after the window) + sql(s"""INSERT INTO $DEFAULT_DATABASE.t_lake_tr_log VALUES (5, "lake")""") + Thread.sleep(200) + + val df = + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_lake_tr_log', '$t1', '$t2') ORDER BY id") + val partitions = lakeInputPartitions(df) + assert(partitions.nonEmpty, "expected at least one Fluss log partition") + assert( + partitions.forall(_.isInstanceOf[FlussAppendInputPartition]), + s"time-range read must be log-only (no lake splits), got: ${partitions.mkString(", ")}" + ) + checkAnswer(df, Row(3, "fluss") :: Row(4, "spark") :: Nil) + } + } + + test("Spark Lake Read: pk table time-range forces log-only (skips lake + kv snapshot)") { + withTable("t_lake_tr_pk") { + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t_lake_tr_pk (id INT, name STRING, score INT) + | TBLPROPERTIES ( + | '${ConfigOptions.TABLE_DATALAKE_ENABLED.key()}' = true, + | '${ConfigOptions.TABLE_DATALAKE_FRESHNESS.key()}' = '1s', + | '${PRIMARY_KEY.key()}' = 'id', + | '${BUCKET_NUMBER.key()}' = 1) + |""".stripMargin) + + // group 1 (before the window) -> tiered to lake, still retained in Fluss changelog + sql(s""" + |INSERT INTO $DEFAULT_DATABASE.t_lake_tr_pk VALUES + |(1, "alice", 90), (2, "bob", 85), (3, "charlie", 95) + |""".stripMargin) + tierToLake("t_lake_tr_pk") + + val t1 = System.currentTimeMillis() + Thread.sleep(50) + // group 2 (inside the window): update id=2, insert id=4 + sql(s""" + |INSERT INTO $DEFAULT_DATABASE.t_lake_tr_pk VALUES + |(2, "bob_updated", 100), (4, "david", 88) + |""".stripMargin) + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + // group 3 (after the window): update id=1, insert id=5 + sql(s""" + |INSERT INTO $DEFAULT_DATABASE.t_lake_tr_pk VALUES + |(1, "alice_updated", 91), (5, "eve", 92) + |""".stripMargin) + Thread.sleep(200) + + val df = + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_lake_tr_pk', '$t1', '$t2') ORDER BY id") + val partitions = lakeInputPartitions(df) + assert(partitions.nonEmpty, "expected at least one Fluss changelog partition") + assert( + partitions.forall { + case p: FlussUpsertInputPartition => p.snapshotId == -1 + case _ => false + }, + s"time-range read must be log-only with no kv/lake snapshot (snapshotId == -1), " + + s"got: ${partitions.mkString(", ")}" + ) + // Only keys inserted/updated within [t1, t2): id=2 (updated), id=4 (inserted). + checkAnswer(df, Row(2, "bob_updated", 100) :: Row(4, "david", 88) :: Nil) + } + } +} + +@SparkLakeTest +class SparkLakePaimonTimeRangeReadTest extends SparkLakeTimeRangeReadTest { + + override protected def dataLakeFormat: DataLakeFormat = DataLakeFormat.PAIMON + + override protected def flussConf: Configuration = { + val conf = super.flussConf + conf.setString("datalake.format", DataLakeFormat.PAIMON.toString) + conf.setString("datalake.paimon.metastore", "filesystem") + conf.setString("datalake.paimon.cache-enabled", "false") + warehousePath = + Files.createTempDirectory("fluss-testing-paimon-timerange-lake").resolve("warehouse").toString + conf.setString("datalake.paimon.warehouse", warehousePath) + conf + } + + override protected def lakeCatalogConf: Configuration = { + val conf = new Configuration() + conf.setString("metastore", "filesystem") + conf.setString("warehouse", warehousePath) + conf + } +} diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala new file mode 100644 index 00000000000..52f8abea401 --- /dev/null +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.spark.read + +import org.apache.fluss.config.Configuration +import org.apache.fluss.spark.SparkFlussConf + +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.assertj.core.api.Assertions.assertThat +import org.scalatest.funsuite.AnyFunSuite + +/** + * Unit tests for how scan options are resolved into offset initializers, and for the retention + * guard of an incremental (time-range) read. The end-to-end behavior is covered by + * [[org.apache.fluss.spark.SparkTimeRangeTvfTest]]. + */ +class FlussOffsetInitializersTest extends AnyFunSuite { + + private def scanOptions(entries: (String, String)*): CaseInsensitiveStringMap = { + val map = new java.util.HashMap[String, String]() + entries.foreach { case (k, v) => map.put(k, v) } + new CaseInsensitiveStringMap(map) + } + + test("incremental read is enabled by the presence of a start timestamp") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + assertThat(FlussOffsetInitializers.isIncrementalRead(scanOptions())).isFalse + assertThat(FlussOffsetInitializers.isIncrementalRead(scanOptions(startKey -> " "))).isFalse + assertThat( + FlussOffsetInitializers.isIncrementalRead(scanOptions(startKey -> "1767225600000"))).isTrue + } + + test("scan.incremental.timestamp.out-of-range toggles fail-fast (default error)") { + val key = SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE.key() + // default (unset) is error -> fail fast + assertThat(FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions())).isTrue + assertThat( + FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "error"))).isTrue + assertThat( + FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "ERROR"))).isTrue + // adjust -> clamp instead of failing + assertThat( + FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "adjust"))).isFalse + } + + test("retention guard decision (isBeforeRetention)") { + // brand-new bucket (earliest == 0) is never flagged, even for a very old start offset + assertThat(FlussOffsetInitializers.isBeforeRetention(0L, 0L)).isFalse + assertThat(FlussOffsetInitializers.isBeforeRetention(5L, 0L)).isFalse + // a trimmed bucket (earliest > 0) is flagged when the start lands at or before earliest + assertThat(FlussOffsetInitializers.isBeforeRetention(10L, 10L)).isTrue + assertThat(FlussOffsetInitializers.isBeforeRetention(3L, 10L)).isTrue + // a start strictly after earliest is within retention + assertThat(FlussOffsetInitializers.isBeforeRetention(11L, 10L)).isFalse + } + + test("TTL-exceeded start fails fast with a table.log.ttl hint") { + // earliest == 0 (brand-new bucket): never rejected, regardless of start + FlussOffsetInitializers.requireStartWithinRetention("fluss.t", null, 0, 0L, 0L) + // start strictly after a trimmed earliest: within retention, no throw + FlussOffsetInitializers.requireStartWithinRetention("fluss.t", null, 0, 11L, 10L) + // start at/before a trimmed earliest (earliest > 0): fail fast with a clear TTL message + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.requireStartWithinRetention("fluss.t", "dt=2026", 2, 5L, 10L) + } + assertThat(ex.getMessage).contains("table.log.ttl") + assertThat(ex.getMessage).contains("bucket 2") + } + + test("invalid start timestamp format fails with the option name") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.incrementalStartOffsetsInitializer( + scanOptions(startKey -> "not-a-timestamp")) + } + assertThat(ex.getMessage).contains(startKey) + } + + test("scan.startup.mode=timestamp is not a batch option") { + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.startOffsetsInitializer( + scanOptions(SparkFlussConf.SCAN_START_UP_MODE.key() -> "timestamp"), + new Configuration()) + } + assertThat(ex.getMessage).contains("Unsupported scan start up mode") + assertThat(ex.getMessage).contains(SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) + } +} diff --git a/website/docs/engine-spark/options.md b/website/docs/engine-spark/options.md index b1f74cbc5ef..784ba8ec1d7 100644 --- a/website/docs/engine-spark/options.md +++ b/website/docs/engine-spark/options.md @@ -14,6 +14,16 @@ The following Spark configurations can be used to control read behavior for both | Option | Default | Description | |--------|---------|-------------| -| `spark.sql.fluss.scan.startup.mode` | `full` | The startup mode when reading a Fluss table. Supported values: **Note:** For Structured Streaming read, only `latest` mode is currently supported. | +| `spark.sql.fluss.scan.startup.mode` | `full` | The startup mode when reading a Fluss table. Supported values: **Note:** This option only affects Structured Streaming reads, and only `latest` mode is currently supported there. Batch reads ignore it: a plain batch read is always the full table, and a time-range batch read is requested per query (see below). | | `spark.sql.fluss.read.optimized` | `false` | If `true`, Spark will only read data from the data lake snapshot or KV snapshot, without merging log changes. This can improve read performance but may return stale data for primary key tables. | | `spark.sql.fluss.scan.poll.timeout` | `10000ms` | The timeout for the log scanner to poll records. | + +## Per-Query Read Options + +The following options configure a single read and are **not** read from session configuration, so a time window can never leak into later reads. In SQL they are set by the `fluss_incremental_between_timestamp(...)` table-valued function; in the DataFrame API by `spark.read.option(...)` (without the `spark.sql.fluss.` prefix). See [Reads](reads.md#time-range-batch-read) for the full semantics. + +| Option | Default | Description | +|--------|---------|-------------| +| `scan.incremental.start.timestamp` | (none) | Enables an incremental (time-range) batch read and sets the **inclusive** lower bound of the window. Accepts epoch milliseconds (e.g. `1678883047356`) or a `yyyy-MM-dd HH:mm:ss` datetime (e.g. `2023-12-09 23:09:12`) interpreted in the Spark session time zone (`spark.sql.session.timeZone`). Batch read only; it has no effect on streaming reads. If the timestamp predates the data still retained by Fluss (bounded by `table.log.ttl`), behavior is controlled by `scan.incremental.timestamp.out-of-range`. | +| `scan.incremental.end.timestamp` | `latest` | The **exclusive** upper bound of an incremental batch read, producing a left-closed right-open `[start, end)` window. `latest` (default) stops at the latest committed data captured at planning time; otherwise the same value format as `scan.incremental.start.timestamp`. Only honored when `scan.incremental.start.timestamp` is set. A timestamp in the future is rejected by the server (`InvalidTimestampException`). | +| `scan.incremental.timestamp.out-of-range` | `error` | Behavior when `scan.incremental.start.timestamp` precedes the earliest data still retained by Fluss (bounded by `table.log.ttl`). | diff --git a/website/docs/engine-spark/reads.md b/website/docs/engine-spark/reads.md index ca48bdb2017..5335bceb322 100644 --- a/website/docs/engine-spark/reads.md +++ b/website/docs/engine-spark/reads.md @@ -254,6 +254,86 @@ INSERT INTO fluss_order_with_lake VALUES SELECT SUM(total_price) AS total_revenue FROM fluss_order_with_lake; ``` +## Time-Range Batch Read + +A time-range batch read returns the data written to Fluss within a `[start, end)` time window (left-closed, right-open on the commit timestamp). This is the building block for incremental pipelines, e.g. an hourly job that reads only the rows written in the past hour. + +The timestamp value is either epoch milliseconds or a `yyyy-MM-dd HH:mm:ss` datetime interpreted in the Spark session time zone (`spark.sql.session.timeZone`). + +### Output semantics + +| Table type | Output of a `[t1, t2)` read | +|------------|-----------------------------| +| **Log table** | Every record appended within the window. | +| **Primary key table** | The rows whose keys were **inserted or updated** within the window, folded to their latest value as of `t2`. Keys that were only deleted in the window are excluded (i.e. the result keeps `+I`/`+U` after-images and drops `-U`/`-D`). | +| **Lake-enabled table** | Same as the underlying log or primary key table, but read **only from Fluss**. A time-range read never unions the lake snapshot, so it is limited to the data still retained in Fluss (see below). | + +### Using the table-valued function (recommended for SQL) + +`fluss_incremental_between_timestamp(table, start[, end])` reads a time window in a single statement. Omit `end` to read up to the latest committed data at planning time. + +```sql title="Spark SQL" +-- Read the past hour on a log table (epoch-millis form) +SELECT * FROM fluss_incremental_between_timestamp('log_table', '1767225600000', '1767312000000') +ORDER BY order_id; +``` + +```sql title="Spark SQL" +-- Incremental changes on a primary key table (datetime form) +-- Returns the latest value of every key changed in the window; deleted keys are excluded +SELECT * FROM fluss_incremental_between_timestamp( + 'pk_table', '2026-01-01 00:00:00', '2026-01-02 00:00:00') +ORDER BY order_id; +``` + +```sql title="Spark SQL" +-- Omit the end to read from a start timestamp up to the latest data +SELECT * FROM fluss_incremental_between_timestamp('log_table', '2026-01-01 00:00:00'); +``` + +The start/end arguments may be any constant expression, so a rolling window does not need to be computed outside SQL: + +```sql title="Spark SQL" +-- The past hour, as epoch milliseconds (unix_timestamp() returns seconds) +SELECT * FROM fluss_incremental_between_timestamp( + 'log_table', + CAST((unix_timestamp() - 3600) * 1000 AS STRING), + CAST(unix_timestamp() * 1000 AS STRING)); + +-- The past hour, as datetime strings +SELECT * FROM fluss_incremental_between_timestamp( + 'log_table', + date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss'), + date_format(now(), 'yyyy-MM-dd HH:mm:ss')); +``` + +The table argument is a string and accepts `table`, `database.table` or `catalog.database.table`; unqualified names resolve against the current catalog and database. The start/end arguments accept a string (epoch milliseconds or `yyyy-MM-dd HH:mm:ss`), an integral epoch-milliseconds value, or a `TIMESTAMP`, and may be produced by constant expressions such as the datetime functions above (column references are not allowed). The result is an ordinary relation, so projection, filters and joins work as usual. + +:::note +The function is provided by the Fluss Spark session extension, so `spark.sql.extensions=org.apache.fluss.spark.FlussSparkSessionExtensions` must be configured (see [Getting Started](getting-started.md)). Its options apply to that single query only. + +Fluss uses `[start, end)` (start inclusive, end exclusive). This differs from Paimon's similarly named `paimon_incremental_between_timestamp`, which is start-exclusive and end-inclusive. +::: + +### Using the DataFrame API + +The same window can be expressed with scan options, which is how it is configured from the DataFrame API. Setting `scan.incremental.start.timestamp` is what turns a batch read into an incremental one; `scan.incremental.end.timestamp` defaults to the reserved value `latest`: + +```scala title="Spark Scala" +spark.read + .option("scan.incremental.start.timestamp", "2026-01-01 00:00:00") + .option("scan.incremental.end.timestamp", "2026-01-02 00:00:00") + .table("fluss_catalog.fluss.log_table") +``` + +:::note +The `scan.incremental.*` options are per-query read options only. Unlike the options in [Options](options.md), they are **not** picked up from session configuration (`SET spark.sql.fluss.scan.incremental...` has no effect), which keeps a time window from silently applying to later reads in the same session. +::: + +:::warning Retention boundary +A time-range read only sees data still retained by Fluss, which is bounded by `table.log.ttl` (default 7 days). If the start timestamp predates the earliest retained data, the default behavior (`scan.incremental.timestamp.out-of-range=error`) **fails fast** with a clear error instead of silently returning a truncated window — narrow the time range or increase `table.log.ttl`. Set `scan.incremental.timestamp.out-of-range=adjust` to instead clamp the start to the earliest retained data and read from there. An end timestamp in the future is rejected by the server. Reading data older than the Fluss retention (including from tiered lake storage) is not supported by this mode. +::: + ## All Data Types Fluss Spark connector supports reading all Fluss data types including nested types: diff --git a/website/docs/engine-spark/structured-streaming.md b/website/docs/engine-spark/structured-streaming.md index 4fcf2fafd71..0f6e3ba9ac7 100644 --- a/website/docs/engine-spark/structured-streaming.md +++ b/website/docs/engine-spark/structured-streaming.md @@ -51,7 +51,7 @@ Fluss supports exactly-once semantics for streaming writes through Spark's check Fluss supports reading data from Fluss tables using Spark Structured Streaming. The streaming source continuously reads new data as it arrives. :::caution Limitations -- Streaming read currently only supports the `latest` startup mode. Other modes (`full`, `earliest`, `timestamp`) are not yet supported and will be available in a future release. +- Streaming read currently only supports the `latest` startup mode. Other modes (`full`, `earliest`) are not yet supported and will be available in a future release. To read a bounded time range, use an incremental batch read instead (see [Reads](reads.md#time-range-batch-read)). ::: ### Read from Log Table From 73c311c3038418c0ca30b56f2398095a503a1f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Thu, 6 Aug 2026 23:27:43 +0800 Subject: [PATCH 02/10] [spark] Treat blank incremental timestamps as unset and clean up time-range tests Blank scan.incremental.* values now count as unset, so a whitespace-only start timestamp no longer enables an incremental read. Test cleanups: merge the redundant datetime-expression TVF case into the timestamp arguments case, drop the future-end case (server-side validation), slim the retention-guard message test, and replace the weak option-scoping case with a session-configuration negative test. Co-Authored-By: Qoder AI-Model: Qoder Auto AI-Contributed/Feature: 7/7 AI-Contributed/UT: 81/81 --- .../spark/read/FlussOffsetInitializers.scala | 7 +- .../fluss/spark/SparkTimeRangeTvfTest.scala | 76 +++++++------------ .../read/FlussOffsetInitializersTest.scala | 5 +- 3 files changed, 35 insertions(+), 53 deletions(-) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala index fc111b08a2c..2745f8ad114 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala @@ -140,11 +140,14 @@ object FlussOffsetInitializers { } } - /** Reads a `scan.incremental.*` option from the scan options, falling back to its default. */ + /** + * Reads a `scan.incremental.*` option from the scan options, falling back to its default. A blank + * value counts as unset, so a whitespace-only start timestamp never enables an incremental read. + */ private def incrementalOption( options: CaseInsensitiveStringMap, option: ConfigOption[String]): Option[String] = - Option(options.getOrDefault(option.key(), option.defaultValue())) + Option(options.getOrDefault(option.key(), option.defaultValue())).filter(_.trim.nonEmpty) private def resolveStartupMode( options: CaseInsensitiveStringMap, diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala index b28c721d78a..d6f903dcaa4 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala @@ -150,7 +150,7 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { } } - test("TVF: options are scoped to the query and do not leak into later reads") { + test("TVF: session-level scan.incremental.* options are ignored") { withTable("t") { createLogTable("t") @@ -159,21 +159,26 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { val t1 = System.currentTimeMillis() Thread.sleep(50) sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") - Thread.sleep(500) - val t2 = System.currentTimeMillis() - Thread.sleep(50) - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (3L, 13L, 103, "a3")""") Thread.sleep(200) - // No SET is required for the TVF to work. - checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), - Row(2L, 12L, 102, "a2") :: Nil) - - // The following plain read must still see the whole table. - checkAnswer( - sql(s"SELECT * FROM $DEFAULT_DATABASE.t ORDER BY orderId"), - Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil) + // A stale window in session configuration must not leak into reads: the scan.incremental.* + // options are only honored as per-query scan options (TVF arguments / DataFrameReader). + withSQLConf( + s"spark.sql.fluss.${SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()}" -> + "2000-01-01 00:00:00", + s"spark.sql.fluss.${SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key()}" -> + "2000-01-02 00:00:00" + ) { + // A plain batch read still returns the full table. + checkAnswer( + sql(s"SELECT * FROM $DEFAULT_DATABASE.t ORDER BY orderId"), + Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Nil) + + // The TVF window is unaffected by the session values. + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1') ORDER BY orderId"), + Row(2L, 12L, 102, "a2") :: Nil) + } } } @@ -195,21 +200,6 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { } } - test("TVF: future end timestamp is rejected") { - withTable("t") { - createLogTable("t") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") - Thread.sleep(200) - - val start = System.currentTimeMillis() - 60000 - val future = System.currentTimeMillis() + 3600000 - val ex = intercept[Exception] { - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$start', '$future')").collect() - } - assertThat(fullMessage(ex)).contains("current timestamp") - } - } - test("TVF: epoch millis, datetime string and TIMESTAMP literal yield the same window") { withTable("t") { createLogTable("t") @@ -246,7 +236,7 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { } } - test("TVF: Spark expressions as epoch-millis arguments") { + test("TVF: Spark expressions as timestamp arguments") { withTable("t") { createLogTable("t") @@ -256,7 +246,8 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { // truncated "now" to keep the window boundaries deterministic. Thread.sleep(1300) - // [now - 1h, now) covers every row written above + // [now - 1h, now) covers every row written above, as epoch milliseconds + // (unix_timestamp() returns seconds) checkAnswer( sql(s"""SELECT * FROM $TVF( | '$DEFAULT_DATABASE.t', @@ -265,30 +256,21 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil ) - // [now, latest) excludes them, proving the expression is really evaluated and applied + // the same window as datetime strings checkAnswer( sql(s"""SELECT * FROM $TVF( | '$DEFAULT_DATABASE.t', - | CAST(unix_timestamp() * 1000 AS STRING))""".stripMargin), - Nil + | date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss'), + | date_format(now(), 'yyyy-MM-dd HH:mm:ss')) ORDER BY orderId""".stripMargin), + Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil ) - } - } - - test("TVF: Spark expressions as datetime-string arguments") { - withTable("t") { - createLogTable("t") - - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES - |(1L, 11L, 101, "a1"), (2L, 12L, 102, "a2")""".stripMargin) - Thread.sleep(1300) + // [now, latest) excludes them, proving the expression is really evaluated and applied checkAnswer( sql(s"""SELECT * FROM $TVF( | '$DEFAULT_DATABASE.t', - | date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss'), - | date_format(now(), 'yyyy-MM-dd HH:mm:ss')) ORDER BY orderId""".stripMargin), - Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Nil + | CAST(unix_timestamp() * 1000 AS STRING))""".stripMargin), + Nil ) } } diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala index 52f8abea401..9e24fcdff2e 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala @@ -70,16 +70,13 @@ class FlussOffsetInitializersTest extends AnyFunSuite { } test("TTL-exceeded start fails fast with a table.log.ttl hint") { - // earliest == 0 (brand-new bucket): never rejected, regardless of start - FlussOffsetInitializers.requireStartWithinRetention("fluss.t", null, 0, 0L, 0L) - // start strictly after a trimmed earliest: within retention, no throw - FlussOffsetInitializers.requireStartWithinRetention("fluss.t", null, 0, 11L, 10L) // start at/before a trimmed earliest (earliest > 0): fail fast with a clear TTL message val ex = intercept[IllegalArgumentException] { FlussOffsetInitializers.requireStartWithinRetention("fluss.t", "dt=2026", 2, 5L, 10L) } assertThat(ex.getMessage).contains("table.log.ttl") assertThat(ex.getMessage).contains("bucket 2") + assertThat(ex.getMessage).contains("partition 'dt=2026'") } test("invalid start timestamp format fails with the option name") { From 4bfc8a0eb0592625e75931fce31401f8feb1c23d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Fri, 7 Aug 2026 15:53:37 +0800 Subject: [PATCH 03/10] [spark] Add changelog-fold scenario tests for incremental TVF on PK tables Cover -U/+U, +I/-D, -D/+I and pure -D folding within the time-range window for primary key tables, including partitioned PK tables. Each test first asserts the raw changelog really contains the claimed change types, so the folded-output assertions cannot pass vacuously. --- .../fluss/spark/SparkTimeRangeTvfTest.scala | 285 +++++++++++++++++- 1 file changed, 279 insertions(+), 6 deletions(-) diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala index d6f903dcaa4..f588670958e 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala @@ -17,12 +17,13 @@ package org.apache.fluss.spark +import org.apache.fluss.client.table.Table import org.apache.fluss.row.{BinaryString, GenericRow} import org.apache.spark.sql.Row import org.assertj.core.api.Assertions.assertThat -import java.time.{Instant, ZoneId} +import java.time.{Duration, Instant, ZoneId} import java.time.format.DateTimeFormatter /** @@ -112,14 +113,25 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { } } + private def createPkTable(name: String): Unit = + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.$name + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) + |TBLPROPERTIES("primary.key" = "orderId", "bucket.num" = 1) + |""".stripMargin) + + private def createPartitionedPkTable(name: String): Unit = + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.$name + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING, dt STRING) + |PARTITIONED BY (dt) + |TBLPROPERTIES("primary.key" = "orderId,dt", "bucket.num" = 1) + |""".stripMargin) + test("TVF: primary key table folds to +I/+U and excludes deletes") { withTable("t") { val tablePath = createTablePath("t") - sql(s""" - |CREATE TABLE $DEFAULT_DATABASE.t - |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) - |TBLPROPERTIES("primary.key" = "orderId", "bucket.num" = 1) - |""".stripMargin) + createPkTable("t") val writer = loadFlussTable(tablePath).newUpsert().createWriter() writer.upsert(row(1L, 11L, 101, "a1")).get() @@ -144,12 +156,215 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { writer.flush() Thread.sleep(200) + val table = loadFlussTable(tablePath) + // evidence: the window changelog really contains -U/+U (key 2), +I (key 4), -D (key 1) + val changes = changelogInWindow(table, t1, t2) + assertThat(changes.filter(_._1 == "-U").map(_._2)).isEqualTo(Seq(2L)) + assertThat(changes.filter(_._1 == "+U").map(_._2)).isEqualTo(Seq(2L)) + assertThat(changes.filter(_._1 == "+I").map(_._2)).isEqualTo(Seq(4L)) + assertThat(changes.filter(_._1 == "-D").map(_._2)).isEqualTo(Seq(1L)) + checkAnswer( sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), Row(2L, 120L, 1002, "a2_upd") :: Row(4L, 14L, 104, "a4") :: Nil) } } + test("TVF: primary key table collapses repeated -U/+U updates into the latest value") { + withTable("t") { + val tablePath = createTablePath("t") + createPkTable("t") + + val writer = loadFlussTable(tablePath).newUpsert().createWriter() + // before the window: keys 1-3 inserted + writer.upsert(row(1L, 11L, 101, "a1")).get() + writer.upsert(row(2L, 12L, 102, "a2")).get() + writer.upsert(row(3L, 13L, 103, "a3")).get() + writer.flush() + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + + // in-window: -U/+U twice on key 1, -U/+U once on key 2, key 3 untouched + writer.upsert(row(1L, 110L, 1001, "a1_v2")).get() + writer.upsert(row(1L, 111L, 1002, "a1_v3")).get() + writer.upsert(row(2L, 120L, 2001, "a2_v2")).get() + writer.flush() + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + + // after the window + writer.upsert(row(1L, 112L, 1003, "a1_v4")).get() + writer.flush() + Thread.sleep(200) + + // evidence: three genuine -U/+U pairs exist in the window changelog (two for key 1, one + // for key 2), so the folding assertions below operate on real -U/+U records + val changes = changelogInWindow(loadFlussTable(tablePath), t1, t2) + assertThat(changes.filter(_._1 == "-U").map(_._2)).isEqualTo(Seq(1L, 1L, 2L)) + assertThat(changes.filter(_._1 == "+U").map(_._2)).isEqualTo(Seq(1L, 1L, 2L)) + + // each updated key appears exactly once, with its last in-window value + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), + Row(1L, 111L, 1002, "a1_v3") :: Row(2L, 120L, 2001, "a2_v2") :: Nil) + + // the two-argument form reads through to the latest state + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1') ORDER BY orderId"), + Row(1L, 112L, 1003, "a1_v4") :: Row(2L, 120L, 2001, "a2_v2") :: Nil) + } + } + + test("TVF: primary key table cancels out +I followed by -D in the window") { + withTable("t") { + val tablePath = createTablePath("t") + createPkTable("t") + + val writer = loadFlussTable(tablePath).newUpsert().createWriter() + // before the window + writer.upsert(row(1L, 11L, 101, "a1")).get() + writer.flush() + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + + // in-window: insert key 2 then delete it again (cancels out), insert key 3 (survives) + writer.upsert(row(2L, 12L, 102, "a2")).get() + writer.delete(deleteKey(2L)).get() + writer.upsert(row(3L, 13L, 103, "a3")).get() + writer.flush() + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + + // evidence: the window changelog holds +I then -D for key 2, and +I for key 3 + val changes = changelogInWindow(loadFlussTable(tablePath), t1, t2) + assertThat(changes.filter(r => r._1 == "+I" || r._1 == "-D")) + .isEqualTo(Seq(("+I", 2L), ("-D", 2L), ("+I", 3L))) + + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), + Row(3L, 13L, 103, "a3") :: Nil) + } + } + + test("TVF: primary key table keeps a key deleted then re-inserted in the window") { + withTable("t") { + val tablePath = createTablePath("t") + createPkTable("t") + + val writer = loadFlussTable(tablePath).newUpsert().createWriter() + // before the window + writer.upsert(row(1L, 11L, 101, "a1")).get() + writer.upsert(row(2L, 12L, 102, "a2")).get() + writer.flush() + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + + // in-window: delete key 1 then re-insert it with new values (-D then +I survives), + // delete key 2 permanently (-D only, excluded) + writer.delete(deleteKey(1L)).get() + writer.upsert(row(1L, 110L, 1001, "a1_new")).get() + writer.delete(deleteKey(2L)).get() + writer.flush() + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + + // evidence: the window changelog holds -D then +I for key 1, and -D for key 2 + assertThat(changelogInWindow(loadFlussTable(tablePath), t1, t2)) + .isEqualTo(Seq(("-D", 1L), ("+I", 1L), ("-D", 2L))) + + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), + Row(1L, 110L, 1001, "a1_new") :: Nil) + } + } + + test("TVF: primary key table window containing only -D returns nothing") { + withTable("t") { + val tablePath = createTablePath("t") + createPkTable("t") + + val writer = loadFlussTable(tablePath).newUpsert().createWriter() + writer.upsert(row(1L, 11L, 101, "a1")).get() + writer.upsert(row(2L, 12L, 102, "a2")).get() + writer.flush() + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + + writer.delete(deleteKey(1L)).get() + writer.delete(deleteKey(2L)).get() + writer.flush() + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + + // evidence: the window changelog holds exactly two -D records, so the empty result below + // reflects genuine delete folding rather than an empty window + val changes = changelogInWindow(loadFlussTable(tablePath), t1, t2) + assertThat(changes.map(_._1)).isEqualTo(Seq("-D", "-D")) + assertThat(changes.map(_._2)).isEqualTo(Seq(1L, 2L)) + + checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2')"), Nil) + } + } + + test("TVF: partitioned primary key table folds changes across partitions") { + withTable("t_pk_part") { + val tablePath = createTablePath("t_pk_part") + createPartitionedPkTable("t_pk_part") + + val writer = loadFlussTable(tablePath).newUpsert().createWriter() + // before the window: one row per partition + writer.upsert(pkRow(1L, 11L, 101, "a1", "2026-01-01")).get() + writer.upsert(pkRow(2L, 12L, 102, "a2", "2026-01-02")).get() + writer.flush() + Thread.sleep(500) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + + // in-window: update key 1 (partition 1), permanent delete of key 2 (partition 2), + // insert key 3 then delete it again in partition 1 (cancels out) + writer.upsert(pkRow(1L, 110L, 1001, "a1_upd", "2026-01-01")).get() + writer.delete(deleteKey(2L, "2026-01-02")).get() + writer.upsert(pkRow(3L, 13L, 103, "a3", "2026-01-01")).get() + writer.delete(deleteKey(3L, "2026-01-01")).get() + writer.flush() + Thread.sleep(500) + val t2 = System.currentTimeMillis() + Thread.sleep(50) + + // after the window + writer.upsert(pkRow(4L, 14L, 104, "a4", "2026-01-01")).get() + writer.flush() + Thread.sleep(200) + + // evidence: the window changelog holds -U/+U (key 1), -D (key 2), +I then -D (key 3); + // the -D comparison sorts first because poll order across partitions is not deterministic + val changes = changelogInWindow(loadFlussTable(tablePath), t1, t2) + assertThat(changes.filter(_._1 == "-U").map(_._2)).isEqualTo(Seq(1L)) + assertThat(changes.filter(_._1 == "+U").map(_._2)).isEqualTo(Seq(1L)) + assertThat(changes.filter(_._1 == "-D").map(_._2).sorted).isEqualTo(Seq(2L, 3L)) + assertThat(changes.filter(_._1 == "+I").map(_._2)).isEqualTo(Seq(3L)) + + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_pk_part', '$t1', '$t2') ORDER BY orderId"), + Row(1L, 110L, 1001, "a1_upd", "2026-01-01") :: Nil) + + // partition filter on top of the TVF relation + checkAnswer( + sql(s"""SELECT orderId FROM $TVF('$DEFAULT_DATABASE.t_pk_part', '$t1') + |WHERE dt = '2026-01-01' ORDER BY orderId""".stripMargin), + Row(1L) :: Row(4L) :: Nil + ) + } + } + test("TVF: session-level scan.incremental.* options are ignored") { withTable("t") { createLogTable("t") @@ -344,6 +559,64 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { Int.box(amount), BinaryString.fromString(address)) + /** + * Raw changelog records of `table` whose commit timestamp falls inside [start, end), as + * (changeType, orderId) pairs in log order. Used to prove the claimed change types (-U/+U/-D/+I) + * really exist in the window, so the folded-output assertions below cannot pass vacuously. + */ + private def changelogInWindow(table: Table, start: Long, end: Long): Seq[(String, Long)] = { + val scanner = table.newScan().createLogScanner() + try { + if (table.getTableInfo.isPartitioned) { + admin.listPartitionInfos(table.getTableInfo.getTablePath).get().forEach { + pi => scanner.subscribeFromBeginning(pi.getPartitionId, 0) + } + } else { + scanner.subscribeFromBeginning(0) + } + val records = scala.collection.mutable.ArrayBuffer[(String, Long)]() + // Poll until records arrive and a poll comes back empty (all caught up), or the deadline. + // Mirrors FlussSparkTestBase.getRowsWithChangeType: the high watermark may advance in + // stages, so a single early empty poll must not end the scan. + val deadline = System.currentTimeMillis() + 10000 + var hasReceivedAny = false + var done = false + while (!done && System.currentTimeMillis() < deadline) { + val polled = scanner.poll(Duration.ofSeconds(1)) + if (!polled.isEmpty) { + hasReceivedAny = true + polled.forEach { + r => + if (r.timestamp() >= start && r.timestamp() < end) { + records += ((r.getChangeType.shortString(), r.getRow.getLong(0))) + } + } + } else if (hasReceivedAny) { + done = true + } + } + records.toSeq + } finally { + scanner.close() + } + } + + private def pkRow( + orderId: Long, + itemId: Long, + amount: Int, + address: String, + dt: String): GenericRow = + GenericRow.of( + Long.box(orderId), + Long.box(itemId), + Int.box(amount), + BinaryString.fromString(address), + BinaryString.fromString(dt)) + private def deleteKey(orderId: Long): GenericRow = GenericRow.of(Long.box(orderId), null, null, null) + + private def deleteKey(orderId: Long, dt: String): GenericRow = + GenericRow.of(Long.box(orderId), null, null, null, BinaryString.fromString(dt)) } From 36ebb30a4fa6c58b3c819fecc02ccf6a3075f43e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Mon, 10 Aug 2026 16:02:34 +0800 Subject: [PATCH 04/10] [spark] Harden incremental batch read options and upsert planning Address review feedback on the time-range incremental read: - failOnTimestampOutOfRange falls back to the option default for blank values and rejects unknown modes with an IllegalArgumentException listing the supported values, instead of a bare NoSuchElementException. The planner call sites become lazy vals so the incremental-only option never breaks plain batch reads. - The incremental upsert planner skips buckets whose resolved [start, stop) range is empty, mirroring the append planner guard, so empty windows cost no Spark task, Fluss connection or RPC. - The incremental upsert planner fails fast when read.optimized is enabled: the combination has no snapshot to read and would silently return zero rows. Co-Authored-By: Qoder AI-Model: Qoder Auto AI-Contributed/Feature: 51/51 AI-Contributed/UT: 54/54 --- .../spark/read/FlussOffsetInitializers.scala | 19 ++++++--- .../fluss/spark/read/SplitPlanner.scala | 32 ++++++++++---- .../fluss/spark/SparkTimeRangeTvfTest.scala | 42 +++++++++++++++++++ .../read/FlussOffsetInitializersTest.scala | 12 ++++++ 4 files changed, 91 insertions(+), 14 deletions(-) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala index 2745f8ad114..751f1f89a29 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala @@ -79,11 +79,20 @@ object FlussOffsetInitializers { */ def failOnTimestampOutOfRange(options: CaseInsensitiveStringMap): Boolean = { val mode = - incrementalOption( - options, - SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE).get.toUpperCase - SparkFlussConf.TimestampOutOfRangeMode.withName(mode) == - SparkFlussConf.TimestampOutOfRangeMode.ERROR + incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE) + .getOrElse(SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE.defaultValue()) + .trim + .toUpperCase + SparkFlussConf.TimestampOutOfRangeMode.values.find(_.toString == mode) match { + case Some(resolved) => resolved == SparkFlussConf.TimestampOutOfRangeMode.ERROR + case None => + throw new IllegalArgumentException( + s"Unsupported value for " + + s"'${SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE.key()}': '$mode'. " + + s"Supported values are " + + s"'${SparkFlussConf.TimestampOutOfRangeMode.values.toList.map(_.toString.toLowerCase).mkString("', '")}'" + + s".") + } } /** diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala index 9414d868091..fd169752f7f 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala @@ -281,7 +281,8 @@ class AppendPlanner( FlussOffsetInitializers.isIncrementalRead(options) // Whether a start timestamp preceding retained data fails (default) or is clamped to earliest. - private val failOnOutOfRange: Boolean = + // Lazy on purpose: the option is incremental-only and must not affect plain batch reads. + private lazy val failOnOutOfRange: Boolean = FlussOffsetInitializers.failOnTimestampOutOfRange(options) // An incremental read starts at scan.incremental.start.timestamp, a plain batch read at the @@ -655,7 +656,8 @@ class UpsertPlanner( FlussOffsetInitializers.isIncrementalRead(options) // Whether a start timestamp preceding retained data fails (default) or is clamped to earliest. - private val failOnOutOfRange: Boolean = + // Lazy on purpose: the option is incremental-only and must not affect plain batch reads. + private lazy val failOnOutOfRange: Boolean = FlussOffsetInitializers.failOnTimestampOutOfRange(options) // Start offset of an incremental read, resolved from scan.incremental.start.timestamp. Lazy on @@ -754,6 +756,13 @@ class UpsertPlanner( // --------------------------------------------------------------------------------------------- private def planIncrementalLogOnly(): Array[InputPartition] = { + if (flussConfig.get(SparkFlussConf.READ_OPTIMIZED_OPTION)) { + throw new IllegalArgumentException( + s"'${SparkFlussConf.READ_OPTIMIZED_OPTION.key()}' must not be enabled for an " + + s"incremental (time-range) read: it skips log changes and reads only snapshots, while " + + s"an incremental read folds only the changelog, so this combination would silently " + + s"return no rows.") + } val bucketOffsetsRetriever = new BucketOffsetsRetrieverImpl(admin, tablePath, true) val buckets = (0 until tableInfo.getNumBuckets).toSeq @@ -797,18 +806,23 @@ class UpsertPlanner( } val tableId = tableInfo.getTableId - buckets.map { + buckets.flatMap { bucketId => val tableBucket = partitionId match { case Some(pid) => new TableBucket(tableId, pid, bucketId) case None => new TableBucket(tableId, bucketId) } - FlussUpsertInputPartition( - tableBucket, - -1L, - Long2long(startBucketOffsets.get(Integer.valueOf(bucketId))), - Long2long(stoppingBucketOffsets.get(Integer.valueOf(bucketId)))) - .asInstanceOf[InputPartition] + val startOffset = Long2long(startBucketOffsets.get(Integer.valueOf(bucketId))) + val stopOffset = Long2long(stoppingBucketOffsets.get(Integer.valueOf(bucketId))) + if (startOffset >= stopOffset) { + // Empty range (e.g. a time-range window with no data, or an empty bucket): emit no + // partition so the upsert reader is not handed an invalid [start, start) range. + None + } else { + Some( + FlussUpsertInputPartition(tableBucket, -1L, startOffset, stopOffset) + .asInstanceOf[InputPartition]) + } }.toArray } diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala index f588670958e..98a110e4999 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala @@ -415,6 +415,48 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { } } + test("TVF: empty window on a primary key table returns no rows") { + withTable("t") { + createPkTable("t") + + val writer = loadFlussTable(createTablePath("t")).newUpsert().createWriter() + writer.upsert(row(1L, 11L, 101, "a1")).get() + writer.flush() + Thread.sleep(400) + val ta = System.currentTimeMillis() + Thread.sleep(300) + val tb = System.currentTimeMillis() + Thread.sleep(300) + // written after the [ta, tb) gap, so the window contains no data + writer.upsert(row(2L, 12L, 102, "a2")).get() + writer.flush() + Thread.sleep(200) + + checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$ta', '$tb')"), Nil) + } + } + + test("TVF: incremental read fails fast when read.optimized is enabled") { + withTable("t") { + createPkTable("t") + + val writer = loadFlussTable(createTablePath("t")).newUpsert().createWriter() + writer.upsert(row(1L, 11L, 101, "a1")).get() + writer.flush() + Thread.sleep(200) + val t1 = System.currentTimeMillis() + + withSQLConf( + s"${SparkFlussConf.SPARK_FLUSS_CONF_PREFIX}${SparkFlussConf.READ_OPTIMIZED_OPTION.key()}" -> + "true") { + val ex = intercept[Exception] { + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1')").collect() + } + assertThat(fullMessage(ex)).contains(SparkFlussConf.READ_OPTIMIZED_OPTION.key()) + } + } + } + test("TVF: epoch millis, datetime string and TIMESTAMP literal yield the same window") { withTable("t") { createLogTable("t") diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala index 9e24fcdff2e..ab75f16363b 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala @@ -53,11 +53,23 @@ class FlussOffsetInitializersTest extends AnyFunSuite { FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "error"))).isTrue assertThat( FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "ERROR"))).isTrue + // a blank value counts as unset and falls back to the default (error) + assertThat(FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> " "))).isTrue // adjust -> clamp instead of failing assertThat( FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "adjust"))).isFalse } + test("invalid scan.incremental.timestamp.out-of-range value fails with supported values") { + val key = SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE.key() + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "warn")) + } + assertThat(ex.getMessage).contains(key) + assertThat(ex.getMessage).contains("WARN") + assertThat(ex.getMessage).contains("'error', 'adjust'") + } + test("retention guard decision (isBeforeRetention)") { // brand-new bucket (earliest == 0) is never flagged, even for a very old start offset assertThat(FlussOffsetInitializers.isBeforeRetention(0L, 0L)).isFalse From 537eee5956d56ff7adfd3884d3555d2deb75f4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Mon, 10 Aug 2026 18:36:01 +0800 Subject: [PATCH 05/10] [spark] Fix NTZ timezone parsing and fail fast on invalid incremental windows Interpret TIMESTAMP_NTZ TVF arguments in the Spark session time zone (matching the string form and the Flink connector convention) instead of pinning them to UTC, and reject invalid window specifications instead of silently changing semantics: a blank start timestamp, an end timestamp set without a start timestamp, and a window whose start is not strictly before its end all fail fast at planning time. --- .../apache/fluss/spark/SparkFlussConf.scala | 3 +- .../logical/FlussTableValuedFunctions.scala | 31 ++++++++- .../spark/read/FlussOffsetInitializers.scala | 50 ++++++++++++-- .../fluss/spark/read/SplitPlanner.scala | 6 ++ .../fluss/spark/SparkTimeRangeTvfTest.scala | 67 +++++++++++++++++++ .../read/FlussOffsetInitializersTest.scala | 42 +++++++++++- website/docs/engine-spark/options.md | 4 +- website/docs/engine-spark/reads.md | 6 +- 8 files changed, 197 insertions(+), 12 deletions(-) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala index aaa8ca1d75f..cb885a441ab 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala @@ -72,7 +72,8 @@ object SparkFlussConf { "left-closed right-open '[start, end)' window. 'latest' (default) stops at the " + "latest committed data captured at planning time; otherwise accepts epoch " + "milliseconds or a 'yyyy-MM-dd HH:mm:ss' datetime string interpreted in the Spark " + - "session time zone. Only honored when 'scan.incremental.start.timestamp' is set.") + "session time zone. Setting it without 'scan.incremental.start.timestamp' fails " + + "fast, as does a window whose start is not strictly before its end.") val SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE: ConfigOption[String] = ConfigBuilder diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala index 0adf8b9248a..317751cedcd 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala @@ -28,9 +28,12 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, Express import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, LongType, ShortType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.sql.util.CaseInsensitiveStringMap +import java.time.{LocalDateTime, ZoneId, ZoneOffset} + import scala.collection.JavaConverters._ import scala.util.control.NonFatal @@ -129,8 +132,11 @@ object FlussTableValuedFunctions { * * A STRING argument is passed through untouched, so both epoch milliseconds and * `yyyy-MM-dd HH:mm:ss` keep being interpreted by the option layer. Integral arguments are epoch - * milliseconds. TIMESTAMP arguments are converted from Spark's internal microseconds, otherwise a - * `TIMESTAMP '...'` literal would silently be read as epoch milliseconds. + * milliseconds. TIMESTAMP (local instant) arguments are converted from Spark's internal + * microseconds, otherwise a `TIMESTAMP '...'` literal would silently be read as epoch + * milliseconds. TIMESTAMP_NTZ arguments hold wall-clock microseconds and are re-interpreted in + * the Spark session time zone, so an NTZ literal resolves to the same window as the same + * `yyyy-MM-dd HH:mm:ss` string literal. * * Any constant expression is accepted, e.g. `CAST(unix_timestamp() * 1000 AS STRING)` or * `date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss')`. @@ -159,7 +165,8 @@ object FlussTableValuedFunctions { evaluable.dataType match { case StringType => value.toString case ShortType | IntegerType | LongType => value.toString - case TimestampType | TimestampNTZType => (value.asInstanceOf[Long] / 1000L).toString + case TimestampType => (value.asInstanceOf[Long] / 1000L).toString + case TimestampNTZType => ntzMicrosToEpochMillis(value.asInstanceOf[Long]).toString case other => throw new IllegalArgumentException( s"Unsupported timestamp argument type $other for $fnName. Use a STRING (epoch " + @@ -167,6 +174,24 @@ object FlussTableValuedFunctions { "TIMESTAMP.") } } + + private val MICROS_PER_SECOND = 1000000L + + /** + * Converts a `TIMESTAMP_NTZ` argument to epoch milliseconds. NTZ values are wall-clock + * microseconds encoded as if in UTC; they are re-interpreted in the Spark session time zone — the + * same convention the `yyyy-MM-dd HH:mm:ss` string form uses (and the same as the Flink + * connector's timestamp options) — so both forms resolve to the same window for the same literal. + */ + private def ntzMicrosToEpochMillis(micros: Long): Long = { + val seconds = Math.floorDiv(micros, MICROS_PER_SECOND) + val nanos = Math.floorMod(micros, MICROS_PER_SECOND) * 1000L + LocalDateTime + .ofEpochSecond(seconds, nanos.toInt, ZoneOffset.UTC) + .atZone(ZoneId.of(SQLConf.get.sessionLocalTimeZone)) + .toInstant + .toEpochMilli + } } /** diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala index 751f1f89a29..d9a17a58978 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala @@ -40,9 +40,19 @@ object FlussOffsetInitializers { * `fluss_incremental_between_timestamp` table-valued function or `DataFrameReader.option` — and * deliberately not from session configuration, so a window can never leak into another query. * Streaming reads ignore them. + * + * An explicitly set but blank start timestamp fails fast instead of silently falling back to a + * full-table batch read. */ def isIncrementalRead(options: CaseInsensitiveStringMap): Boolean = { - incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP).isDefined + val startOption = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP + val rawValue = Option(options.get(startOption.key())) + if (rawValue.exists(_.trim.isEmpty)) { + throw new IllegalArgumentException( + s"'${startOption.key()}' must not be blank. Provide epoch milliseconds or a " + + s"'yyyy-MM-dd HH:mm:ss' timestamp, or omit the option for a full-table batch read.") + } + rawValue.isDefined } /** @@ -131,8 +141,15 @@ object FlussOffsetInitializers { if (!isBatch) { new NoStoppingOffsetsInitializer() } else if (!isIncrementalRead(options)) { - // A plain batch read stops at the latest committed data; an end timestamp alone must not - // truncate it. + val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() + if (Option(options.get(endKey)).isDefined) { + throw new IllegalArgumentException( + s"'$endKey' is set but " + + s"'${SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()}' is missing. An end " + + s"timestamp alone cannot truncate a batch read; set a start timestamp for an " + + s"incremental read, or remove the end option.") + } + // A plain batch read stops at the latest committed data. OffsetsInitializer.latest() } else { val end = @@ -149,9 +166,34 @@ object FlussOffsetInitializers { } } + /** + * Validates the `[start, end)` window of an incremental read: when the end bound is an explicit + * timestamp (not the reserved value `latest`), it must be strictly after the start timestamp. A + * reversed or degenerate window fails fast instead of silently returning no rows. Note this only + * checks the requested timestamps; a bucket that simply has no data inside a valid window still + * yields an empty result. + */ + def requireValidWindow(options: CaseInsensitiveStringMap): Unit = { + val end = incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP) + if (end.exists(!_.trim.equalsIgnoreCase(SparkFlussConf.END_TIMESTAMP_LATEST))) { + val start = incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP).get + val startMillis = + parseTimestamp(start.trim, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) + val endMillis = + parseTimestamp(end.get.trim, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key()) + if (startMillis >= endMillis) { + throw new IllegalArgumentException( + s"Invalid time range for an incremental read: the start timestamp '$start' must be " + + s"strictly before the end timestamp '${end.get.trim}'. The window is left-closed " + + s"right-open '[start, end)'.") + } + } + } + /** * Reads a `scan.incremental.*` option from the scan options, falling back to its default. A blank - * value counts as unset, so a whitespace-only start timestamp never enables an incremental read. + * value counts as unset for the end and out-of-range options; a blank start timestamp is rejected + * by [[isIncrementalRead]] before it can reach here. */ private def incrementalOption( options: CaseInsensitiveStringMap, diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala index fd169752f7f..72bba3e92a6 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala @@ -306,6 +306,9 @@ class AppendPlanner( override def plan(): Array[InputPartition] = try { + if (incrementalMode) { + FlussOffsetInitializers.requireValidWindow(options) + } readableLakeSnapshot match { // An incremental read never unions a lake snapshot; it reads only Fluss. case Some(snap) if !incrementalMode => planLakeUnion(snap) @@ -675,6 +678,9 @@ class UpsertPlanner( override def plan(): Array[InputPartition] = try { + if (incrementalMode) { + FlussOffsetInitializers.requireValidWindow(options) + } readableLakeSnapshot match { // An incremental read reads neither the lake nor the kv snapshot; it folds only the Fluss // changelog within [start, end). diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala index 98a110e4999..72a18df0b6d 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala @@ -493,6 +493,73 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { } } + test("TVF: TIMESTAMP_NTZ argument is interpreted in the session time zone") { + withTable("t") { + createLogTable("t") + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") + Thread.sleep(1500) + val t1 = secondAligned(System.currentTimeMillis()) + waitPast(t1) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") + Thread.sleep(1500) + val t2 = secondAligned(System.currentTimeMillis()) + waitPast(t2) + + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (3L, 13L, 103, "a3")""") + Thread.sleep(200) + + val expected = Row(2L, 12L, 102, "a2") :: Nil + + // Under a non-UTC session time zone, an NTZ literal must resolve to the same window as the + // identical 'yyyy-MM-dd HH:mm:ss' string (parsed in the session time zone); pinning NTZ to + // UTC would shift the window by the zone offset and miss or leak rows. + withSQLConf("spark.sql.session.timeZone" -> "Asia/Shanghai") { + checkAnswer( + sql(s"""SELECT * FROM $TVF('$DEFAULT_DATABASE.t', + |TIMESTAMP_NTZ '${formatTs(t1)}', TIMESTAMP_NTZ '${formatTs(t2)}')""".stripMargin), + expected + ) + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '${formatTs(t1)}', '${formatTs(t2)}')"), + expected) + } + } + } + + test("TVF: blank timestamp arguments fail fast") { + withTable("t") { + createLogTable("t") + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") + Thread.sleep(200) + + // A blank start must fail instead of silently reading the full table. + val ex = intercept[Exception] { + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', ' ')").collect() + } + assertThat(fullMessage(ex)) + .contains(SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) + .contains("must not be blank") + } + } + + test("TVF: reversed time range fails fast") { + withTable("t") { + createLogTable("t") + sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") + Thread.sleep(300) + val t1 = System.currentTimeMillis() + Thread.sleep(50) + val t2 = System.currentTimeMillis() + + val ex = intercept[Exception] { + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t2', '$t1')").collect() + } + assertThat(fullMessage(ex)).contains("strictly before") + } + } + test("TVF: Spark expressions as timestamp arguments") { withTable("t") { createLogTable("t") diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala index ab75f16363b..3cee268c296 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala @@ -40,11 +40,51 @@ class FlussOffsetInitializersTest extends AnyFunSuite { test("incremental read is enabled by the presence of a start timestamp") { val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() assertThat(FlussOffsetInitializers.isIncrementalRead(scanOptions())).isFalse - assertThat(FlussOffsetInitializers.isIncrementalRead(scanOptions(startKey -> " "))).isFalse assertThat( FlussOffsetInitializers.isIncrementalRead(scanOptions(startKey -> "1767225600000"))).isTrue } + test("a blank start timestamp fails fast instead of silently reading the full table") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.isIncrementalRead(scanOptions(startKey -> " ")) + } + assertThat(ex.getMessage).contains(startKey) + assertThat(ex.getMessage).contains("must not be blank") + } + + test("an end timestamp without a start timestamp fails fast") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.stoppingOffsetsInitializer( + true, + scanOptions(endKey -> "1767312000000")) + } + assertThat(ex.getMessage).contains(endKey) + assertThat(ex.getMessage).contains(startKey) + } + + test("a window whose start is not strictly before its end fails fast") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() + // degenerate: start == end + intercept[IllegalArgumentException] { + FlussOffsetInitializers.requireValidWindow( + scanOptions(startKey -> "1767225600000", endKey -> "1767225600000")) + } + // reversed: start > end + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.requireValidWindow( + scanOptions(startKey -> "1767312000000", endKey -> "1767225600000")) + } + assertThat(ex.getMessage).contains("strictly before") + // 'latest' or absent end is always accepted + FlussOffsetInitializers.requireValidWindow(scanOptions(startKey -> "1767225600000")) + FlussOffsetInitializers.requireValidWindow( + scanOptions(startKey -> "1767225600000", endKey -> "latest")) + } + test("scan.incremental.timestamp.out-of-range toggles fail-fast (default error)") { val key = SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE.key() // default (unset) is error -> fail fast diff --git a/website/docs/engine-spark/options.md b/website/docs/engine-spark/options.md index 784ba8ec1d7..43d4929f367 100644 --- a/website/docs/engine-spark/options.md +++ b/website/docs/engine-spark/options.md @@ -24,6 +24,6 @@ The following options configure a single read and are **not** read from session | Option | Default | Description | |--------|---------|-------------| -| `scan.incremental.start.timestamp` | (none) | Enables an incremental (time-range) batch read and sets the **inclusive** lower bound of the window. Accepts epoch milliseconds (e.g. `1678883047356`) or a `yyyy-MM-dd HH:mm:ss` datetime (e.g. `2023-12-09 23:09:12`) interpreted in the Spark session time zone (`spark.sql.session.timeZone`). Batch read only; it has no effect on streaming reads. If the timestamp predates the data still retained by Fluss (bounded by `table.log.ttl`), behavior is controlled by `scan.incremental.timestamp.out-of-range`. | -| `scan.incremental.end.timestamp` | `latest` | The **exclusive** upper bound of an incremental batch read, producing a left-closed right-open `[start, end)` window. `latest` (default) stops at the latest committed data captured at planning time; otherwise the same value format as `scan.incremental.start.timestamp`. Only honored when `scan.incremental.start.timestamp` is set. A timestamp in the future is rejected by the server (`InvalidTimestampException`). | +| `scan.incremental.start.timestamp` | (none) | Enables an incremental (time-range) batch read and sets the **inclusive** lower bound of the window. Accepts epoch milliseconds (e.g. `1678883047356`) or a `yyyy-MM-dd HH:mm:ss` datetime (e.g. `2023-12-09 23:09:12`) interpreted in the Spark session time zone (`spark.sql.session.timeZone`). A blank or unparseable value fails fast instead of falling back to a full-table read. Batch read only; it has no effect on streaming reads. If the timestamp predates the data still retained by Fluss (bounded by `table.log.ttl`), behavior is controlled by `scan.incremental.timestamp.out-of-range`. | +| `scan.incremental.end.timestamp` | `latest` | The **exclusive** upper bound of an incremental batch read, producing a left-closed right-open `[start, end)` window. `latest` (default) stops at the latest committed data captured at planning time; otherwise the same value format as `scan.incremental.start.timestamp`. Setting it without `scan.incremental.start.timestamp` fails fast, as does a window whose start is not strictly before its end. A timestamp in the future is rejected by the server (`InvalidTimestampException`). | | `scan.incremental.timestamp.out-of-range` | `error` | Behavior when `scan.incremental.start.timestamp` precedes the earliest data still retained by Fluss (bounded by `table.log.ttl`).
  • `error` (default): fail fast so a truncated window is never returned silently.
  • `adjust`: clamp the start to the earliest retained offset and read from there.
| diff --git a/website/docs/engine-spark/reads.md b/website/docs/engine-spark/reads.md index 5335bceb322..f797f0304ac 100644 --- a/website/docs/engine-spark/reads.md +++ b/website/docs/engine-spark/reads.md @@ -307,7 +307,7 @@ SELECT * FROM fluss_incremental_between_timestamp( date_format(now(), 'yyyy-MM-dd HH:mm:ss')); ``` -The table argument is a string and accepts `table`, `database.table` or `catalog.database.table`; unqualified names resolve against the current catalog and database. The start/end arguments accept a string (epoch milliseconds or `yyyy-MM-dd HH:mm:ss`), an integral epoch-milliseconds value, or a `TIMESTAMP`, and may be produced by constant expressions such as the datetime functions above (column references are not allowed). The result is an ordinary relation, so projection, filters and joins work as usual. +The table argument is a string and accepts `table`, `database.table` or `catalog.database.table`; unqualified names resolve against the current catalog and database. The start/end arguments accept a string (epoch milliseconds or `yyyy-MM-dd HH:mm:ss`), an integral epoch-milliseconds value, or a `TIMESTAMP`/`TIMESTAMP_NTZ` literal (interpreted in the Spark session time zone, same as the `yyyy-MM-dd HH:mm:ss` string form), and may be produced by constant expressions such as the datetime functions above (column references are not allowed). The result is an ordinary relation, so projection, filters and joins work as usual. :::note The function is provided by the Fluss Spark session extension, so `spark.sql.extensions=org.apache.fluss.spark.FlussSparkSessionExtensions` must be configured (see [Getting Started](getting-started.md)). Its options apply to that single query only. @@ -334,6 +334,10 @@ The `scan.incremental.*` options are per-query read options only. Unlike the opt A time-range read only sees data still retained by Fluss, which is bounded by `table.log.ttl` (default 7 days). If the start timestamp predates the earliest retained data, the default behavior (`scan.incremental.timestamp.out-of-range=error`) **fails fast** with a clear error instead of silently returning a truncated window — narrow the time range or increase `table.log.ttl`. Set `scan.incremental.timestamp.out-of-range=adjust` to instead clamp the start to the earliest retained data and read from there. An end timestamp in the future is rejected by the server. Reading data older than the Fluss retention (including from tiered lake storage) is not supported by this mode. ::: +:::note Invalid windows fail fast +Malformed window specifications are rejected at planning time instead of silently changing semantics: a blank or unparseable start timestamp, an end timestamp set without a start timestamp (it cannot truncate a plain batch read on its own), and a window whose start is not strictly before its end. A bucket that simply has no data inside a valid window still yields an empty result. +::: + ## All Data Types Fluss Spark connector supports reading all Fluss data types including nested types: From 36700a8ddd334e15f8e47d2bac22ffc8e58287f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Wed, 12 Aug 2026 16:41:03 +0800 Subject: [PATCH 06/10] [spark] Pin the incremental window at analysis and filter it while reading A time-range batch read now returns whatever Fluss still retains inside the requested window instead of failing when the start predates retention: the result is always a genuine subset of the window, so the extra earliest-offset lookup and the scan.incremental.timestamp.out-of-range option paid for nothing. Both bounds are resolved while the statement is analyzed, filling an omitted end with the current timestamp, so the window no longer shifts between analysis and planning. The scan positions itself with the start offset but stops at the latest offset, and the reader cuts the window on each record's commit timestamp, which keeps [start, end) exact on segments whose time index is sparse and avoids sending a driver-computed "now" to the server. Co-Authored-By: Qoder AI-Model: Qoder Auto AI-Contributed/Feature: 225/233 AI-Contributed/UT: 250/250 --- .../apache/fluss/spark/SparkFlussConf.scala | 30 +- .../org/apache/fluss/spark/SparkTable.scala | 22 +- .../logical/FlussTableValuedFunctions.scala | 133 +-- .../read/FlussAppendPartitionReader.scala | 40 +- .../spark/read/FlussInputPartition.scala | 33 +- .../spark/read/FlussOffsetInitializers.scala | 119 +-- .../apache/fluss/spark/read/FlussScan.scala | 16 +- .../read/FlussUpsertPartitionReader.scala | 19 +- .../fluss/spark/read/SplitPlanner.scala | 96 +- .../fluss/spark/SparkTimeRangeTvfTest.scala | 844 +++++------------- .../read/FlussOffsetInitializersTest.scala | 70 +- website/docs/engine-spark/options.md | 7 +- website/docs/engine-spark/reads.md | 14 +- 13 files changed, 483 insertions(+), 960 deletions(-) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala index cb885a441ab..24fbfb20483 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkFlussConf.scala @@ -37,13 +37,6 @@ object SparkFlussConf { val FULL, EARLIEST, LATEST, TIMESTAMP = Value } - object TimestampOutOfRangeMode extends Enumeration { - val ERROR, ADJUST = Value - } - - /** Reserved value of [[SCAN_INCREMENTAL_END_TIMESTAMP]] meaning "the latest committed data". */ - val END_TIMESTAMP_LATEST = "latest" - val SCAN_START_UP_MODE: ConfigOption[String] = ConfigBuilder .key("scan.startup.mode") @@ -66,25 +59,14 @@ object SparkFlussConf { ConfigBuilder .key("scan.incremental.end.timestamp") .stringType() - .defaultValue(END_TIMESTAMP_LATEST) + .noDefaultValue() .withDescription( "The exclusive upper bound of an incremental (time-range) batch read, yielding a " + - "left-closed right-open '[start, end)' window. 'latest' (default) stops at the " + - "latest committed data captured at planning time; otherwise accepts epoch " + - "milliseconds or a 'yyyy-MM-dd HH:mm:ss' datetime string interpreted in the Spark " + - "session time zone. Setting it without 'scan.incremental.start.timestamp' fails " + - "fast, as does a window whose start is not strictly before its end.") - - val SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE: ConfigOption[String] = - ConfigBuilder - .key("scan.incremental.timestamp.out-of-range") - .stringType() - .defaultValue(TimestampOutOfRangeMode.ERROR.toString) - .withDescription( - "Behavior when 'scan.incremental.start.timestamp' precedes the earliest data still " + - "retained by Fluss (bounded by 'table.log.ttl'). 'error' (default): fail fast so a " + - "truncated window is never returned silently. 'adjust': clamp the start to the " + - "earliest retained offset and read from there.") + "left-closed right-open '[start, end)' window. Accepts epoch milliseconds or a " + + "'yyyy-MM-dd HH:mm:ss' datetime string interpreted in the Spark session time zone; " + + "when unset the read runs up to the latest committed data. Setting it without " + + "'scan.incremental.start.timestamp' fails fast, as does a window whose start is not " + + "strictly before its end.") val SCAN_POLL_TIMEOUT: ConfigOption[Duration] = ConfigBuilder diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkTable.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkTable.scala index 144db03aeaa..b2bd2fdf508 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkTable.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/SparkTable.scala @@ -41,30 +41,36 @@ class SparkTable( with SupportsWrite with SQLConfHelper { - private def populateSparkConf(flussConfig: FlussConfiguration): Unit = { + /** + * Merges the current `spark.sql.fluss.*` session configuration onto a copy of the catalog + * configuration, which is shared by every table of this catalog and must not be mutated. + */ + private def configWithSessionConfs(): FlussConfiguration = { + val merged = new FlussConfiguration(flussConfig) conf.getAllConfs .filter(_._1.startsWith(SparkFlussConf.SPARK_FLUSS_CONF_PREFIX)) .foreach { case (k, v) => - flussConfig.setString(k.substring(SparkFlussConf.SPARK_FLUSS_CONF_PREFIX.length), v) + merged.setString(k.substring(SparkFlussConf.SPARK_FLUSS_CONF_PREFIX.length), v) } + merged } override def newWriteBuilder(logicalWriteInfo: LogicalWriteInfo): WriteBuilder = { - populateSparkConf(flussConfig) + val config = configWithSessionConfs() if (tableInfo.getPrimaryKeys.isEmpty) { - new FlussAppendWriteBuilder(tablePath, logicalWriteInfo.schema(), flussConfig) + new FlussAppendWriteBuilder(tablePath, logicalWriteInfo.schema(), config) } else { - new FlussUpsertWriteBuilder(tablePath, logicalWriteInfo.schema(), flussConfig) + new FlussUpsertWriteBuilder(tablePath, logicalWriteInfo.schema(), config) } } override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { - populateSparkConf(flussConfig) + val config = configWithSessionConfs() if (tableInfo.getPrimaryKeys.isEmpty) { - new FlussAppendScanBuilder(tablePath, tableInfo, options, flussConfig) + new FlussAppendScanBuilder(tablePath, tableInfo, options, config) } else { - new FlussUpsertScanBuilder(tablePath, tableInfo, options, flussConfig) + new FlussUpsertScanBuilder(tablePath, tableInfo, options, config) } } } diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala index 317751cedcd..01e2a5886b1 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala @@ -19,31 +19,29 @@ package org.apache.fluss.spark.catalyst.plans.logical import org.apache.fluss.spark.{SparkFlussConf, SparkTable} import org.apache.fluss.spark.catalyst.plans.logical.FlussTableValuedFunctions._ +import org.apache.fluss.spark.read.FlussOffsetInitializers import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.FunctionRegistryBase import org.apache.spark.sql.catalyst.analysis.TableFunctionRegistry.TableFunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExpressionInfo, RuntimeReplaceable} +import org.apache.spark.sql.catalyst.expressions.{Attribute, CurrentTimestamp, Expression, ExpressionInfo, RuntimeReplaceable} import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{IntegerType, LongType, ShortType, StringType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.types.{DateType, IntegerType, LongType, ShortType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.sql.util.CaseInsensitiveStringMap -import java.time.{LocalDateTime, ZoneId, ZoneOffset} +import java.time.{LocalDate, LocalDateTime, ZoneId, ZoneOffset} import scala.collection.JavaConverters._ import scala.util.control.NonFatal /** - * Fluss table-valued functions (TVFs), usable from pure SQL. - * - * A TVF is only sugar over per-relation scan options: the function arguments are translated into - * the same `scan.*` options the DataFrame API accepts, and the call is then resolved into a plain - * [[DataSourceV2Relation]]. Consequently projection, filter push down and metrics keep working, and - * the options are scoped to the single query instead of leaking through session configuration. + * Fluss table-valued functions (TVFs), usable from pure SQL. A call is translated into the `scan.*` + * options the DataFrame API accepts and resolved into a plain [[DataSourceV2Relation]], so it is + * scoped to the single query and keeps projection, filter push down and metrics working. */ object FlussTableValuedFunctions { @@ -65,10 +63,7 @@ object FlussTableValuedFunctions { (FunctionIdentifier(fnName), info, builder) } - /** - * Resolves a Fluss TVF call into a [[DataSourceV2Relation]] over the referenced Fluss table, with - * the function arguments translated into scan options. - */ + /** Resolves a TVF call into a relation over the referenced Fluss table. */ def resolveFlussTableValuedFunction( spark: SparkSession, tvf: FlussTableValueFunction): LogicalPlan = { @@ -81,8 +76,7 @@ object FlussTableValuedFunctions { s"${tvf.fnName} requires a table identifier as its first argument.") } - // Parse the remaining arguments first so that an argument error is reported without depending - // on the referenced table being resolvable. + // Parse the arguments first, so an argument error does not depend on the table being resolvable. val options = tvf.parseArgs(args.tail) val tableArg = args.head.eval() @@ -95,14 +89,15 @@ object FlussTableValuedFunctions { val (catalogName, namespace, tableName) = sessionState.sqlParser.parseMultipartIdentifier(tableIdentifier) match { case Seq(table) => - (catalogManager.currentCatalog.name(), catalogManager.currentNamespace.head, table) - case Seq(db, table) => (catalogManager.currentCatalog.name(), db, table) - case Seq(catalog, db, table) => (catalog, db, table) + (catalogManager.currentCatalog.name(), catalogManager.currentNamespace, table) + case Seq(db, table) => (catalogManager.currentCatalog.name(), Array(db), table) + case Seq(catalog, db, table) => (catalog, Array(db), table) case _ => throw new IllegalArgumentException( s"Invalid table identifier '$tableIdentifier' for ${tvf.fnName}. Expected " + "'table', 'database.table' or 'catalog.database.table'.") } + val fullTableIdentifier = (catalogName +: namespace :+ tableName).mkString(".") val catalogPlugin = catalogManager.catalog(catalogName) if (!catalogPlugin.isInstanceOf[TableCatalog]) { @@ -111,11 +106,11 @@ object FlussTableValuedFunctions { s"${catalogPlugin.getClass.getName}.") } val tableCatalog = catalogPlugin.asInstanceOf[TableCatalog] - val ident = Identifier.of(Array(namespace), tableName) + val ident = Identifier.of(namespace, tableName) val table = tableCatalog.loadTable(ident) if (!table.isInstanceOf[SparkTable]) { throw new IllegalArgumentException( - s"${tvf.fnName} only supports Fluss tables, but '$catalogName.$namespace.$tableName' is " + + s"${tvf.fnName} only supports Fluss tables, but '$fullTableIdentifier' is " + s"backed by ${table.getClass.getName}.") } @@ -127,24 +122,15 @@ object FlussTableValuedFunctions { } /** - * Normalizes a timestamp argument to the string form accepted by the `scan.incremental.*` - * timestamp options. - * - * A STRING argument is passed through untouched, so both epoch milliseconds and - * `yyyy-MM-dd HH:mm:ss` keep being interpreted by the option layer. Integral arguments are epoch - * milliseconds. TIMESTAMP (local instant) arguments are converted from Spark's internal - * microseconds, otherwise a `TIMESTAMP '...'` literal would silently be read as epoch - * milliseconds. TIMESTAMP_NTZ arguments hold wall-clock microseconds and are re-interpreted in - * the Spark session time zone, so an NTZ literal resolves to the same window as the same - * `yyyy-MM-dd HH:mm:ss` string literal. - * - * Any constant expression is accepted, e.g. `CAST(unix_timestamp() * 1000 AS STRING)` or - * `date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss')`. + * Normalizes a timestamp argument, which may be any constant expression, to the string form the + * `scan.incremental.*` options accept: a STRING is passed through (the option layer reads both + * epoch millis and `yyyy-MM-dd HH:mm:ss`), an integral value is epoch millis, a DATE becomes the + * start of that day, and TIMESTAMP / TIMESTAMP_NTZ are converted from Spark's internal + * microseconds. */ private[logical] def toTimestampOptionValue(fnName: String, expr: Expression): String = { - // `RuntimeReplaceable` expressions (such as the `-` in `now() - INTERVAL 1 HOUR`) only become - // evaluable once the optimizer's ReplaceExpressions rule rewrites them, which has not happened - // yet while the analyzer resolves this function. Apply the same rewrite bottom-up here. + // RuntimeReplaceable expressions (e.g. the `-` in `now() - INTERVAL 1 HOUR`) only become + // evaluable once the optimizer rewrites them, which has not happened yet during analysis. val evaluable = expr.transformUp { case r: RuntimeReplaceable => r.replacement } val value = @@ -162,26 +148,43 @@ object FlussTableValuedFunctions { if (value == null) { throw new IllegalArgumentException(s"Timestamp arguments of $fnName must not be null.") } - evaluable.dataType match { + val normalized = evaluable.dataType match { case StringType => value.toString case ShortType | IntegerType | LongType => value.toString + case DateType => dateDaysToEpochMillis(value.asInstanceOf[Int]).toString case TimestampType => (value.asInstanceOf[Long] / 1000L).toString case TimestampNTZType => ntzMicrosToEpochMillis(value.asInstanceOf[Long]).toString case other => throw new IllegalArgumentException( s"Unsupported timestamp argument type $other for $fnName. Use a STRING (epoch " + - "milliseconds or 'yyyy-MM-dd HH:mm:ss'), an integral epoch milliseconds value, or a " + - "TIMESTAMP.") + "milliseconds or 'yyyy-MM-dd HH:mm:ss'), an integral epoch milliseconds value, a " + + "DATE or a TIMESTAMP.") + } + if (normalized.trim.isEmpty) { + throw new IllegalArgumentException( + s"Timestamp arguments of $fnName must not be blank. Provide epoch milliseconds or a " + + "'yyyy-MM-dd HH:mm:ss' timestamp.") } + normalized } private val MICROS_PER_SECOND = 1000000L /** - * Converts a `TIMESTAMP_NTZ` argument to epoch milliseconds. NTZ values are wall-clock - * microseconds encoded as if in UTC; they are re-interpreted in the Spark session time zone — the - * same convention the `yyyy-MM-dd HH:mm:ss` string form uses (and the same as the Flink - * connector's timestamp options) — so both forms resolve to the same window for the same literal. + * Converts a `DATE` argument to epoch milliseconds, at the start of that day in the Spark session + * time zone. + */ + private def dateDaysToEpochMillis(days: Int): Long = + LocalDate + .ofEpochDay(days.toLong) + .atStartOfDay(ZoneId.of(SQLConf.get.sessionLocalTimeZone)) + .toInstant + .toEpochMilli + + /** + * Converts a `TIMESTAMP_NTZ` argument to epoch milliseconds. Its wall-clock microseconds are + * re-interpreted in the Spark session time zone, the same convention the `yyyy-MM-dd HH:mm:ss` + * string form uses, so both forms resolve to the same instant. */ private def ntzMicrosToEpochMillis(micros: Long): Long = { val seconds = Math.floorDiv(micros, MICROS_PER_SECOND) @@ -213,13 +216,10 @@ abstract class FlussTableValueFunction(val fnName: String) extends LeafNode { } /** - * Plan for [[FlussTableValuedFunctions.INCREMENTAL_BETWEEN_TIMESTAMP]]. + * Plan for `fluss_incremental_between_timestamp(table, startTimestamp[, endTimestamp])`. * - * Usage: - * - `fluss_incremental_between_timestamp(table, startTimestamp, endTimestamp)` - * - `fluss_incremental_between_timestamp(table, startTimestamp)` reads up to the latest data - * - * The window is left-closed and right-open, `[start, end)`, on the record commit timestamp. + * The window is left-closed and right-open, `[start, end)`, on the record commit timestamp, and + * covers the data Fluss still retains inside it. An omitted end timestamp means "up to now". */ case class IncrementalBetweenTimestamp(override val args: Seq[Expression]) extends FlussTableValueFunction(INCREMENTAL_BETWEEN_TIMESTAMP) { @@ -234,17 +234,30 @@ case class IncrementalBetweenTimestamp(override val args: Seq[Expression]) } val start = toTimestampOptionValue(INCREMENTAL_BETWEEN_TIMESTAMP, argsWithoutTable.head) - val startOptions = - Map(SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() -> start) - - // The end bound is always written explicitly so the call stays self-contained: options take - // precedence over session configuration, which may still hold a stale end timestamp. - if (argsWithoutTable.size == 2) { - val end = toTimestampOptionValue(INCREMENTAL_BETWEEN_TIMESTAMP, argsWithoutTable.last) - startOptions + (SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() -> end) - } else { - startOptions + - (SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() -> SparkFlussConf.END_TIMESTAMP_LATEST) + // Resolving the end here pins the window at analysis time, so rows committed while the query is + // planned stay out of it and re-executing the same relation reads the same window. + val endArg = if (argsWithoutTable.size == 2) argsWithoutTable.last else CurrentTimestamp() + val end = toTimestampOptionValue(INCREMENTAL_BETWEEN_TIMESTAMP, endArg) + requireValidWindow(start, end) + + Map( + SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() -> start, + SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() -> end) + } + + /** A reversed or degenerate window fails fast instead of silently returning no rows. */ + private def requireValidWindow(start: String, end: String): Unit = { + val startMs = FlussOffsetInitializers.parseTimestamp( + start, + SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) + val endMs = FlussOffsetInitializers.parseTimestamp( + end, + SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key()) + if (startMs >= endMs) { + throw new IllegalArgumentException( + s"Invalid time range for $INCREMENTAL_BETWEEN_TIMESTAMP: the start timestamp '$start' " + + s"must be strictly before the end timestamp '$end'. The window is left-closed " + + s"right-open '[start, end)'.") } } } diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala index 354c9b8f2a4..ae147e9fd08 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala @@ -47,6 +47,12 @@ class FlussAppendPartitionReader( // The latest offset of fluss is -2 private var currentOffset: Long = flussPartition.startOffset.max(0L) + private val timeRange: Option[FlussTimeRange] = flussPartition.timeRange + + // Set once a record at or after the end of the requested window is seen; commit timestamps are + // non-decreasing within a bucket, so no later record can belong to the window either. + private var reachedWindowEnd = false + // initialize log scanner initialize() @@ -60,26 +66,28 @@ class FlussAppendPartitionReader( } override def next0(): Boolean = { - if (closed || currentOffset >= flussPartition.stopOffset) { - return false - } - - if (!currentRecords.hasNext) { - pollMoreRecords() - } + while (!closed && !reachedWindowEnd && currentOffset < flussPartition.stopOffset) { + if (!currentRecords.hasNext) { + pollMoreRecords() + } + if (!currentRecords.hasNext) { + throw new IllegalStateException(s"No more data from fluss server," + + s" but current offset $currentOffset not reach the stop offset ${flussPartition.stopOffset}") + } - // If we have records in current batch, return next one - if (currentRecords.hasNext) { val scanRecord = currentRecords.next() - currentRow = convertToSparkRow(scanRecord) currentOffset = scanRecord.logOffset() + 1 - true - } else if (currentOffset < flussPartition.stopOffset) { - throw new IllegalStateException(s"No more data from fluss server," + - s" but current offset $currentOffset not reach the stop offset ${flussPartition.stopOffset}") - } else { - false + timeRange match { + case Some(range) if range.isAfter(scanRecord.timestamp()) => reachedWindowEnd = true + // The record precedes the requested window: the start offset resolved from the start + // timestamp is only time-index accurate on tiered segments, so it can undershoot. + case Some(range) if !range.contains(scanRecord.timestamp()) => // skip + case _ => + currentRow = convertToSparkRow(scanRecord) + return true + } } + false } override def close0(): Unit = { diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussInputPartition.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussInputPartition.scala index b4ea3b9581a..b126cefc14e 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussInputPartition.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussInputPartition.scala @@ -30,18 +30,39 @@ trait FlussInputPartition extends InputPartition { } +/** + * The `[startMs, endMs)` window a time-range batch read was asked for, on the record commit + * timestamp. The reader applies it because offsets resolved from timestamps are only as accurate as + * the server-side time index. Timestamps are non-decreasing within a bucket, so the first record at + * or after `endMs` also ends the partition. + */ +case class FlussTimeRange(startMs: Long, endMs: Long) { + + def contains(timestampMs: Long): Boolean = timestampMs >= startMs && timestampMs < endMs + + def isAfter(timestampMs: Long): Boolean = timestampMs >= endMs + + override def toString: String = s"TimeRange[$startMs - $endMs]" +} + /** * Represents an input partition for reading data from a Fluss table bucket. * * @param tableBucket * the table bucket to read from + * @param timeRange + * the requested time window, set for a time-range batch read only */ -case class FlussAppendInputPartition(tableBucket: TableBucket, startOffset: Long, stopOffset: Long) +case class FlussAppendInputPartition( + tableBucket: TableBucket, + startOffset: Long, + stopOffset: Long, + timeRange: Option[FlussTimeRange] = None) extends FlussInputPartition { override def toString: String = { s"FlussAppendInputPartition{tableId=${tableBucket.getTableId}, bucketId=${tableBucket.getBucket}," + s" partitionId=${tableBucket.getPartitionId}" + - s" logStartOffset=$startOffset, logStopOffset=$stopOffset" + s" logStartOffset=$startOffset, logStopOffset=$stopOffset, timeRange=$timeRange" } } @@ -58,16 +79,20 @@ case class FlussAppendInputPartition(tableBucket: TableBucket, startOffset: Long * the log offset where incremental reading should start * @param logStoppingOffset * the log offset where incremental reading should end + * @param timeRange + * the requested time window, set for a time-range batch read only */ case class FlussUpsertInputPartition( tableBucket: TableBucket, snapshotId: Long, logStartingOffset: Long, - logStoppingOffset: Long) + logStoppingOffset: Long, + timeRange: Option[FlussTimeRange] = None) extends FlussInputPartition { override def toString: String = { s"FlussUpsertInputPartition{tableId=${tableBucket.getTableId}, bucketId=${tableBucket.getBucket}," + s" partitionId=${tableBucket.getPartitionId}, snapshotId=$snapshotId," + - s" logStartOffset=$logStartingOffset, logStopOffset=$logStoppingOffset}" + s" logStartOffset=$logStartingOffset, logStopOffset=$logStoppingOffset," + + s" timeRange=$timeRange}" } } diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala index d9a17a58978..7f4d9bacfe2 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala @@ -55,56 +55,6 @@ object FlussOffsetInitializers { rawValue.isDefined } - /** - * Whether a resolved start offset predates the data Fluss still retains for a bucket. A bucket - * whose earliest offset is still 0 has dropped nothing and is never flagged. - */ - def isBeforeRetention(startOffset: Long, earliestOffset: Long): Boolean = - earliestOffset > 0 && startOffset <= earliestOffset - - /** - * Rejects a start offset that predates the earliest retained data (see [[isBeforeRetention]]), so - * a truncated window is never returned silently. Callers must pass a concrete earliest offset, - * i.e. from a retriever built with `fetchEarliestOffset = true`. - */ - def requireStartWithinRetention( - tableDescription: String, - partitionName: String, - bucketId: Int, - startOffset: Long, - earliestOffset: Long): Unit = { - if (isBeforeRetention(startOffset, earliestOffset)) { - val partitionDesc = if (partitionName != null) s" partition '$partitionName'" else "" - throw new IllegalArgumentException( - s"The requested start timestamp resolves to log offset $startOffset for bucket " + - s"$bucketId$partitionDesc of table $tableDescription, which is at or before the " + - s"earliest retained offset $earliestOffset. The requested time range exceeds Fluss " + - s"retention (table.log.ttl); narrow the time range or increase table.log.ttl.") - } - } - - /** - * Whether a start timestamp preceding the earliest retained data fails fast (default) instead of - * being clamped to that offset. Controlled by `scan.incremental.timestamp.out-of-range`. - */ - def failOnTimestampOutOfRange(options: CaseInsensitiveStringMap): Boolean = { - val mode = - incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE) - .getOrElse(SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE.defaultValue()) - .trim - .toUpperCase - SparkFlussConf.TimestampOutOfRangeMode.values.find(_.toString == mode) match { - case Some(resolved) => resolved == SparkFlussConf.TimestampOutOfRangeMode.ERROR - case None => - throw new IllegalArgumentException( - s"Unsupported value for " + - s"'${SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE.key()}': '$mode'. " + - s"Supported values are " + - s"'${SparkFlussConf.TimestampOutOfRangeMode.values.toList.map(_.toString.toLowerCase).mkString("', '")}'" + - s".") - } - } - /** * Start offsets of an incremental batch read, resolved from `scan.incremental.start.timestamp`. * Requires that option to be set. @@ -140,60 +90,39 @@ object FlussOffsetInitializers { options: CaseInsensitiveStringMap): OffsetsInitializer = { if (!isBatch) { new NoStoppingOffsetsInitializer() - } else if (!isIncrementalRead(options)) { - val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() - if (Option(options.get(endKey)).isDefined) { - throw new IllegalArgumentException( - s"'$endKey' is set but " + - s"'${SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()}' is missing. An end " + - s"timestamp alone cannot truncate a batch read; set a start timestamp for an " + - s"incremental read, or remove the end option.") - } - // A plain batch read stops at the latest committed data. - OffsetsInitializer.latest() } else { - val end = - incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP).getOrElse("").trim - if ( - end.isEmpty || - end.equalsIgnoreCase(SparkFlussConf.END_TIMESTAMP_LATEST) - ) { - OffsetsInitializer.latest() - } else { - OffsetsInitializer.timestamp( - parseTimestamp(end.trim, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key())) + if (!isIncrementalRead(options)) { + val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() + if (Option(options.get(endKey)).isDefined) { + throw new IllegalArgumentException( + s"'$endKey' is set but " + + s"'${SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()}' is missing. An end " + + s"timestamp alone cannot truncate a batch read; set a start timestamp for an " + + s"incremental read, or remove the end option.") + } } + // A batch read stops at the latest committed data, an incremental one included: its end bound + // is applied by the reader on the record commit timestamp (see [[incrementalTimeRange]]). + OffsetsInitializer.latest() } } - /** - * Validates the `[start, end)` window of an incremental read: when the end bound is an explicit - * timestamp (not the reserved value `latest`), it must be strictly after the start timestamp. A - * reversed or degenerate window fails fast instead of silently returning no rows. Note this only - * checks the requested timestamps; a bucket that simply has no data inside a valid window still - * yields an empty result. - */ - def requireValidWindow(options: CaseInsensitiveStringMap): Unit = { - val end = incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP) - if (end.exists(!_.trim.equalsIgnoreCase(SparkFlussConf.END_TIMESTAMP_LATEST))) { - val start = incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP).get - val startMillis = - parseTimestamp(start.trim, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) - val endMillis = - parseTimestamp(end.get.trim, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key()) - if (startMillis >= endMillis) { - throw new IllegalArgumentException( - s"Invalid time range for an incremental read: the start timestamp '$start' must be " + - s"strictly before the end timestamp '${end.get.trim}'. The window is left-closed " + - s"right-open '[start, end)'.") - } + /** The `[start, end)` window of an incremental read, empty for a plain batch read. */ + def incrementalTimeRange(options: CaseInsensitiveStringMap): Option[FlussTimeRange] = { + if (!isIncrementalRead(options)) { + return None } + val startMs = requiredTimestamp(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP) + val endMs = incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP) + .map(end => parseTimestamp(end.trim, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key())) + .getOrElse(Long.MaxValue) + Some(FlussTimeRange(startMs, endMs)) } /** * Reads a `scan.incremental.*` option from the scan options, falling back to its default. A blank - * value counts as unset for the end and out-of-range options; a blank start timestamp is rejected - * by [[isIncrementalRead]] before it can reach here. + * value counts as unset; a blank start timestamp is rejected by [[isIncrementalRead]] before it + * can reach here. */ private def incrementalOption( options: CaseInsensitiveStringMap, @@ -223,7 +152,7 @@ object FlussOffsetInitializers { * Parses a timestamp option value to epoch milliseconds: a purely numeric string is epoch * milliseconds, otherwise it is parsed as 'yyyy-MM-dd HH:mm:ss' in the Spark session time zone. */ - private def parseTimestamp(timestampStr: String, optionKey: String): Long = { + private[spark] def parseTimestamp(timestampStr: String, optionKey: String): Long = { if (timestampStr.matches("\\d+")) { timestampStr.toLong } else { diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussScan.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussScan.scala index 0b242e0ce2d..96def8ee814 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussScan.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussScan.scala @@ -44,6 +44,8 @@ trait FlussScan extends Scan { def limit: Option[Int] = None + def timeRange: Option[FlussTimeRange] = None + protected def scanType: String override def readSchema(): StructType = { @@ -59,10 +61,16 @@ trait FlussScan extends Scan { case Some(p) => s"$withPushed [PartitionFilter: $p]" case None => withPushed } - limit match { - case Some(l) => s"$withPartition [Limit: $l]" + val withTimeRange = timeRange match { + case Some(r) if r.endMs == Long.MaxValue => + s"$withPartition [TimeRange: [${r.startMs}, latest)]" + case Some(r) => s"$withPartition [TimeRange: [${r.startMs}, ${r.endMs})]" case None => withPartition } + limit match { + case Some(l) => s"$withTimeRange [Limit: $l]" + case None => withTimeRange + } } override def supportedCustomMetrics(): Array[CustomMetric] = @@ -90,6 +98,8 @@ case class FlussAppendScan( override protected lazy val scanType: String = if (planner.hasLakeSnapshot) "LakeAppend" else "Append" + override def timeRange: Option[FlussTimeRange] = planner.timeRange + override def toBatch: Batch = { new FlussAppendBatch( tablePath, @@ -134,6 +144,8 @@ case class FlussUpsertScan( override protected lazy val scanType: String = if (planner.hasLakeSnapshot) "LakeUpsert" else "Upsert" + override def timeRange: Option[FlussTimeRange] = planner.timeRange + override def toBatch: Batch = { new FlussUpsertBatch( tablePath, diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala index 6fc0cd26ddf..16ce95b3d92 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala @@ -59,6 +59,7 @@ class FlussUpsertPartitionReader( private val snapshotId: Long = flussPartition.snapshotId private val logStartingOffset: Long = flussPartition.logStartingOffset private val logStoppingOffset: Long = flussPartition.logStoppingOffset + private val timeRange: Option[FlussTimeRange] = flussPartition.timeRange private val logScanFinished = logStartingOffset >= logStoppingOffset || logStoppingOffset <= 0 private val (projectionWithPks, pkProjection) = { @@ -140,14 +141,20 @@ class FlussUpsertPartitionReader( if (!records.isEmpty) { val flatRecords = records.asScala for (scanRecord <- flatRecords) { - // Maybe data with logStoppingOffset doesn't exist. - if (scanRecord.logOffset() < logStoppingOffset - 1) { - allLogRecords += scanRecord - } else if (scanRecord.logOffset() == logStoppingOffset - 1) { - allLogRecords += scanRecord + if (timeRange.exists(_.isAfter(scanRecord.timestamp()))) { + // Past the end of the requested window, and commit timestamps only grow from here. continue = false } else { - continue = false // Stop if we reach the stopping offset + // Maybe data with logStoppingOffset doesn't exist. + if ( + scanRecord.logOffset() <= logStoppingOffset - 1 && + timeRange.forall(_.contains(scanRecord.timestamp())) + ) { + allLogRecords += scanRecord + } + if (scanRecord.logOffset() >= logStoppingOffset - 1) { + continue = false // Stop if we reach the stopping offset + } } } } diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala index 72bba3e92a6..d552cf45e74 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala @@ -83,6 +83,12 @@ sealed trait SplitPlanner extends AutoCloseable { * snapshots. */ def logTailPredicate: Option[FlussPredicate] + + /** + * The requested window of an incremental read, empty otherwise. The scan positions itself with + * offsets, the reader cuts the window by record commit timestamp. + */ + def timeRange: Option[FlussTimeRange] } /** Marker: planner yields partitions consumable by an append (log-table) reader factory. */ @@ -198,32 +204,6 @@ abstract class AbstractSplitPlanner( .toMap } - /** - * Fail-fast guard for an incremental batch read: rejects a start offset that predates the data - * Fluss still retains (bounded by `table.log.ttl`) instead of silently returning a truncated - * window. Requires a retriever created with `fetchEarliestOffset = true`; otherwise - * `earliestOffsets` returns the EARLIEST_OFFSET sentinel (-2) and the guard is a no-op. - */ - protected def checkTimeRangeWithinRetention( - partitionName: String, - buckets: Seq[Int], - startOffsets: scala.collection.Map[Integer, java.lang.Long], - bucketOffsetsRetriever: BucketOffsetsRetrieverImpl): Unit = { - val earliestOffsets = bucketOffsetsRetriever - .earliestOffsets(partitionName, buckets.map(Integer.valueOf).asJava) - .asScala - buckets.foreach { - bucketId => - val bucket = Integer.valueOf(bucketId) - FlussOffsetInitializers.requireStartWithinRetention( - tablePath.toString, - partitionName, - bucketId, - Long2long(startOffsets(bucket)), - Long2long(earliestOffsets(bucket))) - } - } - /** * Releases the Fluss client connection. Idempotent and null-safe; it never forces the lazily * opened connection into existence, so it is a no-op when no metadata access ever occurred. @@ -280,10 +260,8 @@ class AppendPlanner( private val incrementalMode: Boolean = FlussOffsetInitializers.isIncrementalRead(options) - // Whether a start timestamp preceding retained data fails (default) or is clamped to earliest. - // Lazy on purpose: the option is incremental-only and must not affect plain batch reads. - private lazy val failOnOutOfRange: Boolean = - FlussOffsetInitializers.failOnTimestampOutOfRange(options) + override val timeRange: Option[FlussTimeRange] = + FlussOffsetInitializers.incrementalTimeRange(options) // An incremental read starts at scan.incremental.start.timestamp, a plain batch read at the // beginning of the table (see class scaladoc). @@ -306,9 +284,6 @@ class AppendPlanner( override def plan(): Array[InputPartition] = try { - if (incrementalMode) { - FlussOffsetInitializers.requireValidWindow(options) - } readableLakeSnapshot match { // An incremental read never unions a lake snapshot; it reads only Fluss. case Some(snap) if !incrementalMode => planLakeUnion(snap) @@ -328,13 +303,10 @@ class AppendPlanner( if (value > 0) Some(value) else None } - // Both the retention guard and the max-records splitter need concrete earliest offsets; - // otherwise the earliest sentinel (-2) is enough. + // Only the max-records splitter needs concrete earliest offsets; otherwise the earliest + // sentinel (-2) is enough. val bucketOffsetsRetrieverImpl = - new BucketOffsetsRetrieverImpl( - admin, - tablePath, - maxRecordsPerPartition.isDefined || incrementalMode) + new BucketOffsetsRetrieverImpl(admin, tablePath, maxRecordsPerPartition.isDefined) val buckets = (0 until tableInfo.getNumBuckets).toSeq def splitOffsetRange( @@ -345,7 +317,7 @@ class AppendPlanner( if ( startOffset < 0 || stopOffset <= startOffset || stopOffset <= (startOffset + maxRecords) ) { - return Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset)) + return Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset, timeRange)) } val rangeSize = stopOffset - startOffset val numSplits = ((rangeSize + maxRecords - 1) / maxRecords).toInt @@ -356,7 +328,12 @@ class AppendPlanner( .take(numSplits) .map(i => startOffset + i * step) .map { - from => FlussAppendInputPartition(tableBucket, from, math.min(from + step, stopOffset)) + from => + FlussAppendInputPartition( + tableBucket, + from, + math.min(from + step, stopOffset), + timeRange) } .toSeq } @@ -380,7 +357,8 @@ class AppendPlanner( } maxRecordsPerPartition match { case Some(maxRecs) => splitOffsetRange(tableBucket, startOffset, stopOffset, maxRecs) - case _ => Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset)) + case _ => + Seq(FlussAppendInputPartition(tableBucket, startOffset, stopOffset, timeRange)) } } }.toArray @@ -403,13 +381,6 @@ class AppendPlanner( partitionName, buckets.map(Integer.valueOf).asJava, bucketOffsetsRetrieverImpl) - if (incrementalMode && failOnOutOfRange) { - checkTimeRangeWithinRetention( - partitionName, - buckets, - startBucketOffsets.asScala, - bucketOffsetsRetrieverImpl) - } ( partitionInfo.getPartitionId, startBucketOffsets.asScala.map(e => (e._1, Long2long(e._2))), @@ -432,13 +403,6 @@ class AppendPlanner( null, buckets.map(Integer.valueOf).asJava, bucketOffsetsRetrieverImpl) - if (incrementalMode && failOnOutOfRange) { - checkTimeRangeWithinRetention( - null, - buckets, - startBucketOffsets.asScala, - bucketOffsetsRetrieverImpl) - } createPartitions( None, startBucketOffsets.asScala.map(e => (e._1, Long2long(e._2))).toMap, @@ -658,10 +622,8 @@ class UpsertPlanner( private val incrementalMode: Boolean = FlussOffsetInitializers.isIncrementalRead(options) - // Whether a start timestamp preceding retained data fails (default) or is clamped to earliest. - // Lazy on purpose: the option is incremental-only and must not affect plain batch reads. - private lazy val failOnOutOfRange: Boolean = - FlussOffsetInitializers.failOnTimestampOutOfRange(options) + override val timeRange: Option[FlussTimeRange] = + FlussOffsetInitializers.incrementalTimeRange(options) // Start offset of an incremental read, resolved from scan.incremental.start.timestamp. Lazy on // purpose: resolving it requires that option, while a plain batch scan derives its start from kv @@ -678,9 +640,6 @@ class UpsertPlanner( override def plan(): Array[InputPartition] = try { - if (incrementalMode) { - FlussOffsetInitializers.requireValidWindow(options) - } readableLakeSnapshot match { // An incremental read reads neither the lake nor the kv snapshot; it folds only the Fluss // changelog within [start, end). @@ -769,7 +728,7 @@ class UpsertPlanner( s"an incremental read folds only the changelog, so this combination would silently " + s"return no rows.") } - val bucketOffsetsRetriever = new BucketOffsetsRetrieverImpl(admin, tablePath, true) + val bucketOffsetsRetriever = new BucketOffsetsRetrieverImpl(admin, tablePath) val buckets = (0 until tableInfo.getNumBuckets).toSeq if (tableInfo.isPartitioned) { @@ -803,13 +762,6 @@ class UpsertPlanner( bucketOffsetsRetriever) val stoppingBucketOffsets = stoppingOffsetsInitializer.getBucketOffsets(partitionName, jBuckets, bucketOffsetsRetriever) - if (failOnOutOfRange) { - checkTimeRangeWithinRetention( - partitionName, - buckets, - startBucketOffsets.asScala, - bucketOffsetsRetriever) - } val tableId = tableInfo.getTableId buckets.flatMap { @@ -826,7 +778,7 @@ class UpsertPlanner( None } else { Some( - FlussUpsertInputPartition(tableBucket, -1L, startOffset, stopOffset) + FlussUpsertInputPartition(tableBucket, -1L, startOffset, stopOffset, timeRange) .asInstanceOf[InputPartition]) } }.toArray diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala index 72a18df0b6d..6ebc1114b51 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala @@ -17,700 +17,328 @@ package org.apache.fluss.spark -import org.apache.fluss.client.table.Table import org.apache.fluss.row.{BinaryString, GenericRow} +import org.apache.fluss.spark.read.{FlussOffsetInitializers, FlussTimeRange} import org.apache.spark.sql.Row +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.assertj.core.api.Assertions.assertThat -import java.time.{Duration, Instant, ZoneId} -import java.time.format.DateTimeFormatter - /** * Verifies the `fluss_incremental_between_timestamp` table-valued function. The window is * left-closed, right-open `[start, end)` on the record commit timestamp, and the function's options - * are scoped to the single query. + * are scoped to the single query. Tables are partitioned unless a case says otherwise. */ class SparkTimeRangeTvfTest extends FlussSparkTestBase { private val TVF = "fluss_incremental_between_timestamp" - private def createLogTable(name: String): Unit = - sql(s""" - |CREATE TABLE $DEFAULT_DATABASE.$name - |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) - |""".stripMargin) - - /** Truncates to a whole second so a millis value, a datetime string and a TIMESTAMP agree. */ - private def secondAligned(ms: Long): Long = (ms / 1000L) * 1000L - - private def waitPast(ms: Long): Unit = { - while (System.currentTimeMillis() <= ms) { - Thread.sleep(20) - } - } - - private def formatTs(ms: Long): String = - Instant - .ofEpochMilli(ms) - .atZone(ZoneId.of(spark.sessionState.conf.sessionLocalTimeZone)) - .format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) - - private def fullMessage(t: Throwable): String = { - val sw = new java.io.StringWriter() - t.printStackTrace(new java.io.PrintWriter(sw)) - sw.toString - } - - test("TVF: log table window [t1, t2)") { - withTable("t") { - createLogTable("t") - - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES - |(1L, 11L, 101, "a1"), (2L, 12L, 102, "a2")""".stripMargin) - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) + private val P1 = "2026-01-01" + private val P2 = "2026-01-02" - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES - |(3L, 13L, 103, "a3"), (4L, 14L, 104, "a4")""".stripMargin) - Thread.sleep(500) - val t2 = System.currentTimeMillis() - Thread.sleep(50) + test("TVF: log table window") { + withTable("t_log") { + createLogTable("t_log") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (5L, 15L, 105, "a5")""") - Thread.sleep(200) + insert("t_log", s"""(1L, 11L, 101, "a1", "$P1"), (2L, 12L, 102, "a2", "$P2")""") + val t1 = boundary() + insert("t_log", s"""(3L, 13L, 103, "a3", "$P1"), (4L, 14L, 104, "a4", "$P2")""") + val t2 = boundary() + insert("t_log", s"""(5L, 15L, 105, "a5", "$P1")""") + val t3 = boundary() + val t4 = boundary() + // the window spans every partition checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), - Row(3L, 13L, 103, "a3") :: Row(4L, 14L, 104, "a4") :: Nil) + sql(s"SELECT * FROM ${tvf("t_log", t1, t2)} ORDER BY orderId"), + Row(3L, 13L, 103, "a3", P1) :: Row(4L, 14L, 104, "a4", P2) :: Nil) - // projection and filter still work on top of the TVF relation + // projection, filter and partition pruning still work on top of the TVF relation checkAnswer( - sql(s"""SELECT address FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') - |WHERE amount = 104""".stripMargin), + sql(s"SELECT address FROM ${tvf("t_log", t1, t2)} WHERE amount = 104"), Row("a4") :: Nil) - } - } - - test("TVF: two-argument form reads up to the latest data") { - withTable("t") { - createLogTable("t") - - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) - - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") - Thread.sleep(300) - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (3L, 13L, 103, "a3")""") - Thread.sleep(200) - checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1') ORDER BY orderId"), - Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil) - } - } - - private def createPkTable(name: String): Unit = - sql(s""" - |CREATE TABLE $DEFAULT_DATABASE.$name - |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) - |TBLPROPERTIES("primary.key" = "orderId", "bucket.num" = 1) - |""".stripMargin) - - private def createPartitionedPkTable(name: String): Unit = - sql(s""" - |CREATE TABLE $DEFAULT_DATABASE.$name - |(orderId BIGINT, itemId BIGINT, amount INT, address STRING, dt STRING) - |PARTITIONED BY (dt) - |TBLPROPERTIES("primary.key" = "orderId,dt", "bucket.num" = 1) - |""".stripMargin) + sql(s"SELECT orderId FROM ${tvf("t_log", t1, t2)} WHERE dt = '$P1'"), + Row(3L) :: Nil) - test("TVF: primary key table folds to +I/+U and excludes deletes") { - withTable("t") { - val tablePath = createTablePath("t") - createPkTable("t") + // without an end timestamp the window runs up to the latest data + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_log", t2)} ORDER BY orderId"), + Row(5L, 15L, 105, "a5", P1) :: Nil) - val writer = loadFlussTable(tablePath).newUpsert().createWriter() - writer.upsert(row(1L, 11L, 101, "a1")).get() - writer.upsert(row(2L, 12L, 102, "a2")).get() - writer.upsert(row(3L, 13L, 103, "a3")).get() - writer.flush() - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) + // a window without writes yields nothing + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t3, t4)}"), Nil) - // in-window: update key 2, insert key 4, delete key 1 - writer.upsert(row(2L, 120L, 1002, "a2_upd")).get() - writer.upsert(row(4L, 14L, 104, "a4")).get() - writer.delete(deleteKey(1L)).get() - writer.flush() - Thread.sleep(500) - val t2 = System.currentTimeMillis() - Thread.sleep(50) + // the table argument may be unqualified or fully qualified + checkAnswer(sql(s"SELECT * FROM $TVF('t_log', '$t2')"), Row(5L, 15L, 105, "a5", P1) :: Nil) + checkAnswer( + sql(s"SELECT * FROM $TVF('$DEFAULT_CATALOG.$DEFAULT_DATABASE.t_log', '$t2')"), + Row(5L, 15L, 105, "a5", P1) :: Nil) - // after the window - writer.upsert(row(5L, 15L, 105, "a5")).get() - writer.flush() + // the omitted end bound is pinned when the statement is analyzed, so a row committed before + // the scan is planned stays outside the window + val pinned = sql(s"SELECT * FROM ${tvf("t_log", t2)} ORDER BY orderId") + insert("t_log", s"""(6L, 16L, 106, "a6", "$P1")""") Thread.sleep(200) - - val table = loadFlussTable(tablePath) - // evidence: the window changelog really contains -U/+U (key 2), +I (key 4), -D (key 1) - val changes = changelogInWindow(table, t1, t2) - assertThat(changes.filter(_._1 == "-U").map(_._2)).isEqualTo(Seq(2L)) - assertThat(changes.filter(_._1 == "+U").map(_._2)).isEqualTo(Seq(2L)) - assertThat(changes.filter(_._1 == "+I").map(_._2)).isEqualTo(Seq(4L)) - assertThat(changes.filter(_._1 == "-D").map(_._2)).isEqualTo(Seq(1L)) - - checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), - Row(2L, 120L, 1002, "a2_upd") :: Row(4L, 14L, 104, "a4") :: Nil) + checkAnswer(pinned, Row(5L, 15L, 105, "a5", P1) :: Nil) } } - test("TVF: primary key table collapses repeated -U/+U updates into the latest value") { - withTable("t") { - val tablePath = createTablePath("t") - createPkTable("t") + test("TVF: primary key table folds the window changelog") { + withTable("t_fold") { + createPkTable("t_fold") - val writer = loadFlussTable(tablePath).newUpsert().createWriter() - // before the window: keys 1-3 inserted - writer.upsert(row(1L, 11L, 101, "a1")).get() - writer.upsert(row(2L, 12L, 102, "a2")).get() - writer.upsert(row(3L, 13L, 103, "a3")).get() + val writer = loadFlussTable(createTablePath("t_fold")).newUpsert().createWriter() + writer.upsert(row(1L, 11L, 101, "a1", P1)).get() + writer.upsert(row(2L, 12L, 102, "a2", P2)).get() + writer.upsert(row(3L, 13L, 103, "a3", P1)).get() + writer.upsert(row(4L, 14L, 104, "a4", P2)).get() writer.flush() - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) - - // in-window: -U/+U twice on key 1, -U/+U once on key 2, key 3 untouched - writer.upsert(row(1L, 110L, 1001, "a1_v2")).get() - writer.upsert(row(1L, 111L, 1002, "a1_v3")).get() - writer.upsert(row(2L, 120L, 2001, "a2_v2")).get() + val t1 = boundary() + + // key 1 updated twice, key 2 updated once, key 3 deleted then re-inserted, key 4 deleted, + // key 5 inserted then deleted again + writer.upsert(row(1L, 110L, 1001, "a1_v2", P1)).get() + writer.upsert(row(1L, 111L, 1002, "a1_v3", P1)).get() + writer.upsert(row(2L, 120L, 1002, "a2_upd", P2)).get() + writer.delete(deleteKey(3L, P1)).get() + writer.upsert(row(3L, 130L, 1003, "a3_new", P1)).get() + writer.delete(deleteKey(4L, P2)).get() + writer.upsert(row(5L, 15L, 105, "a5", P2)).get() + writer.delete(deleteKey(5L, P2)).get() writer.flush() - Thread.sleep(500) - val t2 = System.currentTimeMillis() - Thread.sleep(50) + val t2 = boundary() - // after the window - writer.upsert(row(1L, 112L, 1003, "a1_v4")).get() + writer.upsert(row(1L, 112L, 1004, "a1_v4", P1)).get() writer.flush() - Thread.sleep(200) - - // evidence: three genuine -U/+U pairs exist in the window changelog (two for key 1, one - // for key 2), so the folding assertions below operate on real -U/+U records - val changes = changelogInWindow(loadFlussTable(tablePath), t1, t2) - assertThat(changes.filter(_._1 == "-U").map(_._2)).isEqualTo(Seq(1L, 1L, 2L)) - assertThat(changes.filter(_._1 == "+U").map(_._2)).isEqualTo(Seq(1L, 1L, 2L)) + val t3 = boundary() + val t4 = boundary() - // each updated key appears exactly once, with its last in-window value + // each changed key appears once with its last in-window value; deleted keys and an insert + // cancelled by a delete are excluded checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), - Row(1L, 111L, 1002, "a1_v3") :: Row(2L, 120L, 2001, "a2_v2") :: Nil) - - // the two-argument form reads through to the latest state - checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1') ORDER BY orderId"), - Row(1L, 112L, 1003, "a1_v4") :: Row(2L, 120L, 2001, "a2_v2") :: Nil) - } - } - - test("TVF: primary key table cancels out +I followed by -D in the window") { - withTable("t") { - val tablePath = createTablePath("t") - createPkTable("t") - - val writer = loadFlussTable(tablePath).newUpsert().createWriter() - // before the window - writer.upsert(row(1L, 11L, 101, "a1")).get() - writer.flush() - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) - - // in-window: insert key 2 then delete it again (cancels out), insert key 3 (survives) - writer.upsert(row(2L, 12L, 102, "a2")).get() - writer.delete(deleteKey(2L)).get() - writer.upsert(row(3L, 13L, 103, "a3")).get() - writer.flush() - Thread.sleep(500) - val t2 = System.currentTimeMillis() - Thread.sleep(50) - - // evidence: the window changelog holds +I then -D for key 2, and +I for key 3 - val changes = changelogInWindow(loadFlussTable(tablePath), t1, t2) - assertThat(changes.filter(r => r._1 == "+I" || r._1 == "-D")) - .isEqualTo(Seq(("+I", 2L), ("-D", 2L), ("+I", 3L))) + sql(s"SELECT * FROM ${tvf("t_fold", t1, t2)} ORDER BY orderId"), + Row(1L, 111L, 1002, "a1_v3", P1) :: + Row(2L, 120L, 1002, "a2_upd", P2) :: + Row(3L, 130L, 1003, "a3_new", P1) :: Nil + ) checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), - Row(3L, 13L, 103, "a3") :: Nil) - } - } - - test("TVF: primary key table keeps a key deleted then re-inserted in the window") { - withTable("t") { - val tablePath = createTablePath("t") - createPkTable("t") - - val writer = loadFlussTable(tablePath).newUpsert().createWriter() - // before the window - writer.upsert(row(1L, 11L, 101, "a1")).get() - writer.upsert(row(2L, 12L, 102, "a2")).get() - writer.flush() - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) - - // in-window: delete key 1 then re-insert it with new values (-D then +I survives), - // delete key 2 permanently (-D only, excluded) - writer.delete(deleteKey(1L)).get() - writer.upsert(row(1L, 110L, 1001, "a1_new")).get() - writer.delete(deleteKey(2L)).get() - writer.flush() - Thread.sleep(500) - val t2 = System.currentTimeMillis() - Thread.sleep(50) - - // evidence: the window changelog holds -D then +I for key 1, and -D for key 2 - assertThat(changelogInWindow(loadFlussTable(tablePath), t1, t2)) - .isEqualTo(Seq(("-D", 1L), ("+I", 1L), ("-D", 2L))) + sql(s"SELECT * FROM ${tvf("t_fold", t1)} ORDER BY orderId"), + Row(1L, 112L, 1004, "a1_v4", P1) :: + Row(2L, 120L, 1002, "a2_upd", P2) :: + Row(3L, 130L, 1003, "a3_new", P1) :: Nil + ) checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2') ORDER BY orderId"), - Row(1L, 110L, 1001, "a1_new") :: Nil) - } - } + sql(s"SELECT orderId FROM ${tvf("t_fold", t1, t2)} WHERE dt = '$P1' ORDER BY orderId"), + Row(1L) :: Row(3L) :: Nil) - test("TVF: primary key table window containing only -D returns nothing") { - withTable("t") { - val tablePath = createTablePath("t") - createPkTable("t") - - val writer = loadFlussTable(tablePath).newUpsert().createWriter() - writer.upsert(row(1L, 11L, 101, "a1")).get() - writer.upsert(row(2L, 12L, 102, "a2")).get() - writer.flush() - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) - - writer.delete(deleteKey(1L)).get() - writer.delete(deleteKey(2L)).get() - writer.flush() - Thread.sleep(500) - val t2 = System.currentTimeMillis() - Thread.sleep(50) - - // evidence: the window changelog holds exactly two -D records, so the empty result below - // reflects genuine delete folding rather than an empty window - val changes = changelogInWindow(loadFlussTable(tablePath), t1, t2) - assertThat(changes.map(_._1)).isEqualTo(Seq("-D", "-D")) - assertThat(changes.map(_._2)).isEqualTo(Seq(1L, 2L)) - - checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2')"), Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_fold", t3, t4)}"), Nil) } } - test("TVF: partitioned primary key table folds changes across partitions") { - withTable("t_pk_part") { - val tablePath = createTablePath("t_pk_part") - createPartitionedPkTable("t_pk_part") + test("TVF: non-partitioned tables") { + withTable("t_np", "t_np_pk") { + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t_np + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) + |""".stripMargin) + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.t_np_pk + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING) + |TBLPROPERTIES("primary.key" = "orderId", "bucket.num" = 1) + |""".stripMargin) - val writer = loadFlussTable(tablePath).newUpsert().createWriter() - // before the window: one row per partition - writer.upsert(pkRow(1L, 11L, 101, "a1", "2026-01-01")).get() - writer.upsert(pkRow(2L, 12L, 102, "a2", "2026-01-02")).get() - writer.flush() - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) - - // in-window: update key 1 (partition 1), permanent delete of key 2 (partition 2), - // insert key 3 then delete it again in partition 1 (cancels out) - writer.upsert(pkRow(1L, 110L, 1001, "a1_upd", "2026-01-01")).get() - writer.delete(deleteKey(2L, "2026-01-02")).get() - writer.upsert(pkRow(3L, 13L, 103, "a3", "2026-01-01")).get() - writer.delete(deleteKey(3L, "2026-01-01")).get() + val writer = loadFlussTable(createTablePath("t_np_pk")).newUpsert().createWriter() + insert("t_np", """(1L, 11L, 101, "a1")""") + writer.upsert(unpartitionedRow(1L, 11L, 101, "a1")).get() writer.flush() - Thread.sleep(500) - val t2 = System.currentTimeMillis() - Thread.sleep(50) + val t1 = boundary() - // after the window - writer.upsert(pkRow(4L, 14L, 104, "a4", "2026-01-01")).get() + insert("t_np", """(2L, 12L, 102, "a2")""") + writer.upsert(unpartitionedRow(1L, 110L, 1001, "a1_upd")).get() + writer.upsert(unpartitionedRow(2L, 12L, 102, "a2")).get() writer.flush() Thread.sleep(200) - // evidence: the window changelog holds -U/+U (key 1), -D (key 2), +I then -D (key 3); - // the -D comparison sorts first because poll order across partitions is not deterministic - val changes = changelogInWindow(loadFlussTable(tablePath), t1, t2) - assertThat(changes.filter(_._1 == "-U").map(_._2)).isEqualTo(Seq(1L)) - assertThat(changes.filter(_._1 == "+U").map(_._2)).isEqualTo(Seq(1L)) - assertThat(changes.filter(_._1 == "-D").map(_._2).sorted).isEqualTo(Seq(2L, 3L)) - assertThat(changes.filter(_._1 == "+I").map(_._2)).isEqualTo(Seq(3L)) - - checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_pk_part', '$t1', '$t2') ORDER BY orderId"), - Row(1L, 110L, 1001, "a1_upd", "2026-01-01") :: Nil) - - // partition filter on top of the TVF relation + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t1)}"), Row(2L, 12L, 102, "a2") :: Nil) checkAnswer( - sql(s"""SELECT orderId FROM $TVF('$DEFAULT_DATABASE.t_pk_part', '$t1') - |WHERE dt = '2026-01-01' ORDER BY orderId""".stripMargin), - Row(1L) :: Row(4L) :: Nil - ) + sql(s"SELECT * FROM ${tvf("t_np_pk", t1)} ORDER BY orderId"), + Row(1L, 110L, 1001, "a1_upd") :: Row(2L, 12L, 102, "a2") :: Nil) } } - test("TVF: session-level scan.incremental.* options are ignored") { - withTable("t") { - createLogTable("t") - - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") - Thread.sleep(200) - - // A stale window in session configuration must not leak into reads: the scan.incremental.* - // options are only honored as per-query scan options (TVF arguments / DataFrameReader). - withSQLConf( - s"spark.sql.fluss.${SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()}" -> - "2000-01-01 00:00:00", - s"spark.sql.fluss.${SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key()}" -> - "2000-01-02 00:00:00" - ) { - // A plain batch read still returns the full table. - checkAnswer( - sql(s"SELECT * FROM $DEFAULT_DATABASE.t ORDER BY orderId"), - Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Nil) - - // The TVF window is unaffected by the session values. - checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1') ORDER BY orderId"), - Row(2L, 12L, 102, "a2") :: Nil) + test("TVF: timestamp argument forms resolve to the same window") { + withTable("t_ts") { + createLogTable("t_ts") + + // every accepted form of the same instant must resolve to the same window; asserted on the + // options the reader consumes, so no data has to be written + def assertAllFormsAgree(): Unit = { + val startTs = "2026-01-01 00:00:00" + val endTs = "2026-01-02 00:00:00" + val expected = FlussTimeRange(parseTs(startTs), parseTs(endTs)) + Seq( + s"'${expected.startMs}', '${expected.endMs}'", + s"${expected.startMs}L, ${expected.endMs}L", + s"'$startTs', '$endTs'", + s"TIMESTAMP '$startTs', TIMESTAMP '$endTs'", + s"TIMESTAMP_NTZ '$startTs', TIMESTAMP_NTZ '$endTs'", + "DATE '2026-01-01', DATE '2026-01-02'" + ).foreach(args => assertThat(resolvedWindow("t_ts", args)).isEqualTo(expected)) } - } - } - - test("TVF: empty window returns no rows") { - withTable("t") { - createLogTable("t") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") - Thread.sleep(400) - val ta = System.currentTimeMillis() - Thread.sleep(300) - val tb = System.currentTimeMillis() - Thread.sleep(300) - // written after the [ta, tb) gap, so the window contains no data - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") - Thread.sleep(200) - - checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$ta', '$tb')"), Nil) + assertAllFormsAgree() + // the session time zone applies to the datetime, TIMESTAMP_NTZ and DATE forms alike + withSQLConf("spark.sql.session.timeZone" -> "Asia/Shanghai")(assertAllFormsAgree()) + + // an omitted end bound is pinned to the analysis time + val before = System.currentTimeMillis() + val openEnded = resolvedWindow("t_ts", "'2026-01-01 00:00:00'") + assertThat(openEnded.endMs).isBetween(before, System.currentTimeMillis()) + + // constant expressions are evaluated during analysis + val lastHour = resolvedWindow( + "t_ts", + "date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss'), " + + "CAST(unix_timestamp() * 1000 AS STRING)") + assertThat(lastHour.endMs - lastHour.startMs).isBetween(3590000L, 3610000L) + + val aroundToday = + resolvedWindow("t_ts", "current_date() - INTERVAL 1 DAY, current_date() + INTERVAL 1 DAY") + assertThat(aroundToday.startMs).isLessThan(before) + assertThat(aroundToday.endMs).isGreaterThan(before) } } - test("TVF: empty window on a primary key table returns no rows") { - withTable("t") { - createPkTable("t") + test("TVF: invalid usage fails fast") { + withTable("t_bad", "t_bad_pk") { + createLogTable("t_bad") + createPkTable("t_bad_pk") - val writer = loadFlussTable(createTablePath("t")).newUpsert().createWriter() - writer.upsert(row(1L, 11L, 101, "a1")).get() + val writer = loadFlussTable(createTablePath("t_bad_pk")).newUpsert().createWriter() + insert("t_bad", s"""(1L, 11L, 101, "a1", "$P1")""") + writer.upsert(row(1L, 11L, 101, "a1", P1)).get() writer.flush() - Thread.sleep(400) - val ta = System.currentTimeMillis() - Thread.sleep(300) - val tb = System.currentTimeMillis() Thread.sleep(300) - // written after the [ta, tb) gap, so the window contains no data - writer.upsert(row(2L, 12L, 102, "a2")).get() - writer.flush() - Thread.sleep(200) - - checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$ta', '$tb')"), Nil) - } - } - - test("TVF: incremental read fails fast when read.optimized is enabled") { - withTable("t") { - createPkTable("t") - - val writer = loadFlussTable(createTablePath("t")).newUpsert().createWriter() - writer.upsert(row(1L, 11L, 101, "a1")).get() - writer.flush() - Thread.sleep(200) val t1 = System.currentTimeMillis() - withSQLConf( - s"${SparkFlussConf.SPARK_FLUSS_CONF_PREFIX}${SparkFlussConf.READ_OPTIMIZED_OPTION.key()}" -> - "true") { - val ex = intercept[Exception] { - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1')").collect() - } - assertThat(fullMessage(ex)).contains(SparkFlussConf.READ_OPTIMIZED_OPTION.key()) - } - } - } - - test("TVF: epoch millis, datetime string and TIMESTAMP literal yield the same window") { - withTable("t") { - createLogTable("t") + // a blank start must not silently read the full table + assertThat(failureOf(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_bad', ' ')")) + .contains(TVF) + .contains("must not be blank") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") - Thread.sleep(1500) - val t1 = secondAligned(System.currentTimeMillis()) - waitPast(t1) + assertThat(failureOf(s"SELECT * FROM ${tvf("t_bad", t1 + 1000, t1)}")) + .contains("strictly before") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") - Thread.sleep(1500) - val t2 = secondAligned(System.currentTimeMillis()) - waitPast(t2) + assertThat(failureOf(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_bad')")) + .contains("endTimestamp") + assertThat(failureOf(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_bad', '1', '2', '3')")) + .contains("endTimestamp") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (3L, 13L, 103, "a3")""") - Thread.sleep(200) + assertThat(failureOf(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.not_exist', '1', '2')")) + .contains("not_exist") - val expected = Row(2L, 12L, 102, "a2") :: Nil - - // epoch milliseconds as a string - checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t1', '$t2')"), expected) - // epoch milliseconds as an integral literal - checkAnswer(sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', ${t1}L, ${t2}L)"), expected) - // 'yyyy-MM-dd HH:mm:ss' in the session time zone - checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '${formatTs(t1)}', '${formatTs(t2)}')"), - expected) - // TIMESTAMP literals - checkAnswer( - sql(s"""SELECT * FROM $TVF('$DEFAULT_DATABASE.t', - |TIMESTAMP '${formatTs(t1)}', TIMESTAMP '${formatTs(t2)}')""".stripMargin), - expected - ) + // an incremental read reconciles the changelog and cannot serve a read-optimized scan + withSQLConf(sessionKey(SparkFlussConf.READ_OPTIMIZED_OPTION.key()) -> "true") { + assertThat(failureOf(s"SELECT * FROM ${tvf("t_bad_pk", t1)}")) + .contains(SparkFlussConf.READ_OPTIMIZED_OPTION.key()) + } } } - test("TVF: TIMESTAMP_NTZ argument is interpreted in the session time zone") { - withTable("t") { - createLogTable("t") + test("TVF: window bounds are never read from session configuration") { + withTable("t_conf", "t_conf_pk") { + createLogTable("t_conf") + createPkTable("t_conf_pk") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") - Thread.sleep(1500) - val t1 = secondAligned(System.currentTimeMillis()) - waitPast(t1) - - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (2L, 12L, 102, "a2")""") - Thread.sleep(1500) - val t2 = secondAligned(System.currentTimeMillis()) - waitPast(t2) + val writer = loadFlussTable(createTablePath("t_conf_pk")).newUpsert().createWriter() + insert("t_conf", s"""(1L, 11L, 101, "a1", "$P1")""") + writer.upsert(row(1L, 11L, 101, "a1", P1)).get() + writer.flush() + val t1 = boundary() - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (3L, 13L, 103, "a3")""") + insert("t_conf", s"""(2L, 12L, 102, "a2", "$P2")""") + writer.upsert(row(2L, 12L, 102, "a2", P2)).get() + writer.flush() Thread.sleep(200) - val expected = Row(2L, 12L, 102, "a2") :: Nil + val window = Row(2L, 12L, 102, "a2", P2) :: Nil - // Under a non-UTC session time zone, an NTZ literal must resolve to the same window as the - // identical 'yyyy-MM-dd HH:mm:ss' string (parsed in the session time zone); pinning NTZ to - // UTC would shift the window by the zone offset and miss or leak rows. - withSQLConf("spark.sql.session.timeZone" -> "Asia/Shanghai") { - checkAnswer( - sql(s"""SELECT * FROM $TVF('$DEFAULT_DATABASE.t', - |TIMESTAMP_NTZ '${formatTs(t1)}', TIMESTAMP_NTZ '${formatTs(t2)}')""".stripMargin), - expected - ) + withSQLConf( + sessionKey(SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) -> "2000-01-01 00:00:00", + sessionKey(SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key()) -> "2000-01-02 00:00:00" + ) { checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '${formatTs(t1)}', '${formatTs(t2)}')"), - expected) - } - } - } - - test("TVF: blank timestamp arguments fail fast") { - withTable("t") { - createLogTable("t") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") - Thread.sleep(200) - - // A blank start must fail instead of silently reading the full table. - val ex = intercept[Exception] { - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', ' ')").collect() + sql(s"SELECT * FROM $DEFAULT_DATABASE.t_conf ORDER BY orderId"), + Row(1L, 11L, 101, "a1", P1) :: Row(2L, 12L, 102, "a2", P2) :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_conf", t1)}"), window) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_conf_pk", t1)}"), window) } - assertThat(fullMessage(ex)) - .contains(SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) - .contains("must not be blank") - } - } - - test("TVF: reversed time range fails fast") { - withTable("t") { - createLogTable("t") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") - Thread.sleep(300) - val t1 = System.currentTimeMillis() - Thread.sleep(50) - val t2 = System.currentTimeMillis() - - val ex = intercept[Exception] { - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '$t2', '$t1')").collect() - } - assertThat(fullMessage(ex)).contains("strictly before") - } - } - - test("TVF: Spark expressions as timestamp arguments") { - withTable("t") { - createLogTable("t") - - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES - |(1L, 11L, 101, "a1"), (2L, 12L, 102, "a2"), (3L, 13L, 103, "a3")""".stripMargin) - // unix_timestamp() has second granularity, so make every row strictly older than the - // truncated "now" to keep the window boundaries deterministic. - Thread.sleep(1300) - - // [now - 1h, now) covers every row written above, as epoch milliseconds - // (unix_timestamp() returns seconds) - checkAnswer( - sql(s"""SELECT * FROM $TVF( - | '$DEFAULT_DATABASE.t', - | CAST((unix_timestamp() - 3600) * 1000 AS STRING), - | CAST(unix_timestamp() * 1000 AS STRING)) ORDER BY orderId""".stripMargin), - Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil - ) - - // the same window as datetime strings - checkAnswer( - sql(s"""SELECT * FROM $TVF( - | '$DEFAULT_DATABASE.t', - | date_format(now() - INTERVAL 1 HOUR, 'yyyy-MM-dd HH:mm:ss'), - | date_format(now(), 'yyyy-MM-dd HH:mm:ss')) ORDER BY orderId""".stripMargin), - Row(1L, 11L, 101, "a1") :: Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil - ) - - // [now, latest) excludes them, proving the expression is really evaluated and applied - checkAnswer( - sql(s"""SELECT * FROM $TVF( - | '$DEFAULT_DATABASE.t', - | CAST(unix_timestamp() * 1000 AS STRING))""".stripMargin), - Nil - ) } } - test("TVF: partitioned log table window read") { - withTable("t_part") { - sql(s""" - |CREATE TABLE $DEFAULT_DATABASE.t_part - |(orderId BIGINT, itemId BIGINT, amount INT, dt STRING) - |PARTITIONED BY (dt) - |""".stripMargin) + private def createLogTable(name: String): Unit = + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.$name + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING, dt STRING) + |PARTITIONED BY (dt) + |""".stripMargin) - sql(s"""INSERT INTO $DEFAULT_DATABASE.t_part VALUES - |(1L, 11L, 101, "2026-01-01"), (2L, 12L, 102, "2026-01-02")""".stripMargin) - Thread.sleep(500) - val t1 = System.currentTimeMillis() - Thread.sleep(50) + private def createPkTable(name: String): Unit = + sql(s""" + |CREATE TABLE $DEFAULT_DATABASE.$name + |(orderId BIGINT, itemId BIGINT, amount INT, address STRING, dt STRING) + |PARTITIONED BY (dt) + |TBLPROPERTIES("primary.key" = "orderId,dt", "bucket.num" = 1) + |""".stripMargin) - sql(s"""INSERT INTO $DEFAULT_DATABASE.t_part VALUES - |(3L, 13L, 103, "2026-01-01"), (4L, 14L, 104, "2026-01-02")""".stripMargin) - Thread.sleep(500) - val t2 = System.currentTimeMillis() - Thread.sleep(50) + private def insert(table: String, values: String): Unit = + sql(s"INSERT INTO $DEFAULT_DATABASE.$table VALUES $values") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t_part VALUES (5L, 15L, 105, "2026-01-01")""") - Thread.sleep(200) + private def tvf(table: String, timestamps: Long*): String = + s"$TVF('$DEFAULT_DATABASE.$table'${timestamps.map(ts => s", '$ts'").mkString})" - checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_part', '$t1', '$t2') ORDER BY orderId"), - Row(3L, 13L, 103, "2026-01-01") :: Row(4L, 14L, 104, "2026-01-02") :: Nil - ) + private def sessionKey(option: String): String = + s"${SparkFlussConf.SPARK_FLUSS_CONF_PREFIX}$option" - // partition filter on top of the TVF relation - checkAnswer( - sql(s"""SELECT orderId FROM $TVF('$DEFAULT_DATABASE.t_part', '$t1', '$t2') - |WHERE dt = '2026-01-01'""".stripMargin), - Row(3L) :: Nil) - } + /** A timestamp after everything written so far and before anything written next. */ + private def boundary(): Long = { + Thread.sleep(500) + val ms = System.currentTimeMillis() + Thread.sleep(50) + ms } - test("TVF: wrong argument count fails with a usage hint") { - withTable("t") { - createLogTable("t") - sql(s"""INSERT INTO $DEFAULT_DATABASE.t VALUES (1L, 11L, 101, "a1")""") - - // only the table identifier - val tooFew = intercept[Exception] { - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t')").collect() - } - assertThat(fullMessage(tooFew)).contains("endTimestamp") - - // one argument too many - val tooMany = intercept[Exception] { - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t', '1', '2', '3')").collect() - } - assertThat(fullMessage(tooMany)).contains("endTimestamp") - } - } - - test("TVF: unknown table fails") { - val ex = intercept[Exception] { - sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.not_exist_tvf_table', '1', '2')").collect() - } - assertThat(fullMessage(ex)).contains("not_exist_tvf_table") + /** + * The window the reader would apply for a TVF call, taken from the scan options of the analyzed + * relation. Only the table metadata is touched; no data is read. + */ + private def resolvedWindow(table: String, args: String): FlussTimeRange = { + val plan = + sql(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.$table', $args)").queryExecution.analyzed + val options = plan + .collectFirst { case relation: DataSourceV2Relation => relation.options } + .getOrElse(fail(s"no Fluss relation resolved for $TVF($args)")) + FlussOffsetInitializers.incrementalTimeRange(options).get } - private def row(orderId: Long, itemId: Long, amount: Int, address: String): GenericRow = - GenericRow.of( - Long.box(orderId), - Long.box(itemId), - Int.box(amount), - BinaryString.fromString(address)) + /** Parses a `yyyy-MM-dd HH:mm:ss` string the way the scan options do. */ + private def parseTs(datetime: String): Long = + FlussOffsetInitializers.parseTimestamp( + datetime, + SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) - /** - * Raw changelog records of `table` whose commit timestamp falls inside [start, end), as - * (changeType, orderId) pairs in log order. Used to prove the claimed change types (-U/+U/-D/+I) - * really exist in the window, so the folded-output assertions below cannot pass vacuously. - */ - private def changelogInWindow(table: Table, start: Long, end: Long): Seq[(String, Long)] = { - val scanner = table.newScan().createLogScanner() - try { - if (table.getTableInfo.isPartitioned) { - admin.listPartitionInfos(table.getTableInfo.getTablePath).get().forEach { - pi => scanner.subscribeFromBeginning(pi.getPartitionId, 0) - } - } else { - scanner.subscribeFromBeginning(0) - } - val records = scala.collection.mutable.ArrayBuffer[(String, Long)]() - // Poll until records arrive and a poll comes back empty (all caught up), or the deadline. - // Mirrors FlussSparkTestBase.getRowsWithChangeType: the high watermark may advance in - // stages, so a single early empty poll must not end the scan. - val deadline = System.currentTimeMillis() + 10000 - var hasReceivedAny = false - var done = false - while (!done && System.currentTimeMillis() < deadline) { - val polled = scanner.poll(Duration.ofSeconds(1)) - if (!polled.isEmpty) { - hasReceivedAny = true - polled.forEach { - r => - if (r.timestamp() >= start && r.timestamp() < end) { - records += ((r.getChangeType.shortString(), r.getRow.getLong(0))) - } - } - } else if (hasReceivedAny) { - done = true - } - } - records.toSeq - } finally { - scanner.close() - } + /** The full stack trace of the failure raised by `query`, which must fail. */ + private def failureOf(query: String): String = { + val ex = intercept[Exception](sql(query).collect()) + val sw = new java.io.StringWriter() + ex.printStackTrace(new java.io.PrintWriter(sw)) + sw.toString } - private def pkRow( + private def row( orderId: Long, itemId: Long, amount: Int, @@ -723,9 +351,17 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { BinaryString.fromString(address), BinaryString.fromString(dt)) - private def deleteKey(orderId: Long): GenericRow = - GenericRow.of(Long.box(orderId), null, null, null) - private def deleteKey(orderId: Long, dt: String): GenericRow = GenericRow.of(Long.box(orderId), null, null, null, BinaryString.fromString(dt)) + + private def unpartitionedRow( + orderId: Long, + itemId: Long, + amount: Int, + address: String): GenericRow = + GenericRow.of( + Long.box(orderId), + Long.box(itemId), + Int.box(amount), + BinaryString.fromString(address)) } diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala index 3cee268c296..5886f60140a 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala @@ -65,70 +65,20 @@ class FlussOffsetInitializersTest extends AnyFunSuite { assertThat(ex.getMessage).contains(startKey) } - test("a window whose start is not strictly before its end fails fast") { + test("the time range carried to the reader mirrors the requested window") { val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() - // degenerate: start == end - intercept[IllegalArgumentException] { - FlussOffsetInitializers.requireValidWindow( - scanOptions(startKey -> "1767225600000", endKey -> "1767225600000")) - } - // reversed: start > end - val ex = intercept[IllegalArgumentException] { - FlussOffsetInitializers.requireValidWindow( - scanOptions(startKey -> "1767312000000", endKey -> "1767225600000")) - } - assertThat(ex.getMessage).contains("strictly before") - // 'latest' or absent end is always accepted - FlussOffsetInitializers.requireValidWindow(scanOptions(startKey -> "1767225600000")) - FlussOffsetInitializers.requireValidWindow( - scanOptions(startKey -> "1767225600000", endKey -> "latest")) - } - - test("scan.incremental.timestamp.out-of-range toggles fail-fast (default error)") { - val key = SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE.key() - // default (unset) is error -> fail fast - assertThat(FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions())).isTrue - assertThat( - FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "error"))).isTrue + // a plain batch read has no window to apply + assertThat(FlussOffsetInitializers.incrementalTimeRange(scanOptions()).isDefined).isFalse assertThat( - FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "ERROR"))).isTrue - // a blank value counts as unset and falls back to the default (error) - assertThat(FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> " "))).isTrue - // adjust -> clamp instead of failing + FlussOffsetInitializers + .incrementalTimeRange(scanOptions(startKey -> "1767225600000", endKey -> "1767312000000")) + .get) + .isEqualTo(FlussTimeRange(1767225600000L, 1767312000000L)) + // an unset end leaves the upper bound open assertThat( - FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "adjust"))).isFalse - } - - test("invalid scan.incremental.timestamp.out-of-range value fails with supported values") { - val key = SparkFlussConf.SCAN_INCREMENTAL_TIMESTAMP_OUT_OF_RANGE.key() - val ex = intercept[IllegalArgumentException] { - FlussOffsetInitializers.failOnTimestampOutOfRange(scanOptions(key -> "warn")) - } - assertThat(ex.getMessage).contains(key) - assertThat(ex.getMessage).contains("WARN") - assertThat(ex.getMessage).contains("'error', 'adjust'") - } - - test("retention guard decision (isBeforeRetention)") { - // brand-new bucket (earliest == 0) is never flagged, even for a very old start offset - assertThat(FlussOffsetInitializers.isBeforeRetention(0L, 0L)).isFalse - assertThat(FlussOffsetInitializers.isBeforeRetention(5L, 0L)).isFalse - // a trimmed bucket (earliest > 0) is flagged when the start lands at or before earliest - assertThat(FlussOffsetInitializers.isBeforeRetention(10L, 10L)).isTrue - assertThat(FlussOffsetInitializers.isBeforeRetention(3L, 10L)).isTrue - // a start strictly after earliest is within retention - assertThat(FlussOffsetInitializers.isBeforeRetention(11L, 10L)).isFalse - } - - test("TTL-exceeded start fails fast with a table.log.ttl hint") { - // start at/before a trimmed earliest (earliest > 0): fail fast with a clear TTL message - val ex = intercept[IllegalArgumentException] { - FlussOffsetInitializers.requireStartWithinRetention("fluss.t", "dt=2026", 2, 5L, 10L) - } - assertThat(ex.getMessage).contains("table.log.ttl") - assertThat(ex.getMessage).contains("bucket 2") - assertThat(ex.getMessage).contains("partition 'dt=2026'") + FlussOffsetInitializers.incrementalTimeRange(scanOptions(startKey -> "1767225600000")).get) + .isEqualTo(FlussTimeRange(1767225600000L, Long.MaxValue)) } test("invalid start timestamp format fails with the option name") { diff --git a/website/docs/engine-spark/options.md b/website/docs/engine-spark/options.md index 43d4929f367..c9d9a4614a5 100644 --- a/website/docs/engine-spark/options.md +++ b/website/docs/engine-spark/options.md @@ -24,6 +24,7 @@ The following options configure a single read and are **not** read from session | Option | Default | Description | |--------|---------|-------------| -| `scan.incremental.start.timestamp` | (none) | Enables an incremental (time-range) batch read and sets the **inclusive** lower bound of the window. Accepts epoch milliseconds (e.g. `1678883047356`) or a `yyyy-MM-dd HH:mm:ss` datetime (e.g. `2023-12-09 23:09:12`) interpreted in the Spark session time zone (`spark.sql.session.timeZone`). A blank or unparseable value fails fast instead of falling back to a full-table read. Batch read only; it has no effect on streaming reads. If the timestamp predates the data still retained by Fluss (bounded by `table.log.ttl`), behavior is controlled by `scan.incremental.timestamp.out-of-range`. | -| `scan.incremental.end.timestamp` | `latest` | The **exclusive** upper bound of an incremental batch read, producing a left-closed right-open `[start, end)` window. `latest` (default) stops at the latest committed data captured at planning time; otherwise the same value format as `scan.incremental.start.timestamp`. Setting it without `scan.incremental.start.timestamp` fails fast, as does a window whose start is not strictly before its end. A timestamp in the future is rejected by the server (`InvalidTimestampException`). | -| `scan.incremental.timestamp.out-of-range` | `error` | Behavior when `scan.incremental.start.timestamp` precedes the earliest data still retained by Fluss (bounded by `table.log.ttl`).
  • `error` (default): fail fast so a truncated window is never returned silently.
  • `adjust`: clamp the start to the earliest retained offset and read from there.
| +| `scan.incremental.start.timestamp` | (none) | Enables an incremental (time-range) batch read and sets the **inclusive** lower bound of the window. Accepts epoch milliseconds (e.g. `1678883047356`) or a `yyyy-MM-dd HH:mm:ss` datetime (e.g. `2023-12-09 23:09:12`) interpreted in the Spark session time zone (`spark.sql.session.timeZone`). A blank or unparseable value fails fast instead of falling back to a full-table read. Batch read only; it has no effect on streaming reads. Only the data Fluss still retains (bounded by `table.log.ttl`) is returned, so a timestamp predating it simply yields fewer rows. | +| `scan.incremental.end.timestamp` | (none) | The **exclusive** upper bound of an incremental batch read, producing a left-closed right-open `[start, end)` window. Same value format as `scan.incremental.start.timestamp`; when unset the read runs up to the latest committed data. The table-valued function always writes a concrete timestamp, pinning the bound when the statement is analyzed. Setting it without `scan.incremental.start.timestamp` fails fast, as does a window whose start is not strictly before its end. | + +Both bounds are compared against the record commit timestamp while reading, so the window is exact even for data already tiered to remote storage, where resolving a timestamp to a log offset is only as accurate as the server-side time index. diff --git a/website/docs/engine-spark/reads.md b/website/docs/engine-spark/reads.md index f797f0304ac..38b4a09122e 100644 --- a/website/docs/engine-spark/reads.md +++ b/website/docs/engine-spark/reads.md @@ -270,7 +270,7 @@ The timestamp value is either epoch milliseconds or a `yyyy-MM-dd HH:mm:ss` date ### Using the table-valued function (recommended for SQL) -`fluss_incremental_between_timestamp(table, start[, end])` reads a time window in a single statement. Omit `end` to read up to the latest committed data at planning time. +`fluss_incremental_between_timestamp(table, start[, end])` reads a time window in a single statement. Omit `end` to read up to the moment the statement is analyzed. ```sql title="Spark SQL" -- Read the past hour on a log table (epoch-millis form) @@ -287,7 +287,7 @@ ORDER BY order_id; ``` ```sql title="Spark SQL" --- Omit the end to read from a start timestamp up to the latest data +-- Omit the end to read from a start timestamp up to the analysis time SELECT * FROM fluss_incremental_between_timestamp('log_table', '2026-01-01 00:00:00'); ``` @@ -307,7 +307,7 @@ SELECT * FROM fluss_incremental_between_timestamp( date_format(now(), 'yyyy-MM-dd HH:mm:ss')); ``` -The table argument is a string and accepts `table`, `database.table` or `catalog.database.table`; unqualified names resolve against the current catalog and database. The start/end arguments accept a string (epoch milliseconds or `yyyy-MM-dd HH:mm:ss`), an integral epoch-milliseconds value, or a `TIMESTAMP`/`TIMESTAMP_NTZ` literal (interpreted in the Spark session time zone, same as the `yyyy-MM-dd HH:mm:ss` string form), and may be produced by constant expressions such as the datetime functions above (column references are not allowed). The result is an ordinary relation, so projection, filters and joins work as usual. +The table argument is a string and accepts `table`, `database.table` or `catalog.database.table`; unqualified names resolve against the current catalog and database. The start/end arguments accept a string (epoch milliseconds or `yyyy-MM-dd HH:mm:ss`), an integral epoch-milliseconds value, a `DATE` (the start of that day), or a `TIMESTAMP`/`TIMESTAMP_NTZ` literal — all interpreted in the Spark session time zone — and may be produced by constant expressions such as the datetime functions above (column references are not allowed). The result is an ordinary relation, so projection, filters and joins work as usual. :::note The function is provided by the Fluss Spark session extension, so `spark.sql.extensions=org.apache.fluss.spark.FlussSparkSessionExtensions` must be configured (see [Getting Started](getting-started.md)). Its options apply to that single query only. @@ -317,7 +317,7 @@ Fluss uses `[start, end)` (start inclusive, end exclusive). This differs from Pa ### Using the DataFrame API -The same window can be expressed with scan options, which is how it is configured from the DataFrame API. Setting `scan.incremental.start.timestamp` is what turns a batch read into an incremental one; `scan.incremental.end.timestamp` defaults to the reserved value `latest`: +The same window can be expressed with scan options, which is how it is configured from the DataFrame API. Setting `scan.incremental.start.timestamp` is what turns a batch read into an incremental one; `scan.incremental.end.timestamp` is optional and the read runs up to the latest committed data when it is left unset: ```scala title="Spark Scala" spark.read @@ -331,11 +331,13 @@ The `scan.incremental.*` options are per-query read options only. Unlike the opt ::: :::warning Retention boundary -A time-range read only sees data still retained by Fluss, which is bounded by `table.log.ttl` (default 7 days). If the start timestamp predates the earliest retained data, the default behavior (`scan.incremental.timestamp.out-of-range=error`) **fails fast** with a clear error instead of silently returning a truncated window — narrow the time range or increase `table.log.ttl`. Set `scan.incremental.timestamp.out-of-range=adjust` to instead clamp the start to the earliest retained data and read from there. An end timestamp in the future is rejected by the server. Reading data older than the Fluss retention (including from tiered lake storage) is not supported by this mode. +A time-range read returns the data Fluss still retains, which is bounded by `table.log.ttl` (default 7 days). A window reaching further back is **not** an error: the part that has already been dropped simply yields fewer rows, or none at all, so the result is always a subset of the requested window. Increase `table.log.ttl` if you need to read further back. Data available only in tiered lake storage is not read by this mode. + +The start timestamp positions the scan through the server's timestamp-to-offset lookup, and both bounds are then applied on each record's commit timestamp, so the window stays exact even for data already tiered to remote storage. ::: :::note Invalid windows fail fast -Malformed window specifications are rejected at planning time instead of silently changing semantics: a blank or unparseable start timestamp, an end timestamp set without a start timestamp (it cannot truncate a plain batch read on its own), and a window whose start is not strictly before its end. A bucket that simply has no data inside a valid window still yields an empty result. +Malformed window specifications are rejected instead of silently changing semantics: a blank or unparseable start timestamp, and an end timestamp set without a start timestamp (it cannot truncate a plain batch read on its own). The table-valued function additionally rejects a window whose start is not strictly before its end while the statement is analyzed; the same window given through DataFrame options simply returns no rows. A bucket that has no data inside a valid window also yields an empty result. ::: ## All Data Types From 2883e7dc3610e9bad4e9fcba1392d0f829a3a054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Thu, 13 Aug 2026 11:58:19 +0800 Subject: [PATCH 07/10] [spark] Tolerate an incremental window that reads no log records An incremental read scans up to the latest offset and applies its end bound on the record commit timestamp while reading, so a non-empty offset range can still leave zero records behind: this happens whenever the window itself is empty but data was committed after it, because the start timestamp then resolves to a record past the end bound. The upsert reader handed that empty batch to LogChangesIterator, which seeds its cursor from the first record and threw NoSuchElementException. Return an empty iterator instead, and correct the planner comment that claimed its start >= stop guard already ruled this out. Cover it in the TVF suite, which now asserts several adjacent windows per table, empty leading, gap and trailing windows, a delete-only window that folds to nothing, a primary key table spread over three buckets, and an explicitly triggered kv snapshot that a window must never serve. Co-Authored-By: Qoder AI-Model: Qoder Auto AI-Contributed/Feature: 8/16 AI-Contributed/UT: 195/195 --- .../read/FlussUpsertPartitionReader.scala | 11 +- .../fluss/spark/read/SplitPlanner.scala | 5 +- .../fluss/spark/SparkTimeRangeTvfTest.scala | 195 ++++++++++++++++-- 3 files changed, 186 insertions(+), 25 deletions(-) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala index 16ce95b3d92..af55cca3e26 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala @@ -123,7 +123,7 @@ class FlussUpsertPartitionReader( } } - def createLogChangesIterator(): LogChangesIterator = { + def createLogChangesIterator(): CloseableIterator[KeyValueRow] = { // Initialize the log scanner logScanner = table.newScan().project(projectionWithPks).createLogScanner() if (tableBucket.getPartitionId == null) { @@ -160,7 +160,14 @@ class FlussUpsertPartitionReader( } } - LogChangesIterator(allLogRecords.toArray, pkProjection, comparator) + // An incremental read scans up to the latest offset and applies its end bound above, so a + // non-empty offset range can still leave nothing behind. LogChangesIterator seeds its cursor + // from the first record, so it must not be handed an empty batch. + if (allLogRecords.isEmpty) { + CloseableIterator.emptyIterator[KeyValueRow]() + } else { + LogChangesIterator(allLogRecords.toArray, pkProjection, comparator) + } } def createSnapshotIterator(): CloseableIterator[LogRecord] = { diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala index d552cf45e74..1a9b3ade0d4 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala @@ -773,8 +773,9 @@ class UpsertPlanner( val startOffset = Long2long(startBucketOffsets.get(Integer.valueOf(bucketId))) val stopOffset = Long2long(stoppingBucketOffsets.get(Integer.valueOf(bucketId))) if (startOffset >= stopOffset) { - // Empty range (e.g. a time-range window with no data, or an empty bucket): emit no - // partition so the upsert reader is not handed an invalid [start, start) range. + // The start timestamp resolved past the end of the log, so there is nothing to read. + // A window that is empty only in time still yields start < stop, because the stop offset + // is the latest offset and the end bound is applied by the reader. None } else { Some( diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala index 6ebc1114b51..9011434832e 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkTimeRangeTvfTest.scala @@ -17,6 +17,7 @@ package org.apache.fluss.spark +import org.apache.fluss.config.Configuration import org.apache.fluss.row.{BinaryString, GenericRow} import org.apache.fluss.spark.read.{FlussOffsetInitializers, FlussTimeRange} @@ -31,6 +32,9 @@ import org.assertj.core.api.Assertions.assertThat */ class SparkTimeRangeTvfTest extends FlussSparkTestBase { + /** Snapshots are triggered explicitly, so no window can straddle an automatic one. */ + override protected def flussConf: Configuration = new Configuration() + private val TVF = "fluss_incremental_between_timestamp" private val P1 = "2026-01-01" @@ -40,18 +44,35 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { withTable("t_log") { createLogTable("t_log") + val t0 = boundary() insert("t_log", s"""(1L, 11L, 101, "a1", "$P1"), (2L, 12L, 102, "a2", "$P2")""") val t1 = boundary() insert("t_log", s"""(3L, 13L, 103, "a3", "$P1"), (4L, 14L, 104, "a4", "$P2")""") val t2 = boundary() insert("t_log", s"""(5L, 15L, 105, "a5", "$P1")""") val t3 = boundary() + // nothing is committed in [t3, t4), but rows follow it val t4 = boundary() - - // the window spans every partition + insert("t_log", s"""(6L, 16L, 106, "a6", "$P2"), (7L, 17L, 107, "a7", "$P1")""") + val t5 = boundary() + val t6 = boundary() + + val batch1 = Row(1L, 11L, 101, "a1", P1) :: Row(2L, 12L, 102, "a2", P2) :: Nil + val batch2 = Row(3L, 13L, 103, "a3", P1) :: Row(4L, 14L, 104, "a4", P2) :: Nil + val batch3 = Row(5L, 15L, 105, "a5", P1) :: Nil + val batch4 = Row(6L, 16L, 106, "a6", P2) :: Row(7L, 17L, 107, "a7", P1) :: Nil + + // consecutive windows partition the table exactly, each one spanning every partition + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t0, t1)} ORDER BY orderId"), batch1) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t1, t2)} ORDER BY orderId"), batch2) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t2, t3)} ORDER BY orderId"), batch3) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t4, t5)} ORDER BY orderId"), batch4) + + // a window may span several writes, and one covering everything matches a plain batch read + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t1, t4)} ORDER BY orderId"), batch2 ::: batch3) checkAnswer( - sql(s"SELECT * FROM ${tvf("t_log", t1, t2)} ORDER BY orderId"), - Row(3L, 13L, 103, "a3", P1) :: Row(4L, 14L, 104, "a4", P2) :: Nil) + sql(s"SELECT * FROM ${tvf("t_log", t0, t6)} ORDER BY orderId"), + batch1 ::: batch2 ::: batch3 ::: batch4) // projection, filter and partition pruning still work on top of the TVF relation checkAnswer( @@ -62,25 +83,25 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { Row(3L) :: Nil) // without an end timestamp the window runs up to the latest data - checkAnswer( - sql(s"SELECT * FROM ${tvf("t_log", t2)} ORDER BY orderId"), - Row(5L, 15L, 105, "a5", P1) :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t2)} ORDER BY orderId"), batch3 ::: batch4) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t4)} ORDER BY orderId"), batch4) - // a window without writes yields nothing + // a window without writes yields nothing, whether it precedes, splits or trails the data checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t3, t4)}"), Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_log", t5, t6)}"), Nil) // the table argument may be unqualified or fully qualified - checkAnswer(sql(s"SELECT * FROM $TVF('t_log', '$t2')"), Row(5L, 15L, 105, "a5", P1) :: Nil) + checkAnswer(sql(s"SELECT * FROM $TVF('t_log', '$t4')"), batch4) checkAnswer( - sql(s"SELECT * FROM $TVF('$DEFAULT_CATALOG.$DEFAULT_DATABASE.t_log', '$t2')"), - Row(5L, 15L, 105, "a5", P1) :: Nil) + sql(s"SELECT * FROM $TVF('$DEFAULT_CATALOG.$DEFAULT_DATABASE.t_log', '$t4')"), + batch4) // the omitted end bound is pinned when the statement is analyzed, so a row committed before // the scan is planned stays outside the window - val pinned = sql(s"SELECT * FROM ${tvf("t_log", t2)} ORDER BY orderId") - insert("t_log", s"""(6L, 16L, 106, "a6", "$P1")""") + val pinned = sql(s"SELECT * FROM ${tvf("t_log", t4)} ORDER BY orderId") + insert("t_log", s"""(8L, 18L, 108, "a8", "$P1")""") Thread.sleep(200) - checkAnswer(pinned, Row(5L, 15L, 105, "a5", P1) :: Nil) + checkAnswer(pinned, batch4) } } @@ -112,8 +133,19 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { writer.upsert(row(1L, 112L, 1004, "a1_v4", P1)).get() writer.flush() val t3 = boundary() + // nothing is committed in [t3, t4), but changes follow it: the start offset then resolves to + // a record past the end bound instead of to the end of the log val t4 = boundary() + writer.delete(deleteKey(2L, P2)).get() + writer.flush() + val t5 = boundary() + + writer.upsert(row(6L, 16L, 106, "a6", P1)).get() + writer.flush() + val t6 = boundary() + val t7 = boundary() + // each changed key appears once with its last in-window value; deleted keys and an insert // cancelled by a delete are excluded checkAnswer( @@ -123,18 +155,110 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { Row(3L, 130L, 1003, "a3_new", P1) :: Nil ) + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_fold", t2, t3)} ORDER BY orderId"), + Row(1L, 112L, 1004, "a1_v4", P1) :: Nil) + + // a window whose only change is a delete folds to nothing, even though it does read records + checkAnswer(sql(s"SELECT * FROM ${tvf("t_fold", t4, t5)}"), Nil) + + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_fold", t5, t6)} ORDER BY orderId"), + Row(6L, 16L, 106, "a6", P1) :: Nil) + + // spanning several batches: key 1 keeps its last value, key 2 is dropped by the later delete + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_fold", t2, t6)} ORDER BY orderId"), + Row(1L, 112L, 1004, "a1_v4", P1) :: Row(6L, 16L, 106, "a6", P1) :: Nil) + checkAnswer( sql(s"SELECT * FROM ${tvf("t_fold", t1)} ORDER BY orderId"), Row(1L, 112L, 1004, "a1_v4", P1) :: - Row(2L, 120L, 1002, "a2_upd", P2) :: - Row(3L, 130L, 1003, "a3_new", P1) :: Nil + Row(3L, 130L, 1003, "a3_new", P1) :: + Row(6L, 16L, 106, "a6", P1) :: Nil ) checkAnswer( sql(s"SELECT orderId FROM ${tvf("t_fold", t1, t2)} WHERE dt = '$P1' ORDER BY orderId"), Row(1L) :: Row(3L) :: Nil) + // windows without any change yield nothing, whether they split or trail the changelog checkAnswer(sql(s"SELECT * FROM ${tvf("t_fold", t3, t4)}"), Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_fold", t6, t7)}"), Nil) + } + } + + test("TVF: primary key table with several buckets") { + withTable("t_multi") { + createPkTable("t_multi", buckets = 3) + + val writer = loadFlussTable(createTablePath("t_multi")).newUpsert().createWriter() + (1 to 6).foreach { + k => + val dt = if (k % 2 == 0) P2 else P1 + writer.upsert(row(k.toLong, (10 + k).toLong, 100 + k, s"a$k", dt)).get() + } + writer.flush() + val t1 = boundary() + + // the window touches only some of the six bucket/partition pairs, so the rest are planned + // and read down to nothing + writer.upsert(row(2L, 120L, 1002, "a2_upd", P2)).get() + writer.delete(deleteKey(5L, P1)).get() + writer.upsert(row(7L, 17L, 107, "a7", P1)).get() + writer.flush() + val t2 = boundary() + + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_multi", t1, t2)} ORDER BY orderId"), + Row(2L, 120L, 1002, "a2_upd", P2) :: Row(7L, 17L, 107, "a7", P1) :: Nil) + } + } + + test("TVF: primary key table never serves the kv snapshot") { + withTable("t_snap") { + createPkTable("t_snap") + val tablePath = createTablePath("t_snap") + val writer = loadFlussTable(tablePath).newUpsert().createWriter() + + val t0 = boundary() + writer.upsert(row(1L, 11L, 101, "a1", P1)).get() + writer.upsert(row(2L, 12L, 102, "a2", P2)).get() + writer.flush() + // key 1 and key 2 now live in a kv snapshot, so any read that consults it sees them + flussServer.triggerAndWaitSnapshot(tablePath) + val t1 = boundary() + + writer.upsert(row(1L, 110L, 1001, "a1_upd", P1)).get() + writer.upsert(row(3L, 13L, 103, "a3", P1)).get() + writer.flush() + val t2 = boundary() + // the second batch is snapshotted as well, so the window's own rows are in the snapshot too + flussServer.triggerAndWaitSnapshot(tablePath) + val t3 = boundary() + + // a read-optimized full read serves the snapshot alone, which is how we know it is there + withSQLConf(sessionKey(SparkFlussConf.READ_OPTIMIZED_OPTION.key()) -> "true") { + checkAnswer( + sql(s"SELECT * FROM $DEFAULT_DATABASE.t_snap ORDER BY orderId"), + Row(1L, 110L, 1001, "a1_upd", P1) :: + Row(2L, 12L, 102, "a2", P2) :: + Row(3L, 13L, 103, "a3", P1) :: Nil + ) + } + + // key 2 changed before the window and only survives in the snapshot, so it must not appear + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_snap", t1, t2)} ORDER BY orderId"), + Row(1L, 110L, 1001, "a1_upd", P1) :: Row(3L, 13L, 103, "a3", P1) :: Nil) + + // an earlier window still folds from the changelog, at the values it held back then + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_snap", t0, t1)} ORDER BY orderId"), + Row(1L, 11L, 101, "a1", P1) :: Row(2L, 12L, 102, "a2", P2) :: Nil) + + // a window with no changes stays empty instead of falling back to the whole snapshot + checkAnswer(sql(s"SELECT * FROM ${tvf("t_snap", t2, t3)}"), Nil) } } @@ -151,6 +275,7 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { |""".stripMargin) val writer = loadFlussTable(createTablePath("t_np_pk")).newUpsert().createWriter() + val t0 = boundary() insert("t_np", """(1L, 11L, 101, "a1")""") writer.upsert(unpartitionedRow(1L, 11L, 101, "a1")).get() writer.flush() @@ -160,12 +285,35 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { writer.upsert(unpartitionedRow(1L, 110L, 1001, "a1_upd")).get() writer.upsert(unpartitionedRow(2L, 12L, 102, "a2")).get() writer.flush() - Thread.sleep(200) + val t2 = boundary() + + // nothing is committed in [t2, t3), but writes follow it + val t3 = boundary() + + insert("t_np", """(3L, 13L, 103, "a3")""") + writer.delete(unpartitionedDeleteKey(2L)).get() + writer.flush() + val t4 = boundary() + + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t0, t1)}"), Row(1L, 11L, 101, "a1") :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np_pk", t0, t1)}"), Row(1L, 11L, 101, "a1") :: Nil) - checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t1)}"), Row(2L, 12L, 102, "a2") :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t1, t2)}"), Row(2L, 12L, 102, "a2") :: Nil) checkAnswer( - sql(s"SELECT * FROM ${tvf("t_np_pk", t1)} ORDER BY orderId"), + sql(s"SELECT * FROM ${tvf("t_np_pk", t1, t2)} ORDER BY orderId"), Row(1L, 110L, 1001, "a1_upd") :: Row(2L, 12L, 102, "a2") :: Nil) + + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t2, t3)}"), Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np_pk", t2, t3)}"), Nil) + + // the pk window only deletes, so it folds to nothing while the log table keeps its append + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np", t3, t4)}"), Row(3L, 13L, 103, "a3") :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np_pk", t3, t4)}"), Nil) + + checkAnswer( + sql(s"SELECT * FROM ${tvf("t_np", t1)} ORDER BY orderId"), + Row(2L, 12L, 102, "a2") :: Row(3L, 13L, 103, "a3") :: Nil) + checkAnswer(sql(s"SELECT * FROM ${tvf("t_np_pk", t1)}"), Row(1L, 110L, 1001, "a1_upd") :: Nil) } } @@ -231,6 +379,8 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { assertThat(failureOf(s"SELECT * FROM ${tvf("t_bad", t1 + 1000, t1)}")) .contains("strictly before") + assertThat(failureOf(s"SELECT * FROM ${tvf("t_bad", t1, t1)}")) + .contains("strictly before") assertThat(failureOf(s"SELECT * FROM $TVF('$DEFAULT_DATABASE.t_bad')")) .contains("endTimestamp") @@ -286,12 +436,12 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { |PARTITIONED BY (dt) |""".stripMargin) - private def createPkTable(name: String): Unit = + private def createPkTable(name: String, buckets: Int = 1): Unit = sql(s""" |CREATE TABLE $DEFAULT_DATABASE.$name |(orderId BIGINT, itemId BIGINT, amount INT, address STRING, dt STRING) |PARTITIONED BY (dt) - |TBLPROPERTIES("primary.key" = "orderId,dt", "bucket.num" = 1) + |TBLPROPERTIES("primary.key" = "orderId,dt", "bucket.num" = $buckets) |""".stripMargin) private def insert(table: String, values: String): Unit = @@ -364,4 +514,7 @@ class SparkTimeRangeTvfTest extends FlussSparkTestBase { Long.box(itemId), Int.box(amount), BinaryString.fromString(address)) + + private def unpartitionedDeleteKey(orderId: Long): GenericRow = + GenericRow.of(Long.box(orderId), null, null, null) } From 995485d1bc39d7a2355f11d8ec40f0ddd4f53c44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Thu, 13 Aug 2026 14:16:31 +0800 Subject: [PATCH 08/10] [spark] Name the log-only sentinel snapshot id in incremental planning The incremental branch marked its bucket splits as log-only by passing a bare -1 as the snapshot id, and the lake time-range test compared against the same magic number. Reuse TableBucketSnapshot.NO_SNAPSHOT_ID on both sides so the check reads the same way as every other snapshot guard in the codebase, and update the block comment along with it. Co-Authored-By: Qoder AI-Model: Qoder Auto AI-Contributed/Feature: 7/7 AI-Contributed/UT: 5/5 --- .../scala/org/apache/fluss/spark/read/SplitPlanner.scala | 9 +++++++-- .../fluss/spark/lake/SparkLakeTimeRangeReadTest.scala | 9 +++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala index 1a9b3ade0d4..c871ef5ec24 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala @@ -715,7 +715,7 @@ class UpsertPlanner( // --------------------------------------------------------------------------------------------- // Incremental branch: fold the Fluss changelog within [start, end) per bucket, with no kv - // snapshot and no lake. Emitting snapshotId = -1 makes FlussUpsertPartitionReader skip the + // snapshot and no lake. Emitting NO_SNAPSHOT_ID makes FlussUpsertPartitionReader skip the // snapshot and fold only the log range; SortMergeReader drops delete rows, so the output is the // surviving +I/+U rows (keys inserted or updated in the window; deleted keys excluded). // --------------------------------------------------------------------------------------------- @@ -779,7 +779,12 @@ class UpsertPlanner( None } else { Some( - FlussUpsertInputPartition(tableBucket, -1L, startOffset, stopOffset, timeRange) + FlussUpsertInputPartition( + tableBucket, + TableBucketSnapshot.NO_SNAPSHOT_ID, + startOffset, + stopOffset, + timeRange) .asInstanceOf[InputPartition]) } }.toArray diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala index 84d9ba9e9eb..78dbf58a097 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/lake/SparkLakeTimeRangeReadTest.scala @@ -18,7 +18,7 @@ package org.apache.fluss.spark.lake import org.apache.fluss.config.{ConfigOptions, Configuration} -import org.apache.fluss.metadata.DataLakeFormat +import org.apache.fluss.metadata.{DataLakeFormat, TableBucketSnapshot} import org.apache.fluss.spark.SparkConnectorOptions.{BUCKET_NUMBER, PRIMARY_KEY} import org.apache.fluss.spark.read.{FlussAppendInputPartition, FlussUpsertInputPartition} @@ -114,11 +114,12 @@ abstract class SparkLakeTimeRangeReadTest extends SparkLakeTableReadTestBase { assert(partitions.nonEmpty, "expected at least one Fluss changelog partition") assert( partitions.forall { - case p: FlussUpsertInputPartition => p.snapshotId == -1 + case p: FlussUpsertInputPartition => + p.snapshotId == TableBucketSnapshot.NO_SNAPSHOT_ID case _ => false }, - s"time-range read must be log-only with no kv/lake snapshot (snapshotId == -1), " + - s"got: ${partitions.mkString(", ")}" + s"time-range read must be log-only with no kv/lake snapshot " + + s"(snapshotId == NO_SNAPSHOT_ID), got: ${partitions.mkString(", ")}" ) // Only keys inserted/updated within [t1, t2): id=2 (updated), id=4 (inserted). checkAnswer(df, Row(2, "bob_updated", 100) :: Row(4, "david", 88) :: Nil) From c7a758f5af96ae38679586caadaf2d40ccdcf817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Fri, 14 Aug 2026 12:07:08 +0800 Subject: [PATCH 09/10] [spark] Fail fast on reversed incremental windows from scan options A window whose start timestamp is not strictly before its end was only rejected on the table-valued function path; the same window given through scan.incremental.* options slipped through planning and silently yielded no rows (or tripped the reader's Invalid offset range guard). Move the window check into FlussOffsetInitializers.incrementalTimeRange, which both entry points pass through, so an invalid specification fails fast with the same message regardless of how it was expressed; the TVF keeps its analysis-time check by delegating to the shared validation. Co-Authored-By: Qoder AI-Model: Qoder Auto AI-Contributed/Feature: 42/42 AI-Contributed/UT: 13/13 --- .../logical/FlussTableValuedFunctions.scala | 7 +--- .../spark/read/FlussOffsetInitializers.scala | 33 +++++++++++++++---- .../read/FlussOffsetInitializersTest.scala | 13 ++++++++ website/docs/engine-spark/reads.md | 2 +- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala index 01e2a5886b1..c8001ef6a26 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/catalyst/plans/logical/FlussTableValuedFunctions.scala @@ -253,11 +253,6 @@ case class IncrementalBetweenTimestamp(override val args: Seq[Expression]) val endMs = FlussOffsetInitializers.parseTimestamp( end, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key()) - if (startMs >= endMs) { - throw new IllegalArgumentException( - s"Invalid time range for $INCREMENTAL_BETWEEN_TIMESTAMP: the start timestamp '$start' " + - s"must be strictly before the end timestamp '$end'. The window is left-closed " + - s"right-open '[start, end)'.") - } + FlussOffsetInitializers.requireValidWindow(start, end, startMs, endMs) } } diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala index 7f4d9bacfe2..4f3eae36dcc 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala @@ -59,9 +59,11 @@ object FlussOffsetInitializers { * Start offsets of an incremental batch read, resolved from `scan.incremental.start.timestamp`. * Requires that option to be set. */ - def incrementalStartOffsetsInitializer(options: CaseInsensitiveStringMap): OffsetsInitializer = + def incrementalStartOffsetsInitializer(options: CaseInsensitiveStringMap): OffsetsInitializer = { + val start = requiredTimestamp(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP) OffsetsInitializer.timestamp( - requiredTimestamp(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP)) + parseTimestamp(start, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key())) + } /** * Start offsets of a streaming read, driven by `scan.startup.mode`. Batch reads ignore this @@ -112,13 +114,30 @@ object FlussOffsetInitializers { if (!isIncrementalRead(options)) { return None } - val startMs = requiredTimestamp(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP) - val endMs = incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP) - .map(end => parseTimestamp(end.trim, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key())) + val start = requiredTimestamp(options, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP) + val startMs = parseTimestamp(start, SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key()) + val end = incrementalOption(options, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP) + .map(_.trim) + val endMs = end + .map(parseTimestamp(_, SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key())) .getOrElse(Long.MaxValue) + end.foreach(requireValidWindow(start, _, startMs, endMs)) Some(FlussTimeRange(startMs, endMs)) } + /** Rejects a reversed or degenerate window instead of silently reading nothing. */ + private[spark] def requireValidWindow( + start: String, + end: String, + startMs: Long, + endMs: Long): Unit = { + if (startMs >= endMs) { + throw new IllegalArgumentException( + s"Invalid incremental time range: the start timestamp '$start' must be strictly before " + + s"the end timestamp '$end'. The window is left-closed right-open '[start, end)'.") + } + } + /** * Reads a `scan.incremental.*` option from the scan options, falling back to its default. A blank * value counts as unset; a blank start timestamp is rejected by [[isIncrementalRead]] before it @@ -138,14 +157,14 @@ object FlussOffsetInitializers { private def requiredTimestamp( options: CaseInsensitiveStringMap, - option: ConfigOption[String]): Long = { + option: ConfigOption[String]): String = { val value = incrementalOption(options, option) if (value.getOrElse("").isEmpty) { throw new IllegalArgumentException( s"'${option.key()}' must not be empty. Provide epoch milliseconds or a " + s"'yyyy-MM-dd HH:mm:ss' timestamp.") } - parseTimestamp(value.get.trim, option.key()) + value.get.trim } /** diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala index 5886f60140a..37846e666ee 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala @@ -81,6 +81,19 @@ class FlussOffsetInitializersTest extends AnyFunSuite { .isEqualTo(FlussTimeRange(1767225600000L, Long.MaxValue)) } + test("a window whose start is not strictly before its end fails fast") { + val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() + val endKey = SparkFlussConf.SCAN_INCREMENTAL_END_TIMESTAMP.key() + for (end <- Seq("1767225600000", "1767139200000")) { + val ex = intercept[IllegalArgumentException] { + FlussOffsetInitializers.incrementalTimeRange( + scanOptions(startKey -> "1767225600000", endKey -> end)) + } + assertThat(ex.getMessage).contains("must be strictly before") + assertThat(ex.getMessage).contains("1767225600000") + } + } + test("invalid start timestamp format fails with the option name") { val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() val ex = intercept[IllegalArgumentException] { diff --git a/website/docs/engine-spark/reads.md b/website/docs/engine-spark/reads.md index 38b4a09122e..2df7be86157 100644 --- a/website/docs/engine-spark/reads.md +++ b/website/docs/engine-spark/reads.md @@ -337,7 +337,7 @@ The start timestamp positions the scan through the server's timestamp-to-offset ::: :::note Invalid windows fail fast -Malformed window specifications are rejected instead of silently changing semantics: a blank or unparseable start timestamp, and an end timestamp set without a start timestamp (it cannot truncate a plain batch read on its own). The table-valued function additionally rejects a window whose start is not strictly before its end while the statement is analyzed; the same window given through DataFrame options simply returns no rows. A bucket that has no data inside a valid window also yields an empty result. +Malformed window specifications are rejected instead of silently changing semantics: a blank or unparseable start timestamp, an end timestamp set without a start timestamp (it cannot truncate a plain batch read on its own), and a window whose start is not strictly before its end — whether given through the table-valued function (rejected while the statement is analyzed) or through DataFrame options. A bucket that has no data inside a valid window yields an empty result. ::: ## All Data Types From 763b412f6eb4a0ce9e000d5cae49e4b99eea8b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BE=8A=E5=B7=9D?= Date: Fri, 14 Aug 2026 12:21:49 +0800 Subject: [PATCH 10/10] [spark] Warn when an incremental window predates log retention An incremental read whose start timestamp is older than table.log.ttl can only return a subset of the requested window, but nothing surfaced that the window reached past retention. Log a WARN on the driver while planning, on both the append and upsert paths, comparing the window start against now minus the table's current table.log.ttl. Stays a warning rather than an error: TTL is a lower bound because expired segments are deleted lazily, so the data may still be there, and a partial window is the documented behavior of this read mode. The check lives in a pure predatesRetention helper so it is testable with a fixed clock. Co-Authored-By: Qoder AI-Model: Qoder Auto AI-Contributed/Feature: 38/38 AI-Contributed/UT: 13/13 --- .../spark/read/FlussOffsetInitializers.scala | 30 +++++++++++++++++-- .../fluss/spark/read/SplitPlanner.scala | 6 ++++ .../read/FlussOffsetInitializersTest.scala | 13 ++++++++ website/docs/engine-spark/reads.md | 2 ++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala index 4f3eae36dcc..22ae841b531 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussOffsetInitializers.scala @@ -19,15 +19,17 @@ package org.apache.fluss.spark.read import org.apache.fluss.client.initializer.{NoStoppingOffsetsInitializer, OffsetsInitializer} import org.apache.fluss.config.{ConfigOption, Configuration} +import org.apache.fluss.metadata.TablePath import org.apache.fluss.spark.SparkFlussConf +import org.apache.spark.internal.Logging import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.util.CaseInsensitiveStringMap -import java.time.{LocalDateTime, ZoneId} +import java.time.{Duration, LocalDateTime, ZoneId} import java.time.format.DateTimeFormatter -object FlussOffsetInitializers { +object FlussOffsetInitializers extends Logging { private val DATE_TIME_FORMATTER: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") @@ -138,6 +140,30 @@ object FlussOffsetInitializers { } } + /** + * Warns when the requested window starts before what the table is still guaranteed to retain: the + * log below `table.log.ttl` has been dropped, so the read can only return a subset of the window. + * + * Stays a warning rather than an error: TTL is a lower bound — expired segments are deleted + * lazily — so the data may in fact still be there, and a partial window is the documented + * behavior of this read mode. + */ + def warnIfWindowPredatesRetention( + tablePath: TablePath, + range: FlussTimeRange, + logTtlMs: Long): Unit = { + if (predatesRetention(range.startMs, logTtlMs, System.currentTimeMillis())) { + logWarning( + s"Incremental read of $tablePath starts at ${range.startMs}, which is earlier than the " + + s"table is guaranteed to retain (table.log.ttl = ${Duration.ofMillis(logTtlMs)}). " + + s"Records that already expired cannot be returned, so the result may be a subset of " + + s"the requested window. Increase table.log.ttl or move the window forward.") + } + } + + private[spark] def predatesRetention(startMs: Long, logTtlMs: Long, nowMs: Long): Boolean = + startMs < nowMs - logTtlMs + /** * Reads a `scan.incremental.*` option from the scan options, falling back to its default. A blank * value counts as unset; a blank start timestamp is rejected by [[isIncrementalRead]] before it diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala index c871ef5ec24..e8386c94ffa 100644 --- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala +++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/SplitPlanner.scala @@ -284,6 +284,9 @@ class AppendPlanner( override def plan(): Array[InputPartition] = try { + timeRange.foreach( + FlussOffsetInitializers + .warnIfWindowPredatesRetention(tablePath, _, tableInfo.getTableConfig.getLogTTLMs)) readableLakeSnapshot match { // An incremental read never unions a lake snapshot; it reads only Fluss. case Some(snap) if !incrementalMode => planLakeUnion(snap) @@ -640,6 +643,9 @@ class UpsertPlanner( override def plan(): Array[InputPartition] = try { + timeRange.foreach( + FlussOffsetInitializers + .warnIfWindowPredatesRetention(tablePath, _, tableInfo.getTableConfig.getLogTTLMs)) readableLakeSnapshot match { // An incremental read reads neither the lake nor the kv snapshot; it folds only the Fluss // changelog within [start, end). diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala index 37846e666ee..f5c53261eec 100644 --- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala +++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/read/FlussOffsetInitializersTest.scala @@ -94,6 +94,19 @@ class FlussOffsetInitializersTest extends AnyFunSuite { } } + test("predatesRetention compares the window start against now minus table.log.ttl") { + val nowMs = 1767225600000L + val ttlMs = 7L * 24 * 60 * 60 * 1000 + // a start well before the retention horizon predates it + assertThat(FlussOffsetInitializers.predatesRetention(nowMs - ttlMs - 1, ttlMs, nowMs)).isTrue + // a start inside the retention horizon does not + assertThat(FlussOffsetInitializers.predatesRetention(nowMs - ttlMs + 1, ttlMs, nowMs)).isFalse + // a start exactly at the horizon is not warned about + assertThat(FlussOffsetInitializers.predatesRetention(nowMs - ttlMs, ttlMs, nowMs)).isFalse + // a ttl larger than the elapsed time retains everything since epoch 0 + assertThat(FlussOffsetInitializers.predatesRetention(0L, nowMs + 1, nowMs)).isFalse + } + test("invalid start timestamp format fails with the option name") { val startKey = SparkFlussConf.SCAN_INCREMENTAL_START_TIMESTAMP.key() val ex = intercept[IllegalArgumentException] { diff --git a/website/docs/engine-spark/reads.md b/website/docs/engine-spark/reads.md index 2df7be86157..9ee5dbf852b 100644 --- a/website/docs/engine-spark/reads.md +++ b/website/docs/engine-spark/reads.md @@ -334,6 +334,8 @@ The `scan.incremental.*` options are per-query read options only. Unlike the opt A time-range read returns the data Fluss still retains, which is bounded by `table.log.ttl` (default 7 days). A window reaching further back is **not** an error: the part that has already been dropped simply yields fewer rows, or none at all, so the result is always a subset of the requested window. Increase `table.log.ttl` if you need to read further back. Data available only in tiered lake storage is not read by this mode. The start timestamp positions the scan through the server's timestamp-to-offset lookup, and both bounds are then applied on each record's commit timestamp, so the window stays exact even for data already tiered to remote storage. + +When the window starts before the table is guaranteed to retain, planning logs a `WARN` on the driver. This is a conservative hint derived from the table's current `table.log.ttl`, not an exact statement about what was deleted: it may over-report (expired segments are deleted lazily, so the data may still be readable) and may under-report (it cannot know that a window predates the table's creation, or that whole partitions were dropped by `table.auto-partition.num-retention`), and it reflects only the current value of `table.log.ttl`, not past changes to it. ::: :::note Invalid windows fail fast