diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java index bd846cec28d..e23bae37023 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java @@ -23,6 +23,7 @@ import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.lake.lakestorage.LakeCatalog; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; @@ -39,7 +40,6 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,27 +53,21 @@ import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonSchemaChanges; import static org.apache.fluss.lake.paimon.utils.PaimonTableValidation.checkTableIsEmpty; import static org.apache.fluss.lake.paimon.utils.PaimonTableValidation.isPaimonSchemaCompatible; -import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; -import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; -import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; /** A Paimon implementation of {@link LakeCatalog}. */ public class PaimonLakeCatalog implements LakeCatalog { private static final Logger LOG = LoggerFactory.getLogger(PaimonLakeCatalog.class); private static final String PAIMON_PATH_KEY = "paimon.path"; - public static final LinkedHashMap SYSTEM_COLUMNS = new LinkedHashMap<>(); - - static { - // We need __bucket system column to filter out the given bucket - // for paimon bucket-unaware append only table. - // It's not required for paimon bucket-aware table like primary key table - // and bucket-aware append only table, but we always add the system column - // for consistent behavior - SYSTEM_COLUMNS.put(BUCKET_COLUMN_NAME, DataTypes.INT()); - SYSTEM_COLUMNS.put(OFFSET_COLUMN_NAME, DataTypes.BIGINT()); - SYSTEM_COLUMNS.put(TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS()); - } + + /** + * The three Fluss system columns and their Paimon types, kept for readers, writers and the + * lookuper to recognise legacy tables by column name. Retained as an alias of {@link + * PaimonSystemColumns#SYSTEM_COLUMNS}; under FIP-27 these columns are no longer added to newly + * created (clean) tables. + */ + public static final LinkedHashMap SYSTEM_COLUMNS = + PaimonSystemColumns.SYSTEM_COLUMNS; private final Catalog paimonCatalog; @@ -127,13 +121,15 @@ public void alterTable(TablePath tablePath, List tableChanges, Cont } Schema currentPaimonSchema = fileStoreTable.schema().toSchema(); + PaimonSystemColumns.LakeLayout lakeLayout = + PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType()); List paimonSchemaChanges; if (isPaimonSchemaCompatible( currentPaimonSchema, toPaimonSchema(context.getCurrentTable()))) { // if the paimon schema is same as current fluss schema, directly apply all the // changes. - paimonSchemaChanges = toPaimonSchemaChanges(changesToApply); + paimonSchemaChanges = toPaimonSchemaChanges(changesToApply, lakeLayout); } else if (isPaimonSchemaCompatible( currentPaimonSchema, toPaimonSchema(context.getExpectedTable()))) { // if the schema is same as applied fluss schema , skip adding columns. @@ -144,7 +140,8 @@ currentPaimonSchema, toPaimonSchema(context.getExpectedTable()))) { tableChange -> !(tableChange instanceof TableChange.AddColumn)) - .collect(Collectors.toList())); + .collect(Collectors.toList()), + lakeLayout); } else { throw new InvalidAlterTableException( String.format( diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java index 3af7467cfab..577b17658db 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java @@ -19,6 +19,8 @@ package org.apache.fluss.lake.paimon.source; import org.apache.fluss.lake.paimon.utils.PaimonRowAsFlussRow; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.lake.source.RecordReader; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.GenericRecord; @@ -46,6 +48,14 @@ /** Record reader for paimon table. */ public class PaimonRecordReader implements RecordReader { + /** + * Sentinel log offset / timestamp emitted for rows read from a clean lake table, which does not + * store the {@code __offset} / {@code __timestamp} system columns. A negative offset is + * interpreted downstream as "no valid offset" (snapshot phase), see {@code + * LakeRecordRecordEmitter}. + */ + private static final long NO_SYSTEM_COLUMN_VALUE = -1L; + protected PaimonRowAsFlussRecordIterator iterator; protected @Nullable int[][] project; protected RowType paimonRowType; @@ -56,10 +66,12 @@ public PaimonRecordReader( @Nullable int[][] project, @Nullable Predicate predicate) throws IOException { + LakeLayout lakeLayout = + PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType()); ReadBuilder readBuilder = fileStoreTable.newReadBuilder(); RowType paimonFullRowType = fileStoreTable.rowType(); if (project != null) { - readBuilder = applyProject(readBuilder, project, paimonFullRowType); + readBuilder = applyProject(readBuilder, project, paimonFullRowType, lakeLayout); } if (predicate != null) { @@ -71,13 +83,15 @@ public PaimonRecordReader( if (split == null) { iterator = new PaimonRecordReader.PaimonRowAsFlussRecordIterator( - org.apache.paimon.utils.CloseableIterator.empty(), paimonRowType); + org.apache.paimon.utils.CloseableIterator.empty(), + paimonRowType, + lakeLayout); } else { org.apache.paimon.reader.RecordReader recordReader = tableRead.createReader(split.dataSplit()); iterator = new PaimonRecordReader.PaimonRowAsFlussRecordIterator( - recordReader.toCloseableIterator(), paimonRowType); + recordReader.toCloseableIterator(), paimonRowType, lakeLayout); } } @@ -87,9 +101,19 @@ public CloseableIterator read() throws IOException { } private ReadBuilder applyProject( - ReadBuilder readBuilder, int[][] projects, RowType paimonFullRowType) { + ReadBuilder readBuilder, + int[][] projects, + RowType paimonFullRowType, + LakeLayout lakeLayout) { int[] projectIds = Arrays.stream(projects).mapToInt(project -> project[0]).toArray(); + if (lakeLayout == LakeLayout.CLEAN) { + // Clean tables have no system columns to read, so project the business columns only. + return readBuilder.withProjection(projectIds); + } + + // Legacy tables carry __offset/__timestamp, which the iterator needs to recover the log + // offset and timestamp of each record; append them to the projection. int offsetFieldPos = paimonFullRowType.getFieldIndex(OFFSET_COLUMN_NAME); int timestampFieldPos = paimonFullRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME); @@ -115,13 +139,28 @@ public static class PaimonRowAsFlussRecordIterator implements CloseableIterator< public PaimonRowAsFlussRecordIterator( org.apache.paimon.utils.CloseableIterator paimonRowIterator, - RowType paimonRowType) { + RowType paimonRowType, + LakeLayout lakeLayout) { this.paimonRowIterator = paimonRowIterator; - this.logOffsetColIndex = paimonRowType.getFieldIndex(OFFSET_COLUMN_NAME); - this.timestampColIndex = paimonRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME); - int[] project = IntStream.range(0, paimonRowType.getFieldCount() - 2).toArray(); - projectedRow = ProjectedRow.from(project); + int fieldCount = paimonRowType.getFieldCount(); + if (lakeLayout == LakeLayout.CLEAN) { + // No system columns are read; all projected fields are business fields, and the + // log offset / timestamp are not available from the lake table. + this.logOffsetColIndex = -1; + this.timestampColIndex = -1; + projectedRow = ProjectedRow.from(IntStream.range(0, fieldCount).toArray()); + } else { + // Legacy layout: applyProject appended exactly __offset and __timestamp (not + // __bucket) as the last two projected fields, so the business fields are all fields + // except those trailing two. + this.logOffsetColIndex = paimonRowType.getFieldIndex(OFFSET_COLUMN_NAME); + this.timestampColIndex = paimonRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME); + int[] project = IntStream.range(0, fieldCount - 2).toArray(); + projectedRow = ProjectedRow.from(project); + } + // The wrapped row is only ever accessed by index through projectedRow, which already + // drops the system columns, so no trailing-system-column trimming is needed here. paimonRowAsFlussRow = new PaimonRowAsFlussRow(); } @@ -143,8 +182,14 @@ public boolean hasNext() { public LogRecord next() { InternalRow paimonRow = paimonRowIterator.next(); ChangeType changeType = toChangeType(paimonRow.getRowKind()); - long offset = paimonRow.getLong(logOffsetColIndex); - long timestamp = paimonRow.getTimestamp(timestampColIndex, 6).getMillisecond(); + long offset = + logOffsetColIndex < 0 + ? NO_SYSTEM_COLUMN_VALUE + : paimonRow.getLong(logOffsetColIndex); + long timestamp = + timestampColIndex < 0 + ? NO_SYSTEM_COLUMN_VALUE + : paimonRow.getTimestamp(timestampColIndex, 6).getMillisecond(); return new GenericRecord( offset, diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java index bc030301037..0705ac4ed99 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.paimon.tiering; import org.apache.fluss.lake.paimon.source.FlussRowAsPaimonRow; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.record.LogRecord; import org.apache.paimon.data.InternalRow; @@ -34,6 +35,7 @@ public class FlussRecordAsPaimonRow extends FlussRowAsPaimonRow { private final int bucket; + private final LakeLayout lakeLayout; private LogRecord logRecord; private int originRowFieldCount; private final int businessFieldCount; @@ -41,13 +43,23 @@ public class FlussRecordAsPaimonRow extends FlussRowAsPaimonRow { private final int offsetFieldIndex; private final int timestampFieldIndex; - public FlussRecordAsPaimonRow(int bucket, RowType tableTowType) { + public FlussRecordAsPaimonRow(int bucket, RowType tableTowType, LakeLayout lakeLayout) { super(tableTowType); this.bucket = bucket; - this.businessFieldCount = tableRowType.getFieldCount() - SYSTEM_COLUMNS.size(); - this.bucketFieldIndex = businessFieldCount; - this.offsetFieldIndex = businessFieldCount + 1; - this.timestampFieldIndex = businessFieldCount + 2; + this.lakeLayout = lakeLayout; + if (lakeLayout == LakeLayout.LEGACY) { + // Legacy tables append the three system columns after the business columns. + this.businessFieldCount = tableRowType.getFieldCount() - SYSTEM_COLUMNS.size(); + this.bucketFieldIndex = businessFieldCount; + this.offsetFieldIndex = businessFieldCount + 1; + this.timestampFieldIndex = businessFieldCount + 2; + } else { + // Clean tables contain only business columns; there are no system fields to emit. + this.businessFieldCount = tableRowType.getFieldCount(); + this.bucketFieldIndex = -1; + this.offsetFieldIndex = -1; + this.timestampFieldIndex = -1; + } } public void setFlussRecord(LogRecord logRecord) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java index 2a38e388a06..cb82976c9bb 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java @@ -21,6 +21,8 @@ import org.apache.fluss.lake.batch.RecordBatch; import org.apache.fluss.lake.paimon.tiering.append.AppendOnlyWriter; import org.apache.fluss.lake.paimon.tiering.mergetree.MergeTreeWriter; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.lake.writer.LakeWriter; import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; @@ -58,6 +60,12 @@ public PaimonLakeWriter( List partitionKeys = fileStoreTable.partitionKeys(); RowType flussRowType = writerInitContext.tableInfo().getRowType(); + // FIP-27: detect whether the target Paimon table is a clean table (only user columns) or a + // legacy table (carrying the three Fluss system columns). Writers emit system columns only + // for legacy tables. + LakeLayout lakeLayout = + PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType()); + this.recordWriter = fileStoreTable.primaryKeys().isEmpty() ? new AppendOnlyWriter( @@ -65,14 +73,16 @@ public PaimonLakeWriter( writerInitContext.tableBucket(), writerInitContext.partition(), partitionKeys, - flussRowType) + flussRowType, + lakeLayout) : new MergeTreeWriter( fileStoreTable, writerInitContext.tableBucket(), writerInitContext.partition(), partitionKeys, flussRowType, - writerInitContext.ioTmpDirs()); + writerInitContext.ioTmpDirs(), + lakeLayout); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java index 173407bb9c7..dad3f7e8f04 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java @@ -17,6 +17,7 @@ package org.apache.fluss.lake.paimon.tiering; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.LogRecord; @@ -49,7 +50,8 @@ public RecordWriter( TableBucket tableBucket, @Nullable String partition, List partitionKeys, - org.apache.fluss.types.RowType flussRowType) { + org.apache.fluss.types.RowType flussRowType, + LakeLayout lakeLayout) { this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = tableBucket.getBucket(); @@ -62,7 +64,7 @@ public RecordWriter( this.partition = resolvePartition(partition, partitionKeys, flussRowType); } this.flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket.getBucket(), tableRowType); + new FlussRecordAsPaimonRow(tableBucket.getBucket(), tableRowType, lakeLayout); } public abstract void write(LogRecord record) throws Exception; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java index 4fdb8bce79b..4036ef96396 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java @@ -17,6 +17,7 @@ package org.apache.fluss.lake.paimon.tiering.append; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.record.ArrowBatchData; @@ -57,6 +58,7 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { private final TableWriteImpl tableWrite; private final RowType tableRowType; private final int bucket; + private final LakeLayout lakeLayout; private static final Field BUCKET_FIELD = new Field( @@ -88,11 +90,13 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable { FileStoreTable fileStoreTable, TableWriteImpl tableWrite, RowType tableRowType, - int bucket) { + int bucket, + LakeLayout lakeLayout) { this.fileStoreTable = fileStoreTable; this.tableWrite = tableWrite; this.tableRowType = tableRowType; this.bucket = bucket; + this.lakeLayout = lakeLayout; } /** @@ -107,6 +111,16 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws } VectorSchemaRoot originalRoot = arrowBatchData.getVectorSchemaRoot(); + + if (lakeLayout == LakeLayout.CLEAN) { + // Clean tables contain only user columns, so the incoming Arrow batch already matches + // the Paimon table schema. Write it directly without enriching system columns. + ArrowBundleRecords cleanRecords = + new ArrowBundleRecords(originalRoot, tableRowType, false); + tableWrite.writeBundle(partition, writtenBucket, cleanRecords); + return; + } + long baseOffset = arrowBatchData.getBaseLogOffset(); long timestamp = arrowBatchData.getTimestamp(); int rowCount = originalRoot.getRowCount(); diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java index 0caaed97c4b..601afc531ea 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyWriter.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.paimon.tiering.append; import org.apache.fluss.lake.paimon.tiering.RecordWriter; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.ArrowBatchData; import org.apache.fluss.record.LogRecord; @@ -46,12 +47,15 @@ public class AppendOnlyWriter extends RecordWriter { */ @Nullable private AutoCloseable arrowBatchHelper; + private final LakeLayout lakeLayout; + public AppendOnlyWriter( FileStoreTable fileStoreTable, TableBucket tableBucket, @Nullable String partition, List partitionKeys, - RowType flussRowType) { + RowType flussRowType, + LakeLayout lakeLayout) { //noinspection unchecked super( (TableWriteImpl) @@ -61,8 +65,10 @@ public AppendOnlyWriter( tableBucket, partition, partitionKeys, - flussRowType); + flussRowType, + lakeLayout); this.fileStoreTable = fileStoreTable; + this.lakeLayout = lakeLayout; } @Override @@ -90,7 +96,7 @@ public void writeArrowBatch(ArrowBatchData arrowBatchData) throws Exception { if (arrowBatchHelper == null) { helper = new AppendOnlyArrowBatchHelper( - fileStoreTable, tableWrite, tableRowType, bucket); + fileStoreTable, tableWrite, tableRowType, bucket, lakeLayout); arrowBatchHelper = helper; } else { helper = (AppendOnlyArrowBatchHelper) arrowBatchHelper; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java index b3e1ecdeed8..9b0b2c0f744 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/mergetree/MergeTreeWriter.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.paimon.tiering.mergetree; import org.apache.fluss.lake.paimon.tiering.RecordWriter; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.LogRecord; import org.apache.fluss.types.RowType; @@ -49,8 +50,16 @@ public MergeTreeWriter( TableBucket tableBucket, @Nullable String partition, List partitionKeys, - RowType flussRowType) { - this(fileStoreTable, tableBucket, partition, partitionKeys, flussRowType, (String[]) null); + RowType flussRowType, + LakeLayout lakeLayout) { + this( + fileStoreTable, + tableBucket, + partition, + partitionKeys, + flussRowType, + (String[]) null, + lakeLayout); } public MergeTreeWriter( @@ -59,14 +68,16 @@ public MergeTreeWriter( @Nullable String partition, List partitionKeys, RowType flussRowType, - @Nullable String[] ioTmpDirs) { + @Nullable String[] ioTmpDirs, + LakeLayout lakeLayout) { this( fileStoreTable, createIOManager(ioTmpDirs), tableBucket, partition, partitionKeys, - flussRowType); + flussRowType, + lakeLayout); } MergeTreeWriter( @@ -75,14 +86,16 @@ public MergeTreeWriter( TableBucket tableBucket, @Nullable String partition, List partitionKeys, - RowType flussRowType) { + RowType flussRowType, + LakeLayout lakeLayout) { super( createTableWrite(fileStoreTable, ioManager), fileStoreTable.rowType(), tableBucket, partition, partitionKeys, - flussRowType); + flussRowType, + lakeLayout); this.rowKeyExtractor = fileStoreTable.createRowKeyExtractor(); this.ioManager = ioManager; } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java index becf8a2f056..c90f0e91438 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonConversions.java @@ -51,6 +51,7 @@ import java.util.function.Function; import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; +import static org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import static org.apache.fluss.utils.Preconditions.checkState; /** Utils for conversion between Paimon and Fluss. */ @@ -172,7 +173,8 @@ public static BinaryRow toPaimonPartition( return partitionExtractor.apply(new FlussRowAsPaimonRow(partitionRow, paimonRowType)); } - public static List toPaimonSchemaChanges(List tableChanges) { + public static List toPaimonSchemaChanges( + List tableChanges, LakeLayout lakeLayout) { List schemaChanges = new ArrayList<>(tableChanges.size()); for (TableChange tableChange : tableChanges) { @@ -203,14 +205,27 @@ public static List toPaimonSchemaChanges(List tableCh org.apache.paimon.types.DataType paimonDataType = flussDataType.accept(FlussDataTypeToPaimonDataType.INSTANCE); - String firstSystemColumnName = SYSTEM_COLUMNS.keySet().iterator().next(); - schemaChanges.add( - SchemaChange.addColumn( - addColumn.getName(), - paimonDataType, - addColumn.getComment(), - SchemaChange.Move.before( - addColumn.getName(), firstSystemColumnName))); + if (lakeLayout == LakeLayout.LEGACY) { + // Legacy tables keep the three system columns as the last physical columns, so + // a new business column must be inserted right before the first system column. + String firstSystemColumnName = SYSTEM_COLUMNS.keySet().iterator().next(); + schemaChanges.add( + SchemaChange.addColumn( + addColumn.getName(), + paimonDataType, + addColumn.getComment(), + SchemaChange.Move.before( + addColumn.getName(), firstSystemColumnName))); + } else { + // Clean tables have no trailing system columns, so a new business column is + // simply appended at the end. + schemaChanges.add( + SchemaChange.addColumn( + addColumn.getName(), + paimonDataType, + addColumn.getComment(), + null)); + } } else { throw new UnsupportedOperationException( "Unsupported table change: " + tableChange.getClass()); @@ -264,10 +279,10 @@ public static Schema toPaimonSchema(TableDescriptor tableDescriptor) { column.getComment().orElse(null)); } - // add system metadata columns to schema - for (Map.Entry systemColumn : SYSTEM_COLUMNS.entrySet()) { - schemaBuilder.column(systemColumn.getKey(), systemColumn.getValue()); - } + // FIP-27: newly created lake tables use a clean physical schema containing only + // user-defined columns. The three Fluss system columns (__bucket, __offset, __timestamp) + // are no longer added. Existing legacy tables that still carry these columns remain + // readable and writable, see PaimonSystemColumns#detectLayout. // set pk if (tableDescriptor.hasPrimaryKey()) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java index fe956561a69..9e8be46c65b 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonRowAsFlussRow.java @@ -17,6 +17,7 @@ package org.apache.fluss.lake.paimon.utils; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.row.BinaryString; import org.apache.fluss.row.Decimal; import org.apache.fluss.row.InternalArray; @@ -27,17 +28,28 @@ import org.apache.paimon.data.Timestamp; -import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; - /** Adapter for paimon row as fluss row. */ public class PaimonRowAsFlussRow implements InternalRow { private org.apache.paimon.data.InternalRow paimonRow; - public PaimonRowAsFlussRow() {} + // Number of trailing Fluss system columns carried by the wrapped Paimon row that must be + // excluded from the exposed field count. This is only non-zero for a legacy table's top-level + // physical row; clean tables and nested/projected rows carry no system columns. + private final int trailingSystemColumns; + + public PaimonRowAsFlussRow() { + this.trailingSystemColumns = 0; + } + + public PaimonRowAsFlussRow(LakeLayout lakeLayout) { + this.trailingSystemColumns = + lakeLayout == LakeLayout.LEGACY ? PaimonSystemColumns.systemColumnCount() : 0; + } public PaimonRowAsFlussRow(org.apache.paimon.data.InternalRow paimonRow) { this.paimonRow = paimonRow; + this.trailingSystemColumns = 0; } public PaimonRowAsFlussRow replaceRow(org.apache.paimon.data.InternalRow paimonRow) { @@ -47,7 +59,7 @@ public PaimonRowAsFlussRow replaceRow(org.apache.paimon.data.InternalRow paimonR @Override public int getFieldCount() { - return paimonRow.getFieldCount() - SYSTEM_COLUMNS.size(); + return paimonRow.getFieldCount() - trailingSystemColumns; } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java new file mode 100644 index 00000000000..35e4a265f52 --- /dev/null +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSystemColumns.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.paimon.utils; + +import org.apache.fluss.exception.InvalidTableException; + +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME; +import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME; +import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; + +/** + * Utilities describing the two physical layouts a Paimon lake table can have under FIP-27, and the + * single place that detects which layout a given Paimon table uses. + * + *
    + *
  • {@link LakeLayout#CLEAN} - the table only contains user-defined columns. This is the layout + * of every newly created lake table. + *
  • {@link LakeLayout#LEGACY} - the table was created before FIP-27 and carries the three + * mandatory Fluss system columns {@code __bucket}, {@code __offset}, {@code __timestamp} as + * its last three physical columns. + *
+ * + *

Detection is based purely on the physical Paimon schema, so no extra metadata or table + * property is needed and existing tables are never migrated. A table that carries only some of the + * system columns, or carries them with an unexpected type, is neither a clean nor a valid legacy + * table and is rejected with a clear error. + */ +public class PaimonSystemColumns { + + /** + * The three mandatory Fluss system columns and their expected Paimon types, in physical order. + * The {@code __timestamp} type is compared with relaxed precision (see {@link + * #isSystemTimestampType}) to stay compatible with legacy tables written by older clusters. + */ + public static final LinkedHashMap SYSTEM_COLUMNS = new LinkedHashMap<>(); + + static { + // We need __bucket system column to filter out the given bucket + // for paimon bucket-unaware append only table. + // It's not required for paimon bucket-aware table like primary key table + // and bucket-aware append only table, but legacy tables always carry the system column + // for consistent behavior. + SYSTEM_COLUMNS.put(BUCKET_COLUMN_NAME, DataTypes.INT()); + SYSTEM_COLUMNS.put(OFFSET_COLUMN_NAME, DataTypes.BIGINT()); + SYSTEM_COLUMNS.put(TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS()); + } + + /** The physical layout of a Paimon lake table with respect to Fluss system columns. */ + public enum LakeLayout { + /** Only user-defined columns are present (FIP-27 default for new tables). */ + CLEAN, + /** The three Fluss system columns are appended as the last physical columns. */ + LEGACY + } + + private PaimonSystemColumns() {} + + /** Returns the number of system columns carried by a {@link LakeLayout#LEGACY} table. */ + public static int systemColumnCount() { + return SYSTEM_COLUMNS.size(); + } + + public static boolean isSystemColumn(String columnName) { + return SYSTEM_COLUMNS.containsKey(columnName); + } + + /** + * Detects whether a Paimon table with the given physical row type uses the clean or the legacy + * layout. + * + *

The detection tolerates the {@code __timestamp} precision difference between old + * (precision 6) and new (precision 3) clusters, mirroring {@link + * PaimonTableValidation#equalIgnoreSystemColumnTimestampPrecision}. + * + * @throws InvalidTableException if the table carries only some of the system columns, carries + * them out of order, with an incompatible type, or embeds a system column name among the + * business columns. Such a table is neither clean nor a valid legacy table. + */ + public static LakeLayout detectLayout(RowType paimonRowType) { + List fields = paimonRowType.getFields(); + + int firstSystemColumnPos = -1; + for (int i = 0; i < fields.size(); i++) { + if (SYSTEM_COLUMNS.containsKey(fields.get(i).name())) { + firstSystemColumnPos = i; + break; + } + } + + // No system column anywhere -> clean layout. + if (firstSystemColumnPos < 0) { + return LakeLayout.CLEAN; + } + + // A system column exists. For a valid legacy table, all three must appear, in the canonical + // order, as the very last physical columns, each with a compatible type. + int businessFieldCount = fields.size() - SYSTEM_COLUMNS.size(); + if (firstSystemColumnPos != businessFieldCount) { + throw partialLayoutException(paimonRowType); + } + + int pos = businessFieldCount; + for (Map.Entry systemColumn : SYSTEM_COLUMNS.entrySet()) { + DataField field = fields.get(pos); + if (!field.name().equals(systemColumn.getKey()) + || !isSystemColumnTypeCompatible(field.name(), field.type())) { + throw partialLayoutException(paimonRowType); + } + pos++; + } + + return LakeLayout.LEGACY; + } + + private static boolean isSystemColumnTypeCompatible(String name, DataType actualType) { + if (TIMESTAMP_COLUMN_NAME.equals(name)) { + // Old clusters wrote precision 6, new clusters write precision 3; both are accepted. + return isSystemTimestampType(actualType); + } + // Compare the type family and precision, ignoring nullability: legacy system columns were + // written as non-null, but we only care that the physical type matches. + DataType expected = SYSTEM_COLUMNS.get(name); + return actualType.copy(true).equalsIgnoreFieldId(expected.copy(true)); + } + + private static boolean isSystemTimestampType(DataType actualType) { + // Legacy tables carry __timestamp with varying timestamp types depending on the cluster + // that created them: with or without local time zone, and precision 3 (new clusters) or 6 + // (old clusters). Accept the whole timestamp family and let the reader handle the + // precision, mirroring the relaxed check in + // PaimonTableValidation#equalIgnoreSystemColumnTimestampPrecision. + switch (actualType.getTypeRoot()) { + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return true; + default: + return false; + } + } + + private static InvalidTableException partialLayoutException(RowType paimonRowType) { + return new InvalidTableException( + String.format( + "The Paimon table has an incompatible system-column layout. A table must " + + "either contain none of the Fluss system columns (clean layout) or " + + "contain all of %s as its last columns, in this order, with " + + "compatible types (legacy layout). Actual schema: %s.", + SYSTEM_COLUMNS.keySet(), paimonRowType)); + } +} diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java index 0b8f56052f9..b10e485dc85 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonTableValidation.java @@ -34,12 +34,23 @@ import static org.apache.fluss.lake.paimon.utils.PaimonConversions.PAIMON_UNSETTABLE_OPTIONS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.PARTITION_GENERATE_LEGACY_NAME_OPTION_KEY; +import static org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME; /** Utils to verify whether the existing Paimon table is compatible with the table to be created. */ public class PaimonTableValidation { public static boolean isPaimonSchemaCompatible(Schema existingSchema, Schema newSchema) { + // FIP-27: newly generated schemas are always clean (no system columns). When the existing + // table is a legacy table that still carries the three system columns, re-enabling lake + // tiering must keep that physical layout. Enrich the clean new schema with the trailing + // system columns before comparison, so an existing legacy table is recognised as + // compatible and its layout is preserved. Detection also rejects a partial/type-mismatched + // legacy layout with a clear error. + if (PaimonSystemColumns.detectLayout(existingSchema.rowType()) == LakeLayout.LEGACY) { + newSchema = appendSystemColumns(newSchema); + } + if (!equalPhysicalSchema(existingSchema, newSchema)) { // Allow different precisions for __timestamp column for backward compatibility, // old cluster will use precision 6, but new cluster will use precision 3, @@ -50,6 +61,32 @@ public static boolean isPaimonSchemaCompatible(Schema existingSchema, Schema new return true; } + /** + * Returns a copy of {@code cleanSchema} with the three Fluss system columns appended as the + * last physical columns, so a clean schema generated by the current cluster can be compared + * against an existing legacy table. Partition keys, primary keys, options and comment are + * preserved. System-column field ids continue the existing id sequence to avoid collisions; + * they are irrelevant to {@link #equalPhysicalSchema}, which ignores field ids. + */ + private static Schema appendSystemColumns(Schema cleanSchema) { + List fields = new ArrayList<>(cleanSchema.fields()); + int nextFieldId = 0; + for (DataField field : fields) { + nextFieldId = Math.max(nextFieldId, field.id() + 1); + } + for (Map.Entry systemColumn : + PaimonSystemColumns.SYSTEM_COLUMNS.entrySet()) { + fields.add( + new DataField(nextFieldId++, systemColumn.getKey(), systemColumn.getValue())); + } + return new Schema( + fields, + cleanSchema.partitionKeys(), + cleanSchema.primaryKeys(), + cleanSchema.options(), + cleanSchema.comment()); + } + /** * Check if the {@code existingSchema} is compatible with {@code newSchema} by ignoring the * precision difference of the system column {@code __timestamp}. @@ -67,6 +104,11 @@ public static boolean isPaimonSchemaCompatible(Schema existingSchema, Schema new public static boolean equalIgnoreSystemColumnTimestampPrecision( Schema existingSchema, Schema newSchema) { List existingFields = new ArrayList<>(existingSchema.fields()); + // Only legacy tables carry a trailing __timestamp system column. Clean tables have no + // system columns, so there is no precision to relax and we compare them directly. + if (existingFields.isEmpty()) { + return equalPhysicalSchema(existingSchema, newSchema); + } DataField systemTimestampField = existingFields.get(existingFields.size() - 1); if (systemTimestampField.name().equals(TIMESTAMP_COLUMN_NAME) && systemTimestampField diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java index f1d75d27810..fafc95c5871 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java @@ -44,7 +44,6 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; -import org.apache.paimon.data.Timestamp; import org.apache.paimon.options.Options; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.FileStoreTable; @@ -172,19 +171,9 @@ void testCreateLakeEnabledTable() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "log_c1", - "log_c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"log_c1", "log_c2"}), "log_c1,log_c2", BUCKET_NUM); @@ -210,19 +199,9 @@ void testCreateLakeEnabledTable() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "log_c1", - "log_c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"log_c1", "log_c2"}), null, BUCKET_NUM); @@ -249,19 +228,9 @@ void testCreateLakeEnabledTable() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT().notNull(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "pk_c1", - "pk_c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"pk_c1", "pk_c2"}), "pk_c1", BUCKET_NUM); @@ -292,20 +261,9 @@ void testCreateLakeEnabledTable() throws Exception { new DataType[] { org.apache.paimon.types.DataTypes.INT().notNull(), org.apache.paimon.types.DataTypes.STRING(), - org.apache.paimon.types.DataTypes.STRING().notNull(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING().notNull() }, - new String[] { - "c1", - "c2", - "c3", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2", "c3"}), "c1", BUCKET_NUM); } @@ -359,32 +317,12 @@ void testCreateLakeEnabledTableWithAllTypes() throws Exception { org.apache.paimon.types.DataTypes.DATE(), org.apache.paimon.types.DataTypes.TIME(), org.apache.paimon.types.DataTypes.TIMESTAMP(), - org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE() }, new String[] { - "log_c1", - "log_c2", - "log_c3", - "log_c4", - "log_c5", - "log_c6", - "log_c7", - "log_c8", - "log_c9", - "log_c10", - "log_c11", - "log_c12", - "log_c13", - "log_c14", - "log_c15", - "log_c16", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME + "log_c1", "log_c2", "log_c3", "log_c4", "log_c5", "log_c6", "log_c7", + "log_c8", "log_c9", "log_c10", "log_c11", "log_c12", "log_c13", + "log_c14", "log_c15", "log_c16" }), null, BUCKET_NUM); @@ -599,10 +537,9 @@ void testCreateLakeEnableTableWithExistLakeTable() throws Exception { .hasMessageContaining( "The table `fluss`.`log_table_with_exist_lake_table` already exists in Paimon catalog, but the table schema is not compatible.") .hasMessageContaining( - "Existing schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `__bucket` INT, `__offset` BIGINT, `__timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE]") + "Existing schema: UpdateSchema{fields=[`c1` STRING, `c2` INT]") .hasMessageContaining("options={bucket=-1") - .hasMessageContaining( - "new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `__bucket` INT, `__offset` BIGINT, `__timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE]") + .hasMessageContaining("new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT]") .hasMessageContaining("options={bucket=3") .hasMessageContaining("bucket-key=c1,c2") .hasMessageEndingWith( @@ -624,9 +561,9 @@ void testCreateLakeEnableTableWithExistLakeTable() throws Exception { .hasMessageContaining( "The table `fluss`.`log_table_with_exist_lake_table` already exists in Paimon catalog, but the table schema is not compatible.") .hasMessageContaining( - "Existing schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `__bucket` INT, `__offset` BIGINT, `__timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE]") + "Existing schema: UpdateSchema{fields=[`c1` STRING, `c2` INT]") .hasMessageContaining( - "new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `c3` STRING, `__bucket` INT, `__offset` BIGINT, `__timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE]") + "new schema: UpdateSchema{fields=[`c1` STRING, `c2` INT, `c3` STRING]") .hasMessageEndingWith( "Please first drop the table in Paimon catalog or use a new table name."); @@ -781,19 +718,9 @@ void testAlterLakeEnabledLogTable() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "log_c1", - "log_c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"log_c1", "log_c2"}), "log_c1,log_c2", BUCKET_NUM); @@ -888,19 +815,9 @@ void testAlterLakeEnabledTableProperties() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -919,19 +836,9 @@ void testAlterLakeEnabledTableProperties() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -966,19 +873,9 @@ void testAlterLakeEnabledTableProperties() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -1047,19 +944,9 @@ void testEnableLakeTableAfterAlterTableProperties() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -1100,19 +987,9 @@ void testAlterLakeEnabledTableSchema() throws Exception { RowType.of( new DataType[] { org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.STRING(), - // for __bucket, __offset, __timestamp - org.apache.paimon.types.DataTypes.INT(), - org.apache.paimon.types.DataTypes.BIGINT(), - org.apache.paimon.types.DataTypes.TIMESTAMP_LTZ_MILLIS() + org.apache.paimon.types.DataTypes.STRING() }, - new String[] { - "c1", - "c2", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME - }), + new String[] {"c1", "c2"}), "c1,c2", BUCKET_NUM); @@ -1131,15 +1008,8 @@ void testAlterLakeEnabledTableSchema() throws Exception { paimonCatalog.getTable(Identifier.create(DATABASE, tablePath.getTableName())); // Verify the new column c3 with comment was added to Paimon table RowType alteredRowType = alteredPaimonTable.rowType(); - assertThat(alteredRowType.getFieldCount()).isEqualTo(6); - assertThat(alteredRowType.getFieldNames()) - .containsExactly( - "c1", - "c2", - "c3", - BUCKET_COLUMN_NAME, - OFFSET_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME); + assertThat(alteredRowType.getFieldCount()).isEqualTo(3); + assertThat(alteredRowType.getFieldNames()).containsExactly("c1", "c2", "c3"); // Verify c3 column has the correct type and comment assertThat(alteredRowType.getField("c3").type()) .isEqualTo(org.apache.paimon.types.DataTypes.INT()); @@ -1159,12 +1029,21 @@ void testEnableLakeTableWithLegacySystemTimestampColumn() throws Exception { Identifier paimonIdentifier = Identifier.create(DATABASE, tablePath.getTableName()); - // alter to TIMESTAMP_WITH_LOCAL_TIME_ZONE to mock the legacy behavior + // FIP-27: a newly created table is clean (no system columns). To exercise the legacy + // compatibility path, first turn it into a legacy table by appending the three trailing + // system columns, using TIMESTAMP_WITH_LOCAL_TIME_ZONE for __timestamp to mock the + // precision-6 layout written by an old cluster. paimonCatalog.alterTable( paimonIdentifier, - SchemaChange.updateColumnType( - TIMESTAMP_COLUMN_NAME, - org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()), + Arrays.asList( + SchemaChange.addColumn( + BUCKET_COLUMN_NAME, org.apache.paimon.types.DataTypes.INT()), + SchemaChange.addColumn( + OFFSET_COLUMN_NAME, org.apache.paimon.types.DataTypes.BIGINT()), + SchemaChange.addColumn( + TIMESTAMP_COLUMN_NAME, + org.apache.paimon.types.DataTypes + .TIMESTAMP_WITH_LOCAL_TIME_ZONE())), false); // disable data lake @@ -1310,13 +1189,7 @@ private void writeData(Table table) throws Exception { BatchTableCommit commit = writeBuilder.newCommit()) { for (int i = 0; i < 10; i++) { - GenericRow row = - GenericRow.of( - i, - BinaryString.fromString("row-" + i), - 0, - (long) i, - Timestamp.fromEpochMillis(System.currentTimeMillis())); + GenericRow row = GenericRow.of(i, BinaryString.fromString("row-" + i)); write.write(row); } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/PaimonLakeCatalogTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/PaimonLakeCatalogTest.java index 1683533e506..1e67a92a281 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/PaimonLakeCatalogTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/PaimonLakeCatalogTest.java @@ -160,15 +160,7 @@ void testAlterTableAddColumnLastNullable() throws Exception { Table table = flussPaimonCatalog.getPaimonCatalog().getTable(identifier); assertThat(table.rowType().getFieldNames()) - .containsSequence( - "id", - "name", - "amount", - "address", - "new_col", - "__bucket", - "__offset", - "__timestamp"); + .containsSequence("id", "name", "amount", "address", "new_col"); } @Test @@ -205,15 +197,7 @@ void testAlterTableAddColumnIgnoresPaimonCommentsAndOptions() throws Exception { assertThat(((FileStoreTable) table).schema().toSchema().comment()).isEqualTo(""); assertThat(table.options().get("fluss.key")).isEqualTo("value"); assertThat(table.rowType().getFieldNames()) - .containsSequence( - "id", - "name", - "amount", - "address", - "is_direct_play", - "__bucket", - "__offset", - "__timestamp"); + .containsSequence("id", "name", "amount", "address", "is_direct_play"); } @Test diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java index 81fa9b05018..f91e56c27ea 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/flink/FlinkUnionReadPrimaryKeyTableITCase.java @@ -123,7 +123,9 @@ void testUnionReadFullType(Boolean isPartitioned) throws Exception { CollectionUtil.iteratorToList(tableResult.collect()).stream() .map( row -> { - int userColumnCount = row.getArity() - 3; + // FIP-27: a clean lake table exposes only user columns via + // $lake, so there are no trailing system columns to strip. + int userColumnCount = row.getArity(); Object[] fields = new Object[userColumnCount]; for (int i = 0; i < userColumnCount; i++) { fields[i] = row.getField(i); @@ -277,7 +279,9 @@ void testUnionReadFullType(Boolean isPartitioned) throws Exception { .stream() .map( row -> { - int columnCount = row.getArity() - 3; + // FIP-27: a clean lake table exposes only user columns via + // $lake, so there are no trailing system columns to strip. + int columnCount = row.getArity(); Object[] fields = new Object[columnCount]; for (int i = 0; i < columnCount; i++) { fields[i] = row.getField(i); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java index e085b694ba0..dacb0479c80 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRowTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.lake.paimon.tiering; +import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout; import org.apache.fluss.record.GenericRecord; import org.apache.fluss.record.LogRecord; import org.apache.fluss.row.BinaryString; @@ -75,7 +76,7 @@ void testLogTableRecordAllTypes() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(14); @@ -143,7 +144,7 @@ void testPrimaryKeyTableRecord() { new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -185,7 +186,7 @@ void testArrayTypeWithIntElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 10; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(2); @@ -224,7 +225,7 @@ void testArrayTypeWithStringElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 5; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -259,7 +260,7 @@ void testNestedArrayType() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -313,7 +314,7 @@ void testArrayWithAllPrimitiveTypes() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(7); @@ -386,7 +387,7 @@ void testArrayWithDecimalElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -420,7 +421,7 @@ void testArrayWithTimestampElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -454,7 +455,7 @@ void testArrayWithBinaryElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -483,7 +484,7 @@ void testNullArray() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -508,7 +509,7 @@ void testArrayWithNullableElements() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -537,7 +538,7 @@ void testEmptyArray() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -564,7 +565,7 @@ void testPaimonSchemaWiderThanFlussRecord() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 7L; long timeStamp = System.currentTimeMillis(); @@ -595,7 +596,7 @@ void testFlussRecordWiderThanPaimonSchema() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 7L; long timeStamp = System.currentTimeMillis(); @@ -690,7 +691,7 @@ void testNestedRowType() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(8); @@ -918,7 +919,7 @@ void testMapWithAllTypes() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, nestedMapRowType); + new FlussRecordAsPaimonRow(tableBucket, nestedMapRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow nestedMapGenericRow = new GenericRow(1); @@ -971,7 +972,8 @@ private void testMapType( new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); - FlussRecordAsPaimonRow flussRow = new FlussRecordAsPaimonRow(tableBucket, rowType); + FlussRecordAsPaimonRow flussRow = + new FlussRecordAsPaimonRow(tableBucket, rowType, LakeLayout.LEGACY); GenericRow genericRow = new GenericRow(1); genericRow.setField(0, new GenericMap(mapData)); LogRecord logRecord = new GenericRecord(logOffset, timeStamp, APPEND_ONLY, genericRow); @@ -997,7 +999,7 @@ void testNullMap() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -1023,7 +1025,7 @@ void testMapWithNullableValues() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -1058,7 +1060,7 @@ void testEmptyMap() { new org.apache.paimon.types.LocalZonedTimestampType(3)); FlussRecordAsPaimonRow flussRecordAsPaimonRow = - new FlussRecordAsPaimonRow(tableBucket, tableRowType); + new FlussRecordAsPaimonRow(tableBucket, tableRowType, LakeLayout.LEGACY); long logOffset = 0; long timeStamp = System.currentTimeMillis(); GenericRow genericRow = new GenericRow(1); @@ -1082,7 +1084,7 @@ void testAccessRowBeforeSetThrowsIllegalState() { new org.apache.paimon.types.IntType(), new org.apache.paimon.types.BigIntType(), new org.apache.paimon.types.LocalZonedTimestampType(3)); - FlussRecordAsPaimonRow row = new FlussRecordAsPaimonRow(0, rowType); + FlussRecordAsPaimonRow row = new FlussRecordAsPaimonRow(0, rowType, LakeLayout.LEGACY); assertThatThrownBy(row::getRowKind) .isInstanceOf(IllegalStateException.class) .hasMessageContaining(expectedMsg); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java index 3d4da4fe50c..7101e0069df 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/tiering/PaimonTieringITCase.java @@ -504,9 +504,9 @@ private void checkDataInPaimonAppendOnlyTable( InternalRow flussRow = flussRowIterator.next(); assertThat(row.getInt(0)).isEqualTo(flussRow.getInt(0)); assertThat(row.getString(1).toString()).isEqualTo(flussRow.getString(1).toString()); - // system columns are always the last three: __bucket, __offset, __timestamp - int offsetIndex = row.getFieldCount() - 2; - assertThat(row.getLong(offsetIndex)).isEqualTo(startingOffset++); + // FIP-27: a clean lake table only stores user columns, so there are no trailing + // __bucket/__offset/__timestamp columns to verify here. + assertThat(row.getFieldCount()).isEqualTo(2); } assertThat(flussRowIterator.hasNext()).isFalse(); } @@ -526,8 +526,9 @@ private void checkDataInPaimonAppendOnlyPartitionedTable( assertThat(row.getInt(0)).isEqualTo(flussRow.getInt(0)); assertThat(row.getString(1).toString()).isEqualTo(flussRow.getString(1).toString()); assertThat(row.getString(2).toString()).isEqualTo(flussRow.getString(2).toString()); - // the idx 3 is __bucket, so use 4 - assertThat(row.getLong(4)).isEqualTo(startingOffset++); + // FIP-27: a clean lake table only stores user columns, so there are no trailing + // __bucket/__offset/__timestamp columns to verify here. + assertThat(row.getFieldCount()).isEqualTo(3); } assertThat(flussRowIterator.hasNext()).isFalse(); } @@ -594,9 +595,8 @@ void testTieringWithAddColumn() throws Exception { FileStoreTable paimonTable = (FileStoreTable) paimonCatalog.getTable(tableIdentifier); List fieldNames = paimonTable.rowType().getFieldNames(); - // Should have exact fields in order: a, b, c3, __bucket, __offset, __timestamp - assertThat(fieldNames) - .containsExactly("a", "b", "c3", "__bucket", "__offset", "__timestamp"); + // FIP-27: a clean lake table only stores user columns, in order: a, b, c3. + assertThat(fieldNames).containsExactly("a", "b", "c3"); // 9. Verify both schema evolution and data correctness // For initial rows (before ADD COLUMN), c3 should be NULL