[lake/paimon] Support clean and legacy Paimon lake table schemas

fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeCatalog.java

@@ -23,6 +23,7 @@
 import org.apache.fluss.exception.TableAlreadyExistException;
 import org.apache.fluss.exception.TableNotExistException;
 import org.apache.fluss.lake.lakestorage.LakeCatalog;
+import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns;
 import org.apache.fluss.metadata.TableChange;
 import org.apache.fluss.metadata.TableDescriptor;
 import org.apache.fluss.metadata.TablePath;
@@ -39,7 +40,6 @@
 import org.apache.paimon.table.FileStoreTable;
 import org.apache.paimon.table.Table;
 import org.apache.paimon.types.DataType;
-import org.apache.paimon.types.DataTypes;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,27 +53,21 @@
 import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonSchemaChanges;
 import static org.apache.fluss.lake.paimon.utils.PaimonTableValidation.checkTableIsEmpty;
 import static org.apache.fluss.lake.paimon.utils.PaimonTableValidation.isPaimonSchemaCompatible;
-import static org.apache.fluss.metadata.TableDescriptor.BUCKET_COLUMN_NAME;
-import static org.apache.fluss.metadata.TableDescriptor.OFFSET_COLUMN_NAME;
-import static org.apache.fluss.metadata.TableDescriptor.TIMESTAMP_COLUMN_NAME;
 
 /** A Paimon implementation of {@link LakeCatalog}. */
 public class PaimonLakeCatalog implements LakeCatalog {
 
     private static final Logger LOG = LoggerFactory.getLogger(PaimonLakeCatalog.class);
     private static final String PAIMON_PATH_KEY = "paimon.path";
-    public static final LinkedHashMap<String, DataType> SYSTEM_COLUMNS = new LinkedHashMap<>();
-
-    static {
-        // We need __bucket system column to filter out the given bucket
-        // for paimon bucket-unaware append only table.
-        // It's not required for paimon bucket-aware table like primary key table
-        // and bucket-aware append only table, but we always add the system column
-        // for consistent behavior
-        SYSTEM_COLUMNS.put(BUCKET_COLUMN_NAME, DataTypes.INT());
-        SYSTEM_COLUMNS.put(OFFSET_COLUMN_NAME, DataTypes.BIGINT());
-        SYSTEM_COLUMNS.put(TIMESTAMP_COLUMN_NAME, DataTypes.TIMESTAMP_LTZ_MILLIS());
-    }
+
+    /**
+     * The three Fluss system columns and their Paimon types, kept for readers, writers and the
+     * lookuper to recognise legacy tables by column name. Retained as an alias of {@link
+     * PaimonSystemColumns#SYSTEM_COLUMNS}; under FIP-27 these columns are no longer added to newly
+     * created (clean) tables.
+     */
+    public static final LinkedHashMap<String, DataType> SYSTEM_COLUMNS =
+            PaimonSystemColumns.SYSTEM_COLUMNS;
 
     private final Catalog paimonCatalog;
 
@@ -127,13 +121,15 @@ public void alterTable(TablePath tablePath, List<TableChange> tableChanges, Cont
             }
 
             Schema currentPaimonSchema = fileStoreTable.schema().toSchema();
+            PaimonSystemColumns.LakeLayout lakeLayout =
+                    PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType());
 
             List<SchemaChange> paimonSchemaChanges;
             if (isPaimonSchemaCompatible(
                     currentPaimonSchema, toPaimonSchema(context.getCurrentTable()))) {
                 // if the paimon schema is same as current fluss schema, directly apply all the
                 // changes.
-                paimonSchemaChanges = toPaimonSchemaChanges(changesToApply);
+                paimonSchemaChanges = toPaimonSchemaChanges(changesToApply, lakeLayout);
             } else if (isPaimonSchemaCompatible(
                     currentPaimonSchema, toPaimonSchema(context.getExpectedTable()))) {
                 // if the schema is same as applied fluss schema , skip adding columns.
@@ -144,7 +140,8 @@ currentPaimonSchema, toPaimonSchema(context.getExpectedTable()))) {
                                                 tableChange ->
                                                         !(tableChange
                                                                 instanceof TableChange.AddColumn))
-                                        .collect(Collectors.toList()));
+                                        .collect(Collectors.toList()),
+                                lakeLayout);
             } else {
                 throw new InvalidAlterTableException(
                         String.format(

fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/PaimonRecordReader.java

@@ -19,6 +19,8 @@
 package org.apache.fluss.lake.paimon.source;
 
 import org.apache.fluss.lake.paimon.utils.PaimonRowAsFlussRow;
+import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns;
+import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout;
 import org.apache.fluss.lake.source.RecordReader;
 import org.apache.fluss.record.ChangeType;
 import org.apache.fluss.record.GenericRecord;
@@ -46,6 +48,14 @@
 /** Record reader for paimon table. */
 public class PaimonRecordReader implements RecordReader {
 
+    /**
+     * Sentinel log offset / timestamp emitted for rows read from a clean lake table, which does not
+     * store the {@code __offset} / {@code __timestamp} system columns. A negative offset is
+     * interpreted downstream as "no valid offset" (snapshot phase), see {@code
+     * LakeRecordRecordEmitter}.
+     */
+    private static final long NO_SYSTEM_COLUMN_VALUE = -1L;
+
     protected PaimonRowAsFlussRecordIterator iterator;
     protected @Nullable int[][] project;
     protected RowType paimonRowType;
@@ -56,10 +66,12 @@ public PaimonRecordReader(
             @Nullable int[][] project,
             @Nullable Predicate predicate)
             throws IOException {
+        LakeLayout lakeLayout =
+                PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType());
         ReadBuilder readBuilder = fileStoreTable.newReadBuilder();
         RowType paimonFullRowType = fileStoreTable.rowType();
         if (project != null) {
-            readBuilder = applyProject(readBuilder, project, paimonFullRowType);
+            readBuilder = applyProject(readBuilder, project, paimonFullRowType, lakeLayout);
         }
 
         if (predicate != null) {
@@ -71,13 +83,15 @@ public PaimonRecordReader(
         if (split == null) {
             iterator =
                     new PaimonRecordReader.PaimonRowAsFlussRecordIterator(
-                            org.apache.paimon.utils.CloseableIterator.empty(), paimonRowType);
+                            org.apache.paimon.utils.CloseableIterator.empty(),
+                            paimonRowType,
+                            lakeLayout);
         } else {
             org.apache.paimon.reader.RecordReader<InternalRow> recordReader =
                     tableRead.createReader(split.dataSplit());
             iterator =
                     new PaimonRecordReader.PaimonRowAsFlussRecordIterator(
-                            recordReader.toCloseableIterator(), paimonRowType);
+                            recordReader.toCloseableIterator(), paimonRowType, lakeLayout);
         }
     }
 
@@ -87,9 +101,19 @@ public CloseableIterator<LogRecord> read() throws IOException {
     }
 
     private ReadBuilder applyProject(
-            ReadBuilder readBuilder, int[][] projects, RowType paimonFullRowType) {
+            ReadBuilder readBuilder,
+            int[][] projects,
+            RowType paimonFullRowType,
+            LakeLayout lakeLayout) {
         int[] projectIds = Arrays.stream(projects).mapToInt(project -> project[0]).toArray();
 
+        if (lakeLayout == LakeLayout.CLEAN) {
+            // Clean tables have no system columns to read, so project the business columns only.
+            return readBuilder.withProjection(projectIds);
+        }
+
+        // Legacy tables carry __offset/__timestamp, which the iterator needs to recover the log
+        // offset and timestamp of each record; append them to the projection.
         int offsetFieldPos = paimonFullRowType.getFieldIndex(OFFSET_COLUMN_NAME);
         int timestampFieldPos = paimonFullRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME);
 
@@ -115,13 +139,28 @@ public static class PaimonRowAsFlussRecordIterator implements CloseableIterator<
 
         public PaimonRowAsFlussRecordIterator(
                 org.apache.paimon.utils.CloseableIterator<InternalRow> paimonRowIterator,
-                RowType paimonRowType) {
+                RowType paimonRowType,
+                LakeLayout lakeLayout) {
             this.paimonRowIterator = paimonRowIterator;
-            this.logOffsetColIndex = paimonRowType.getFieldIndex(OFFSET_COLUMN_NAME);
-            this.timestampColIndex = paimonRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME);
 
-            int[] project = IntStream.range(0, paimonRowType.getFieldCount() - 2).toArray();
-            projectedRow = ProjectedRow.from(project);
+            int fieldCount = paimonRowType.getFieldCount();
+            if (lakeLayout == LakeLayout.CLEAN) {
+                // No system columns are read; all projected fields are business fields, and the
+                // log offset / timestamp are not available from the lake table.
+                this.logOffsetColIndex = -1;
+                this.timestampColIndex = -1;
+                projectedRow = ProjectedRow.from(IntStream.range(0, fieldCount).toArray());
+            } else {
+                // Legacy layout: applyProject appended exactly __offset and __timestamp (not
+                // __bucket) as the last two projected fields, so the business fields are all fields
+                // except those trailing two.
+                this.logOffsetColIndex = paimonRowType.getFieldIndex(OFFSET_COLUMN_NAME);
+                this.timestampColIndex = paimonRowType.getFieldIndex(TIMESTAMP_COLUMN_NAME);
+                int[] project = IntStream.range(0, fieldCount - 2).toArray();
+                projectedRow = ProjectedRow.from(project);
+            }
+            // The wrapped row is only ever accessed by index through projectedRow, which already
+            // drops the system columns, so no trailing-system-column trimming is needed here.
             paimonRowAsFlussRow = new PaimonRowAsFlussRow();
         }
 
@@ -143,8 +182,14 @@ public boolean hasNext() {
         public LogRecord next() {
             InternalRow paimonRow = paimonRowIterator.next();
             ChangeType changeType = toChangeType(paimonRow.getRowKind());
-            long offset = paimonRow.getLong(logOffsetColIndex);
-            long timestamp = paimonRow.getTimestamp(timestampColIndex, 6).getMillisecond();
+            long offset =
+                    logOffsetColIndex < 0
+                            ? NO_SYSTEM_COLUMN_VALUE
+                            : paimonRow.getLong(logOffsetColIndex);
+            long timestamp =
+                    timestampColIndex < 0
+                            ? NO_SYSTEM_COLUMN_VALUE
+                            : paimonRow.getTimestamp(timestampColIndex, 6).getMillisecond();
 
             return new GenericRecord(
                     offset,

fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/FlussRecordAsPaimonRow.java

@@ -18,6 +18,7 @@
 package org.apache.fluss.lake.paimon.tiering;
 
 import org.apache.fluss.lake.paimon.source.FlussRowAsPaimonRow;
+import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout;
 import org.apache.fluss.record.LogRecord;
 
 import org.apache.paimon.data.InternalRow;
@@ -34,20 +35,31 @@
 public class FlussRecordAsPaimonRow extends FlussRowAsPaimonRow {
 
     private final int bucket;
+    private final LakeLayout lakeLayout;
     private LogRecord logRecord;
     private int originRowFieldCount;
     private final int businessFieldCount;
     private final int bucketFieldIndex;
     private final int offsetFieldIndex;
     private final int timestampFieldIndex;
 
-    public FlussRecordAsPaimonRow(int bucket, RowType tableTowType) {
+    public FlussRecordAsPaimonRow(int bucket, RowType tableTowType, LakeLayout lakeLayout) {
         super(tableTowType);
         this.bucket = bucket;
-        this.businessFieldCount = tableRowType.getFieldCount() - SYSTEM_COLUMNS.size();
-        this.bucketFieldIndex = businessFieldCount;
-        this.offsetFieldIndex = businessFieldCount + 1;
-        this.timestampFieldIndex = businessFieldCount + 2;
+        this.lakeLayout = lakeLayout;
+        if (lakeLayout == LakeLayout.LEGACY) {
+            // Legacy tables append the three system columns after the business columns.
+            this.businessFieldCount = tableRowType.getFieldCount() - SYSTEM_COLUMNS.size();
+            this.bucketFieldIndex = businessFieldCount;
+            this.offsetFieldIndex = businessFieldCount + 1;
+            this.timestampFieldIndex = businessFieldCount + 2;
+        } else {
+            // Clean tables contain only business columns; there are no system fields to emit.
+            this.businessFieldCount = tableRowType.getFieldCount();
+            this.bucketFieldIndex = -1;
+            this.offsetFieldIndex = -1;
+            this.timestampFieldIndex = -1;
+        }
     }
 
     public void setFlussRecord(LogRecord logRecord) {

fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeWriter.java

@@ -21,6 +21,8 @@
 import org.apache.fluss.lake.batch.RecordBatch;
 import org.apache.fluss.lake.paimon.tiering.append.AppendOnlyWriter;
 import org.apache.fluss.lake.paimon.tiering.mergetree.MergeTreeWriter;
+import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns;
+import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout;
 import org.apache.fluss.lake.writer.LakeWriter;
 import org.apache.fluss.lake.writer.SupportsRecordBatchWrite;
 import org.apache.fluss.lake.writer.WriterInitContext;
@@ -58,21 +60,29 @@ public PaimonLakeWriter(
         List<String> partitionKeys = fileStoreTable.partitionKeys();
         RowType flussRowType = writerInitContext.tableInfo().getRowType();
 
+        // FIP-27: detect whether the target Paimon table is a clean table (only user columns) or a
+        // legacy table (carrying the three Fluss system columns). Writers emit system columns only
+        // for legacy tables.
+        LakeLayout lakeLayout =
+                PaimonSystemColumns.detectLayout(fileStoreTable.schema().logicalRowType());
+
         this.recordWriter =
                 fileStoreTable.primaryKeys().isEmpty()
                         ? new AppendOnlyWriter(
                                 fileStoreTable,
                                 writerInitContext.tableBucket(),
                                 writerInitContext.partition(),
                                 partitionKeys,
-                                flussRowType)
+                                flussRowType,
+                                lakeLayout)
                         : new MergeTreeWriter(
                                 fileStoreTable,
                                 writerInitContext.tableBucket(),
                                 writerInitContext.partition(),
                                 partitionKeys,
                                 flussRowType,
-                                writerInitContext.ioTmpDirs());
+                                writerInitContext.ioTmpDirs(),
+                                lakeLayout);
     }
 
     @Override

fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/RecordWriter.java

@@ -17,6 +17,7 @@
 
 package org.apache.fluss.lake.paimon.tiering;
 
+import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout;
 import org.apache.fluss.metadata.ResolvedPartitionSpec;
 import org.apache.fluss.metadata.TableBucket;
 import org.apache.fluss.record.LogRecord;
@@ -49,7 +50,8 @@ public RecordWriter(
             TableBucket tableBucket,
             @Nullable String partition,
             List<String> partitionKeys,
-            org.apache.fluss.types.RowType flussRowType) {
+            org.apache.fluss.types.RowType flussRowType,
+            LakeLayout lakeLayout) {
         this.tableWrite = tableWrite;
         this.tableRowType = tableRowType;
         this.bucket = tableBucket.getBucket();
@@ -62,7 +64,7 @@ public RecordWriter(
             this.partition = resolvePartition(partition, partitionKeys, flussRowType);
         }
         this.flussRecordAsPaimonRow =
-                new FlussRecordAsPaimonRow(tableBucket.getBucket(), tableRowType);
+                new FlussRecordAsPaimonRow(tableBucket.getBucket(), tableRowType, lakeLayout);
     }
 
     public abstract void write(LogRecord record) throws Exception;

fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/append/AppendOnlyArrowBatchHelper.java

@@ -17,6 +17,7 @@
 
 package org.apache.fluss.lake.paimon.tiering.append;
 
+import org.apache.fluss.lake.paimon.utils.PaimonSystemColumns.LakeLayout;
 import org.apache.fluss.metadata.TableDescriptor;
 import org.apache.fluss.record.ArrowBatchData;
 
@@ -57,6 +58,7 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable {
     private final TableWriteImpl<InternalRow> tableWrite;
     private final RowType tableRowType;
     private final int bucket;
+    private final LakeLayout lakeLayout;
 
     private static final Field BUCKET_FIELD =
             new Field(
@@ -88,11 +90,13 @@ class AppendOnlyArrowBatchHelper implements AutoCloseable {
             FileStoreTable fileStoreTable,
             TableWriteImpl<InternalRow> tableWrite,
             RowType tableRowType,
-            int bucket) {
+            int bucket,
+            LakeLayout lakeLayout) {
         this.fileStoreTable = fileStoreTable;
         this.tableWrite = tableWrite;
         this.tableRowType = tableRowType;
         this.bucket = bucket;
+        this.lakeLayout = lakeLayout;
     }
 
     /**
@@ -107,6 +111,16 @@ void writeArrowBatch(ArrowBatchData arrowBatchData, BinaryRow partition) throws
         }
 
         VectorSchemaRoot originalRoot = arrowBatchData.getVectorSchemaRoot();
+
+        if (lakeLayout == LakeLayout.CLEAN) {
+            // Clean tables contain only user columns, so the incoming Arrow batch already matches
+            // the Paimon table schema. Write it directly without enriching system columns.
+            ArrowBundleRecords cleanRecords =
+                    new ArrowBundleRecords(originalRoot, tableRowType, false);
+            tableWrite.writeBundle(partition, writtenBucket, cleanRecords);
+            return;
+        }
+
         long baseOffset = arrowBatchData.getBaseLogOffset();
         long timestamp = arrowBatchData.getTimestamp();
         int rowCount = originalRoot.getRowCount();