From 323ab463713851edf475599d5006f25d0d4b6eeb Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 5 Aug 2026 07:32:59 +0800 Subject: [PATCH 01/15] [server] Prepare historical lookup cache generations Allow active historical lookup generations to finish after replacement, refresh lookupers when lake configuration changes, and use table-scoped local cache directories. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 148/173 AI-Contributed/UT: 7/7 --- .../org/apache/fluss/utils/FlussPaths.java | 25 ++- .../replica/HistoricalLakeLookupManager.java | 147 ++++++++++++++---- .../fluss/server/replica/ReplicaManager.java | 1 + .../HistoricalLakeLookupManagerTest.java | 7 +- 4 files changed, 147 insertions(+), 33 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java index 1c75663ba39..6d210706680 100644 --- a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java +++ b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java @@ -37,7 +37,8 @@ import static org.apache.fluss.utils.Preconditions.checkState; /** - * Central place for defining all the paths of kv and log local/remote files/directories. + * Central place for defining all the paths of kv, log, and historical lookup local/remote + * files/directories. * *

All the local file/directories returns the {@link File java.io.File} interface. * @@ -147,6 +148,28 @@ public static File kvTabletDir( return tabletParentDir.resolve(KV_TABLET_DIR_PREFIX + tableBucket.getBucket()).toFile(); } + /** + * Returns the local directory path for storing historical lookup files for a table. + * + *

The path contract: + * + * 

+     * {$lookupRoot}/{databaseName}/{tableName}-{tableId}
+     *
+ * + * @param lookupRoot the historical lookup root directory + * @param tablePath the table path + * @param tableId the table ID + */ + public static File historicalLookupTableDir( + File lookupRoot, TablePath tablePath, long tableId) { + return Paths.get( + lookupRoot.getAbsolutePath(), + tablePath.getDatabaseName(), + tablePath.getTableName() + "-" + tableId) + .toFile(); + } + private static Path tabletParentDir( File dataDir, PhysicalTablePath tablePath, TableBucket tableBucket) { String dbName = tablePath.getDatabaseName(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index fe1b8028d95..977d6d66a41 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -41,6 +41,7 @@ import org.apache.fluss.server.entity.LookupDataForBucket; import org.apache.fluss.utils.ExecutorUtils; import org.apache.fluss.utils.FileUtils; +import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.IOUtils; import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; import org.apache.fluss.utils.concurrent.Scheduler; @@ -49,6 +50,8 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.RemovalCause; import com.github.benmanes.caffeine.cache.Ticker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; @@ -59,6 +62,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -82,7 +86,9 @@ *

Creating a lake table lookuper may initialize catalog, table, and query state and allocate * local lookup files, so lookupers are cached and reused. The cache is keyed by table ID rather * than table path to prevent a deleted and recreated table from reusing the old table's lookuper. A - * cached lookuper is replaced when its schema ID no longer matches the requested table schema. + * cached lookuper is replaced when its schema ID or lake configuration version no longer matches + * the current request. Active lookups can finish on the old lookuper, which is closed after its + * last lookup releases it. * *

A lookuper is closed when replaced, explicitly invalidated by a replica lifecycle event, * evicted after the cache reaches ten tables, the manager shuts down, or after three hours without @@ -91,6 +97,8 @@ */ class HistoricalLakeLookupManager implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(HistoricalLakeLookupManager.class); + private static final String PAIMON_LOOKUP_DIR_NAME = "paimon-lookup"; private static final String LOOKUPER_CACHE_EXPIRATION_TASK_NAME = "historical-lookuper-cache-expiration"; @@ -107,7 +115,8 @@ class HistoricalLakeLookupManager implements AutoCloseable { // table lookupers because fixed per-table limits can underutilize cache for hot tables while // reserving too much for cold tables. - private final Configuration conf; + private volatile Configuration conf; + private volatile long lakeConfigVersion; private final @Nullable PluginManager pluginManager; private final int serverId; private final Semaphore lookupPermits; @@ -115,7 +124,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { private final Set> pendingLookups; private final Cache lakeTableLookupers; private final ExecutorService historicalPartitionExecutor; - private @Nullable String paimonLookupTempDir; + private @Nullable File paimonLookupTempDir; HistoricalLakeLookupManager( Configuration conf, @@ -259,12 +268,25 @@ void invalidateTableLookuper(long tableId) { lakeTableLookupers.invalidate(tableId); } + synchronized void reconfigure(Configuration newConf) { + checkNotNull(newConf, "newConf must not be null."); + boolean lakeConfigChanged = hasLakeConfigChanged(conf, newConf); + // Publish the configuration before its version. A lookup that observes the new version + // must also observe the matching configuration snapshot. + conf = newConf; + if (lakeConfigChanged) { + lakeConfigVersion++; + } + } + private LookupResultForBucket lookupInternal( LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo) { TableBucket tableBucket = lookupData.tableBucket(); CachedLakeTableLookuper cachedLookuper = null; try { LookupContext context = createLookupContext(lookupData, tableInfo, schemaInfo); + long currentLakeConfigVersion = lakeConfigVersion; + Configuration currentConf = conf; cachedLookuper = lakeTableLookupers .asMap() @@ -273,18 +295,18 @@ private LookupResultForBucket lookupInternal( (ignored, currentLookuper) -> { CachedLakeTableLookuper selectedLookuper = currentLookuper; // Create the lookuper lazily, and recreate it after schema - // evolution so it reloads lake table/query state and - // encodes values with the requested schema. + // or lake configuration changes so it reloads lake + // table/query state and uses the current configuration. if (selectedLookuper == null - || selectedLookuper.schemaId != context.schemaId) { - LakeTableLookuper newLookuper = - createLakeTableLookuper( - context.tablePath, - getOrPreparePaimonLookupTempDir(), - tableInfo.getTableConfig()); + || selectedLookuper.schemaId != context.schemaId + || selectedLookuper.lakeConfigVersion + != currentLakeConfigVersion) { selectedLookuper = - new CachedLakeTableLookuper( - context.schemaId, newLookuper); + createCachedLookuper( + context, + tableInfo.getTableConfig(), + currentConf, + currentLakeConfigVersion); } // Pin the lookuper before leaving the atomic cache update. // Eviction or invalidation can then defer closing it until @@ -311,6 +333,26 @@ private LookupResultForBucket lookupInternal( } } + private CachedLakeTableLookuper createCachedLookuper( + LookupContext context, + TableConfig tableConfig, + Configuration clusterConf, + long currentLakeConfigVersion) { + File tableLookupDir = + FlussPaths.historicalLookupTableDir( + getOrPreparePaimonLookupTempDir(clusterConf), + context.tablePath, + context.tableId); + LakeTableLookuper lookuper = + createLakeTableLookuper( + context.tablePath, + tableLookupDir.getAbsolutePath(), + tableConfig, + clusterConf); + return new CachedLakeTableLookuper( + context.schemaId, currentLakeConfigVersion, tableLookupDir, lookuper); + } + private LookupContext createLookupContext( LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo) { TableBucket tableBucket = lookupData.tableBucket(); @@ -345,8 +387,11 @@ private LookupContext createLookupContext( } LakeTableLookuper createLakeTableLookuper( - TablePath tablePath, String ioTmpDir, TableConfig tableConfig) { - DataLakeFormat dataLakeFormat = conf.get(ConfigOptions.DATALAKE_FORMAT); + TablePath tablePath, + String ioTmpDir, + TableConfig tableConfig, + Configuration clusterConf) { + DataLakeFormat dataLakeFormat = clusterConf.get(ConfigOptions.DATALAKE_FORMAT); if (dataLakeFormat == null) { throw new LakeStorageNotConfiguredException( "Historical lookup requires cluster lake storage to be configured."); @@ -358,7 +403,7 @@ LakeTableLookuper createLakeTableLookuper( dataLakeFormat)); } - Map lakeProperties = extractLakeProperties(conf); + Map lakeProperties = extractLakeProperties(clusterConf); if (lakeProperties == null) { throw new LakeStorageNotConfiguredException( "Historical lookup requires cluster lake storage properties to be configured."); @@ -372,14 +417,21 @@ LakeTableLookuper createLakeTableLookuper( tablePath, new LakeStorage.LookuperContext(ioTmpDir, tableConfig)); } - private synchronized String getOrPreparePaimonLookupTempDir() { + private static boolean hasLakeConfigChanged(Configuration currentConf, Configuration newConf) { + return currentConf.get(ConfigOptions.DATALAKE_FORMAT) + != newConf.get(ConfigOptions.DATALAKE_FORMAT) + || !Objects.equals( + extractLakeProperties(currentConf), extractLakeProperties(newConf)); + } + + private synchronized File getOrPreparePaimonLookupTempDir(Configuration clusterConf) { if (paimonLookupTempDir == null) { - paimonLookupTempDir = preparePaimonLookupTempDir(conf, serverId); + paimonLookupTempDir = preparePaimonLookupTempDir(clusterConf, serverId); } return paimonLookupTempDir; } - private static String preparePaimonLookupTempDir(Configuration conf, int serverId) { + private static File preparePaimonLookupTempDir(Configuration conf, int serverId) { File paimonLookupTempDir = new File( new File(conf.get(ConfigOptions.SERVER_IO_TMP_DIR), PAIMON_LOOKUP_DIR_NAME), @@ -390,7 +442,7 @@ private static String preparePaimonLookupTempDir(Configuration conf, int serverI // lookuper; cleaning in each table lookuper would delete files used by other tables. FileUtils.deleteDirectory(paimonLookupTempDir); Files.createDirectories(paimonLookupTempDir.toPath()); - return paimonLookupTempDir.getAbsolutePath(); + return paimonLookupTempDir; } catch (IOException e) { throw new FlussRuntimeException( "Failed to prepare Paimon lookup temporary directory: " + paimonLookupTempDir, @@ -399,7 +451,24 @@ private static String preparePaimonLookupTempDir(Configuration conf, int serverI } private static void closeLookuper(CachedLakeTableLookuper cachedLookuper) { - IOUtils.closeQuietly(cachedLookuper.lookuper, "historical lake table lookuper"); + try { + IOUtils.closeQuietly(cachedLookuper.lookuper, "historical lake table lookuper"); + } finally { + deleteTableLookupDirIfEmpty(cachedLookuper.tableLookupDir); + } + } + + private static void deleteTableLookupDirIfEmpty(File tableLookupDir) { + if (FileUtils.isDirectoryEmpty(tableLookupDir)) { + try { + Files.deleteIfExists(tableLookupDir.toPath()); + } catch (IOException e) { + LOG.debug( + "Failed to delete empty historical lookup directory {}.", + tableLookupDir, + e); + } + } } private static final class LookupContext { @@ -422,13 +491,21 @@ private LookupContext( private static final class CachedLakeTableLookuper { private final int schemaId; + private final long lakeConfigVersion; + private final File tableLookupDir; private final LakeTableLookuper lookuper; private int activeLookups; private boolean invalidated; private boolean closed; - private CachedLakeTableLookuper(int schemaId, LakeTableLookuper lookuper) { + private CachedLakeTableLookuper( + int schemaId, + long lakeConfigVersion, + File tableLookupDir, + LakeTableLookuper lookuper) { this.schemaId = schemaId; + this.lakeConfigVersion = lakeConfigVersion; + this.tableLookupDir = tableLookupDir; this.lookuper = lookuper; } @@ -439,22 +516,32 @@ private synchronized void acquire() { activeLookups++; } - private synchronized void release() { - if (activeLookups <= 0) { - throw new IllegalStateException("Lake table lookuper is not acquired."); + private void release() { + synchronized (this) { + if (activeLookups <= 0) { + throw new IllegalStateException("Lake table lookuper is not acquired."); + } + activeLookups--; } - activeLookups--; closeIfUnused(); } - private synchronized void invalidate() { - invalidated = true; + private void invalidate() { + synchronized (this) { + invalidated = true; + } closeIfUnused(); } private void closeIfUnused() { - if (invalidated && activeLookups == 0 && !closed) { - closed = true; + boolean shouldClose; + synchronized (this) { + shouldClose = invalidated && activeLookups == 0 && !closed; + if (shouldClose) { + closed = true; + } + } + if (shouldClose) { closeLookuper(this); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 5b0bebfc3e2..2d00371b007 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -415,6 +415,7 @@ public void validate(Configuration newConfig) throws ConfigException { @Override public void reconfigure(Configuration newConfig) { + historicalLakeLookupManager.reconfigure(newConfig); int newMinInSyncReplicas = newConfig.get(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER); if (newMinInSyncReplicas == minInSyncReplicas) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index ecf3a5e4e2e..cc9d31c2330 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -204,7 +204,7 @@ void testLazilyCleansPaimonLookupTempDirectory() throws Exception { lookupAndRun(manager, executor, PARTITION_TABLE_INFO); assertThat(staleLookupFile).doesNotExist(); assertThat(serverLookupDir).isDirectory(); - assertThat(manager.createdIoTmpDirs).containsExactly(serverLookupDir.getAbsolutePath()); + assertThat(manager.createdIoTmpDirs.get(0)).startsWith(serverLookupDir.getAbsolutePath()); } @Test @@ -417,7 +417,10 @@ private TestingHistoricalLakeLookupManager( @Override LakeTableLookuper createLakeTableLookuper( - TablePath tablePath, String ioTmpDir, TableConfig tableConfig) { + TablePath tablePath, + String ioTmpDir, + TableConfig tableConfig, + Configuration clusterConf) { TestingLakeTableLookuper lookuper = new TestingLakeTableLookuper(); createdLookupers.add(lookuper); createdIoTmpDirs.add(ioTmpDir); From 546300155ef0704a3cde96c2c1f00f43e5a0dfb9 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 5 Aug 2026 11:46:58 +0800 Subject: [PATCH 02/15] [server] Manage historical lookup cache disk capacity Add server and table cache capacity settings, validate table limits, reserve configured capacity atomically, and evict lookupers in best-effort LRU order. Expose cached table and eviction metrics and pass per-table limits to Paimon. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 404/638 AI-Contributed/UT: 256/396 --- .../apache/fluss/config/ConfigOptions.java | 27 ++ .../apache/fluss/config/FlussConfigUtils.java | 19 + .../org/apache/fluss/config/TableConfig.java | 6 + .../org/apache/fluss/metrics/MetricNames.java | 5 + .../fluss/config/FlussConfigUtilsTest.java | 34 ++ .../apache/fluss/config/TableConfigTest.java | 14 + .../lookup/PaimonLakeTableLookuper.java | 18 +- .../coordinator/CoordinatorService.java | 2 +- .../server/coordinator/MetadataManager.java | 12 +- .../replica/HistoricalLakeLookupManager.java | 339 +++++++++++++++--- .../HistoricalLookupCacheBudgetManager.java | 167 +++++++++ .../fluss/server/replica/ReplicaManager.java | 6 + .../utils/TableDescriptorValidation.java | 37 +- .../HistoricalLakeLookupManagerTest.java | 222 +++++++++++- ...istoricalLookupCacheBudgetManagerTest.java | 73 ++++ ...istoricalPartitionTableValidationTest.java | 53 ++- 16 files changed, 951 insertions(+), 83 deletions(-) create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 23c50097ba6..f3609c810cd 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -428,6 +428,24 @@ public class ConfigOptions { "The maximum number of threads used for historical partition operations, such as lake lookups and writes. " + "Threads are started lazily and released after the keep-alive timeout when idle."); + public static final ConfigOption + SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE = + key("server.historical-partition.lookup-cache.max-disk-size") + .memoryType() + .defaultValue(MemorySize.parse("80gb")) + .withDescription( + "The total configured disk capacity available to current and creating historical partition lookup caches on a TabletServer. " + + "Retired cache generations that are still serving active lookups are not included in this limit."); + + public static final ConfigOption + SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS = + key("server.historical-partition.lookuper-cache.expire-after-access") + .durationType() + .defaultValue(Duration.ofHours(3)) + .withDescription( + "The duration after which an idle historical partition table lookuper is removed from the cache. " + + "This option requires a TabletServer restart to take effect."); + public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = key("server.data-disk.write-limit-ratio") .doubleType() @@ -1858,6 +1876,15 @@ public class ConfigOptions { + "to look up historical partition data so that their clients load the " + "updated table configuration."); + public static final ConfigOption + TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE = + key("table.datalake.historical-partition.lookup-cache.max-disk-size") + .memoryType() + .defaultValue(MemorySize.parse("8gb")) + .withDescription( + "The maximum local disk capacity reserved for this table's historical partition lookup cache on each TabletServer. " + + "The value must be greater than zero and no greater than the TabletServer historical lookup cache limit."); + public static final ConfigOption TABLE_DATALAKE_FORMAT = key("table.datalake.format") .enumType(DataLakeFormat.class) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index 593b0d61741..2e1899b812c 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -222,6 +222,11 @@ protected static void validateServerConfigs(Configuration conf) { validMinValue(conf, ConfigOptions.SERVER_IO_POOL_SIZE, 1); validMinValue(conf, ConfigOptions.BACKGROUND_THREADS, 1); validMinDuration(conf, ConfigOptions.LOG_RETENTION_CHECK_INTERVAL, 1); + validMinDuration( + conf, + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS, + 1); + validateHistoricalLookupCacheLimit(conf); if (conf.get(ConfigOptions.LOG_SEGMENT_FILE_SIZE).getBytes() > Integer.MAX_VALUE) { throw new IllegalConfigurationException( @@ -231,6 +236,20 @@ protected static void validateServerConfigs(Configuration conf) { } } + private static void validateHistoricalLookupCacheLimit(Configuration conf) { + MemorySize historicalLookupCacheMaxSize = + conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); + MemorySize defaultTableHistoricalLookupCacheSize = + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .defaultValue(); + if (historicalLookupCacheMaxSize.compareTo(defaultTableHistoricalLookupCacheSize) < 0) { + throw new IllegalConfigurationException( + "Invalid configuration for %s, it must be greater than or equal to the default table cache size %s.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key(), + defaultTableHistoricalLookupCacheSize); + } + } + private static void validMinValue( Configuration conf, ConfigOption option, int minValue) { validMinValue(option, conf.get(option), minValue); diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index 931c33e9a75..4e72aedd100 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -105,6 +105,12 @@ public boolean isHistoricalPartitionEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED); } + /** Gets the maximum local disk size of the historical partition lookup cache. */ + public MemorySize getHistoricalPartitionLookupCacheMaxDiskSize() { + return config.get( + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); + } + /** * Return the data lake format of the table. It'll be the datalake format configured in Fluss * whiling creating the table. Return empty if no datalake format configured while creating. diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java index 7a9c50a86d9..d973933f211 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java +++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java @@ -102,6 +102,11 @@ public class MetricNames { public static final String DISK_USAGE_RATIO = "diskUsageRatio"; public static final String DISK_WRITE_LOCKED = "diskWriteLocked"; + // for historical lookup cache + public static final String HISTORICAL_LOOKUP_CACHED_TABLE_COUNT = + "historicalLookupCachedTableCount"; + public static final String HISTORICAL_LOOKUP_CACHE_EVICTIONS = "historicalLookupCacheEvictions"; + // -------------------------------------------------------------------------------------------- // metrics for user // -------------------------------------------------------------------------------------------- diff --git a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java index 2b9e815ee7d..3ef097b7632 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java @@ -215,6 +215,40 @@ void testValidateLogRetentionCheckInterval() { validateCoordinatorConfigs(conf); } + @Test + void testValidateHistoricalLookupCacheConfigs() { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.REMOTE_DATA_DIR, "s3://bucket/path"); + conf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + MemorySize.parse("4gb")); + + assertThatThrownBy(() -> validateCoordinatorConfigs(conf)) + .isInstanceOf(IllegalConfigurationException.class) + .hasMessageContaining( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key()) + .hasMessageContaining("8 gb"); + + conf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + MemorySize.parse("8gb")); + conf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS, + Duration.ZERO); + assertThatThrownBy(() -> validateCoordinatorConfigs(conf)) + .isInstanceOf(IllegalConfigurationException.class) + .hasMessageContaining( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS + .key()); + + assertThat( + FlussConfigUtils.isAlterableTableOption( + ConfigOptions + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key())) + .isFalse(); + } + @Test void testValidateClientConfigs() { // valid defaults should pass diff --git a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java index 5d18fcd1c97..9b9312543f5 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java @@ -44,4 +44,18 @@ void testDeleteBehavior() { TableConfig tableConfig3 = new TableConfig(conf); assertThat(tableConfig3.getDeleteBehavior()).hasValue(DeleteBehavior.IGNORE); } + + @Test + void testHistoricalPartitionLookupCacheMaxDiskSize() { + Configuration conf = new Configuration(); + TableConfig tableConfig = new TableConfig(conf); + assertThat(tableConfig.getHistoricalPartitionLookupCacheMaxDiskSize()) + .isEqualTo(MemorySize.parse("8gb")); + + conf.set( + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + MemorySize.parse("16gb")); + assertThat(new TableConfig(conf).getHistoricalPartitionLookupCacheMaxDiskSize()) + .isEqualTo(MemorySize.parse("16gb")); + } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index d15a182ed18..ee1e56bdf5e 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -41,7 +41,6 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.memory.MemorySegment; -import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.query.LocalTableQuery; @@ -86,13 +85,6 @@ */ public class PaimonLakeTableLookuper implements LakeTableLookuper { - // Each TabletServer caches at most ten table lookupers, bounding their retained lookup cache - // capacity to 20GB. See HistoricalLakeLookupManager for the follow-up to make this configurable - // and use a global Paimon IOManager limit. - private static final String LOOKUP_CACHE_MAX_DISK_SIZE = "2gb"; - private static final MemorySize LOOKUP_CACHE_MAX_DISK_MEMORY_SIZE = - MemorySize.parse(LOOKUP_CACHE_MAX_DISK_SIZE); - private final Configuration paimonConfig; private final TablePath tablePath; private final String ioTmpDir; @@ -239,14 +231,8 @@ private void ensureInitialized(RowType valueRowType) throws Exception { private FileStoreTable withLookupCacheOptions(FileStoreTable table) { String key = CoreOptions.LOOKUP_CACHE_MAX_DISK_SIZE.key(); - String configuredMaxDiskSize = table.options().get(key); - if (configuredMaxDiskSize != null - && MemorySize.parse(configuredMaxDiskSize) - .compareTo(LOOKUP_CACHE_MAX_DISK_MEMORY_SIZE) - <= 0) { - return table; - } - return table.copy(Collections.singletonMap(key, LOOKUP_CACHE_MAX_DISK_SIZE)); + String maxDiskSize = tableConfig.getHistoricalPartitionLookupCacheMaxDiskSize().toString(); + return table.copy(Collections.singletonMap(key, maxDiskSize)); } private static IOManager createIOManager(String ioTmpDir) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index 04119bb0fd9..e35d03f20b8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -486,7 +486,7 @@ public CompletableFuture createTable(CreateTableRequest req // validate table descriptor before creating table in lake or fluss metadata, // to avoid orphaned lake tables when validation fails - metadataManager.validateTableDescriptor(tableDescriptor); + metadataManager.validateTableDescriptor(tablePath, tableDescriptor); // the distribution and bucket count must be set now //noinspection OptionalGetWithoutIsPresent diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 43d98434252..c489fed8671 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.DatabaseAlreadyExistException; import org.apache.fluss.exception.DatabaseNotEmptyException; import org.apache.fluss.exception.DatabaseNotExistException; @@ -87,6 +88,7 @@ public class MetadataManager { private final ZooKeeperClient zookeeperClient; private final int maxPartitionNum; private final int maxBucketNum; + private final MemorySize historicalLookupCacheMaxSize; private final LakeCatalogDynamicLoader lakeCatalogDynamicLoader; public static final Set SENSITIVE_TABLE_OPTIONS = new HashSet<>(); @@ -110,15 +112,19 @@ public MetadataManager( this.zookeeperClient = zookeeperClient; this.maxPartitionNum = conf.get(ConfigOptions.MAX_PARTITION_NUM); this.maxBucketNum = conf.get(ConfigOptions.MAX_BUCKET_NUM); + this.historicalLookupCacheMaxSize = + conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); this.lakeCatalogDynamicLoader = lakeCatalogDynamicLoader; } /** Validates the table descriptor. */ - public void validateTableDescriptor(TableDescriptor tableDescriptor) { + public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDescriptor) { TableDescriptorValidation.validateTableDescriptor( tableDescriptor, maxBucketNum, - lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat()); + lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat(), + tablePath, + historicalLookupCacheMaxSize); } public void createDatabase( @@ -551,7 +557,7 @@ public void alterTableProperties( } // reuse the same validate logic with the createTable() method - validateTableDescriptor(newDescriptor); + validateTableDescriptor(tablePath, newDescriptor); beforeUpdate.accept(tableInfo, newDescriptor); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index 977d6d66a41..bb7f2ab07dc 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -35,10 +35,13 @@ import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.metrics.Counter; +import org.apache.fluss.metrics.ThreadSafeSimpleCounter; import org.apache.fluss.plugin.PluginManager; import org.apache.fluss.rpc.entity.LookupResultForBucket; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.server.entity.LookupDataForBucket; +import org.apache.fluss.server.replica.HistoricalLookupCacheBudgetManager.Reservation; import org.apache.fluss.utils.ExecutorUtils; import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; @@ -60,6 +63,7 @@ import java.nio.file.Files; import java.time.Duration; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -91,9 +95,10 @@ * last lookup releases it. * *

A lookuper is closed when replaced, explicitly invalidated by a replica lifecycle event, - * evicted after the cache reaches ten tables, the manager shuts down, or after three hours without - * access. Caffeine expiration is scheduled on the shared TabletServer scheduler, allowing idle - * resources to be released even if no subsequent lookup accesses the cache. + * evicted to admit another table within the configured disk budget, the manager shuts down, or + * after the configured idle expiration. Caffeine expiration is scheduled on the shared TabletServer + * scheduler, allowing idle resources to be released even if no subsequent lookup accesses the + * cache. */ class HistoricalLakeLookupManager implements AutoCloseable { @@ -102,23 +107,18 @@ class HistoricalLakeLookupManager implements AutoCloseable { private static final String PAIMON_LOOKUP_DIR_NAME = "paimon-lookup"; private static final String LOOKUPER_CACHE_EXPIRATION_TASK_NAME = "historical-lookuper-cache-expiration"; - private static final Duration LOOKUPER_CACHE_EXPIRATION = Duration.ofHours(3); private static final Duration HISTORICAL_PARTITION_THREAD_KEEP_ALIVE = Duration.ofMinutes(10); private static final Duration HISTORICAL_PARTITION_EXECUTOR_SHUTDOWN_TIMEOUT = Duration.ofSeconds(10); - private static final int MAX_CACHED_LOOKUPERS = 10; private static final String HISTORICAL_PARTITION_THREAD_NAME_PREFIX = "historical-partition-io"; - // TODO: MAX_CACHED_LOOKUPERS and the 2GB per-table limit configured by - // PaimonLakeTableLookuper through Paimon's "lookup.cache-max-disk-size" option are hard-coded. - // Make them configurable, and prefer a Paimon IOManager-level global disk limit shared by all - // table lookupers because fixed per-table limits can underutilize cache for hot tables while - // reserving too much for cold tables. - private volatile Configuration conf; private volatile long lakeConfigVersion; private final @Nullable PluginManager pluginManager; private final int serverId; + private final Ticker ticker; + private final HistoricalLookupCacheBudgetManager budgetManager; + private final Counter capacityEvictions; private final Semaphore lookupPermits; // Accepted lookup futures tracked so close() can cancel tasks left after executor shutdown. private final Set> pendingLookups; @@ -151,6 +151,14 @@ class HistoricalLakeLookupManager implements AutoCloseable { this.conf = checkNotNull(conf, "conf must not be null."); this.pluginManager = pluginManager; this.serverId = serverId; + this.ticker = checkNotNull(ticker, "ticker must not be null."); + this.budgetManager = + new HistoricalLookupCacheBudgetManager( + conf.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE) + .getBytes()); + this.capacityEvictions = new ThreadSafeSimpleCounter(); int maxQueuedHistoricalRequests = conf.get(ConfigOptions.NETTY_SERVER_MAX_QUEUED_HISTORICAL_REQUESTS); checkArgument( @@ -169,9 +177,11 @@ class HistoricalLakeLookupManager implements AutoCloseable { : historicalPartitionExecutor; this.lakeTableLookupers = Caffeine.newBuilder() - .expireAfterAccess(LOOKUPER_CACHE_EXPIRATION) - .maximumSize(MAX_CACHED_LOOKUPERS) - .ticker(checkNotNull(ticker, "ticker must not be null.")) + .expireAfterAccess( + conf.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS)) + .ticker(this.ticker) .scheduler(checkNotNull(cacheScheduler, "cacheScheduler must not be null.")) .executor(Runnable::run) .removalListener( @@ -179,7 +189,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { CachedLakeTableLookuper cachedLookuper, RemovalCause ignoredCause) -> { if (cachedLookuper != null) { - cachedLookuper.invalidate(); + onLookuperRemoved(cachedLookuper); } }) .build(); @@ -268,6 +278,14 @@ void invalidateTableLookuper(long tableId) { lakeTableLookupers.invalidate(tableId); } + int cachedTableCount() { + return lakeTableLookupers.asMap().size(); + } + + Counter capacityEvictions() { + return capacityEvictions; + } + synchronized void reconfigure(Configuration newConf) { checkNotNull(newConf, "newConf must not be null."); boolean lakeConfigChanged = hasLakeConfigChanged(conf, newConf); @@ -287,33 +305,16 @@ private LookupResultForBucket lookupInternal( LookupContext context = createLookupContext(lookupData, tableInfo, schemaInfo); long currentLakeConfigVersion = lakeConfigVersion; Configuration currentConf = conf; + TableConfig tableConfig = tableInfo.getTableConfig(); + long cacheSizeBytes = + tableConfig.getHistoricalPartitionLookupCacheMaxDiskSize().getBytes(); cachedLookuper = - lakeTableLookupers - .asMap() - .compute( - context.tableId, - (ignored, currentLookuper) -> { - CachedLakeTableLookuper selectedLookuper = currentLookuper; - // Create the lookuper lazily, and recreate it after schema - // or lake configuration changes so it reloads lake - // table/query state and uses the current configuration. - if (selectedLookuper == null - || selectedLookuper.schemaId != context.schemaId - || selectedLookuper.lakeConfigVersion - != currentLakeConfigVersion) { - selectedLookuper = - createCachedLookuper( - context, - tableInfo.getTableConfig(), - currentConf, - currentLakeConfigVersion); - } - // Pin the lookuper before leaving the atomic cache update. - // Eviction or invalidation can then defer closing it until - // this lookup releases it. - selectedLookuper.acquire(); - return selectedLookuper; - }); + acquireCachedLookuper( + context, + tableConfig, + currentConf, + currentLakeConfigVersion, + cacheSizeBytes); List values = new ArrayList<>(lookupData.keys().size()); for (byte[] key : lookupData.keys()) { values.add(cachedLookuper.lookuper.lookup(key, context.lookupContext)); @@ -333,24 +334,235 @@ private LookupResultForBucket lookupInternal( } } - private CachedLakeTableLookuper createCachedLookuper( + private CachedLakeTableLookuper acquireCachedLookuper( LookupContext context, TableConfig tableConfig, Configuration clusterConf, + long currentLakeConfigVersion, + long cacheSizeBytes) { + CachedLakeTableLookuper cachedLookuper = + tryAcquireCachedLookuper( + context, + tableConfig, + clusterConf, + currentLakeConfigVersion, + cacheSizeBytes); + if (cachedLookuper != null) { + return cachedLookuper; + } + + int maxEvictions = lakeTableLookupers.asMap().size(); + for (int evictions = 0; evictions < maxEvictions; evictions++) { + // Evict only after compute releases the target table's cache lock. Updating a + // different table mapping from inside compute can deadlock with a concurrent + // replacement performing the inverse update. + if (!evictLeastRecentlyUsed(context.tableId)) { + break; + } + cachedLookuper = + tryAcquireCachedLookuper( + context, + tableConfig, + clusterConf, + currentLakeConfigVersion, + cacheSizeBytes); + if (cachedLookuper != null) { + return cachedLookuper; + } + } + throw capacityThrottledException(context, cacheSizeBytes); + } + + /** + * Makes one atomic attempt to acquire a matching cached lookuper without evicting other tables. + * + * @return the acquired lookuper, or {@code null} if its capacity cannot be reserved + */ + private @Nullable CachedLakeTableLookuper tryAcquireCachedLookuper( + LookupContext context, + TableConfig tableConfig, + Configuration clusterConf, + long currentLakeConfigVersion, + long cacheSizeBytes) { + CachedLakeTableLookuper cachedLookuper = + lakeTableLookupers + .asMap() + .compute( + context.tableId, + (ignored, currentLookuper) -> { + CachedLakeTableLookuper selectedLookuper = currentLookuper; + // Create the lookuper lazily, and recreate it after schema or + // lake configuration changes so it reloads lake table/query + // state and uses the current configuration. + if (!matchesLookupConfiguration( + selectedLookuper, context, currentLakeConfigVersion)) { + selectedLookuper = + tryCreateCachedLookuper( + context, + tableConfig, + clusterConf, + currentLakeConfigVersion, + cacheSizeBytes, + currentLookuper); + if (selectedLookuper == null) { + // Preserve the current mapping and leave compute before + // attempting to evict another table. + return currentLookuper; + } + } + // Pin the lookuper before leaving the atomic cache update. + // Eviction or invalidation can then defer closing it until this + // lookup releases it. + selectedLookuper.acquire(ticker.read()); + return selectedLookuper; + }); + return matchesLookupConfiguration(cachedLookuper, context, currentLakeConfigVersion) + ? cachedLookuper + : null; + } + + private static boolean matchesLookupConfiguration( + @Nullable CachedLakeTableLookuper cachedLookuper, + LookupContext context, long currentLakeConfigVersion) { + return cachedLookuper != null + && cachedLookuper.schemaId == context.schemaId + && cachedLookuper.lakeConfigVersion == currentLakeConfigVersion; + } + + /** + * Creates a lookuper after atomically reserving its configured cache capacity. + * + * @return the new cached lookuper, or {@code null} if the capacity cannot be reserved + */ + private @Nullable CachedLakeTableLookuper tryCreateCachedLookuper( + LookupContext context, + TableConfig tableConfig, + Configuration clusterConf, + long currentLakeConfigVersion, + long cacheSizeBytes, + @Nullable CachedLakeTableLookuper currentLookuper) { File tableLookupDir = FlussPaths.historicalLookupTableDir( getOrPreparePaimonLookupTempDir(clusterConf), context.tablePath, context.tableId); + if (currentLookuper == null) { + // A cache miss must obtain capacity before creating any local lookup resources. + Reservation reservation = budgetManager.tryReserve(context.tableId, cacheSizeBytes); + if (reservation == null) { + return null; + } + try { + LakeTableLookuper lookuper = + createLakeTableLookuper( + context.tablePath, + tableLookupDir.getAbsolutePath(), + tableConfig, + clusterConf); + return new CachedLakeTableLookuper( + context.tableId, + context.tablePath, + context.schemaId, + currentLakeConfigVersion, + cacheSizeBytes, + tableLookupDir, + reservation, + lookuper); + } catch (Throwable throwable) { + budgetManager.release(reservation); + throw throwable; + } + } + + // Build the replacement first so a creation failure leaves the current lookuper and its + // reservation unchanged in the cache. LakeTableLookuper lookuper = createLakeTableLookuper( context.tablePath, tableLookupDir.getAbsolutePath(), tableConfig, clusterConf); + // Replace the reservation atomically: the old and replacement cache sizes never count + // against the global budget at the same time. + Reservation reservation = + budgetManager.tryReplace(currentLookuper.reservation, cacheSizeBytes); + if (reservation == null) { + // The candidate was never published, while the current lookuper remains usable. + closeLookuper(lookuper, tableLookupDir); + return null; + } return new CachedLakeTableLookuper( - context.schemaId, currentLakeConfigVersion, tableLookupDir, lookuper); + context.tableId, + context.tablePath, + context.schemaId, + currentLakeConfigVersion, + cacheSizeBytes, + tableLookupDir, + reservation, + lookuper); + } + + /** + * Evicts one eligible cached lookuper using best-effort LRU order. + * + *

Candidates are ordered by their last-access timestamps without blocking concurrent + * lookups. A candidate accessed after it is ordered may therefore still be evicted. + */ + private boolean evictLeastRecentlyUsed(long excludedTableId) { + List candidates = + new ArrayList<>(lakeTableLookupers.asMap().values()); + candidates.sort(Comparator.comparingLong(CachedLakeTableLookuper::lastAccessNanos)); + for (CachedLakeTableLookuper candidate : candidates) { + if (candidate.tableId == excludedTableId) { + continue; + } + // Claim this victim so concurrent admission threads cannot evict it twice. A false + // result means it was already invalidated or claimed by another eviction. + if (!candidate.markEvictionPending()) { + continue; + } + + // The snapshot may be stale after expiration or replacement. Compare-and-remove + // prevents this eviction from removing a newer lookuper for the same table. + boolean removed = lakeTableLookupers.asMap().remove(candidate.tableId, candidate); + if (!removed) { + candidate.clearEvictionPending(); + continue; + } + + // The direct Caffeine executor normally invokes the listener inline. Repeat the + // transition explicitly so admission does not depend on listener scheduling. + onLookuperRemoved(candidate); + capacityEvictions.inc(); + LOG.info( + "Evicted historical lookup cache for table {} (table ID {}, cache size {} bytes, reserved {} of {} bytes).", + candidate.tablePath, + candidate.tableId, + candidate.cacheSizeBytes, + budgetManager.reservedBytes(), + budgetManager.maxBytes()); + return true; + } + return false; + } + + private HistoricalPartitionThrottledException capacityThrottledException( + LookupContext context, long cacheSizeBytes) { + return new HistoricalPartitionThrottledException( + String.format( + "Historical lookup cache capacity is unavailable for table %s (table ID %s): requested %s bytes, reserved %s of %s bytes across %s cached tables.", + context.tablePath, + context.tableId, + cacheSizeBytes, + budgetManager.reservedBytes(), + budgetManager.maxBytes(), + cachedTableCount())); + } + + private void onLookuperRemoved(CachedLakeTableLookuper cachedLookuper) { + budgetManager.release(cachedLookuper.reservation); + cachedLookuper.invalidate(); } private LookupContext createLookupContext( @@ -451,10 +663,14 @@ private static File preparePaimonLookupTempDir(Configuration conf, int serverId) } private static void closeLookuper(CachedLakeTableLookuper cachedLookuper) { + closeLookuper(cachedLookuper.lookuper, cachedLookuper.tableLookupDir); + } + + private static void closeLookuper(LakeTableLookuper lookuper, File tableLookupDir) { try { - IOUtils.closeQuietly(cachedLookuper.lookuper, "historical lake table lookuper"); + IOUtils.closeQuietly(lookuper, "historical lake table lookuper"); } finally { - deleteTableLookupDirIfEmpty(cachedLookuper.tableLookupDir); + deleteTableLookupDirIfEmpty(tableLookupDir); } } @@ -490,32 +706,63 @@ private LookupContext( } private static final class CachedLakeTableLookuper { + private final long tableId; + private final TablePath tablePath; private final int schemaId; private final long lakeConfigVersion; + private final long cacheSizeBytes; private final File tableLookupDir; + private final Reservation reservation; private final LakeTableLookuper lookuper; + private long lastAccessNanos; private int activeLookups; + private boolean evictionPending; private boolean invalidated; private boolean closed; private CachedLakeTableLookuper( + long tableId, + TablePath tablePath, int schemaId, long lakeConfigVersion, + long cacheSizeBytes, File tableLookupDir, + Reservation reservation, LakeTableLookuper lookuper) { + this.tableId = tableId; + this.tablePath = tablePath; this.schemaId = schemaId; this.lakeConfigVersion = lakeConfigVersion; + this.cacheSizeBytes = cacheSizeBytes; this.tableLookupDir = tableLookupDir; + this.reservation = reservation; this.lookuper = lookuper; } - private synchronized void acquire() { + private synchronized void acquire(long accessNanos) { if (invalidated) { throw new IllegalStateException("Lake table lookuper has been invalidated."); } + lastAccessNanos = accessNanos; activeLookups++; } + private synchronized long lastAccessNanos() { + return lastAccessNanos; + } + + private synchronized boolean markEvictionPending() { + if (invalidated || evictionPending) { + return false; + } + evictionPending = true; + return true; + } + + private synchronized void clearEvictionPending() { + evictionPending = false; + } + private void release() { synchronized (this) { if (activeLookups <= 0) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java new file mode 100644 index 00000000000..da1a4f16d6c --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** + * Tracks the configured disk capacity reserved by historical lookupers. + * + *

This manager accounts for configured cache capacity, not the bytes currently present on disk. + * A reservation belongs to a table's current or creating lookuper. Once that lookuper is removed + * from the cache mapping, its reservation is released immediately even if active requests keep the + * retired lookuper alive for a short time. + * + *

All mutable state is protected by this instance's monitor. The following invariants therefore + * hold after every operation: + * + *

+ */ +@ThreadSafe +final class HistoricalLookupCacheBudgetManager { + + // The configured limit is immutable in this version, so readers do not need synchronization. + private final long maxBytes; + + // Contains only reservations that currently count against the budget. Retired lookupers are + // deliberately absent even when they are still serving an already acquired lookup. + @GuardedBy("this") + private final Map reservationsByTableId = new HashMap<>(); + + @GuardedBy("this") + private long reservedBytes; + + /** Creates a budget manager with the given positive capacity limit. */ + HistoricalLookupCacheBudgetManager(long maxBytes) { + checkArgument(maxBytes > 0, "maxBytes must be greater than 0."); + this.maxBytes = maxBytes; + } + + /** + * Tries to reserve capacity for a new table lookuper. + * + *

The check and reservation insertion are one atomic operation. A request fails when the + * table already owns a reservation or the remaining budget is too small. Subtraction is used + * for the capacity check to avoid overflowing {@code reservedBytes + bytes}. + * + * @return the new reservation, or {@code null} if the table already has a reservation or there + * is insufficient remaining capacity + */ + synchronized @Nullable Reservation tryReserve(long tableId, long bytes) { + checkArgument(bytes > 0, "bytes must be greater than 0."); + if (reservationsByTableId.containsKey(tableId) || bytes > maxBytes - reservedBytes) { + return null; + } + + Reservation reservation = new Reservation(tableId, bytes); + reservationsByTableId.put(tableId, reservation); + reservedBytes = Math.addExact(reservedBytes, bytes); + return reservation; + } + + /** + * Atomically replaces a table's current reservation for a replacement lookuper. + * + *

The supplied reservation must still be the table's current reservation. A stale object can + * be observed after expiration, LRU eviction, or another replacement and must not overwrite the + * newer reservation. If the identity or capacity check fails, the old reservation remains + * unchanged. + * + * @return the replacement reservation, or {@code null} if the supplied reservation is no longer + * current or the replacement does not fit within the capacity limit + */ + synchronized @Nullable Reservation tryReplace(Reservation oldReservation, long newBytes) { + checkArgument(newBytes > 0, "newBytes must be greater than 0."); + Reservation currentReservation = reservationsByTableId.get(oldReservation.getTableId()); + if (currentReservation != oldReservation) { + return null; + } + + long reservedWithoutOld = Math.subtractExact(reservedBytes, oldReservation.getBytes()); + if (newBytes > maxBytes - reservedWithoutOld) { + return null; + } + + Reservation newReservation = new Reservation(oldReservation.getTableId(), newBytes); + reservationsByTableId.put(oldReservation.getTableId(), newReservation); + reservedBytes = Math.addExact(reservedWithoutOld, newBytes); + return newReservation; + } + + /** + * Releases a reservation if it is still the table's current reservation. + * + *

Removal listeners, creation cleanup, and delayed retired-lookuper callbacks can all + * attempt a release. Comparing the reservation object identity makes those calls idempotent and + * prevents an old lookuper from releasing its replacement's capacity. + */ + synchronized void release(Reservation reservation) { + Reservation currentReservation = reservationsByTableId.get(reservation.getTableId()); + if (currentReservation != reservation) { + return; + } + + reservationsByTableId.remove(reservation.getTableId()); + reservedBytes = Math.subtractExact(reservedBytes, reservation.getBytes()); + } + + /** Returns the capacity currently reserved by current and creating lookupers. */ + synchronized long reservedBytes() { + return reservedBytes; + } + + /** Returns the configured capacity limit. */ + long maxBytes() { + return maxBytes; + } + + /** + * An immutable capacity reservation for one cached lookuper. + * + *

Each reserve or replace operation creates a new instance. The budget manager compares + * object identity so delayed callbacks carrying an older instance are harmless. + */ + static final class Reservation { + private final long tableId; + private final long bytes; + + private Reservation(long tableId, long bytes) { + this.tableId = tableId; + this.bytes = bytes; + } + + long getTableId() { + return tableId; + } + + long getBytes() { + return bytes; + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 2d00371b007..9ccd2a9a536 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -444,6 +444,12 @@ private void registerMetrics() { serverMetricGroup.gauge(MetricNames.UNDER_REPLICATED, this::underReplicatedCount); serverMetricGroup.gauge(MetricNames.UNDER_MIN_ISR, this::underMinIsrCount); serverMetricGroup.gauge(MetricNames.AT_MIN_ISR, this::atMinIsrCount); + serverMetricGroup.gauge( + MetricNames.HISTORICAL_LOOKUP_CACHED_TABLE_COUNT, + historicalLakeLookupManager::cachedTableCount); + serverMetricGroup.counter( + MetricNames.HISTORICAL_LOOKUP_CACHE_EVICTIONS, + historicalLakeLookupManager.capacityEvictions()); MetricGroup logicalStorage = serverMetricGroup.addGroup("logicalStorage"); logicalStorage.gauge( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index e8ba9714dd6..36bdc0439e7 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -22,6 +22,7 @@ import org.apache.fluss.config.ConfigOption; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.ReadableConfig; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.InvalidAlterTableException; @@ -38,6 +39,7 @@ import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypeRoot; import org.apache.fluss.types.RowType; @@ -90,7 +92,9 @@ public class TableDescriptorValidation { public static void validateTableDescriptor( TableDescriptor tableDescriptor, int maxBucketNum, - @Nullable DataLakeFormat clusterDataLakeFormat) { + @Nullable DataLakeFormat clusterDataLakeFormat, + TablePath tablePath, + MemorySize historicalLookupCacheMaxSize) { Schema schema = tableDescriptor.getSchema(); boolean hasPrimaryKey = schema.getPrimaryKey().isPresent(); Configuration tableConf = Configuration.fromMap(tableDescriptor.getProperties()); @@ -128,6 +132,7 @@ public static void validateTableDescriptor( checkDeleteBehavior(tableConf, hasPrimaryKey); checkTieredLog(tableConf); checkHistoricalPartition(tableDescriptor, tableConf); + checkHistoricalLookupCacheSize(tableConf, tablePath, historicalLookupCacheMaxSize); checkPartition(tableConf, tableDescriptor.getPartitionKeys(), schema.getRowType()); checkSystemColumns(schema.getRowType()); validateStatisticsConfig(tableDescriptor); @@ -227,6 +232,36 @@ private static void checkHistoricalPartition( } } + private static void checkHistoricalLookupCacheSize( + Configuration tableConf, TablePath tablePath, MemorySize historicalLookupCacheMaxSize) { + MemorySize tableCacheSize = + tableConf.get( + ConfigOptions + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); + if (tableCacheSize.getBytes() == 0) { + throw new InvalidConfigException( + String.format( + "'%s' for table '%s' must be greater than 0 bytes.", + ConfigOptions + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key(), + tablePath)); + } + if (tableCacheSize.compareTo(historicalLookupCacheMaxSize) > 0) { + throw new InvalidConfigException( + String.format( + "'%s' (%s) for table '%s' must be less than or equal to '%s' (%s).", + ConfigOptions + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key(), + tableCacheSize, + tablePath, + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key(), + historicalLookupCacheMaxSize)); + } + } + public static void validateAlterTableProperties( TableInfo currentTable, Set tableKeysToChange) { TableConfig currentConfig = currentTable.getTableConfig(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index cc9d31c2330..9b50c88bed8 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; @@ -49,10 +50,14 @@ import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -300,12 +305,15 @@ void testExpiresIdleLookuperWithoutAnotherLookup() throws Exception { }; TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager( - conf(1), executor, tickerNanos::get, cacheScheduler); + confWithExpiration(Duration.ofHours(1)), + executor, + tickerNanos::get, + cacheScheduler); lookupAndRun(manager, executor, PARTITION_TABLE_INFO); TestingLakeTableLookuper expiredLookuper = manager.createdLookupers.get(0); - tickerNanos.addAndGet(Duration.ofHours(4).toNanos()); + tickerNanos.addAndGet(Duration.ofHours(2).toNanos()); assertThat(expirationTask.get()).isNotNull(); expirationTask.get().run(); @@ -315,19 +323,118 @@ void testExpiresIdleLookuperWithoutAnotherLookup() throws Exception { } @Test - void testLimitsCachedLookupersToTen() throws Exception { + void testEvictsLeastRecentlyUsedLookuperWhenCapacityIsFull() throws Exception { ManualExecutor executor = new ManualExecutor(); - TestingHistoricalLakeLookupManager manager = createTestingManager(executor); + AtomicLong tickerNanos = new AtomicLong(); + Configuration conf = conf(1); + conf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + MemorySize.parse("16gb")); + TestingHistoricalLakeLookupManager manager = + new TestingHistoricalLakeLookupManager( + conf, executor, tickerNanos::get, Scheduler.disabledScheduler()); - for (int i = 0; i < 11; i++) { - lookupAndRun( - manager, - executor, - tableInfo(PARTITION_TABLE_ID + i, PARTITION_TABLE_INFO.getSchemaId())); + TableInfo first = tableInfo(PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId()); + TableInfo second = tableInfo(PARTITION_TABLE_ID + 1, PARTITION_TABLE_INFO.getSchemaId()); + TableInfo third = tableInfo(PARTITION_TABLE_ID + 2, PARTITION_TABLE_INFO.getSchemaId()); + + lookupAndRun(manager, executor, first); + tickerNanos.incrementAndGet(); + lookupAndRun(manager, executor, second); + tickerNanos.incrementAndGet(); + lookupAndRun(manager, executor, first); + tickerNanos.incrementAndGet(); + lookupAndRun(manager, executor, third); + + assertThat(manager.createdLookupers).hasSize(3); + assertThat(manager.createdLookupers.get(0).closed).isFalse(); + assertThat(manager.createdLookupers.get(1).closed).isTrue(); + assertThat(manager.createdLookupers.get(2).closed).isFalse(); + assertThat(manager.cachedTableCount()).isEqualTo(2); + assertThat(manager.capacityEvictions().getCount()).isEqualTo(1); + } + + @Test + void testEvictsOutsideConcurrentTableReplacements() throws Exception { + ExecutorService executor = + Executors.newFixedThreadPool( + 2, + runnable -> { + Thread thread = + new Thread(runnable, "historical-lookup-replacement-test"); + thread.setDaemon(true); + return thread; + }); + MemorySize initialCacheSize = MemorySize.parse("4gb"); + MemorySize replacementCacheSize = MemorySize.parse("8gb"); + Configuration conf = conf(2); + conf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + replacementCacheSize); + CoordinatedReplacementManager manager = + new CoordinatedReplacementManager(conf, executor, replacementCacheSize); + + TableInfo first = + tableInfoWithCacheSize( + PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId(), initialCacheSize); + TableInfo second = + tableInfoWithCacheSize( + PARTITION_TABLE_ID + 1, + PARTITION_TABLE_INFO.getSchemaId(), + initialCacheSize); + try { + assertThat(lookup(manager, first).get(5, TimeUnit.SECONDS).failed()).isFalse(); + assertThat(lookup(manager, second).get(5, TimeUnit.SECONDS).failed()).isFalse(); + + // Both replacements hold their own table's compute lock before admission fails. LRU + // eviction must happen after those locks are released to avoid cross-key deadlock. + TableInfo firstReplacement = + tableInfoWithCacheSize( + first.getTableId(), first.getSchemaId() + 1, replacementCacheSize); + TableInfo secondReplacement = + tableInfoWithCacheSize( + second.getTableId(), second.getSchemaId() + 1, replacementCacheSize); + CompletableFuture firstResult = + lookup(manager, firstReplacement); + CompletableFuture secondResult = + lookup(manager, secondReplacement); + + LookupResultForBucket firstReplacementResult = firstResult.get(5, TimeUnit.SECONDS); + LookupResultForBucket secondReplacementResult = secondResult.get(5, TimeUnit.SECONDS); + assertThat(firstReplacementResult.getError().error()) + .isIn(Errors.NONE, Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(secondReplacementResult.getError().error()) + .isIn(Errors.NONE, Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(firstReplacementResult.failed() && secondReplacementResult.failed()) + .isFalse(); + manager.close(); + } finally { + executor.shutdownNow(); } + } - assertThat(manager.createdLookupers).hasSize(11); - assertThat(manager.createdLookupers).filteredOn(lookuper -> lookuper.closed).hasSize(1); + @Test + void testThrottlesWhenTableCacheSizeExceedsRuntimeLimit() throws Exception { + ManualExecutor executor = new ManualExecutor(); + Configuration conf = conf(1); + conf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + MemorySize.parse("8gb")); + TestingHistoricalLakeLookupManager manager = + new TestingHistoricalLakeLookupManager(conf, executor); + + LookupResultForBucket result = + lookupResultAndRun( + manager, + executor, + tableInfoWithCacheSize( + PARTITION_TABLE_ID, + PARTITION_TABLE_INFO.getSchemaId(), + MemorySize.parse("16gb"))); + + assertThat(result.getError().error()).isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(manager.createdLookupers).isEmpty(); + assertThat(manager.cachedTableCount()).isZero(); } private HistoricalLakeLookupManager createManager( @@ -354,11 +461,27 @@ private Configuration conf(int maxQueuedHistoricalRequests) { return conf; } + private Configuration confWithExpiration(Duration expiration) { + Configuration conf = conf(1); + conf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS, + expiration); + return conf; + } + private static LookupDataForBucket lookupData(TableBucket tableBucket) { return new LookupDataForBucket( tableBucket, Collections.singletonList(new byte[] {1}), "2024"); } + private static CompletableFuture lookup( + HistoricalLakeLookupManager manager, TableInfo tableInfo) { + return manager.lookup( + lookupData(new TableBucket(tableInfo.getTableId(), 1L, 0)), + tableInfo, + tableInfo.getSchemaInfo()); + } + private static TableInfo tableInfo(long tableId, int schemaId) { return TableInfo.of( PARTITION_TABLE_INFO.getTablePath(), @@ -370,6 +493,25 @@ private static TableInfo tableInfo(long tableId, int schemaId) { PARTITION_TABLE_INFO.getModifiedTime()); } + private static TableInfo tableInfoWithCacheSize( + long tableId, int schemaId, MemorySize cacheSize) { + TableDescriptor descriptor = + TableDescriptor.builder(PARTITION_TABLE_INFO.toTableDescriptor()) + .property( + ConfigOptions + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + cacheSize) + .build(); + return TableInfo.of( + PARTITION_TABLE_INFO.getTablePath(), + tableId, + schemaId, + descriptor, + PARTITION_TABLE_INFO.getRemoteDataDir(), + PARTITION_TABLE_INFO.getCreatedTime(), + PARTITION_TABLE_INFO.getModifiedTime()); + } + private static void lookupAndRun( HistoricalLakeLookupManager manager, ManualExecutor executor, TableInfo tableInfo) throws Exception { @@ -382,13 +524,28 @@ private static void lookupAndRun( TableInfo tableInfo, SchemaInfo schemaInfo) throws Exception { + LookupResultForBucket result = lookupResultAndRun(manager, executor, tableInfo, schemaInfo); + assertThat(result.failed()).isFalse(); + assertThat(result.originalPartitionName()).isEqualTo("2024"); + } + + private static LookupResultForBucket lookupResultAndRun( + HistoricalLakeLookupManager manager, ManualExecutor executor, TableInfo tableInfo) + throws Exception { + return lookupResultAndRun(manager, executor, tableInfo, tableInfo.getSchemaInfo()); + } + + private static LookupResultForBucket lookupResultAndRun( + HistoricalLakeLookupManager manager, + ManualExecutor executor, + TableInfo tableInfo, + SchemaInfo schemaInfo) + throws Exception { TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 1L, 0); CompletableFuture future = manager.lookup(lookupData(tableBucket), tableInfo, schemaInfo); executor.runNext(); - LookupResultForBucket result = future.get(1, TimeUnit.SECONDS); - assertThat(result.failed()).isFalse(); - assertThat(result.originalPartitionName()).isEqualTo("2024"); + return future.get(1, TimeUnit.SECONDS); } private static final class TestingHistoricalLakeLookupManager @@ -448,6 +605,43 @@ public void close() { } } + private static final class CoordinatedReplacementManager extends HistoricalLakeLookupManager { + private final MemorySize replacementCacheSize; + private final CyclicBarrier replacementBarrier = new CyclicBarrier(2); + private final AtomicInteger coordinatedCreations = new AtomicInteger(); + + private CoordinatedReplacementManager( + Configuration conf, ExecutorService executor, MemorySize replacementCacheSize) { + super( + conf, + null, + executor, + SERVER_ID, + Ticker.systemTicker(), + Scheduler.disabledScheduler()); + this.replacementCacheSize = replacementCacheSize; + } + + @Override + LakeTableLookuper createLakeTableLookuper( + TablePath tablePath, + String ioTmpDir, + TableConfig tableConfig, + Configuration clusterConf) { + if (tableConfig + .getHistoricalPartitionLookupCacheMaxDiskSize() + .equals(replacementCacheSize) + && coordinatedCreations.getAndIncrement() < 2) { + try { + replacementBarrier.await(5, TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException("Failed to coordinate lookuper replacements.", e); + } + } + return new TestingLakeTableLookuper(); + } + } + private static final class ManualExecutor extends AbstractExecutorService { private final BlockingQueue tasks = new LinkedBlockingQueue<>(); private volatile boolean shutdown; diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java new file mode 100644 index 00000000000..68d74bd53b5 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica; + +import org.apache.fluss.server.replica.HistoricalLookupCacheBudgetManager.Reservation; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link HistoricalLookupCacheBudgetManager}. */ +class HistoricalLookupCacheBudgetManagerTest { + + @Test + void testReserveAndReleaseWithinLimit() { + HistoricalLookupCacheBudgetManager manager = new HistoricalLookupCacheBudgetManager(10); + + Reservation first = manager.tryReserve(1, 4); + Reservation second = manager.tryReserve(2, 6); + assertThat(first).isNotNull(); + assertThat(second).isNotNull(); + assertThat(manager.reservedBytes()).isEqualTo(10); + assertThat(manager.tryReserve(3, 1)).isNull(); + assertThat(manager.tryReserve(1, 1)).isNull(); + + manager.release(first); + manager.release(first); + assertThat(manager.reservedBytes()).isEqualTo(6); + assertThat(manager.tryReserve(3, 4)).isNotNull(); + assertThat(manager.reservedBytes()).isEqualTo(10); + } + + @Test + void testReplaceReservationAtomically() { + HistoricalLookupCacheBudgetManager manager = new HistoricalLookupCacheBudgetManager(12); + Reservation oldReservation = manager.tryReserve(1, 4); + Reservation otherReservation = manager.tryReserve(2, 6); + assertThat(oldReservation).isNotNull(); + assertThat(otherReservation).isNotNull(); + + Reservation replacement = manager.tryReplace(oldReservation, 5); + assertThat(replacement).isNotNull(); + assertThat(replacement.getTableId()).isEqualTo(1); + assertThat(replacement.getBytes()).isEqualTo(5); + assertThat(manager.reservedBytes()).isEqualTo(11); + + // Releasing the retired reservation must not affect the replacement. + manager.release(oldReservation); + assertThat(manager.reservedBytes()).isEqualTo(11); + assertThat(manager.tryReplace(oldReservation, 1)).isNull(); + + // A failed replacement leaves the current reservation unchanged. + assertThat(manager.tryReplace(replacement, 7)).isNull(); + assertThat(manager.reservedBytes()).isEqualTo(11); + manager.release(replacement); + assertThat(manager.reservedBytes()).isEqualTo(6); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index cb427c23ec8..ea7b188e6dd 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -18,10 +18,12 @@ package org.apache.fluss.server.utils; import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.InvalidConfigException; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; import org.apache.fluss.types.DataTypes; import org.junit.jupiter.api.Test; @@ -30,6 +32,8 @@ class HistoricalPartitionTableValidationTest { + private static final TablePath TABLE_PATH = TablePath.of("test_db", "test_table"); + @Test void testReportsAllUnmetHistoricalPartitionRequirements() { // Case 1: Report disabled options, a missing format, and missing table keys together. @@ -46,7 +50,11 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { TableDescriptorValidation.validateTableDescriptor( allRequirementsMissingDescriptor, 100, - DataLakeFormat.PAIMON)) + DataLakeFormat.PAIMON, + TABLE_PATH, + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .defaultValue())) .isInstanceOf(InvalidConfigException.class) .hasMessage( "'table.datalake.historical-partition.enabled' has unmet requirements: " @@ -74,7 +82,11 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { TableDescriptorValidation.validateTableDescriptor( relatedValidationFailuresDescriptor, 100, - DataLakeFormat.PAIMON)) + DataLakeFormat.PAIMON, + TABLE_PATH, + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .defaultValue())) .isInstanceOf(InvalidConfigException.class) .hasMessage( "'table.datalake.historical-partition.enabled' has unmet requirements: " @@ -83,4 +95,41 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); } + + @Test + void testRejectInvalidHistoricalLookupCacheSize() { + TableDescriptor zeroSizeDescriptor = descriptorWithCacheSize(MemorySize.ZERO); + assertThatThrownBy(() -> validate(zeroSizeDescriptor, MemorySize.parse("80gb"))) + .isInstanceOf(InvalidConfigException.class) + .hasMessageContaining( + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key()) + .hasMessageContaining(TABLE_PATH.toString()) + .hasMessageContaining("greater than 0 bytes"); + + TableDescriptor oversizedDescriptor = descriptorWithCacheSize(MemorySize.parse("16gb")); + assertThatThrownBy(() -> validate(oversizedDescriptor, MemorySize.parse("8gb"))) + .isInstanceOf(InvalidConfigException.class) + .hasMessageContaining("16 gb") + .hasMessageContaining( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key()) + .hasMessageContaining("8 gb"); + } + + private static TableDescriptor descriptorWithCacheSize(MemorySize cacheSize) { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) + .property( + ConfigOptions + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + cacheSize) + .build(); + } + + private static void validate(TableDescriptor descriptor, MemorySize globalCacheSize) { + TableDescriptorValidation.validateTableDescriptor( + descriptor, 100, DataLakeFormat.PAIMON, TABLE_PATH, globalCacheSize); + } } From a181cf375777ec0a39bea2e89abb1a55a451eebc Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 5 Aug 2026 12:16:43 +0800 Subject: [PATCH 03/15] [server] Support dynamic historical lookup cache size Allow ALTER TABLE to update the effective historical lookup cache size. Propagate the latest table configuration to TabletServer replicas and recreate cached lookupers only when the effective size changes. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 72/74 AI-Contributed/UT: 106/106 --- .../apache/fluss/config/FlussConfigUtils.java | 2 + .../fluss/config/FlussConfigUtilsTest.java | 2 +- .../replica/HistoricalLakeLookupManager.java | 40 ++++--- .../apache/fluss/server/replica/Replica.java | 11 +- .../fluss/server/replica/ReplicaManager.java | 21 +++- .../HistoricalLakeLookupManagerTest.java | 104 ++++++++++++++++-- 6 files changed, 154 insertions(+), 26 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index 2e1899b812c..ef249671816 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -49,6 +49,8 @@ public class FlussConfigUtils { Arrays.asList( ConfigOptions.TABLE_DATALAKE_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED.key(), + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key(), ConfigOptions.TABLE_DATALAKE_FRESHNESS.key(), ConfigOptions.TABLE_DATALAKE_AUTO_COMPACTION.key(), ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.key(), diff --git a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java index 3ef097b7632..d65d5172fa6 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java @@ -246,7 +246,7 @@ void testValidateHistoricalLookupCacheConfigs() { ConfigOptions .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE .key())) - .isFalse(); + .isTrue(); } @Test diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index bb7f2ab07dc..b9f75bbdab6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -210,7 +210,10 @@ private static com.github.benmanes.caffeine.cache.Scheduler createCacheScheduler } CompletableFuture lookup( - LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo) { + LookupDataForBucket lookupData, + TableInfo tableInfo, + SchemaInfo schemaInfo, + TableConfig tableConfig) { TableBucket tableBucket = lookupData.tableBucket(); if (!lookupPermits.tryAcquire()) { return CompletableFuture.completedFuture( @@ -227,7 +230,7 @@ CompletableFuture lookup( CompletableFuture future; try { - future = submitLookup(lookupData, tableInfo, schemaInfo); + future = submitLookup(lookupData, tableInfo, schemaInfo, tableConfig); } catch (RuntimeException e) { lookupPermits.release(); throw e; @@ -252,10 +255,13 @@ public void close() { } private CompletableFuture submitLookup( - LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo) { + LookupDataForBucket lookupData, + TableInfo tableInfo, + SchemaInfo schemaInfo, + TableConfig tableConfig) { CompletableFuture future = CompletableFuture.supplyAsync( - () -> lookupInternal(lookupData, tableInfo, schemaInfo), + () -> lookupInternal(lookupData, tableInfo, schemaInfo, tableConfig), historicalPartitionExecutor); pendingLookups.add(future); return future; @@ -298,14 +304,16 @@ synchronized void reconfigure(Configuration newConf) { } private LookupResultForBucket lookupInternal( - LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo) { + LookupDataForBucket lookupData, + TableInfo tableInfo, + SchemaInfo schemaInfo, + TableConfig tableConfig) { TableBucket tableBucket = lookupData.tableBucket(); CachedLakeTableLookuper cachedLookuper = null; try { LookupContext context = createLookupContext(lookupData, tableInfo, schemaInfo); long currentLakeConfigVersion = lakeConfigVersion; Configuration currentConf = conf; - TableConfig tableConfig = tableInfo.getTableConfig(); long cacheSizeBytes = tableConfig.getHistoricalPartitionLookupCacheMaxDiskSize().getBytes(); cachedLookuper = @@ -391,11 +399,14 @@ private CachedLakeTableLookuper acquireCachedLookuper( context.tableId, (ignored, currentLookuper) -> { CachedLakeTableLookuper selectedLookuper = currentLookuper; - // Create the lookuper lazily, and recreate it after schema or - // lake configuration changes so it reloads lake table/query - // state and uses the current configuration. + // Create the lookuper lazily, and recreate it after schema, + // lake configuration, or effective cache size changes so it + // reloads lake table/query state and uses the current settings. if (!matchesLookupConfiguration( - selectedLookuper, context, currentLakeConfigVersion)) { + selectedLookuper, + context, + currentLakeConfigVersion, + cacheSizeBytes)) { selectedLookuper = tryCreateCachedLookuper( context, @@ -416,7 +427,8 @@ private CachedLakeTableLookuper acquireCachedLookuper( selectedLookuper.acquire(ticker.read()); return selectedLookuper; }); - return matchesLookupConfiguration(cachedLookuper, context, currentLakeConfigVersion) + return matchesLookupConfiguration( + cachedLookuper, context, currentLakeConfigVersion, cacheSizeBytes) ? cachedLookuper : null; } @@ -424,10 +436,12 @@ private CachedLakeTableLookuper acquireCachedLookuper( private static boolean matchesLookupConfiguration( @Nullable CachedLakeTableLookuper cachedLookuper, LookupContext context, - long currentLakeConfigVersion) { + long currentLakeConfigVersion, + long cacheSizeBytes) { return cachedLookuper != null && cachedLookuper.schemaId == context.schemaId - && cachedLookuper.lakeConfigVersion == currentLakeConfigVersion; + && cachedLookuper.lakeConfigVersion == currentLakeConfigVersion + && cachedLookuper.cacheSizeBytes == cacheSizeBytes; } /** diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index 5192cdeb0b6..29d30546eea 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -186,7 +186,8 @@ public final class Replica { private final SchemaGetter schemaGetter; private final TableInfo tableInfo; - private final TableConfig tableConfig; + // Metadata updates replace this snapshot after applying configuration-specific side effects. + private volatile TableConfig tableConfig; // logFormat and arrowCompressionInfo are used in hot-path, so cache them here. private final LogFormat logFormat; private final ArrowCompressionInfo arrowCompressionInfo; @@ -346,6 +347,14 @@ public TableInfo getTableInfo() { return tableInfo; } + TableConfig getTableConfig() { + return tableConfig; + } + + void updateTableConfig(TableConfig tableConfig) { + this.tableConfig = checkNotNull(tableConfig, "tableConfig"); + } + public @Nullable Integer getLeaderId() { return leaderReplicaIdOpt.get(); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 9ccd2a9a536..534e15af68e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -605,11 +605,18 @@ public void maybeUpdateMetadataCache(int coordinatorEpoch, ClusterMetadata clust private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { Map tableIdToLakeFlag = new HashMap<>(); Map tableIdToTieredLogLocalSegments = new HashMap<>(); + Map tableIdToTableConfig = new HashMap<>(); for (TableMetadata tableMetadata : clusterMetadata.getTableMetadataList()) { TableInfo tableInfo = tableMetadata.getTableInfo(); long tableId = tableInfo.getTableId(); + // Deleted-table markers do not carry authoritative table configuration. + if (tableId != TableMetadata.DELETED_TABLE_ID + && !tableInfo.getTablePath().equals(TableMetadata.DELETED_TABLE_PATH)) { + tableIdToTableConfig.put(tableId, tableInfo.getTableConfig()); + } + // Collect datalake enabled configuration if (tableInfo.getTableConfig().getDataLakeFormat().isPresent()) { boolean dataLakeEnabled = tableInfo.getTableConfig().isDataLakeEnabled(); @@ -621,7 +628,9 @@ private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { tableIdToTieredLogLocalSegments.put(tableId, tieredLogLocalSegments); } - if (tableIdToLakeFlag.isEmpty() && tableIdToTieredLogLocalSegments.isEmpty()) { + if (tableIdToLakeFlag.isEmpty() + && tableIdToTieredLogLocalSegments.isEmpty() + && tableIdToTableConfig.isEmpty()) { return; } @@ -641,6 +650,11 @@ private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { replica.updateTieredLogLocalSegments( tableIdToTieredLogLocalSegments.get(tableId)); } + + // Publish the new snapshot after applying configuration-specific side effects. + if (tableIdToTableConfig.containsKey(tableId)) { + replica.updateTableConfig(tableIdToTableConfig.get(tableId)); + } } } } @@ -844,7 +858,10 @@ public void historicalLookups( SchemaInfo latestSchemaInfo = replica.getSchemaGetter().getLatestSchemaInfo(); lookupFuture = historicalLakeLookupManager.lookup( - data, replica.getTableInfo(), latestSchemaInfo); + data, + replica.getTableInfo(), + latestSchemaInfo, + replica.getTableConfig()); } catch (Exception e) { lookupFuture = CompletableFuture.completedFuture( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index 9b50c88bed8..34574d1d29f 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -83,7 +83,8 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { manager.lookup( lookupData(HISTORICAL_BUCKET), PARTITION_TABLE_INFO, - PARTITION_TABLE_INFO.getSchemaInfo()); + PARTITION_TABLE_INFO.getSchemaInfo(), + PARTITION_TABLE_INFO.getTableConfig()); assertThat(first).isNotDone(); assertThat(executor.numQueuedTasks()).isEqualTo(1); @@ -92,7 +93,8 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { manager.lookup( lookupData(secondBucket), PARTITION_TABLE_INFO, - PARTITION_TABLE_INFO.getSchemaInfo()) + PARTITION_TABLE_INFO.getSchemaInfo(), + PARTITION_TABLE_INFO.getTableConfig()) .get(1, TimeUnit.SECONDS); assertThat(second.failed()).isTrue(); @@ -111,7 +113,8 @@ void testHistoricalLookupReleasesPermitOnFailure() throws Exception { manager.lookup( lookupData(HISTORICAL_BUCKET), PARTITION_TABLE_INFO, - PARTITION_TABLE_INFO.getSchemaInfo()); + PARTITION_TABLE_INFO.getSchemaInfo(), + PARTITION_TABLE_INFO.getTableConfig()); executor.runNext(); LookupResultForBucket firstResult = first.get(1, TimeUnit.SECONDS); assertThat(firstResult.failed()).isTrue(); @@ -122,7 +125,8 @@ void testHistoricalLookupReleasesPermitOnFailure() throws Exception { manager.lookup( lookupData(HISTORICAL_BUCKET), PARTITION_TABLE_INFO, - PARTITION_TABLE_INFO.getSchemaInfo()); + PARTITION_TABLE_INFO.getSchemaInfo(), + PARTITION_TABLE_INFO.getTableConfig()); assertThat(second).isNotDone(); assertThat(executor.numQueuedTasks()).isEqualTo(1); } @@ -136,17 +140,20 @@ void testHistoricalLookupMaxQueuedRequestsUsesExplicitConfig() throws Exception manager.lookup( lookupData(new TableBucket(PARTITION_TABLE_ID, 1L, 0)), PARTITION_TABLE_INFO, - PARTITION_TABLE_INFO.getSchemaInfo()); + PARTITION_TABLE_INFO.getSchemaInfo(), + PARTITION_TABLE_INFO.getTableConfig()); CompletableFuture second = manager.lookup( lookupData(new TableBucket(PARTITION_TABLE_ID, 2L, 0)), PARTITION_TABLE_INFO, - PARTITION_TABLE_INFO.getSchemaInfo()); + PARTITION_TABLE_INFO.getSchemaInfo(), + PARTITION_TABLE_INFO.getTableConfig()); LookupResultForBucket third = manager.lookup( lookupData(new TableBucket(PARTITION_TABLE_ID, 3L, 0)), PARTITION_TABLE_INFO, - PARTITION_TABLE_INFO.getSchemaInfo()) + PARTITION_TABLE_INFO.getSchemaInfo(), + PARTITION_TABLE_INFO.getTableConfig()) .get(1, TimeUnit.SECONDS); assertThat(first).isNotDone(); @@ -287,6 +294,47 @@ void testInvalidatesLookuperOnSchemaAndLifecycleChanges() throws Exception { assertThat(manager.createdLookupers).hasSize(3); } + @Test + void testReplacesLookuperOnlyWhenEffectiveCacheSizeChanges() throws Exception { + ManualExecutor executor = new ManualExecutor(); + TestingHistoricalLakeLookupManager manager = createTestingManager(executor); + + lookupAndRun( + manager, + executor, + PARTITION_TABLE_INFO, + tableConfigWithCacheSize(MemorySize.parse("8gb"))); + TestingLakeTableLookuper initialLookuper = manager.createdLookupers.get(0); + + Configuration unrelatedChange = new Configuration(); + unrelatedChange.set( + ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS, + ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.defaultValue() + 1); + lookupAndRun(manager, executor, PARTITION_TABLE_INFO, new TableConfig(unrelatedChange)); + lookupAndRun( + manager, + executor, + PARTITION_TABLE_INFO, + tableConfigWithCacheSize(MemorySize.parse("8gb"))); + + assertThat(manager.createdLookupers).hasSize(1); + assertThat(initialLookuper.closed).isFalse(); + + lookupAndRun( + manager, + executor, + PARTITION_TABLE_INFO, + tableConfigWithCacheSize(MemorySize.parse("4gb"))); + + assertThat(initialLookuper.closed).isTrue(); + assertThat(manager.createdLookupers).hasSize(2); + assertThat( + manager.createdTableConfigs + .get(1) + .getHistoricalPartitionLookupCacheMaxDiskSize()) + .isEqualTo(MemorySize.parse("4gb")); + } + @Test void testExpiresIdleLookuperWithoutAnotherLookup() throws Exception { ManualExecutor executor = new ManualExecutor(); @@ -476,10 +524,16 @@ private static LookupDataForBucket lookupData(TableBucket tableBucket) { private static CompletableFuture lookup( HistoricalLakeLookupManager manager, TableInfo tableInfo) { + return lookup(manager, tableInfo, tableInfo.getTableConfig()); + } + + private static CompletableFuture lookup( + HistoricalLakeLookupManager manager, TableInfo tableInfo, TableConfig tableConfig) { return manager.lookup( lookupData(new TableBucket(tableInfo.getTableId(), 1L, 0)), tableInfo, - tableInfo.getSchemaInfo()); + tableInfo.getSchemaInfo(), + tableConfig); } private static TableInfo tableInfo(long tableId, int schemaId) { @@ -512,12 +566,33 @@ private static TableInfo tableInfoWithCacheSize( PARTITION_TABLE_INFO.getModifiedTime()); } + private static TableConfig tableConfigWithCacheSize(MemorySize cacheSize) { + Configuration conf = new Configuration(); + conf.set( + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + cacheSize); + return new TableConfig(conf); + } + private static void lookupAndRun( HistoricalLakeLookupManager manager, ManualExecutor executor, TableInfo tableInfo) throws Exception { lookupAndRun(manager, executor, tableInfo, tableInfo.getSchemaInfo()); } + private static void lookupAndRun( + HistoricalLakeLookupManager manager, + ManualExecutor executor, + TableInfo tableInfo, + TableConfig tableConfig) + throws Exception { + LookupResultForBucket result = + lookupResultAndRun( + manager, executor, tableInfo, tableInfo.getSchemaInfo(), tableConfig); + assertThat(result.failed()).isFalse(); + assertThat(result.originalPartitionName()).isEqualTo("2024"); + } + private static void lookupAndRun( HistoricalLakeLookupManager manager, ManualExecutor executor, @@ -541,9 +616,20 @@ private static LookupResultForBucket lookupResultAndRun( TableInfo tableInfo, SchemaInfo schemaInfo) throws Exception { + return lookupResultAndRun( + manager, executor, tableInfo, schemaInfo, tableInfo.getTableConfig()); + } + + private static LookupResultForBucket lookupResultAndRun( + HistoricalLakeLookupManager manager, + ManualExecutor executor, + TableInfo tableInfo, + SchemaInfo schemaInfo, + TableConfig tableConfig) + throws Exception { TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 1L, 0); CompletableFuture future = - manager.lookup(lookupData(tableBucket), tableInfo, schemaInfo); + manager.lookup(lookupData(tableBucket), tableInfo, schemaInfo, tableConfig); executor.runNext(); return future.get(1, TimeUnit.SECONDS); } From 257cc11025b3489fbb6cd9078981110e2b90a39b Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 5 Aug 2026 14:39:25 +0800 Subject: [PATCH 04/15] [server] Support dynamic historical lookup cache capacity Allow the global historical lookup cache capacity to be updated through cluster configuration and apply reductions lazily during admission. Keep coordinator table validation synchronized through a dedicated config updater. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 61/151 AI-Contributed/UT: 63/123 --- .../apache/fluss/config/ConfigOptions.java | 5 +- .../apache/fluss/config/FlussConfigUtils.java | 10 +--- .../fluss/config/FlussConfigUtilsTest.java | 6 +- .../procedure/SetClusterConfigsProcedure.java | 1 + .../flink/procedure/FlinkProcedureITCase.java | 36 ++++++++++-- .../fluss/server/DynamicServerConfig.java | 2 + .../server/coordinator/CoordinatorServer.java | 1 + .../HistoricalLookupCacheConfigUpdater.java | 56 ++++++++++++++++++ .../server/coordinator/MetadataManager.java | 6 +- .../replica/HistoricalLakeLookupManager.java | 50 ++++++++++++---- .../HistoricalLookupCacheBudgetManager.java | 20 +++++-- .../HistoricalLakeLookupManagerTest.java | 57 +++++++++++++++++++ ...istoricalLookupCacheBudgetManagerTest.java | 24 +++++++- 13 files changed, 237 insertions(+), 37 deletions(-) create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index f3609c810cd..72f210538b3 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -435,7 +435,8 @@ public class ConfigOptions { .defaultValue(MemorySize.parse("80gb")) .withDescription( "The total configured disk capacity available to current and creating historical partition lookup caches on a TabletServer. " - + "Retired cache generations that are still serving active lookups are not included in this limit."); + + "Retired cache generations that are still serving active lookups are not included in this limit. " + + "The value must be greater than zero."); public static final ConfigOption SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS = @@ -1883,7 +1884,7 @@ public class ConfigOptions { .defaultValue(MemorySize.parse("8gb")) .withDescription( "The maximum local disk capacity reserved for this table's historical partition lookup cache on each TabletServer. " - + "The value must be greater than zero and no greater than the TabletServer historical lookup cache limit."); + + "When the table is created or altered, the value must be greater than zero and no greater than the current TabletServer historical lookup cache limit."); public static final ConfigOption TABLE_DATALAKE_FORMAT = key("table.datalake.format") diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index ef249671816..dd351841d37 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -241,14 +241,10 @@ protected static void validateServerConfigs(Configuration conf) { private static void validateHistoricalLookupCacheLimit(Configuration conf) { MemorySize historicalLookupCacheMaxSize = conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); - MemorySize defaultTableHistoricalLookupCacheSize = - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE - .defaultValue(); - if (historicalLookupCacheMaxSize.compareTo(defaultTableHistoricalLookupCacheSize) < 0) { + if (historicalLookupCacheMaxSize.getBytes() == 0) { throw new IllegalConfigurationException( - "Invalid configuration for %s, it must be greater than or equal to the default table cache size %s.", - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key(), - defaultTableHistoricalLookupCacheSize); + "Invalid configuration for %s, it must be greater than 0 bytes.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key()); } } diff --git a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java index d65d5172fa6..66678fc7a1f 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java @@ -221,17 +221,17 @@ void testValidateHistoricalLookupCacheConfigs() { conf.set(ConfigOptions.REMOTE_DATA_DIR, "s3://bucket/path"); conf.set( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.parse("4gb")); + MemorySize.ZERO); assertThatThrownBy(() -> validateCoordinatorConfigs(conf)) .isInstanceOf(IllegalConfigurationException.class) .hasMessageContaining( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key()) - .hasMessageContaining("8 gb"); + .hasMessageContaining("greater than 0 bytes"); conf.set( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.parse("8gb")); + MemorySize.parse("4gb")); conf.set( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS, Duration.ZERO); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/procedure/SetClusterConfigsProcedure.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/procedure/SetClusterConfigsProcedure.java index 7246706af8b..ece70d8b003 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/procedure/SetClusterConfigsProcedure.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/procedure/SetClusterConfigsProcedure.java @@ -46,6 +46,7 @@ * -- Set a configuration * CALL sys.set_cluster_configs('kv.rocksdb.shared-rate-limiter.bytes-per-sec', '200MB'); * CALL sys.set_cluster_configs('datalake.format', 'paimon'); + * CALL sys.set_cluster_configs('server.historical-partition.lookup-cache.max-disk-size', '96GB'); * * -- Set multiple configurations at one time * CALL sys.set_cluster_configs('kv.rocksdb.shared-rate-limiter.bytes-per-sec', '200MB','datalake.format', 'paimon'); diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/procedure/FlinkProcedureITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/procedure/FlinkProcedureITCase.java index 872932f3ca9..3870cc0bbbd 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/procedure/FlinkProcedureITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/procedure/FlinkProcedureITCase.java @@ -404,13 +404,16 @@ void testSetClusterConfigs() throws Exception { try (CloseableIterator resultIterator = tEnv.executeSql( String.format( - "Call %s.sys.set_cluster_configs('%s', '300MB', '%s', 'paimon')", + "Call %s.sys.set_cluster_configs('%s', '300MB', '%s', 'paimon', '%s', '96GB')", CATALOG_NAME, ConfigOptions.KV_SHARED_RATE_LIMITER_BYTES_PER_SEC.key(), - ConfigOptions.DATALAKE_FORMAT.key())) + ConfigOptions.DATALAKE_FORMAT.key(), + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key())) .collect()) { List results = CollectionUtil.iteratorToList(resultIterator); - assertThat(results).hasSize(2); + assertThat(results).hasSize(3); assertThat(results.get(0).getField(0)) .asString() .contains("Successfully set to '300MB'") @@ -420,6 +423,13 @@ void testSetClusterConfigs() throws Exception { .asString() .contains("Successfully set to 'paimon'") .contains(ConfigOptions.DATALAKE_FORMAT.key()); + + assertThat(results.get(2).getField(0)) + .asString() + .contains("Successfully set to '96GB'") + .contains( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key()); } // Verify the config was actually set @@ -435,13 +445,29 @@ void testSetClusterConfigs() throws Exception { assertThat(results.get(0).getField(1)).isEqualTo("300MB"); } + try (CloseableIterator resultIterator = + tEnv.executeSql( + String.format( + "Call %s.sys.get_cluster_configs('%s')", + CATALOG_NAME, + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key())) + .collect()) { + List results = CollectionUtil.iteratorToList(resultIterator); + assertThat(results).hasSize(1); + assertThat(results.get(0).getField(1)).isEqualTo("96GB"); + } + // reset cluster configs. tEnv.executeSql( String.format( - "Call %s.sys.reset_cluster_configs('%s', '%s')", + "Call %s.sys.reset_cluster_configs('%s', '%s', '%s')", CATALOG_NAME, ConfigOptions.KV_SHARED_RATE_LIMITER_BYTES_PER_SEC.key(), - ConfigOptions.DATALAKE_FORMAT.key())) + ConfigOptions.DATALAKE_FORMAT.key(), + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key())) .await(); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java index 37ae794f538..367fa69c9d9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java @@ -56,6 +56,7 @@ import static org.apache.fluss.config.ConfigOptions.REMOTE_DATA_DIRS_WEIGHTS; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_RECOVER_RATIO; +import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_CREDENTIALS; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG; import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock; @@ -82,6 +83,7 @@ class DynamicServerConfig { KV_SNAPSHOT_INTERVAL.key(), SERVER_DATA_DISK_WRITE_RECOVER_RATIO.key(), SERVER_DATA_DISK_WRITE_LIMIT_RATIO.key(), + SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key(), // Config options for remote.data.dirs REMOTE_DATA_DIRS.key(), REMOTE_DATA_DIRS_STRATEGY.key(), diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index 0f610eb42a3..5b73fbc8aa2 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -305,6 +305,7 @@ protected void initCoordinatorStandby() throws Exception { dynamicConfigManager.register(lakeCatalogDynamicLoader); dynamicConfigManager.register(remoteDirDynamicLoader); dynamicConfigManager.register(replicaCapacityController); + dynamicConfigManager.register(new HistoricalLookupCacheConfigUpdater(metadataManager)); // Register stateless validators for coordinator-side upfront validation dynamicConfigManager.register(new DiskWriteLimitConfigValidator()); rpcServer.getServerReconfigurables().forEach(dynamicConfigManager::register); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java new file mode 100644 index 00000000000..b3a106a3e1b --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.coordinator; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.MemorySize; +import org.apache.fluss.config.cluster.ServerReconfigurable; +import org.apache.fluss.exception.ConfigException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Applies dynamic historical lookup cache capacity changes to the metadata manager. */ +final class HistoricalLookupCacheConfigUpdater implements ServerReconfigurable { + + private final MetadataManager metadataManager; + + HistoricalLookupCacheConfigUpdater(MetadataManager metadataManager) { + this.metadataManager = checkNotNull(metadataManager, "metadataManager must not be null."); + } + + @Override + public void validate(Configuration newConfig) throws ConfigException { + MemorySize newMaxSize = + newConfig.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); + if (newMaxSize.getBytes() == 0) { + throw new ConfigException( + String.format( + "Invalid configuration for %s, it must be greater than 0 bytes.", + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .key())); + } + } + + @Override + public void reconfigure(Configuration newConfig) { + metadataManager.updateHistoricalLookupCacheMaxSize( + newConfig.get( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE)); + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index c489fed8671..7f2c5df3544 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -88,7 +88,7 @@ public class MetadataManager { private final ZooKeeperClient zookeeperClient; private final int maxPartitionNum; private final int maxBucketNum; - private final MemorySize historicalLookupCacheMaxSize; + private volatile MemorySize historicalLookupCacheMaxSize; private final LakeCatalogDynamicLoader lakeCatalogDynamicLoader; public static final Set SENSITIVE_TABLE_OPTIONS = new HashSet<>(); @@ -127,6 +127,10 @@ public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDe historicalLookupCacheMaxSize); } + void updateHistoricalLookupCacheMaxSize(MemorySize newMaxSize) { + historicalLookupCacheMaxSize = newMaxSize; + } + public void createDatabase( String databaseName, DatabaseDescriptor databaseDescriptor, boolean ignoreIfExists) throws DatabaseAlreadyExistException { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index b9f75bbdab6..b2fafb98e12 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -292,14 +292,25 @@ Counter capacityEvictions() { return capacityEvictions; } - synchronized void reconfigure(Configuration newConf) { + void reconfigure(Configuration newConf) { checkNotNull(newConf, "newConf must not be null."); - boolean lakeConfigChanged = hasLakeConfigChanged(conf, newConf); - // Publish the configuration before its version. A lookup that observes the new version - // must also observe the matching configuration snapshot. - conf = newConf; - if (lakeConfigChanged) { - lakeConfigVersion++; + synchronized (this) { + long newMaxBytes = + newConf.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE) + .getBytes(); + if (newMaxBytes != budgetManager.maxBytes()) { + budgetManager.updateGlobalLimit(newMaxBytes); + } + + boolean lakeConfigChanged = hasLakeConfigChanged(conf, newConf); + // Publish the configuration before its version. A lookup that observes the new version + // must also observe the matching configuration snapshot. + conf = newConf; + if (lakeConfigChanged) { + lakeConfigVersion++; + } } } @@ -447,7 +458,7 @@ private static boolean matchesLookupConfiguration( /** * Creates a lookuper after atomically reserving its configured cache capacity. * - * @return the new cached lookuper, or {@code null} if the capacity cannot be reserved + * @return the new cached lookuper, or {@code null} if its capacity cannot be reserved */ private @Nullable CachedLakeTableLookuper tryCreateCachedLookuper( LookupContext context, @@ -524,10 +535,15 @@ private static boolean matchesLookupConfiguration( * lookups. A candidate accessed after it is ordered may therefore still be evicted. */ private boolean evictLeastRecentlyUsed(long excludedTableId) { - List candidates = - new ArrayList<>(lakeTableLookupers.asMap().values()); - candidates.sort(Comparator.comparingLong(CachedLakeTableLookuper::lastAccessNanos)); - for (CachedLakeTableLookuper candidate : candidates) { + List candidates = new ArrayList<>(); + for (CachedLakeTableLookuper cachedLookuper : lakeTableLookupers.asMap().values()) { + // Snapshot the timestamp so concurrent accesses cannot change comparator inputs while + // the list is being sorted. + candidates.add(new EvictionCandidate(cachedLookuper, cachedLookuper.lastAccessNanos())); + } + candidates.sort(Comparator.comparingLong(candidate -> candidate.lastAccessNanos)); + for (EvictionCandidate evictionCandidate : candidates) { + CachedLakeTableLookuper candidate = evictionCandidate.cachedLookuper; if (candidate.tableId == excludedTableId) { continue; } @@ -719,6 +735,16 @@ private LookupContext( } } + private static final class EvictionCandidate { + private final CachedLakeTableLookuper cachedLookuper; + private final long lastAccessNanos; + + private EvictionCandidate(CachedLakeTableLookuper cachedLookuper, long lastAccessNanos) { + this.cachedLookuper = cachedLookuper; + this.lastAccessNanos = lastAccessNanos; + } + } + private static final class CachedLakeTableLookuper { private final long tableId; private final TablePath tablePath; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java index da1a4f16d6c..6bf27301503 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java @@ -40,14 +40,16 @@ *

    *
  • Each table ID has at most one current reservation. *
  • {@code reservedBytes} is the sum of the reservations in {@code reservationsByTableId}. - *
  • {@code 0 <= reservedBytes <= maxBytes}. + *
  • {@code reservedBytes >= 0}. After a dynamic limit reduction, existing reservations may + * exceed {@code maxBytes}; the caller may evict cached lookupers before retrying the next + * admission. *
*/ @ThreadSafe final class HistoricalLookupCacheBudgetManager { - // The configured limit is immutable in this version, so readers do not need synchronization. - private final long maxBytes; + @GuardedBy("this") + private long maxBytes; // Contains only reservations that currently count against the budget. Retired lookupers are // deliberately absent even when they are still serving an already acquired lookup. @@ -114,6 +116,12 @@ final class HistoricalLookupCacheBudgetManager { return newReservation; } + /** Updates the limit used by subsequent attempts without modifying existing reservations. */ + synchronized void updateGlobalLimit(long newMaxBytes) { + checkArgument(newMaxBytes > 0, "newMaxBytes must be greater than 0."); + maxBytes = newMaxBytes; + } + /** * Releases a reservation if it is still the table's current reservation. * @@ -137,14 +145,14 @@ synchronized long reservedBytes() { } /** Returns the configured capacity limit. */ - long maxBytes() { + synchronized long maxBytes() { return maxBytes; } /** - * An immutable capacity reservation for one cached lookuper. + * A capacity reservation for one cached lookuper. * - *

Each reserve or replace operation creates a new instance. The budget manager compares + *

Each reserve or replace operation creates an immutable instance. The manager compares * object identity so delayed callbacks carrying an older instance are harmless. */ static final class Reservation { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index 34574d1d29f..5425fcd8597 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -402,6 +402,63 @@ void testEvictsLeastRecentlyUsedLookuperWhenCapacityIsFull() throws Exception { assertThat(manager.capacityEvictions().getCount()).isEqualTo(1); } + @Test + void testReconfiguresGlobalCapacityLazily() throws Exception { + ManualExecutor executor = new ManualExecutor(); + Configuration conf = conf(1); + conf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + MemorySize.parse("96gb")); + TestingHistoricalLakeLookupManager manager = + new TestingHistoricalLakeLookupManager(conf, executor); + + for (int i = 0; i < 12; i++) { + lookupAndRun( + manager, + executor, + tableInfo(PARTITION_TABLE_ID + i, PARTITION_TABLE_INFO.getSchemaId())); + } + + assertThat(manager.createdLookupers).hasSize(12); + assertThat(manager.cachedTableCount()).isEqualTo(12); + assertThat(manager.capacityEvictions().getCount()).isZero(); + + // Changing only the global limit must not recreate an existing lookuper. + Configuration increasedConf = new Configuration(conf); + increasedConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + MemorySize.parse("104gb")); + manager.reconfigure(increasedConf); + lookupAndRun( + manager, + executor, + tableInfo(PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId())); + + assertThat(manager.createdLookupers).hasSize(12); + assertThat(manager.cachedTableCount()).isEqualTo(12); + assertThat(manager.capacityEvictions().getCount()).isZero(); + + // A reduction is lazy: cached lookupers remain until another admission needs capacity. + Configuration reducedConf = new Configuration(increasedConf); + reducedConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, + MemorySize.parse("88gb")); + manager.reconfigure(reducedConf); + + assertThat(manager.cachedTableCount()).isEqualTo(12); + assertThat(manager.capacityEvictions().getCount()).isZero(); + + lookupAndRun( + manager, + executor, + tableInfo(PARTITION_TABLE_ID + 12, PARTITION_TABLE_INFO.getSchemaId())); + + assertThat(manager.createdLookupers).hasSize(13); + assertThat(manager.cachedTableCount()).isEqualTo(11); + assertThat(manager.createdLookupers).filteredOn(lookuper -> lookuper.closed).hasSize(2); + assertThat(manager.capacityEvictions().getCount()).isEqualTo(2); + } + @Test void testEvictsOutsideConcurrentTableReplacements() throws Exception { ExecutorService executor = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java index 68d74bd53b5..6ef7fefa9e1 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java @@ -41,7 +41,8 @@ void testReserveAndReleaseWithinLimit() { manager.release(first); manager.release(first); assertThat(manager.reservedBytes()).isEqualTo(6); - assertThat(manager.tryReserve(3, 4)).isNotNull(); + Reservation third = manager.tryReserve(3, 4); + assertThat(third).isNotNull(); assertThat(manager.reservedBytes()).isEqualTo(10); } @@ -70,4 +71,25 @@ void testReplaceReservationAtomically() { manager.release(replacement); assertThat(manager.reservedBytes()).isEqualTo(6); } + + @Test + void testReducedLimitAppliesToSubsequentReservations() { + HistoricalLookupCacheBudgetManager manager = new HistoricalLookupCacheBudgetManager(10); + Reservation first = manager.tryReserve(1, 6); + Reservation second = manager.tryReserve(2, 4); + assertThat(first).isNotNull(); + assertThat(second).isNotNull(); + + // Shrinking is lazy: existing reservations remain even though their total exceeds the new + // limit. + manager.updateGlobalLimit(7); + assertThat(manager.maxBytes()).isEqualTo(7); + assertThat(manager.reservedBytes()).isEqualTo(10); + + // Subsequent reservations use the reduced limit and succeed only after capacity is freed. + assertThat(manager.tryReserve(3, 1)).isNull(); + + manager.release(second); + assertThat(manager.tryReserve(3, 1)).isNotNull(); + } } From 8ce38e33c090270a60aafa54e5368b9c3c0d0fe3 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 5 Aug 2026 15:39:04 +0800 Subject: [PATCH 05/15] [server] Improve historical lookup cache lifecycle and metrics Expose request, table, in-flight, and Paimon materialization metrics for historical lookups.\n\nInvalidate cached lookupers when lake configuration changes and initialize Paimon lookup state only from active data files.\n\nCo-Authored-By: Codex \nAI-Model: gpt-5\nAI-Contributed/Feature: 444/444\nAI-Contributed/UT: 58/58 AI-Contributed/Feature: 378/444 AI-Contributed/UT: 58/58 --- .../lake/lakestorage/LakeTableLookuper.java | 42 +++++ .../org/apache/fluss/metrics/MetricNames.java | 4 + .../lookup/PaimonLakeTableLookuper.java | 77 +++++++++- .../lookup/PaimonLakeTableLookuperTest.java | 27 +++- .../rpc/netty/server/NettyServerHandler.java | 7 +- .../rpc/netty/server/RequestsMetrics.java | 11 +- .../fluss/rpc/util/CommonRpcMessageUtils.java | 12 ++ .../metrics/group/TableMetricGroup.java | 145 +++++++++++++++++- .../group/TabletServerMetricGroup.java | 36 +++++ .../replica/HistoricalLakeLookupManager.java | 65 ++++++-- .../fluss/server/replica/ReplicaManager.java | 31 +++- .../fluss/server/tablet/TabletService.java | 14 +- .../HistoricalLakeLookupManagerTest.java | 31 ++++ 13 files changed, 457 insertions(+), 45 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java index b4dbb046273..25e5960ea5b 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java @@ -33,6 +33,19 @@ @PublicEvolving public interface LakeTableLookuper extends AutoCloseable { + /** Records metrics for a lake table point lookup. */ + @FunctionalInterface + interface LookupMetricRecorder { + + /** + * Records a completed lake table point lookup. + * + * @param lookupTimeNanos time spent on the lake table point lookup, in nanoseconds + * @param lookupFileMaterialization whether the lookup triggered lookup file materialization + */ + void recordLookup(long lookupTimeNanos, boolean lookupFileMaterialization); + } + /** * Looks up one key from the lake table. * @@ -48,10 +61,14 @@ public interface LakeTableLookuper extends AutoCloseable { /** Context for a lake table point lookup. */ final class LookupContext { + private static final LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = + (lookupTimeNanos, lookupFileMaterialization) -> {}; + private final ResolvedPartitionSpec partitionSpec; private final int bucketId; private final short schemaId; private final RowType valueRowType; + private final LookupMetricRecorder lookupMetricRecorder; /** * Creates a lookup context. @@ -66,10 +83,30 @@ public LookupContext( int bucketId, short schemaId, RowType valueRowType) { + this(partitionSpec, bucketId, schemaId, valueRowType, NO_OP_LOOKUP_METRIC_RECORDER); + } + + /** + * Creates a lookup context. + * + * @param partitionSpec resolved Fluss partition spec for the lookup + * @param bucketId target bucket id in the lake table + * @param schemaId schema id to encode the returned Fluss value with + * @param valueRowType row type to encode the returned Fluss value with + * @param lookupMetricRecorder recorder for lake table point lookup metrics + */ + public LookupContext( + ResolvedPartitionSpec partitionSpec, + int bucketId, + short schemaId, + RowType valueRowType, + LookupMetricRecorder lookupMetricRecorder) { this.partitionSpec = checkNotNull(partitionSpec, "partitionSpec must not be null."); this.bucketId = bucketId; this.schemaId = schemaId; this.valueRowType = checkNotNull(valueRowType, "valueRowType must not be null."); + this.lookupMetricRecorder = + checkNotNull(lookupMetricRecorder, "lookupMetricRecorder must not be null."); } /** Returns the resolved Fluss partition spec for the lookup. */ @@ -91,5 +128,10 @@ public short schemaId() { public RowType valueRowType() { return valueRowType; } + + /** Returns the recorder for lake table point lookup metrics. */ + public LookupMetricRecorder lookupMetricRecorder() { + return lookupMetricRecorder; + } } } diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java index d973933f211..332b445a3ad 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java +++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java @@ -92,6 +92,8 @@ public class MetricNames { "delayedFetchFromFollowerExpiresPerSecond"; public static final String DELAYED_FETCH_FROM_CLIENT_EXPIRES_RATE = "delayedFetchFromClientExpiresPerSecond"; + public static final String HISTORICAL_PARTITION_INFLIGHT_REQUESTS = + "historicalPartitionInflightRequests"; public static final String SERVER_LOGICAL_STORAGE_LOG_SIZE = "logSize"; public static final String SERVER_LOGICAL_STORAGE_KV_SIZE = "kvSize"; @@ -135,6 +137,8 @@ public class MetricNames { public static final String TOTAL_LOOKUP_REQUESTS_RATE = "totalLookupRequestsPerSecond"; public static final String FAILED_LOOKUP_REQUESTS_RATE = "failedLookupRequestsPerSecond"; + public static final String LAKE_LOOKUPS_RATE = "lakeLookupsPerSecond"; + public static final String LAKE_LOOKUP_TIME_MS = "lakeLookupTimeMs"; public static final String TOTAL_PUT_KV_REQUESTS_RATE = "totalPutKvRequestsPerSecond"; public static final String FAILED_PUT_KV_REQUESTS_RATE = "failedPutKvRequestsPerSecond"; public static final String TOTAL_LIMIT_SCAN_REQUESTS_RATE = "totalLimitScanRequestsPerSecond"; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index ee1e56bdf5e..eb7b7337526 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -38,6 +38,9 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.disk.BufferFileReader; +import org.apache.paimon.disk.BufferFileWriter; +import org.apache.paimon.disk.FileIOChannel; import org.apache.paimon.disk.IOManager; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.memory.MemorySegment; @@ -98,6 +101,7 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private @Nullable LocalTableQuery localTableQuery; private @Nullable RowPartitionKeyExtractor partitionKeyExtractor; private int primaryKeyFieldCount; + private long lookupFileMaterializationCount; // Both encoders are initialized only for a kv-format-v2 table whose bucket key differs from // its physical primary key. They remain null when the incoming Fluss key already uses Paimon's @@ -135,9 +139,19 @@ public PaimonLakeTableLookuper( org.apache.paimon.data.BinaryRow keyRow = toPaimonLookupKey(key); initializeFilesIfNeeded(partition, context.bucketId()); - org.apache.paimon.data.InternalRow paimonRow = - lookupWithFileRefresh( - partition, context.bucketId(), keyRow, context.valueRowType()); + long materializationCountBeforeLookup = lookupFileMaterializationCount; + long lookupStartNanos = System.nanoTime(); + org.apache.paimon.data.InternalRow paimonRow; + try { + paimonRow = + lookupWithFileRefresh( + partition, context.bucketId(), keyRow, context.valueRowType()); + } finally { + context.lookupMetricRecorder() + .recordLookup( + System.nanoTime() - lookupStartNanos, + lookupFileMaterializationCount > materializationCountBeforeLookup); + } if (paimonRow == null) { return null; } @@ -235,8 +249,8 @@ private FileStoreTable withLookupCacheOptions(FileStoreTable table) { return table.copy(Collections.singletonMap(key, maxDiskSize)); } - private static IOManager createIOManager(String ioTmpDir) { - return IOManager.create(ioTmpDir); + private IOManager createIOManager(String ioTmpDir) { + return new TrackingIOManager(IOManager.create(ioTmpDir)); } private static int[] businessFieldProjection(FileStoreTable fileStoreTable) { @@ -290,7 +304,6 @@ private void initializeFilesIfNeeded(org.apache.paimon.data.BinaryRow partition, return; } - LinkedHashMap beforeFilesByName = new LinkedHashMap<>(); LinkedHashMap dataFilesByName = new LinkedHashMap<>(); InnerTableScan tableScan = @@ -303,7 +316,6 @@ private void initializeFilesIfNeeded(org.apache.paimon.data.BinaryRow partition, continue; } DataSplit dataSplit = (DataSplit) split; - addFilesByName(beforeFilesByName, dataSplit.beforeFiles()); addFilesByName(dataFilesByName, dataSplit.dataFiles()); } @@ -312,11 +324,13 @@ private void initializeFilesIfNeeded(org.apache.paimon.data.BinaryRow partition, // been dropped. This PR does not support writes to expired partitions, so no new rows are // expected and initializing the file set once is sufficient. Compaction-related missing // files are handled by the IOException refresh path below. + // This partition-bucket has no registered lookup levels yet, so there are no old files to + // remove when building its lookup state from the active data files. localTableQuery() .refreshFiles( partition, bucketId, - new ArrayList<>(beforeFilesByName.values()), + Collections.emptyList(), new ArrayList<>(dataFilesByName.values())); initializedBuckets.add(partitionBucket); } @@ -422,4 +436,51 @@ private LocalTableQuery localTableQuery() { private RowPartitionKeyExtractor partitionKeyExtractor() { return checkNotNull(partitionKeyExtractor, "partitionKeyExtractor must be initialized."); } + + private final class TrackingIOManager implements IOManager { + + private final IOManager delegate; + + private TrackingIOManager(IOManager delegate) { + this.delegate = delegate; + } + + @Override + public FileIOChannel.ID createChannel() { + return delegate.createChannel(); + } + + @Override + public FileIOChannel.ID createChannel(String prefix) { + lookupFileMaterializationCount++; + return delegate.createChannel(prefix); + } + + @Override + public String[] tempDirs() { + return delegate.tempDirs(); + } + + @Override + public FileIOChannel.Enumerator createChannelEnumerator() { + return delegate.createChannelEnumerator(); + } + + @Override + public BufferFileWriter createBufferFileWriter(FileIOChannel.ID channelID) + throws IOException { + return delegate.createBufferFileWriter(channelID); + } + + @Override + public BufferFileReader createBufferFileReader(FileIOChannel.ID channelID) + throws IOException { + return delegate.createBufferFileReader(channelID); + } + + @Override + public void close() throws Exception { + delegate.close(); + } + } } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index 5faaacb5f25..907ba5b9481 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -121,8 +121,15 @@ void testLookupPartitionedPrimaryKeyTable() throws Exception { tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED))) { + List lookupFileMaterializations = new ArrayList<>(); LakeTableLookuper.LookupContext context = - lookupContext(schema, "20240101", 0, SCHEMA_ID); + lookupContext( + schema, + "20240101", + 0, + SCHEMA_ID, + (lookupTimeNanos, lookupFileMaterialization) -> + lookupFileMaterializations.add(lookupFileMaterialization)); byte[] value = lookuper.lookup(paimonKey(schema, 1, "20240101"), context); BinaryValue decodedValue = decodeValue(value, SCHEMA_ID, schema); @@ -136,6 +143,9 @@ void testLookupPartitionedPrimaryKeyTable() throws Exception { lookupContext(schema, "20240101", 1, SCHEMA_ID))) .isNull(); assertThat(lookuper.lookup(compactedKey(schema, 1, "20240101"), context)).isNull(); + + // The first lookup creates the local lookup file, while subsequent lookups reuse it. + assertThat(lookupFileMaterializations).containsExactly(true, false, false); } } @@ -602,6 +612,21 @@ private static LakeTableLookuper.LookupContext lookupContext( schema.getRowType()); } + private static LakeTableLookuper.LookupContext lookupContext( + Schema schema, + String partitionName, + int bucket, + short schemaId, + LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { + return new LakeTableLookuper.LookupContext( + ResolvedPartitionSpec.fromPartitionName( + Collections.singletonList("dt"), partitionName), + bucket, + schemaId, + schema.getRowType(), + lookupMetricRecorder); + } + private static List dataFiles( FileStoreTable table, BinaryRow partition, int bucket) { List files = new ArrayList<>(); diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java index 3d22c35238e..deb3367ec7d 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java @@ -26,6 +26,7 @@ import org.apache.fluss.rpc.messages.AuthenticateRequest; import org.apache.fluss.rpc.messages.AuthenticateResponse; import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.messages.LookupRequest; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.rpc.protocol.ApiKeys; import org.apache.fluss.rpc.protocol.ApiManager; @@ -55,6 +56,7 @@ import static org.apache.fluss.rpc.protocol.MessageCodec.encodeErrorResponse; import static org.apache.fluss.rpc.protocol.MessageCodec.encodeServerFailure; import static org.apache.fluss.rpc.protocol.MessageCodec.encodeSuccessResponse; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalLookup; /** Implementation of the channel handler to process inbound requests for RPC server. */ public final class NettyServerHandler extends ChannelInboundHandlerAdapter { @@ -298,13 +300,16 @@ private void updateRequestMetrics(FlussRequest request, long requestEndTimeMs) { private Optional getMetrics(FlussRequest request) { boolean isFromFollower = false; + boolean isHistoricalLookup = false; ApiMessage requestMessage = request.getMessage(); if (request.getApiKey() == ApiKeys.FETCH_LOG.id) { // for fetch, we need to identify it's from client or follower FetchLogRequest fetchLogRequest = (FetchLogRequest) requestMessage; isFromFollower = fetchLogRequest.getFollowerServerId() >= 0; + } else if (request.getApiKey() == ApiKeys.LOOKUP.id) { + isHistoricalLookup = hasHistoricalLookup((LookupRequest) requestMessage); } - return requestsMetrics.getMetrics(request.getApiKey(), isFromFollower); + return requestsMetrics.getMetrics(request.getApiKey(), isFromFollower, isHistoricalLookup); } @VisibleForTesting diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/RequestsMetrics.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/RequestsMetrics.java index 592eaf30d91..aebaa473f7e 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/RequestsMetrics.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/RequestsMetrics.java @@ -41,6 +41,8 @@ */ public class RequestsMetrics { + private static final String HISTORICAL_LOOKUP_METRICS_KEY = "historicalLookup"; + // a map from request name to the metrics registered for the request name private final Map metricsByRequest = new HashMap<>(); @@ -59,6 +61,9 @@ private RequestsMetrics(MetricGroup serverMetricsGroup, Collection apiK addMetrics(serverMetricsGroup, toRequestName(apiKey, false)); } } + if (apiKeys.contains(ApiKeys.LOOKUP)) { + addMetrics(serverMetricsGroup, HISTORICAL_LOOKUP_METRICS_KEY); + } this.requestMetricGroup = serverMetricsGroup.addGroup("request"); } @@ -115,7 +120,11 @@ private static String toRequestName(ApiKeys apiKeys, boolean isFromFollower) { } } - public Optional getMetrics(short apiKey, boolean isFromFollower) { + public Optional getMetrics( + short apiKey, boolean isFromFollower, boolean isHistoricalLookup) { + if (apiKey == ApiKeys.LOOKUP.id && isHistoricalLookup) { + return Optional.ofNullable(metricsByRequest.get(HISTORICAL_LOOKUP_METRICS_KEY)); + } String requestName = toRequestName(ApiKeys.forId(apiKey), isFromFollower); return Optional.ofNullable(metricsByRequest.get(requestName)); } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 2cf783a1184..0a7d260db78 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -26,10 +26,12 @@ import org.apache.fluss.remote.RemoteLogFetchInfo; import org.apache.fluss.remote.RemoteLogSegment; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; +import org.apache.fluss.rpc.messages.LookupRequest; import org.apache.fluss.rpc.messages.PbAclFilter; import org.apache.fluss.rpc.messages.PbAclInfo; import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket; import org.apache.fluss.rpc.messages.PbKeyValue; +import org.apache.fluss.rpc.messages.PbLookupReqForBucket; import org.apache.fluss.rpc.messages.PbPartitionSpec; import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; @@ -59,6 +61,16 @@ */ public class CommonRpcMessageUtils { + /** Returns whether the lookup request contains historical partition lookup data. */ + public static boolean hasHistoricalLookup(LookupRequest lookupRequest) { + for (PbLookupReqForBucket bucketRequest : lookupRequest.getBucketsReqsList()) { + if (bucketRequest.hasOriginalPartitionName()) { + return true; + } + } + return false; + } + public static List toPbAclInfos(Collection aclBindings) { return aclBindings.stream() .map(CommonRpcMessageUtils::toPbAclInfo) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java index 68387b94195..19acb2a58a2 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java @@ -21,6 +21,8 @@ import org.apache.fluss.metadata.TablePath; import org.apache.fluss.metrics.CharacterFilter; import org.apache.fluss.metrics.Counter; +import org.apache.fluss.metrics.DescriptiveStatisticsHistogram; +import org.apache.fluss.metrics.Histogram; import org.apache.fluss.metrics.MeterView; import org.apache.fluss.metrics.MetricNames; import org.apache.fluss.metrics.NoOpCounter; @@ -33,6 +35,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import static org.apache.fluss.metrics.utils.MetricGroupUtils.makeScope; @@ -177,7 +180,7 @@ public Counter totalLookupRequests() { if (kvMetrics == null) { return NoOpCounter.INSTANCE; } else { - return kvMetrics.totalLookupRequests; + return kvMetrics.lookupMetrics.totalLookupRequests(); } } @@ -185,7 +188,39 @@ public Counter failedLookupRequests() { if (kvMetrics == null) { return NoOpCounter.INSTANCE; } else { - return kvMetrics.failedLookupRequests; + return kvMetrics.lookupMetrics.failedLookupRequests(); + } + } + + /** Returns the counter for historical lookup requests received by this table. */ + public Counter totalHistoricalLookupRequests() { + if (kvMetrics == null) { + return NoOpCounter.INSTANCE; + } else { + return kvMetrics.historicalLookupMetrics.totalLookupRequests(); + } + } + + /** Returns the counter for failed historical lookup requests for this table. */ + public Counter failedHistoricalLookupRequests() { + if (kvMetrics == null) { + return NoOpCounter.INSTANCE; + } else { + return kvMetrics.historicalLookupMetrics.failedLookupRequests(); + } + } + + /** + * Records a historical lake table point lookup. + * + * @param lookupTimeNanos time spent on the lake table point lookup, in nanoseconds + * @param lookupFileMaterialization whether the lookup triggered lookup file materialization + */ + public void recordHistoricalLakeLookup( + long lookupTimeNanos, boolean lookupFileMaterialization) { + if (kvMetrics != null) { + kvMetrics.historicalLookupMetrics.recordLakeLookup( + lookupTimeNanos, lookupFileMaterialization); } } @@ -528,8 +563,8 @@ protected String getGroupName(CharacterFilter filter) { private static class KvMetricGroup extends TabletMetricGroup { - private final Counter totalLookupRequests; - private final Counter failedLookupRequests; + private final LookupMetricGroup lookupMetrics; + private final HistoricalLookupMetricGroup historicalLookupMetrics; private final Counter totalPutKvRequests; private final Counter failedPutKvRequests; private final Counter totalLimitScanRequests; @@ -541,10 +576,8 @@ public KvMetricGroup(TableMetricGroup tableMetricGroup) { super(tableMetricGroup, TabletType.KV); // for lookup request - totalLookupRequests = new ThreadSafeSimpleCounter(); - meter(MetricNames.TOTAL_LOOKUP_REQUESTS_RATE, new MeterView(totalLookupRequests)); - failedLookupRequests = new ThreadSafeSimpleCounter(); - meter(MetricNames.FAILED_LOOKUP_REQUESTS_RATE, new MeterView(failedLookupRequests)); + lookupMetrics = new LookupMetricGroup(registry, this, "normal"); + historicalLookupMetrics = new HistoricalLookupMetricGroup(registry, this); // for put kv request totalPutKvRequests = new ThreadSafeSimpleCounter(); meter(MetricNames.TOTAL_PUT_KV_REQUESTS_RATE, new MeterView(totalPutKvRequests)); @@ -577,6 +610,102 @@ protected String getGroupName(CharacterFilter filter) { } } + private static class LookupMetricGroup extends AbstractMetricGroup { + private final String lookupType; + private final Counter totalLookupRequests; + private final Counter failedLookupRequests; + + private LookupMetricGroup( + MetricRegistry registry, KvMetricGroup parent, String lookupType) { + super(registry, parent.getScopeComponents(), parent); + this.lookupType = lookupType; + + totalLookupRequests = new ThreadSafeSimpleCounter(); + meter(MetricNames.TOTAL_LOOKUP_REQUESTS_RATE, new MeterView(totalLookupRequests)); + failedLookupRequests = new ThreadSafeSimpleCounter(); + meter(MetricNames.FAILED_LOOKUP_REQUESTS_RATE, new MeterView(failedLookupRequests)); + } + + final Counter totalLookupRequests() { + return totalLookupRequests; + } + + final Counter failedLookupRequests() { + return failedLookupRequests; + } + + @Override + protected void putVariables(Map variables) { + variables.put("lookup_type", lookupType); + } + + @Override + protected String getGroupName(CharacterFilter filter) { + return ""; + } + } + + private static class HistoricalLookupMetricGroup extends LookupMetricGroup { + + private final LookupFileMaterializationMetricGroup materializedLookupMetrics; + private final LookupFileMaterializationMetricGroup nonMaterializedLookupMetrics; + + private HistoricalLookupMetricGroup(MetricRegistry registry, KvMetricGroup parent) { + super(registry, parent, "historical"); + materializedLookupMetrics = + new LookupFileMaterializationMetricGroup(registry, this, true); + nonMaterializedLookupMetrics = + new LookupFileMaterializationMetricGroup(registry, this, false); + } + + private void recordLakeLookup(long lookupTimeNanos, boolean lookupFileMaterialization) { + LookupFileMaterializationMetricGroup metricGroup = + lookupFileMaterialization + ? materializedLookupMetrics + : nonMaterializedLookupMetrics; + metricGroup.recordLookup(lookupTimeNanos); + } + } + + private static class LookupFileMaterializationMetricGroup extends AbstractMetricGroup { + + private static final int WINDOW_SIZE = 1024; + + private final boolean lookupFileMaterialization; + private final Counter lakeLookups; + private final Histogram lakeLookupTimeMs; + + private LookupFileMaterializationMetricGroup( + MetricRegistry registry, + HistoricalLookupMetricGroup parent, + boolean lookupFileMaterialization) { + super(registry, parent.getScopeComponents(), parent); + this.lookupFileMaterialization = lookupFileMaterialization; + + lakeLookups = new ThreadSafeSimpleCounter(); + meter(MetricNames.LAKE_LOOKUPS_RATE, new MeterView(lakeLookups)); + lakeLookupTimeMs = + histogram( + MetricNames.LAKE_LOOKUP_TIME_MS, + new DescriptiveStatisticsHistogram(WINDOW_SIZE)); + } + + private void recordLookup(long lookupTimeNanos) { + lakeLookups.inc(); + lakeLookupTimeMs.update(TimeUnit.NANOSECONDS.toMillis(lookupTimeNanos)); + } + + @Override + protected void putVariables(Map variables) { + variables.put("lookup_file_materialization", String.valueOf(lookupFileMaterialization)); + } + + @Override + protected String getGroupName(CharacterFilter filter) { + return ""; + } + } + private enum TabletType { LOG("log"), KV("kv"), diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java index 22215bc6de9..d176ecac64d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java @@ -23,6 +23,7 @@ import org.apache.fluss.metrics.CharacterFilter; import org.apache.fluss.metrics.Counter; import org.apache.fluss.metrics.DescriptiveStatisticsHistogram; +import org.apache.fluss.metrics.Gauge; import org.apache.fluss.metrics.Histogram; import org.apache.fluss.metrics.MeterView; import org.apache.fluss.metrics.MetricNames; @@ -239,6 +240,20 @@ public Counter failedIsrUpdates() { return failedIsrUpdates; } + /** + * Registers the number of in-flight historical partition requests for an operation. + * + * @param operation historical partition operation + * @param inflightRequests gauge for accepted requests that have not completed + */ + public void registerHistoricalPartitionInflightRequests( + String operation, Gauge inflightRequests) { + HistoricalPartitionOperationMetricGroup operationMetricGroup = + new HistoricalPartitionOperationMetricGroup(registry, this, operation); + operationMetricGroup.gauge( + MetricNames.HISTORICAL_PARTITION_INFLIGHT_REQUESTS, inflightRequests); + } + // ------------------------------------------------------------------------ // table buckets groups // ------------------------------------------------------------------------ @@ -267,4 +282,25 @@ public void removeTableBucketMetricGroup(TablePath tablePath, TableBucket bucket } } } + + private static class HistoricalPartitionOperationMetricGroup extends AbstractMetricGroup { + + private final String operation; + + private HistoricalPartitionOperationMetricGroup( + MetricRegistry registry, TabletServerMetricGroup parent, String operation) { + super(registry, parent.getScopeComponents(), parent); + this.operation = operation; + } + + @Override + protected void putVariables(Map variables) { + variables.put("operation", operation); + } + + @Override + protected String getGroupName(CharacterFilter filter) { + return ""; + } + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index b2fafb98e12..8830dd1383e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -104,6 +104,9 @@ class HistoricalLakeLookupManager implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(HistoricalLakeLookupManager.class); + private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = + (lookupTimeNanos, lookupFileMaterialization) -> {}; + private static final String PAIMON_LOOKUP_DIR_NAME = "paimon-lookup"; private static final String LOOKUPER_CACHE_EXPIRATION_TASK_NAME = "historical-lookuper-cache-expiration"; @@ -119,6 +122,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { private final Ticker ticker; private final HistoricalLookupCacheBudgetManager budgetManager; private final Counter capacityEvictions; + private final int maxQueuedHistoricalRequests; private final Semaphore lookupPermits; // Accepted lookup futures tracked so close() can cancel tasks left after executor shutdown. private final Set> pendingLookups; @@ -159,7 +163,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE) .getBytes()); this.capacityEvictions = new ThreadSafeSimpleCounter(); - int maxQueuedHistoricalRequests = + this.maxQueuedHistoricalRequests = conf.get(ConfigOptions.NETTY_SERVER_MAX_QUEUED_HISTORICAL_REQUESTS); checkArgument( maxQueuedHistoricalRequests > 0, @@ -214,6 +218,15 @@ CompletableFuture lookup( TableInfo tableInfo, SchemaInfo schemaInfo, TableConfig tableConfig) { + return lookup(lookupData, tableInfo, schemaInfo, tableConfig, NO_OP_LOOKUP_METRIC_RECORDER); + } + + CompletableFuture lookup( + LookupDataForBucket lookupData, + TableInfo tableInfo, + SchemaInfo schemaInfo, + TableConfig tableConfig, + LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { TableBucket tableBucket = lookupData.tableBucket(); if (!lookupPermits.tryAcquire()) { return CompletableFuture.completedFuture( @@ -230,7 +243,15 @@ CompletableFuture lookup( CompletableFuture future; try { - future = submitLookup(lookupData, tableInfo, schemaInfo, tableConfig); + future = + submitLookup( + lookupData, + tableInfo, + schemaInfo, + tableConfig, + checkNotNull( + lookupMetricRecorder, + "lookupMetricRecorder must not be null.")); } catch (RuntimeException e) { lookupPermits.release(); throw e; @@ -258,10 +279,17 @@ private CompletableFuture submitLookup( LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo, - TableConfig tableConfig) { + TableConfig tableConfig, + LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { CompletableFuture future = CompletableFuture.supplyAsync( - () -> lookupInternal(lookupData, tableInfo, schemaInfo, tableConfig), + () -> + lookupInternal( + lookupData, + tableInfo, + schemaInfo, + tableConfig, + lookupMetricRecorder), historicalPartitionExecutor); pendingLookups.add(future); return future; @@ -292,8 +320,13 @@ Counter capacityEvictions() { return capacityEvictions; } + int numInflightRequests() { + return maxQueuedHistoricalRequests - lookupPermits.availablePermits(); + } + void reconfigure(Configuration newConf) { checkNotNull(newConf, "newConf must not be null."); + boolean lakeConfigChanged; synchronized (this) { long newMaxBytes = newConf.get( @@ -304,7 +337,7 @@ void reconfigure(Configuration newConf) { budgetManager.updateGlobalLimit(newMaxBytes); } - boolean lakeConfigChanged = hasLakeConfigChanged(conf, newConf); + lakeConfigChanged = hasLakeConfigChanged(conf, newConf); // Publish the configuration before its version. A lookup that observes the new version // must also observe the matching configuration snapshot. conf = newConf; @@ -312,17 +345,27 @@ void reconfigure(Configuration newConf) { lakeConfigVersion++; } } + if (lakeConfigChanged) { + // Do not invalidate while holding this monitor: lookuper creation holds a cache key + // lock before preparing the lookup directory under the same monitor. Inactive + // lookupers close now, while active lookupers close after their last lookup releases + // them. + lakeTableLookupers.invalidateAll(); + lakeTableLookupers.cleanUp(); + } } private LookupResultForBucket lookupInternal( LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo, - TableConfig tableConfig) { + TableConfig tableConfig, + LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { TableBucket tableBucket = lookupData.tableBucket(); CachedLakeTableLookuper cachedLookuper = null; try { - LookupContext context = createLookupContext(lookupData, tableInfo, schemaInfo); + LookupContext context = + createLookupContext(lookupData, tableInfo, schemaInfo, lookupMetricRecorder); long currentLakeConfigVersion = lakeConfigVersion; Configuration currentConf = conf; long cacheSizeBytes = @@ -596,7 +639,10 @@ private void onLookuperRemoved(CachedLakeTableLookuper cachedLookuper) { } private LookupContext createLookupContext( - LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo) { + LookupDataForBucket lookupData, + TableInfo tableInfo, + SchemaInfo schemaInfo, + LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { TableBucket tableBucket = lookupData.tableBucket(); String originalPartitionName = lookupData.originalPartitionName(); if (originalPartitionName == null) { @@ -623,7 +669,8 @@ private LookupContext createLookupContext( originalPartitionSpec, tableBucket.getBucket(), (short) schemaInfo.getSchemaId(), - schemaInfo.getSchema().getRowType()); + schemaInfo.getSchema().getRowType(), + lookupMetricRecorder); return new LookupContext( tableInfo.getTableId(), schemaInfo.getSchemaId(), tablePath, lookupContext); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 534e15af68e..a1a8c0c48f9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -24,6 +24,7 @@ import org.apache.fluss.config.cluster.ServerReconfigurable; import org.apache.fluss.exception.ConfigException; import org.apache.fluss.exception.FencedLeaderEpochException; +import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidColumnProjectionException; import org.apache.fluss.exception.InvalidCoordinatorException; import org.apache.fluss.exception.InvalidPartitionException; @@ -361,6 +362,8 @@ public ReplicaManager( this.scannerManager = checkNotNull(scannerManager, "scannerManager"); this.historicalLakeLookupManager = new HistoricalLakeLookupManager(conf, pluginManager, serverId, scheduler); + serverMetricGroup.registerHistoricalPartitionInflightRequests( + "lookup", historicalLakeLookupManager::numInflightRequests); registerMetrics(); } @@ -843,8 +846,11 @@ public void historicalLookups( AtomicInteger remainingLookups = new AtomicInteger(lookupData.size()); for (LookupDataForBucket data : lookupData) { CompletableFuture lookupFuture; + TableMetricGroup tableMetrics = null; try { Replica replica = getReplicaOrException(data.tableBucket()); + tableMetrics = replica.tableMetrics(); + tableMetrics.totalHistoricalLookupRequests().inc(); if (!replica.isKvTable()) { throw new NonPrimaryKeyTableException( "Historical lookup is only supported for primary key tables, but " @@ -861,7 +867,8 @@ public void historicalLookups( data, replica.getTableInfo(), latestSchemaInfo, - replica.getTableConfig()); + replica.getTableConfig(), + tableMetrics::recordHistoricalLakeLookup); } catch (Exception e) { lookupFuture = CompletableFuture.completedFuture( @@ -871,18 +878,27 @@ public void historicalLookups( data.originalPartitionName(), ApiError.fromThrowable(e))); } + TableMetricGroup historicalLookupMetrics = tableMetrics; lookupFuture.whenComplete( (bucketResult, error) -> { + LookupResultForBucket completedResult; if (error == null) { - result.add(bucketResult); + completedResult = bucketResult; } else { - result.add( + completedResult = new LookupResultForBucket( data.tableBucket(), null, data.originalPartitionName(), - ApiError.fromThrowable(error))); + ApiError.fromThrowable(error)); } + if (historicalLookupMetrics != null + && completedResult.failed() + && isUnexpectedHistoricalLookupException( + completedResult.getError().exception())) { + historicalLookupMetrics.failedHistoricalLookupRequests().inc(); + } + result.add(completedResult); if (remainingLookups.decrementAndGet() == 0) { responseCallback.accept(result); } @@ -1842,6 +1858,13 @@ private boolean isUnexpectedException(Exception e) { || e instanceof StorageBackpressureException); } + private boolean isUnexpectedHistoricalLookupException(Exception e) { + return isUnexpectedException(e) + && !(e instanceof HistoricalPartitionThrottledException + || e instanceof InvalidPartitionException + || e instanceof NonPrimaryKeyTableException); + } + /** * Start the high watermark check point thread to periodically flush the high watermark value * for all buckets to the high watermark checkpoint file. diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index 9452842ab67..f7a004de351 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -62,7 +62,6 @@ import org.apache.fluss.rpc.messages.NotifyLeaderAndIsrResponse; import org.apache.fluss.rpc.messages.NotifyRemoteLogOffsetsRequest; import org.apache.fluss.rpc.messages.NotifyRemoteLogOffsetsResponse; -import org.apache.fluss.rpc.messages.PbLookupReqForBucket; import org.apache.fluss.rpc.messages.PbScanReqForBucket; import org.apache.fluss.rpc.messages.PrefixLookupRequest; import org.apache.fluss.rpc.messages.PrefixLookupResponse; @@ -123,6 +122,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; +import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.hasHistoricalLookup; import static org.apache.fluss.security.acl.OperationType.DESCRIBE; import static org.apache.fluss.security.acl.OperationType.READ; import static org.apache.fluss.security.acl.OperationType.WRITE; @@ -368,18 +368,6 @@ public CompletableFuture prefixLookup(PrefixLookupRequest return response; } - private boolean hasHistoricalLookup(LookupRequest request) { - for (PbLookupReqForBucket lookupReqForBucket : request.getBucketsReqsList()) { - if (lookupReqForBucket.hasOriginalPartitionName()) { - // An original partition name is only set for historical lookups, so route the - // whole request to the historical path. Conversion rejects requests that mix - // normal and historical lookup batches. - return true; - } - } - return false; - } - @Override public CompletableFuture limitScan(LimitScanRequest request) { authorizeTable(READ, request.getTableId()); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index 5425fcd8597..1acc4c5bd62 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -23,6 +23,7 @@ import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; +import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.SchemaInfo; @@ -78,6 +79,7 @@ class HistoricalLakeLookupManagerTest { void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { ManualExecutor executor = new ManualExecutor(); HistoricalLakeLookupManager manager = createManager(1, executor); + assertThat(manager.numInflightRequests()).isZero(); CompletableFuture first = manager.lookup( @@ -87,6 +89,7 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { PARTITION_TABLE_INFO.getTableConfig()); assertThat(first).isNotDone(); assertThat(executor.numQueuedTasks()).isEqualTo(1); + assertThat(manager.numInflightRequests()).isOne(); TableBucket secondBucket = new TableBucket(PARTITION_TABLE_ID, 2L, 0); LookupResultForBucket second = @@ -102,6 +105,7 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { assertThat(second.getError().exception()) .isInstanceOf(HistoricalPartitionThrottledException.class); assertThat(executor.numQueuedTasks()).isEqualTo(1); + assertThat(manager.numInflightRequests()).isOne(); } @Test @@ -120,6 +124,7 @@ void testHistoricalLookupReleasesPermitOnFailure() throws Exception { assertThat(firstResult.failed()).isTrue(); assertThat(firstResult.getError().error()) .isNotEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); + assertThat(manager.numInflightRequests()).isZero(); CompletableFuture second = manager.lookup( @@ -459,6 +464,30 @@ void testReconfiguresGlobalCapacityLazily() throws Exception { assertThat(manager.capacityEvictions().getCount()).isEqualTo(2); } + @Test + void testReconfiguresLakePropertiesAndInvalidatesLookuper() throws Exception { + Configuration initialConf = conf(1); + initialConf.set(ConfigOptions.DATALAKE_FORMAT, DataLakeFormat.PAIMON); + initialConf.setString("datalake.paimon.warehouse", "old-warehouse"); + ManualExecutor executor = new ManualExecutor(); + TestingHistoricalLakeLookupManager manager = + new TestingHistoricalLakeLookupManager(initialConf, executor); + + lookupAndRun(manager, executor, PARTITION_TABLE_INFO); + TestingLakeTableLookuper initialLookuper = manager.createdLookupers.get(0); + + Configuration newConf = new Configuration(initialConf); + newConf.setString("datalake.paimon.warehouse", "new-warehouse"); + manager.reconfigure(newConf); + + assertThat(initialLookuper.closed).isTrue(); + assertThat(manager.cachedTableCount()).isZero(); + lookupAndRun(manager, executor, PARTITION_TABLE_INFO); + assertThat(manager.createdLookupers).hasSize(2); + assertThat(manager.createdClusterConfigs.get(1).toMap()) + .containsEntry("datalake.paimon.warehouse", "new-warehouse"); + } + @Test void testEvictsOutsideConcurrentTableReplacements() throws Exception { ExecutorService executor = @@ -696,6 +725,7 @@ private static final class TestingHistoricalLakeLookupManager private final List createdLookupers = new ArrayList<>(); private final List createdIoTmpDirs = new ArrayList<>(); private final List createdTableConfigs = new ArrayList<>(); + private final List createdClusterConfigs = new ArrayList<>(); private TestingHistoricalLakeLookupManager(Configuration conf, ManualExecutor executor) { super( @@ -725,6 +755,7 @@ LakeTableLookuper createLakeTableLookuper( createdLookupers.add(lookuper); createdIoTmpDirs.add(ioTmpDir); createdTableConfigs.add(tableConfig); + createdClusterConfigs.add(clusterConf); return lookuper; } } From 5b6e92c092b5808ed442f8ec4811d880beccb129 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 5 Aug 2026 17:57:02 +0800 Subject: [PATCH 06/15] [server] Refine historical lookup cache lifecycle Make idle expiration dynamically reconfigurable and use Caffeine access order for capacity eviction. Preserve existing lookup metric labels by exposing historical request rates separately, and validate table cache sizes only when historical lookup is enabled. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 170/197 AI-Contributed/UT: 49/73 --- .../apache/fluss/config/ConfigOptions.java | 4 +- .../lake/lakestorage/LakeTableLookuper.java | 19 ---- .../org/apache/fluss/metrics/MetricNames.java | 4 + .../lookup/PaimonLakeTableLookuper.java | 1 + .../lookup/PaimonLakeTableLookuperTest.java | 11 ++- .../fluss/server/DynamicServerConfig.java | 2 + .../HistoricalLookupCacheConfigUpdater.java | 17 +++- .../metrics/group/TableMetricGroup.java | 53 +++++------ .../replica/HistoricalLakeLookupManager.java | 93 ++++++------------- .../utils/TableDescriptorValidation.java | 4 +- .../HistoricalLakeLookupManagerTest.java | 38 +++++--- ...istoricalPartitionTableValidationTest.java | 24 ++++- 12 files changed, 136 insertions(+), 134 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 72f210538b3..3dab10fd846 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -435,7 +435,6 @@ public class ConfigOptions { .defaultValue(MemorySize.parse("80gb")) .withDescription( "The total configured disk capacity available to current and creating historical partition lookup caches on a TabletServer. " - + "Retired cache generations that are still serving active lookups are not included in this limit. " + "The value must be greater than zero."); public static final ConfigOption @@ -444,8 +443,7 @@ public class ConfigOptions { .durationType() .defaultValue(Duration.ofHours(3)) .withDescription( - "The duration after which an idle historical partition table lookuper is removed from the cache. " - + "This option requires a TabletServer restart to take effect."); + "The duration after which an idle historical partition table lookuper is removed from the cache."); public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = key("server.data-disk.write-limit-ratio") diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java index 25e5960ea5b..ab50acdb8f2 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java @@ -61,31 +61,12 @@ interface LookupMetricRecorder { /** Context for a lake table point lookup. */ final class LookupContext { - private static final LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = - (lookupTimeNanos, lookupFileMaterialization) -> {}; - private final ResolvedPartitionSpec partitionSpec; private final int bucketId; private final short schemaId; private final RowType valueRowType; private final LookupMetricRecorder lookupMetricRecorder; - /** - * Creates a lookup context. - * - * @param partitionSpec resolved Fluss partition spec for the lookup - * @param bucketId target bucket id in the lake table - * @param schemaId schema id to encode the returned Fluss value with - * @param valueRowType row type to encode the returned Fluss value with - */ - public LookupContext( - ResolvedPartitionSpec partitionSpec, - int bucketId, - short schemaId, - RowType valueRowType) { - this(partitionSpec, bucketId, schemaId, valueRowType, NO_OP_LOOKUP_METRIC_RECORDER); - } - /** * Creates a lookup context. * diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java index 332b445a3ad..e80ad687cfe 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java +++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java @@ -137,6 +137,10 @@ public class MetricNames { public static final String TOTAL_LOOKUP_REQUESTS_RATE = "totalLookupRequestsPerSecond"; public static final String FAILED_LOOKUP_REQUESTS_RATE = "failedLookupRequestsPerSecond"; + public static final String TOTAL_HISTORICAL_LOOKUP_REQUESTS_RATE = + "totalHistoricalLookupRequestsPerSecond"; + public static final String FAILED_HISTORICAL_LOOKUP_REQUESTS_RATE = + "failedHistoricalLookupRequestsPerSecond"; public static final String LAKE_LOOKUPS_RATE = "lakeLookupsPerSecond"; public static final String LAKE_LOOKUP_TIME_MS = "lakeLookupTimeMs"; public static final String TOTAL_PUT_KV_REQUESTS_RATE = "totalPutKvRequestsPerSecond"; diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index eb7b7337526..b40f04cf4c9 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -437,6 +437,7 @@ private RowPartitionKeyExtractor partitionKeyExtractor() { return checkNotNull(partitionKeyExtractor, "partitionKeyExtractor must be initialized."); } + /** Tracks creation of Paimon lookup files while delegating all local I/O operations. */ private final class TrackingIOManager implements IOManager { private final IOManager delegate; diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index 907ba5b9481..eea70ee6c61 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -77,6 +77,8 @@ class PaimonLakeTableLookuperTest { private static final String DB = "lookup_db"; private static final short SCHEMA_ID = 1; private static final short EVOLVED_SCHEMA_ID = 2; + private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = + (lookupTimeNanos, lookupFileMaterialization) -> {}; @TempDir private File tempWarehouseDir; @@ -433,7 +435,8 @@ void testLookupWithNonStringPartitionKey() throws Exception { Collections.singletonList("pt"), "7"), 0, SCHEMA_ID, - schema.getRowType()); + schema.getRowType(), + NO_OP_LOOKUP_METRIC_RECORDER); BinaryValue decodedValue = decodeValue( @@ -468,7 +471,8 @@ void testRejectAppendOnlyTable() throws Exception { Collections.emptyList(), Collections.emptyList()), 0, SCHEMA_ID, - schema.getRowType()); + schema.getRowType(), + NO_OP_LOOKUP_METRIC_RECORDER); assertThatThrownBy(() -> lookuper.lookup(new byte[0], context)) .isInstanceOf(UnsupportedOperationException.class) @@ -609,7 +613,8 @@ private static LakeTableLookuper.LookupContext lookupContext( Collections.singletonList("dt"), partitionName), bucket, schemaId, - schema.getRowType()); + schema.getRowType(), + NO_OP_LOOKUP_METRIC_RECORDER); } private static LakeTableLookuper.LookupContext lookupContext( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java index 367fa69c9d9..f5e7caa9db7 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java @@ -56,6 +56,7 @@ import static org.apache.fluss.config.ConfigOptions.REMOTE_DATA_DIRS_WEIGHTS; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_RECOVER_RATIO; +import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_CREDENTIALS; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG; @@ -84,6 +85,7 @@ class DynamicServerConfig { SERVER_DATA_DISK_WRITE_RECOVER_RATIO.key(), SERVER_DATA_DISK_WRITE_LIMIT_RATIO.key(), SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key(), + SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS.key(), // Config options for remote.data.dirs REMOTE_DATA_DIRS.key(), REMOTE_DATA_DIRS_STRATEGY.key(), diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java index b3a106a3e1b..9b5a0b1292a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java @@ -23,9 +23,11 @@ import org.apache.fluss.config.cluster.ServerReconfigurable; import org.apache.fluss.exception.ConfigException; +import java.time.Duration; + import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Applies dynamic historical lookup cache capacity changes to the metadata manager. */ +/** Validates dynamic historical lookup cache settings and updates the metadata manager. */ final class HistoricalLookupCacheConfigUpdater implements ServerReconfigurable { private final MetadataManager metadataManager; @@ -45,6 +47,19 @@ public void validate(Configuration newConfig) throws ConfigException { ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE .key())); } + + Duration newExpiration = + newConfig.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS); + if (newExpiration.toMillis() < 1) { + throw new ConfigException( + String.format( + "Invalid configuration for %s, it must be greater than or equal 1 ms.", + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS + .key())); + } } @Override diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java index 19acb2a58a2..f78fce63b93 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java @@ -180,7 +180,7 @@ public Counter totalLookupRequests() { if (kvMetrics == null) { return NoOpCounter.INSTANCE; } else { - return kvMetrics.lookupMetrics.totalLookupRequests(); + return kvMetrics.totalLookupRequests; } } @@ -188,7 +188,7 @@ public Counter failedLookupRequests() { if (kvMetrics == null) { return NoOpCounter.INSTANCE; } else { - return kvMetrics.lookupMetrics.failedLookupRequests(); + return kvMetrics.failedLookupRequests; } } @@ -563,7 +563,8 @@ protected String getGroupName(CharacterFilter filter) { private static class KvMetricGroup extends TabletMetricGroup { - private final LookupMetricGroup lookupMetrics; + private final Counter totalLookupRequests; + private final Counter failedLookupRequests; private final HistoricalLookupMetricGroup historicalLookupMetrics; private final Counter totalPutKvRequests; private final Counter failedPutKvRequests; @@ -576,7 +577,10 @@ public KvMetricGroup(TableMetricGroup tableMetricGroup) { super(tableMetricGroup, TabletType.KV); // for lookup request - lookupMetrics = new LookupMetricGroup(registry, this, "normal"); + totalLookupRequests = new ThreadSafeSimpleCounter(); + meter(MetricNames.TOTAL_LOOKUP_REQUESTS_RATE, new MeterView(totalLookupRequests)); + failedLookupRequests = new ThreadSafeSimpleCounter(); + meter(MetricNames.FAILED_LOOKUP_REQUESTS_RATE, new MeterView(failedLookupRequests)); historicalLookupMetrics = new HistoricalLookupMetricGroup(registry, this); // for put kv request totalPutKvRequests = new ThreadSafeSimpleCounter(); @@ -610,20 +614,28 @@ protected String getGroupName(CharacterFilter filter) { } } - private static class LookupMetricGroup extends AbstractMetricGroup { - private final String lookupType; + private static class HistoricalLookupMetricGroup extends AbstractMetricGroup { + private final Counter totalLookupRequests; private final Counter failedLookupRequests; + private final LookupFileMaterializationMetricGroup materializedLookupMetrics; + private final LookupFileMaterializationMetricGroup nonMaterializedLookupMetrics; - private LookupMetricGroup( - MetricRegistry registry, KvMetricGroup parent, String lookupType) { + private HistoricalLookupMetricGroup(MetricRegistry registry, KvMetricGroup parent) { super(registry, parent.getScopeComponents(), parent); - this.lookupType = lookupType; totalLookupRequests = new ThreadSafeSimpleCounter(); - meter(MetricNames.TOTAL_LOOKUP_REQUESTS_RATE, new MeterView(totalLookupRequests)); + meter( + MetricNames.TOTAL_HISTORICAL_LOOKUP_REQUESTS_RATE, + new MeterView(totalLookupRequests)); failedLookupRequests = new ThreadSafeSimpleCounter(); - meter(MetricNames.FAILED_LOOKUP_REQUESTS_RATE, new MeterView(failedLookupRequests)); + meter( + MetricNames.FAILED_HISTORICAL_LOOKUP_REQUESTS_RATE, + new MeterView(failedLookupRequests)); + materializedLookupMetrics = + new LookupFileMaterializationMetricGroup(registry, this, true); + nonMaterializedLookupMetrics = + new LookupFileMaterializationMetricGroup(registry, this, false); } final Counter totalLookupRequests() { @@ -634,29 +646,10 @@ final Counter failedLookupRequests() { return failedLookupRequests; } - @Override - protected void putVariables(Map variables) { - variables.put("lookup_type", lookupType); - } - @Override protected String getGroupName(CharacterFilter filter) { return ""; } - } - - private static class HistoricalLookupMetricGroup extends LookupMetricGroup { - - private final LookupFileMaterializationMetricGroup materializedLookupMetrics; - private final LookupFileMaterializationMetricGroup nonMaterializedLookupMetrics; - - private HistoricalLookupMetricGroup(MetricRegistry registry, KvMetricGroup parent) { - super(registry, parent, "historical"); - materializedLookupMetrics = - new LookupFileMaterializationMetricGroup(registry, this, true); - nonMaterializedLookupMetrics = - new LookupFileMaterializationMetricGroup(registry, this, false); - } private void recordLakeLookup(long lookupTimeNanos, boolean lookupFileMaterialization) { LookupFileMaterializationMetricGroup metricGroup = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index 8830dd1383e..3b4425bb5e7 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -63,7 +63,6 @@ import java.nio.file.Files; import java.time.Duration; import java.util.ArrayList; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -104,9 +103,6 @@ class HistoricalLakeLookupManager implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(HistoricalLakeLookupManager.class); - private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = - (lookupTimeNanos, lookupFileMaterialization) -> {}; - private static final String PAIMON_LOOKUP_DIR_NAME = "paimon-lookup"; private static final String LOOKUPER_CACHE_EXPIRATION_TASK_NAME = "historical-lookuper-cache-expiration"; @@ -119,7 +115,6 @@ class HistoricalLakeLookupManager implements AutoCloseable { private volatile long lakeConfigVersion; private final @Nullable PluginManager pluginManager; private final int serverId; - private final Ticker ticker; private final HistoricalLookupCacheBudgetManager budgetManager; private final Counter capacityEvictions; private final int maxQueuedHistoricalRequests; @@ -155,7 +150,6 @@ class HistoricalLakeLookupManager implements AutoCloseable { this.conf = checkNotNull(conf, "conf must not be null."); this.pluginManager = pluginManager; this.serverId = serverId; - this.ticker = checkNotNull(ticker, "ticker must not be null."); this.budgetManager = new HistoricalLookupCacheBudgetManager( conf.get( @@ -185,7 +179,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { conf.get( ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS)) - .ticker(this.ticker) + .ticker(checkNotNull(ticker, "ticker must not be null.")) .scheduler(checkNotNull(cacheScheduler, "cacheScheduler must not be null.")) .executor(Runnable::run) .removalListener( @@ -213,14 +207,6 @@ private static com.github.benmanes.caffeine.cache.Scheduler createCacheScheduler timeUnit.toMillis(delay)); } - CompletableFuture lookup( - LookupDataForBucket lookupData, - TableInfo tableInfo, - SchemaInfo schemaInfo, - TableConfig tableConfig) { - return lookup(lookupData, tableInfo, schemaInfo, tableConfig, NO_OP_LOOKUP_METRIC_RECORDER); - } - CompletableFuture lookup( LookupDataForBucket lookupData, TableInfo tableInfo, @@ -327,6 +313,11 @@ int numInflightRequests() { void reconfigure(Configuration newConf) { checkNotNull(newConf, "newConf must not be null."); boolean lakeConfigChanged; + boolean expirationChanged; + Duration newExpiration = + newConf.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS); synchronized (this) { long newMaxBytes = newConf.get( @@ -338,6 +329,11 @@ void reconfigure(Configuration newConf) { } lakeConfigChanged = hasLakeConfigChanged(conf, newConf); + expirationChanged = + !newExpiration.equals( + conf.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS)); // Publish the configuration before its version. A lookup that observes the new version // must also observe the matching configuration snapshot. conf = newConf; @@ -345,6 +341,13 @@ void reconfigure(Configuration newConf) { lakeConfigVersion++; } } + if (expirationChanged) { + lakeTableLookupers + .policy() + .expireAfterAccess() + .get() + .setExpiresAfter(newExpiration.toMillis(), TimeUnit.MILLISECONDS); + } if (lakeConfigChanged) { // Do not invalidate while holding this monitor: lookuper creation holds a cache key // lock before preparing the lookup directory under the same monitor. Inactive @@ -478,7 +481,7 @@ private CachedLakeTableLookuper acquireCachedLookuper( // Pin the lookuper before leaving the atomic cache update. // Eviction or invalidation can then defer closing it until this // lookup releases it. - selectedLookuper.acquire(ticker.read()); + selectedLookuper.acquire(); return selectedLookuper; }); return matchesLookupConfiguration( @@ -574,33 +577,24 @@ private static boolean matchesLookupConfiguration( /** * Evicts one eligible cached lookuper using best-effort LRU order. * - *

Candidates are ordered by their last-access timestamps without blocking concurrent - * lookups. A candidate accessed after it is ordered may therefore still be evicted. + *

Candidates use Caffeine's expire-after-access order. A candidate accessed after the + * snapshot is taken may therefore still be evicted. */ private boolean evictLeastRecentlyUsed(long excludedTableId) { - List candidates = new ArrayList<>(); - for (CachedLakeTableLookuper cachedLookuper : lakeTableLookupers.asMap().values()) { - // Snapshot the timestamp so concurrent accesses cannot change comparator inputs while - // the list is being sorted. - candidates.add(new EvictionCandidate(cachedLookuper, cachedLookuper.lastAccessNanos())); - } - candidates.sort(Comparator.comparingLong(candidate -> candidate.lastAccessNanos)); - for (EvictionCandidate evictionCandidate : candidates) { - CachedLakeTableLookuper candidate = evictionCandidate.cachedLookuper; + Map candidates = + lakeTableLookupers + .policy() + .expireAfterAccess() + .get() + .oldest(lakeTableLookupers.asMap().size()); + for (CachedLakeTableLookuper candidate : candidates.values()) { if (candidate.tableId == excludedTableId) { continue; } - // Claim this victim so concurrent admission threads cannot evict it twice. A false - // result means it was already invalidated or claimed by another eviction. - if (!candidate.markEvictionPending()) { - continue; - } - // The snapshot may be stale after expiration or replacement. Compare-and-remove // prevents this eviction from removing a newer lookuper for the same table. boolean removed = lakeTableLookupers.asMap().remove(candidate.tableId, candidate); if (!removed) { - candidate.clearEvictionPending(); continue; } @@ -782,16 +776,6 @@ private LookupContext( } } - private static final class EvictionCandidate { - private final CachedLakeTableLookuper cachedLookuper; - private final long lastAccessNanos; - - private EvictionCandidate(CachedLakeTableLookuper cachedLookuper, long lastAccessNanos) { - this.cachedLookuper = cachedLookuper; - this.lastAccessNanos = lastAccessNanos; - } - } - private static final class CachedLakeTableLookuper { private final long tableId; private final TablePath tablePath; @@ -801,9 +785,7 @@ private static final class CachedLakeTableLookuper { private final File tableLookupDir; private final Reservation reservation; private final LakeTableLookuper lookuper; - private long lastAccessNanos; private int activeLookups; - private boolean evictionPending; private boolean invalidated; private boolean closed; @@ -826,30 +808,13 @@ private CachedLakeTableLookuper( this.lookuper = lookuper; } - private synchronized void acquire(long accessNanos) { + private synchronized void acquire() { if (invalidated) { throw new IllegalStateException("Lake table lookuper has been invalidated."); } - lastAccessNanos = accessNanos; activeLookups++; } - private synchronized long lastAccessNanos() { - return lastAccessNanos; - } - - private synchronized boolean markEvictionPending() { - if (invalidated || evictionPending) { - return false; - } - evictionPending = true; - return true; - } - - private synchronized void clearEvictionPending() { - evictionPending = false; - } - private void release() { synchronized (this) { if (activeLookups <= 0) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 36bdc0439e7..c68c8924419 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -132,7 +132,9 @@ public static void validateTableDescriptor( checkDeleteBehavior(tableConf, hasPrimaryKey); checkTieredLog(tableConf); checkHistoricalPartition(tableDescriptor, tableConf); - checkHistoricalLookupCacheSize(tableConf, tablePath, historicalLookupCacheMaxSize); + if (tableConf.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED)) { + checkHistoricalLookupCacheSize(tableConf, tablePath, historicalLookupCacheMaxSize); + } checkPartition(tableConf, tableDescriptor.getPartitionKeys(), schema.getRowType()); checkSystemColumns(schema.getRowType()); validateStatisticsConfig(tableDescriptor); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index 1acc4c5bd62..e3082f93695 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -72,6 +72,8 @@ class HistoricalLakeLookupManagerTest { private static final int SERVER_ID = 1; private static final TableBucket HISTORICAL_BUCKET = new TableBucket(PARTITION_TABLE_ID, 1L, 0); + private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = + (lookupTimeNanos, lookupFileMaterialization) -> {}; @TempDir private File ioTmpDir; @@ -86,7 +88,8 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { lookupData(HISTORICAL_BUCKET), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig()); + PARTITION_TABLE_INFO.getTableConfig(), + NO_OP_LOOKUP_METRIC_RECORDER); assertThat(first).isNotDone(); assertThat(executor.numQueuedTasks()).isEqualTo(1); assertThat(manager.numInflightRequests()).isOne(); @@ -97,7 +100,8 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { lookupData(secondBucket), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig()) + PARTITION_TABLE_INFO.getTableConfig(), + NO_OP_LOOKUP_METRIC_RECORDER) .get(1, TimeUnit.SECONDS); assertThat(second.failed()).isTrue(); @@ -118,7 +122,8 @@ void testHistoricalLookupReleasesPermitOnFailure() throws Exception { lookupData(HISTORICAL_BUCKET), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig()); + PARTITION_TABLE_INFO.getTableConfig(), + NO_OP_LOOKUP_METRIC_RECORDER); executor.runNext(); LookupResultForBucket firstResult = first.get(1, TimeUnit.SECONDS); assertThat(firstResult.failed()).isTrue(); @@ -131,7 +136,8 @@ void testHistoricalLookupReleasesPermitOnFailure() throws Exception { lookupData(HISTORICAL_BUCKET), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig()); + PARTITION_TABLE_INFO.getTableConfig(), + NO_OP_LOOKUP_METRIC_RECORDER); assertThat(second).isNotDone(); assertThat(executor.numQueuedTasks()).isEqualTo(1); } @@ -146,19 +152,22 @@ void testHistoricalLookupMaxQueuedRequestsUsesExplicitConfig() throws Exception lookupData(new TableBucket(PARTITION_TABLE_ID, 1L, 0)), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig()); + PARTITION_TABLE_INFO.getTableConfig(), + NO_OP_LOOKUP_METRIC_RECORDER); CompletableFuture second = manager.lookup( lookupData(new TableBucket(PARTITION_TABLE_ID, 2L, 0)), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig()); + PARTITION_TABLE_INFO.getTableConfig(), + NO_OP_LOOKUP_METRIC_RECORDER); LookupResultForBucket third = manager.lookup( lookupData(new TableBucket(PARTITION_TABLE_ID, 3L, 0)), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig()) + PARTITION_TABLE_INFO.getTableConfig(), + NO_OP_LOOKUP_METRIC_RECORDER) .get(1, TimeUnit.SECONDS); assertThat(first).isNotDone(); @@ -341,7 +350,7 @@ void testReplacesLookuperOnlyWhenEffectiveCacheSizeChanges() throws Exception { } @Test - void testExpiresIdleLookuperWithoutAnotherLookup() throws Exception { + void testDynamicallyUpdatesExpirationAndExpiresIdleLookuper() throws Exception { ManualExecutor executor = new ManualExecutor(); AtomicLong tickerNanos = new AtomicLong(); AtomicReference> expirationTask = new AtomicReference<>(); @@ -366,7 +375,8 @@ void testExpiresIdleLookuperWithoutAnotherLookup() throws Exception { lookupAndRun(manager, executor, PARTITION_TABLE_INFO); TestingLakeTableLookuper expiredLookuper = manager.createdLookupers.get(0); - tickerNanos.addAndGet(Duration.ofHours(2).toNanos()); + manager.reconfigure(confWithExpiration(Duration.ofMinutes(30))); + tickerNanos.addAndGet(Duration.ofMinutes(31).toNanos()); assertThat(expirationTask.get()).isNotNull(); expirationTask.get().run(); @@ -619,7 +629,8 @@ private static CompletableFuture lookup( lookupData(new TableBucket(tableInfo.getTableId(), 1L, 0)), tableInfo, tableInfo.getSchemaInfo(), - tableConfig); + tableConfig, + NO_OP_LOOKUP_METRIC_RECORDER); } private static TableInfo tableInfo(long tableId, int schemaId) { @@ -715,7 +726,12 @@ private static LookupResultForBucket lookupResultAndRun( throws Exception { TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 1L, 0); CompletableFuture future = - manager.lookup(lookupData(tableBucket), tableInfo, schemaInfo, tableConfig); + manager.lookup( + lookupData(tableBucket), + tableInfo, + schemaInfo, + tableConfig, + NO_OP_LOOKUP_METRIC_RECORDER); executor.runNext(); return future.get(1, TimeUnit.SECONDS); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index ea7b188e6dd..eb8942c534a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; class HistoricalPartitionTableValidationTest { @@ -97,7 +98,16 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { } @Test - void testRejectInvalidHistoricalLookupCacheSize() { + void testValidateHistoricalLookupCacheSize() { + TableDescriptor ordinaryTableDescriptor = + TableDescriptor.builder() + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) + .build(); + assertThatCode(() -> validate(ordinaryTableDescriptor, MemorySize.parse("4gb"))) + .doesNotThrowAnyException(); + TableDescriptor zeroSizeDescriptor = descriptorWithCacheSize(MemorySize.ZERO); assertThatThrownBy(() -> validate(zeroSizeDescriptor, MemorySize.parse("80gb"))) .isInstanceOf(InvalidConfigException.class) @@ -118,9 +128,19 @@ void testRejectInvalidHistoricalLookupCacheSize() { private static TableDescriptor descriptorWithCacheSize(MemorySize cacheSize) { return TableDescriptor.builder() - .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .primaryKey("id", "dt") + .build()) + .partitionedBy("dt") .distributedBy(1) .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) + .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) .property( ConfigOptions .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, From 838500486a87173a7b1b7b45d7c2d5f156991792 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 5 Aug 2026 18:35:39 +0800 Subject: [PATCH 07/15] [server] Clean stale historical lookup cache files Clean cache files left by a previous TabletServer process during startup while keeping directory creation lazy. Tighten historical lookup cache configuration validation and isolate temporary directories in replica tests. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 78/86 AI-Contributed/UT: 42/44 --- .../apache/fluss/config/FlussConfigUtils.java | 2 +- .../HistoricalLookupCacheConfigUpdater.java | 4 +- .../replica/HistoricalLakeLookupManager.java | 76 +++++++++++++------ .../fluss/server/replica/ReplicaManager.java | 2 + .../utils/TableDescriptorValidation.java | 2 +- .../HistoricalLakeLookupManagerTest.java | 42 +++++++--- .../fluss/server/replica/ReplicaTestBase.java | 1 + .../fetcher/ReplicaFetcherThreadTest.java | 1 + 8 files changed, 90 insertions(+), 40 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index dd351841d37..c05cbc13237 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -241,7 +241,7 @@ protected static void validateServerConfigs(Configuration conf) { private static void validateHistoricalLookupCacheLimit(Configuration conf) { MemorySize historicalLookupCacheMaxSize = conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); - if (historicalLookupCacheMaxSize.getBytes() == 0) { + if (historicalLookupCacheMaxSize.getBytes() <= 0) { throw new IllegalConfigurationException( "Invalid configuration for %s, it must be greater than 0 bytes.", ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key()); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java index 9b5a0b1292a..94709e366a4 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java @@ -40,7 +40,7 @@ final class HistoricalLookupCacheConfigUpdater implements ServerReconfigurable { public void validate(Configuration newConfig) throws ConfigException { MemorySize newMaxSize = newConfig.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); - if (newMaxSize.getBytes() == 0) { + if (newMaxSize.getBytes() <= 0) { throw new ConfigException( String.format( "Invalid configuration for %s, it must be greater than 0 bytes.", @@ -55,7 +55,7 @@ public void validate(Configuration newConfig) throws ConfigException { if (newExpiration.toMillis() < 1) { throw new ConfigException( String.format( - "Invalid configuration for %s, it must be greater than or equal 1 ms.", + "Invalid configuration for %s, it must be greater than or equal to 1 ms.", ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS .key())); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index 3b4425bb5e7..593e5728e73 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -57,6 +57,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; import java.io.File; import java.io.IOException; @@ -78,6 +79,7 @@ import static org.apache.fluss.server.utils.LakeStorageUtils.extractLakeProperties; import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; /** * Handles server-side point lookup for historical partitions stored in lake storage. @@ -114,7 +116,6 @@ class HistoricalLakeLookupManager implements AutoCloseable { private volatile Configuration conf; private volatile long lakeConfigVersion; private final @Nullable PluginManager pluginManager; - private final int serverId; private final HistoricalLookupCacheBudgetManager budgetManager; private final Counter capacityEvictions; private final int maxQueuedHistoricalRequests; @@ -123,7 +124,12 @@ class HistoricalLakeLookupManager implements AutoCloseable { private final Set> pendingLookups; private final Cache lakeTableLookupers; private final ExecutorService historicalPartitionExecutor; - private @Nullable File paimonLookupTempDir; + private final File paimonLookupTempDir; + + @GuardedBy("this") + private boolean paimonLookupTempDirCreated; + + private volatile boolean started; HistoricalLakeLookupManager( Configuration conf, @@ -149,7 +155,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { com.github.benmanes.caffeine.cache.Scheduler cacheScheduler) { this.conf = checkNotNull(conf, "conf must not be null."); this.pluginManager = pluginManager; - this.serverId = serverId; + this.paimonLookupTempDir = resolvePaimonLookupTempDir(conf, serverId); this.budgetManager = new HistoricalLookupCacheBudgetManager( conf.get( @@ -207,12 +213,34 @@ private static com.github.benmanes.caffeine.cache.Scheduler createCacheScheduler timeUnit.toMillis(delay)); } + /** + * Attempts to clean lookup cache files left by a previous TabletServer process. + * + *

Only this server's directory is removed. It is recreated lazily when the first table + * lookuper is created. + */ + synchronized void startup() { + if (started) { + return; + } + try { + FileUtils.deleteDirectory(paimonLookupTempDir); + } catch (IOException e) { + LOG.warn( + "Failed to clean Paimon lookup temporary directory {}.", + paimonLookupTempDir, + e); + } + started = true; + } + CompletableFuture lookup( LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo, TableConfig tableConfig, LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { + checkState(started, "Historical lake lookup manager has not been started."); TableBucket tableBucket = lookupData.tableBucket(); if (!lookupPermits.tryAcquire()) { return CompletableFuture.completedFuture( @@ -513,11 +541,6 @@ private static boolean matchesLookupConfiguration( long currentLakeConfigVersion, long cacheSizeBytes, @Nullable CachedLakeTableLookuper currentLookuper) { - File tableLookupDir = - FlussPaths.historicalLookupTableDir( - getOrPreparePaimonLookupTempDir(clusterConf), - context.tablePath, - context.tableId); if (currentLookuper == null) { // A cache miss must obtain capacity before creating any local lookup resources. Reservation reservation = budgetManager.tryReserve(context.tableId, cacheSizeBytes); @@ -525,6 +548,11 @@ private static boolean matchesLookupConfiguration( return null; } try { + File tableLookupDir = + FlussPaths.historicalLookupTableDir( + getOrCreatePaimonLookupTempDir(), + context.tablePath, + context.tableId); LakeTableLookuper lookuper = createLakeTableLookuper( context.tablePath, @@ -546,6 +574,9 @@ private static boolean matchesLookupConfiguration( } } + File tableLookupDir = + FlussPaths.historicalLookupTableDir( + getOrCreatePaimonLookupTempDir(), context.tablePath, context.tableId); // Build the replacement first so a creation failure leaves the current lookuper and its // reservation unchanged in the cache. LakeTableLookuper lookuper = @@ -707,30 +738,25 @@ private static boolean hasLakeConfigChanged(Configuration currentConf, Configura extractLakeProperties(currentConf), extractLakeProperties(newConf)); } - private synchronized File getOrPreparePaimonLookupTempDir(Configuration clusterConf) { - if (paimonLookupTempDir == null) { - paimonLookupTempDir = preparePaimonLookupTempDir(clusterConf, serverId); + private synchronized File getOrCreatePaimonLookupTempDir() { + if (paimonLookupTempDirCreated) { + return paimonLookupTempDir; } - return paimonLookupTempDir; - } - - private static File preparePaimonLookupTempDir(Configuration conf, int serverId) { - File paimonLookupTempDir = - new File( - new File(conf.get(ConfigOptions.SERVER_IO_TMP_DIR), PAIMON_LOOKUP_DIR_NAME), - String.valueOf(serverId)); try { - // A crashed server cannot close the Paimon IOManager, so lookup cache files may be - // left behind. Clean only this server's directory before creating the first table - // lookuper; cleaning in each table lookuper would delete files used by other tables. - FileUtils.deleteDirectory(paimonLookupTempDir); Files.createDirectories(paimonLookupTempDir.toPath()); - return paimonLookupTempDir; + paimonLookupTempDirCreated = true; } catch (IOException e) { throw new FlussRuntimeException( - "Failed to prepare Paimon lookup temporary directory: " + paimonLookupTempDir, + "Failed to create Paimon lookup temporary directory: " + paimonLookupTempDir, e); } + return paimonLookupTempDir; + } + + private static File resolvePaimonLookupTempDir(Configuration conf, int serverId) { + return new File( + new File(conf.get(ConfigOptions.SERVER_IO_TMP_DIR), PAIMON_LOOKUP_DIR_NAME), + String.valueOf(serverId)); } private static void closeLookuper(CachedLakeTableLookuper cachedLookuper) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index a1a8c0c48f9..118cee03348 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -369,6 +369,8 @@ public ReplicaManager( } public void startup() { + historicalLakeLookupManager.startup(); + // start up ISR expiration thread. // A follower can log behind leader for up tp configOptions#LOG_REPLICA_MAX_LAG_TIME x 1.5 // before it is removed from ISR. diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index c68c8924419..4687b95128c 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -240,7 +240,7 @@ private static void checkHistoricalLookupCacheSize( tableConf.get( ConfigOptions .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); - if (tableCacheSize.getBytes() == 0) { + if (tableCacheSize.getBytes() <= 0) { throw new InvalidConfigException( String.format( "'%s' for table '%s' must be greater than 0 bytes.", diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index e3082f93695..8171465b2c1 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -216,7 +216,7 @@ void testRejectNonPositiveHistoricalPartitionThreadPoolMaxSize(int maxThreadPool } @Test - void testLazilyCleansPaimonLookupTempDirectory() throws Exception { + void testCleansPaimonLookupTempDirectoryOnStartupAndCreatesItLazily() throws Exception { File serverLookupDir = new File(new File(ioTmpDir, "paimon-lookup"), String.valueOf(SERVER_ID)); assertThat(serverLookupDir.mkdirs()).isTrue(); @@ -224,13 +224,21 @@ void testLazilyCleansPaimonLookupTempDirectory() throws Exception { assertThat(staleLookupFile.createNewFile()).isTrue(); ManualExecutor executor = new ManualExecutor(); - TestingHistoricalLakeLookupManager manager = createTestingManager(executor); + TestingHistoricalLakeLookupManager manager = + new TestingHistoricalLakeLookupManager(conf(1), executor); assertThat(staleLookupFile).exists(); - lookupAndRun(manager, executor, PARTITION_TABLE_INFO); + manager.startup(); assertThat(staleLookupFile).doesNotExist(); + assertThat(serverLookupDir).doesNotExist(); + lookupAndRun(manager, executor, PARTITION_TABLE_INFO); assertThat(serverLookupDir).isDirectory(); assertThat(manager.createdIoTmpDirs.get(0)).startsWith(serverLookupDir.getAbsolutePath()); + + File liveLookupFile = new File(serverLookupDir, "live-lookup-file"); + assertThat(liveLookupFile.createNewFile()).isTrue(); + manager.startup(); + assertThat(liveLookupFile).exists(); } @Test @@ -371,6 +379,7 @@ void testDynamicallyUpdatesExpirationAndExpiresIdleLookuper() throws Exception { executor, tickerNanos::get, cacheScheduler); + manager.startup(); lookupAndRun(manager, executor, PARTITION_TABLE_INFO); TestingLakeTableLookuper expiredLookuper = manager.createdLookupers.get(0); @@ -396,6 +405,7 @@ void testEvictsLeastRecentlyUsedLookuperWhenCapacityIsFull() throws Exception { TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager( conf, executor, tickerNanos::get, Scheduler.disabledScheduler()); + manager.startup(); TableInfo first = tableInfo(PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId()); TableInfo second = tableInfo(PARTITION_TABLE_ID + 1, PARTITION_TABLE_INFO.getSchemaId()); @@ -426,6 +436,7 @@ void testReconfiguresGlobalCapacityLazily() throws Exception { MemorySize.parse("96gb")); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(conf, executor); + manager.startup(); for (int i = 0; i < 12; i++) { lookupAndRun( @@ -482,6 +493,7 @@ void testReconfiguresLakePropertiesAndInvalidatesLookuper() throws Exception { ManualExecutor executor = new ManualExecutor(); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(initialConf, executor); + manager.startup(); lookupAndRun(manager, executor, PARTITION_TABLE_INFO); TestingLakeTableLookuper initialLookuper = manager.createdLookupers.get(0); @@ -517,6 +529,7 @@ void testEvictsOutsideConcurrentTableReplacements() throws Exception { replacementCacheSize); CoordinatedReplacementManager manager = new CoordinatedReplacementManager(conf, executor, replacementCacheSize); + manager.startup(); TableInfo first = tableInfoWithCacheSize( @@ -566,6 +579,7 @@ void testThrottlesWhenTableCacheSizeExceedsRuntimeLimit() throws Exception { MemorySize.parse("8gb")); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(conf, executor); + manager.startup(); LookupResultForBucket result = lookupResultAndRun( @@ -583,17 +597,23 @@ void testThrottlesWhenTableCacheSizeExceedsRuntimeLimit() throws Exception { private HistoricalLakeLookupManager createManager( int maxQueuedHistoricalRequests, ManualExecutor executor) { - return new HistoricalLakeLookupManager( - conf(maxQueuedHistoricalRequests), - null, - executor, - SERVER_ID, - Ticker.systemTicker(), - Scheduler.disabledScheduler()); + HistoricalLakeLookupManager manager = + new HistoricalLakeLookupManager( + conf(maxQueuedHistoricalRequests), + null, + executor, + SERVER_ID, + Ticker.systemTicker(), + Scheduler.disabledScheduler()); + manager.startup(); + return manager; } private TestingHistoricalLakeLookupManager createTestingManager(ManualExecutor executor) { - return new TestingHistoricalLakeLookupManager(conf(1), executor); + TestingHistoricalLakeLookupManager manager = + new TestingHistoricalLakeLookupManager(conf(1), executor); + manager.startup(); + return manager; } private Configuration conf(int maxQueuedHistoricalRequests) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java index 400d64460ab..6b0c45a88fe 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java @@ -199,6 +199,7 @@ public void setup(TestInfo testInfo) throws Exception { } else { conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath()); } + conf.set(ConfigOptions.SERVER_IO_TMP_DIR, tempDir.getAbsolutePath()); conf.setString(ConfigOptions.COORDINATOR_HOST, "localhost"); conf.set(ConfigOptions.REMOTE_DATA_DIR, tempDir.getAbsolutePath() + "/remote_data_dir"); conf.set(ConfigOptions.SERVER_IO_POOL_SIZE, 2); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java index 8a1f6fb41e5..fcbaed882cc 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java @@ -522,6 +522,7 @@ private ReplicaManager createReplicaManager(int serverId, LocalDiskManager local Configuration conf = new Configuration(); conf.set(ConfigOptions.TABLET_SERVER_ID, serverId); conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath() + "/server-" + serverId); + conf.set(ConfigOptions.SERVER_IO_TMP_DIR, tempDir.getAbsolutePath()); conf.set(ConfigOptions.WRITER_ID_EXPIRATION_TIME, Duration.ofHours(12)); Scheduler scheduler = new FlussScheduler(2); scheduler.startup(); From 761a87f2a3b4b7c2f0a67c47f6a8b7a1bb660d24 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 5 Aug 2026 23:41:45 +0800 Subject: [PATCH 08/15] [server] Configure historical lookup cache by disk ratio Derive global and per-table cache capacities from the total size of the first available data volume. Store lookup files under that data directory, keep the cache root out of tablet recovery, and propagate resolved byte limits to Paimon. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 164/256 AI-Contributed/UT: 191/286 --- .../apache/fluss/config/ConfigOptions.java | 29 ++-- .../apache/fluss/config/FlussConfigUtils.java | 17 ++- .../org/apache/fluss/config/TableConfig.java | 6 +- .../fluss/lake/lakestorage/LakeStorage.java | 14 +- .../org/apache/fluss/utils/FlussPaths.java | 14 ++ .../fluss/config/FlussConfigUtilsTest.java | 19 +-- .../apache/fluss/config/TableConfigTest.java | 13 +- .../procedure/SetClusterConfigsProcedure.java | 2 +- .../flink/procedure/FlinkProcedureITCase.java | 15 +- .../fluss/lake/paimon/PaimonLakeStorage.java | 6 +- .../lookup/PaimonLakeTableLookuper.java | 12 +- .../lookup/PaimonLakeTableLookuperTest.java | 29 ++-- .../fluss/server/DynamicServerConfig.java | 4 +- .../fluss/server/TabletManagerBase.java | 4 + .../HistoricalLookupCacheConfigUpdater.java | 16 +- .../server/coordinator/MetadataManager.java | 13 +- .../replica/HistoricalLakeLookupManager.java | 79 ++++++---- .../fluss/server/replica/ReplicaManager.java | 11 +- .../utils/TableDescriptorValidation.java | 29 ++-- .../fluss/server/TabletManagerBaseTest.java | 26 ++++ .../HistoricalLakeLookupManagerTest.java | 143 ++++++++---------- .../fluss/server/replica/ReplicaTestBase.java | 1 - .../fetcher/ReplicaFetcherThreadTest.java | 1 - ...istoricalPartitionTableValidationTest.java | 39 +++-- 24 files changed, 316 insertions(+), 226 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 3dab10fd846..a798a44c424 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -428,14 +428,15 @@ public class ConfigOptions { "The maximum number of threads used for historical partition operations, such as lake lookups and writes. " + "Threads are started lazily and released after the keep-alive timeout when idle."); - public static final ConfigOption - SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE = - key("server.historical-partition.lookup-cache.max-disk-size") - .memoryType() - .defaultValue(MemorySize.parse("80gb")) + public static final ConfigOption + SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO = + key("server.historical-partition.lookup-cache.max-disk-ratio") + .doubleType() + .defaultValue(0.10) .withDescription( - "The total configured disk capacity available to current and creating historical partition lookup caches on a TabletServer. " - + "The value must be greater than zero."); + "The maximum fraction of the total capacity of the volume containing the first available data directory that historical partition lookup caches may reserve on a TabletServer. " + + "Historical lookup cache files are stored under that data directory; additional data volumes are not used. " + + "The valid range is (0.0, 1.0]."); public static final ConfigOption SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS = @@ -1875,14 +1876,14 @@ public class ConfigOptions { + "to look up historical partition data so that their clients load the " + "updated table configuration."); - public static final ConfigOption - TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE = - key("table.datalake.historical-partition.lookup-cache.max-disk-size") - .memoryType() - .defaultValue(MemorySize.parse("8gb")) + public static final ConfigOption + TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO = + key("table.datalake.historical-partition.lookup-cache.max-disk-ratio") + .doubleType() + .defaultValue(0.01) .withDescription( - "The maximum local disk capacity reserved for this table's historical partition lookup cache on each TabletServer. " - + "When the table is created or altered, the value must be greater than zero and no greater than the current TabletServer historical lookup cache limit."); + "The maximum fraction of the total capacity of the volume containing the first available TabletServer data directory reserved for this table's historical partition lookup cache. " + + "When the table is created or altered, the value must be within (0.0, 1.0] and no greater than the current TabletServer historical lookup cache ratio."); public static final ConfigOption TABLE_DATALAKE_FORMAT = key("table.datalake.format") diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index c05cbc13237..f4751af1da0 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -49,7 +49,8 @@ public class FlussConfigUtils { Arrays.asList( ConfigOptions.TABLE_DATALAKE_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED.key(), - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + ConfigOptions + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key(), ConfigOptions.TABLE_DATALAKE_FRESHNESS.key(), ConfigOptions.TABLE_DATALAKE_AUTO_COMPACTION.key(), @@ -228,7 +229,7 @@ protected static void validateServerConfigs(Configuration conf) { conf, ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS, 1); - validateHistoricalLookupCacheLimit(conf); + validateHistoricalLookupCacheRatio(conf); if (conf.get(ConfigOptions.LOG_SEGMENT_FILE_SIZE).getBytes() > Integer.MAX_VALUE) { throw new IllegalConfigurationException( @@ -238,13 +239,13 @@ protected static void validateServerConfigs(Configuration conf) { } } - private static void validateHistoricalLookupCacheLimit(Configuration conf) { - MemorySize historicalLookupCacheMaxSize = - conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); - if (historicalLookupCacheMaxSize.getBytes() <= 0) { + private static void validateHistoricalLookupCacheRatio(Configuration conf) { + double historicalLookupCacheMaxRatio = + conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); + if (!(historicalLookupCacheMaxRatio > 0.0 && historicalLookupCacheMaxRatio <= 1.0)) { throw new IllegalConfigurationException( - "Invalid configuration for %s, it must be greater than 0 bytes.", - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key()); + "Invalid configuration for %s, it must be within (0.0, 1.0].", + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO.key()); } } diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index 4e72aedd100..e7d53ae1b04 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -105,10 +105,10 @@ public boolean isHistoricalPartitionEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED); } - /** Gets the maximum local disk size of the historical partition lookup cache. */ - public MemorySize getHistoricalPartitionLookupCacheMaxDiskSize() { + /** Gets the maximum local disk ratio of the historical partition lookup cache. */ + public double getHistoricalPartitionLookupCacheMaxDiskRatio() { return config.get( - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); } /** diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java index fa0aa1dccd8..7087dd0e36a 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java @@ -23,6 +23,7 @@ import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TablePath; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** @@ -71,16 +72,22 @@ default LakeTableLookuper createLakeTableLookuper( final class LookuperContext { private final String ioTmpDir; private final TableConfig tableConfig; + private final long lookupCacheMaxDiskBytes; /** * Creates a lookuper context. * * @param ioTmpDir local directory for temporary files used by the lookuper * @param tableConfig configuration of the Fluss table + * @param lookupCacheMaxDiskBytes maximum local lookup cache size in bytes */ - public LookuperContext(String ioTmpDir, TableConfig tableConfig) { + public LookuperContext( + String ioTmpDir, TableConfig tableConfig, long lookupCacheMaxDiskBytes) { this.ioTmpDir = checkNotNull(ioTmpDir, "ioTmpDir must not be null."); this.tableConfig = checkNotNull(tableConfig, "tableConfig must not be null."); + checkArgument( + lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); + this.lookupCacheMaxDiskBytes = lookupCacheMaxDiskBytes; } /** Returns the local directory for temporary files used by the lookuper. */ @@ -92,5 +99,10 @@ public String ioTmpDir() { public TableConfig tableConfig() { return tableConfig; } + + /** Returns the maximum local lookup cache size in bytes. */ + public long lookupCacheMaxDiskBytes() { + return lookupCacheMaxDiskBytes; + } } } diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java index 6d210706680..4554d9a49f4 100644 --- a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java +++ b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java @@ -53,6 +53,11 @@ public class FlussPaths { /** Prefix of a local kv tablet directory to store kv files for a specific kv tablet. */ public static final String KV_TABLET_DIR_PREFIX = "kv-"; + /** + * The directory name for historical lookup cache files under the first local data directory. + */ + public static final String HISTORICAL_LOOKUP_CACHE_DIR_NAME = ".historical-lookup-cache"; + /** Prefix for a partition id to distinguish between table id and partition id. */ public static final String PARTITION_DIR_PREFIX = "p"; @@ -148,6 +153,15 @@ public static File kvTabletDir( return tabletParentDir.resolve(KV_TABLET_DIR_PREFIX + tableBucket.getBucket()).toFile(); } + /** + * Returns the historical lookup cache root under the first local data directory. + * + * @param firstDataDir the first available local data directory + */ + public static File historicalLookupRootDir(File firstDataDir) { + return new File(firstDataDir, HISTORICAL_LOOKUP_CACHE_DIR_NAME); + } + /** * Returns the local directory path for storing historical lookup files for a table. * diff --git a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java index 66678fc7a1f..17ec9fd8492 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java @@ -219,19 +219,20 @@ void testValidateLogRetentionCheckInterval() { void testValidateHistoricalLookupCacheConfigs() { Configuration conf = new Configuration(); conf.set(ConfigOptions.REMOTE_DATA_DIR, "s3://bucket/path"); - conf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.ZERO); + conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.0); assertThatThrownBy(() -> validateCoordinatorConfigs(conf)) .isInstanceOf(IllegalConfigurationException.class) .hasMessageContaining( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key()) - .hasMessageContaining("greater than 0 bytes"); + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO.key()) + .hasMessageContaining("within (0.0, 1.0]"); - conf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.parse("4gb")); + conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 1.01); + assertThatThrownBy(() -> validateCoordinatorConfigs(conf)) + .isInstanceOf(IllegalConfigurationException.class) + .hasMessageContaining("within (0.0, 1.0]"); + + conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.1); conf.set( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS, Duration.ZERO); @@ -244,7 +245,7 @@ void testValidateHistoricalLookupCacheConfigs() { assertThat( FlussConfigUtils.isAlterableTableOption( ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key())) .isTrue(); } diff --git a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java index 9b9312543f5..8470cb3f3ad 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java @@ -46,16 +46,15 @@ void testDeleteBehavior() { } @Test - void testHistoricalPartitionLookupCacheMaxDiskSize() { + void testHistoricalPartitionLookupCacheMaxDiskRatio() { Configuration conf = new Configuration(); TableConfig tableConfig = new TableConfig(conf); - assertThat(tableConfig.getHistoricalPartitionLookupCacheMaxDiskSize()) - .isEqualTo(MemorySize.parse("8gb")); + assertThat(tableConfig.getHistoricalPartitionLookupCacheMaxDiskRatio()).isEqualTo(0.01); conf.set( - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.parse("16gb")); - assertThat(new TableConfig(conf).getHistoricalPartitionLookupCacheMaxDiskSize()) - .isEqualTo(MemorySize.parse("16gb")); + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, + 0.02); + assertThat(new TableConfig(conf).getHistoricalPartitionLookupCacheMaxDiskRatio()) + .isEqualTo(0.02); } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/procedure/SetClusterConfigsProcedure.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/procedure/SetClusterConfigsProcedure.java index ece70d8b003..b474a37ae38 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/procedure/SetClusterConfigsProcedure.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/procedure/SetClusterConfigsProcedure.java @@ -46,7 +46,7 @@ * -- Set a configuration * CALL sys.set_cluster_configs('kv.rocksdb.shared-rate-limiter.bytes-per-sec', '200MB'); * CALL sys.set_cluster_configs('datalake.format', 'paimon'); - * CALL sys.set_cluster_configs('server.historical-partition.lookup-cache.max-disk-size', '96GB'); + * CALL sys.set_cluster_configs('server.historical-partition.lookup-cache.max-disk-ratio', '0.12'); * * -- Set multiple configurations at one time * CALL sys.set_cluster_configs('kv.rocksdb.shared-rate-limiter.bytes-per-sec', '200MB','datalake.format', 'paimon'); diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/procedure/FlinkProcedureITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/procedure/FlinkProcedureITCase.java index 3870cc0bbbd..f1c71f0c26c 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/procedure/FlinkProcedureITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/procedure/FlinkProcedureITCase.java @@ -404,12 +404,12 @@ void testSetClusterConfigs() throws Exception { try (CloseableIterator resultIterator = tEnv.executeSql( String.format( - "Call %s.sys.set_cluster_configs('%s', '300MB', '%s', 'paimon', '%s', '96GB')", + "Call %s.sys.set_cluster_configs('%s', '300MB', '%s', 'paimon', '%s', '0.12')", CATALOG_NAME, ConfigOptions.KV_SHARED_RATE_LIMITER_BYTES_PER_SEC.key(), ConfigOptions.DATALAKE_FORMAT.key(), ConfigOptions - .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key())) .collect()) { List results = CollectionUtil.iteratorToList(resultIterator); @@ -426,9 +426,9 @@ void testSetClusterConfigs() throws Exception { assertThat(results.get(2).getField(0)) .asString() - .contains("Successfully set to '96GB'") + .contains("Successfully set to '0.12'") .contains( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key()); } @@ -451,12 +451,12 @@ void testSetClusterConfigs() throws Exception { "Call %s.sys.get_cluster_configs('%s')", CATALOG_NAME, ConfigOptions - .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key())) .collect()) { List results = CollectionUtil.iteratorToList(resultIterator); assertThat(results).hasSize(1); - assertThat(results.get(0).getField(1)).isEqualTo("96GB"); + assertThat(results.get(0).getField(1)).isEqualTo("0.12"); } // reset cluster configs. @@ -466,7 +466,8 @@ void testSetClusterConfigs() throws Exception { CATALOG_NAME, ConfigOptions.KV_SHARED_RATE_LIMITER_BYTES_PER_SEC.key(), ConfigOptions.DATALAKE_FORMAT.key(), - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key())) .await(); } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java index 9b7e97eb4ff..0b7131a6555 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java @@ -57,6 +57,10 @@ public LakeSource createLakeSource(TablePath tablePath) { @Override public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, LookuperContext context) { return new PaimonLakeTableLookuper( - paimonConfig, tablePath, context.ioTmpDir(), context.tableConfig()); + paimonConfig, + tablePath, + context.ioTmpDir(), + context.tableConfig(), + context.lookupCacheMaxDiskBytes()); } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index b40f04cf4c9..5e0a4166466 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.paimon.lookup; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; @@ -67,6 +68,7 @@ import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.SYSTEM_COLUMNS; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonPartition; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** @@ -92,6 +94,7 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private final TablePath tablePath; private final String ioTmpDir; private final TableConfig tableConfig; + private final long lookupCacheMaxDiskBytes; private final Set initializedBuckets; @@ -114,15 +117,20 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private @Nullable InternalRow.FieldGetter[] cachedValueFieldGetters; private boolean closed; + /** Creates a lookuper with the specified local lookup cache limit. */ public PaimonLakeTableLookuper( Configuration paimonConfig, TablePath tablePath, String ioTmpDir, - TableConfig tableConfig) { + TableConfig tableConfig, + long lookupCacheMaxDiskBytes) { this.paimonConfig = checkNotNull(paimonConfig, "paimonConfig must not be null."); this.tablePath = checkNotNull(tablePath, "tablePath must not be null."); this.ioTmpDir = checkNotNull(ioTmpDir, "ioTmpDir must not be null."); this.tableConfig = checkNotNull(tableConfig, "tableConfig must not be null."); + checkArgument( + lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); + this.lookupCacheMaxDiskBytes = lookupCacheMaxDiskBytes; this.initializedBuckets = new HashSet<>(); } @@ -245,7 +253,7 @@ private void ensureInitialized(RowType valueRowType) throws Exception { private FileStoreTable withLookupCacheOptions(FileStoreTable table) { String key = CoreOptions.LOOKUP_CACHE_MAX_DISK_SIZE.key(); - String maxDiskSize = tableConfig.getHistoricalPartitionLookupCacheMaxDiskSize().toString(); + String maxDiskSize = new MemorySize(lookupCacheMaxDiskBytes).toString(); return table.copy(Collections.singletonMap(key, maxDiskSize)); } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index eea70ee6c61..9f871140b3c 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.lakestorage.TestingLakeCatalogContext; @@ -77,6 +78,7 @@ class PaimonLakeTableLookuperTest { private static final String DB = "lookup_db"; private static final short SCHEMA_ID = 1; private static final short EVOLVED_SCHEMA_ID = 2; + private static final long LOOKUP_CACHE_MAX_DISK_BYTES = MemorySize.parse("8gb").getBytes(); private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = (lookupTimeNanos, lookupFileMaterialization) -> {}; @@ -122,7 +124,8 @@ void testLookupPartitionedPrimaryKeyTable() throws Exception { paimonConfig, tablePath, tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.COMPACTED))) { + tableConfig(KvFormat.COMPACTED), + LOOKUP_CACHE_MAX_DISK_BYTES)) { List lookupFileMaterializations = new ArrayList<>(); LakeTableLookuper.LookupContext context = lookupContext( @@ -181,7 +184,8 @@ void testLookupPartitionsWithSameHashCode() throws Exception { paimonConfig, tablePath, tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.COMPACTED))) { + tableConfig(KvFormat.COMPACTED), + LOOKUP_CACHE_MAX_DISK_BYTES)) { BinaryValue firstValue = decodeValue( lookuper.lookup( @@ -221,7 +225,8 @@ void testLookupWithIndexedKvFormat() throws Exception { paimonConfig, tablePath, tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.INDEXED))) { + tableConfig(KvFormat.INDEXED), + LOOKUP_CACHE_MAX_DISK_BYTES)) { LakeTableLookuper.LookupContext context = lookupContext(schema, "20240101", 0, SCHEMA_ID); @@ -265,7 +270,8 @@ void testLookupKvFormatV2WithNonDefaultBucketKey() throws Exception { paimonConfig, tablePath, tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.COMPACTED, KV_FORMAT_VERSION_2))) { + tableConfig(KvFormat.COMPACTED, KV_FORMAT_VERSION_2), + LOOKUP_CACHE_MAX_DISK_BYTES)) { LakeTableLookuper.LookupContext context = lookupContext(schema, "20240101", 0, SCHEMA_ID); byte[] compactedKey = @@ -320,7 +326,8 @@ void testRetriesInitializationAfterLookupKeyConverterFailure() throws Exception paimonConfig, tablePath, tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.COMPACTED, KV_FORMAT_VERSION_2))) { + tableConfig(KvFormat.COMPACTED, KV_FORMAT_VERSION_2), + LOOKUP_CACHE_MAX_DISK_BYTES)) { // Inject a late initialization failure: the Paimon table requires sub_id in its // lookup key, but the first lookup's value row type deliberately omits that field. assertThatThrownBy( @@ -362,7 +369,8 @@ void testRefreshFilesAfterCompactionAndSnapshotExpiration() throws Exception { paimonConfig, tablePath, tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.COMPACTED))) { + tableConfig(KvFormat.COMPACTED), + LOOKUP_CACHE_MAX_DISK_BYTES)) { LakeTableLookuper.LookupContext context = lookupContext(schema, "20240101", 0, SCHEMA_ID); assertThat(lookuper.lookup(paimonKey(schema, 5, "20240101"), context)).isNotNull(); @@ -428,7 +436,8 @@ void testLookupWithNonStringPartitionKey() throws Exception { paimonConfig, tablePath, tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.COMPACTED))) { + tableConfig(KvFormat.COMPACTED), + LOOKUP_CACHE_MAX_DISK_BYTES)) { LakeTableLookuper.LookupContext context = new LakeTableLookuper.LookupContext( ResolvedPartitionSpec.fromPartitionName( @@ -464,7 +473,8 @@ void testRejectAppendOnlyTable() throws Exception { paimonConfig, tablePath, tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.COMPACTED))) { + tableConfig(KvFormat.COMPACTED), + LOOKUP_CACHE_MAX_DISK_BYTES)) { LakeTableLookuper.LookupContext context = new LakeTableLookuper.LookupContext( new ResolvedPartitionSpec( @@ -521,7 +531,8 @@ void testLookupAfterSchemaEvolutionPadsNewColumnsWithNull() throws Exception { paimonConfig, tablePath, tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.COMPACTED))) { + tableConfig(KvFormat.COMPACTED), + LOOKUP_CACHE_MAX_DISK_BYTES)) { BinaryValue oldSchemaValue = decodeValue( lookuper.lookup( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java index f5e7caa9db7..98d721e152e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java @@ -57,7 +57,7 @@ import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_DATA_DISK_WRITE_RECOVER_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS; -import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE; +import static org.apache.fluss.config.ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_CREDENTIALS; import static org.apache.fluss.config.ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG; import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock; @@ -84,7 +84,7 @@ class DynamicServerConfig { KV_SNAPSHOT_INTERVAL.key(), SERVER_DATA_DISK_WRITE_RECOVER_RATIO.key(), SERVER_DATA_DISK_WRITE_LIMIT_RATIO.key(), - SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key(), + SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO.key(), SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS.key(), // Config options for remote.data.dirs REMOTE_DATA_DIRS.key(), diff --git a/fluss-server/src/main/java/org/apache/fluss/server/TabletManagerBase.java b/fluss-server/src/main/java/org/apache/fluss/server/TabletManagerBase.java index 6ab15dc8397..0c476b10902 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/TabletManagerBase.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/TabletManagerBase.java @@ -56,6 +56,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; +import static org.apache.fluss.utils.FlussPaths.HISTORICAL_LOOKUP_CACHE_DIR_NAME; import static org.apache.fluss.utils.FlussPaths.KV_TABLET_DIR_PREFIX; import static org.apache.fluss.utils.FlussPaths.LOG_TABLET_DIR_PREFIX; import static org.apache.fluss.utils.FlussPaths.isPartitionDir; @@ -115,6 +116,9 @@ protected List listTabletsToLoad(File dataDir) { // Get all database directory. File[] dbDirs = FileUtils.listDirectories(dataDir); for (File dbDir : dbDirs) { + if (dbDir.getName().equals(HISTORICAL_LOOKUP_CACHE_DIR_NAME)) { + continue; + } // Get all table path directory. File[] tableDirs = FileUtils.listDirectories(dbDir); for (File tableDir : tableDirs) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java index 94709e366a4..5b9e5c55e25 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java @@ -19,7 +19,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.cluster.ServerReconfigurable; import org.apache.fluss.exception.ConfigException; @@ -38,13 +37,14 @@ final class HistoricalLookupCacheConfigUpdater implements ServerReconfigurable { @Override public void validate(Configuration newConfig) throws ConfigException { - MemorySize newMaxSize = - newConfig.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); - if (newMaxSize.getBytes() <= 0) { + double newMaxRatio = + newConfig.get( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); + if (!(newMaxRatio > 0.0 && newMaxRatio <= 1.0)) { throw new ConfigException( String.format( - "Invalid configuration for %s, it must be greater than 0 bytes.", - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + "Invalid configuration for %s, it must be within (0.0, 1.0].", + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key())); } @@ -64,8 +64,8 @@ public void validate(Configuration newConfig) throws ConfigException { @Override public void reconfigure(Configuration newConfig) { - metadataManager.updateHistoricalLookupCacheMaxSize( + metadataManager.updateHistoricalLookupCacheMaxRatio( newConfig.get( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE)); + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO)); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 7f2c5df3544..b197054581e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -19,7 +19,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.DatabaseAlreadyExistException; import org.apache.fluss.exception.DatabaseNotEmptyException; import org.apache.fluss.exception.DatabaseNotExistException; @@ -88,7 +87,7 @@ public class MetadataManager { private final ZooKeeperClient zookeeperClient; private final int maxPartitionNum; private final int maxBucketNum; - private volatile MemorySize historicalLookupCacheMaxSize; + private volatile double historicalLookupCacheMaxRatio; private final LakeCatalogDynamicLoader lakeCatalogDynamicLoader; public static final Set SENSITIVE_TABLE_OPTIONS = new HashSet<>(); @@ -112,8 +111,8 @@ public MetadataManager( this.zookeeperClient = zookeeperClient; this.maxPartitionNum = conf.get(ConfigOptions.MAX_PARTITION_NUM); this.maxBucketNum = conf.get(ConfigOptions.MAX_BUCKET_NUM); - this.historicalLookupCacheMaxSize = - conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); + this.historicalLookupCacheMaxRatio = + conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); this.lakeCatalogDynamicLoader = lakeCatalogDynamicLoader; } @@ -124,11 +123,11 @@ public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDe maxBucketNum, lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat(), tablePath, - historicalLookupCacheMaxSize); + historicalLookupCacheMaxRatio); } - void updateHistoricalLookupCacheMaxSize(MemorySize newMaxSize) { - historicalLookupCacheMaxSize = newMaxSize; + void updateHistoricalLookupCacheMaxRatio(double newMaxRatio) { + historicalLookupCacheMaxRatio = newMaxRatio; } public void createDatabase( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index 593e5728e73..cba2eef9800 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -105,7 +105,6 @@ class HistoricalLakeLookupManager implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(HistoricalLakeLookupManager.class); - private static final String PAIMON_LOOKUP_DIR_NAME = "paimon-lookup"; private static final String LOOKUPER_CACHE_EXPIRATION_TASK_NAME = "historical-lookuper-cache-expiration"; private static final Duration HISTORICAL_PARTITION_THREAD_KEEP_ALIVE = Duration.ofMinutes(10); @@ -124,23 +123,26 @@ class HistoricalLakeLookupManager implements AutoCloseable { private final Set> pendingLookups; private final Cache lakeTableLookupers; private final ExecutorService historicalPartitionExecutor; - private final File paimonLookupTempDir; + private final File historicalLookupCacheRootDir; + private final long lookupVolumeBytes; @GuardedBy("this") - private boolean paimonLookupTempDirCreated; + private boolean historicalLookupCacheRootDirCreated; private volatile boolean started; HistoricalLakeLookupManager( Configuration conf, @Nullable PluginManager pluginManager, - int serverId, + File firstDataDir, + long lookupVolumeBytes, Scheduler scheduler) { this( conf, pluginManager, null, - serverId, + firstDataDir, + lookupVolumeBytes, Ticker.systemTicker(), createCacheScheduler(scheduler)); } @@ -150,18 +152,23 @@ class HistoricalLakeLookupManager implements AutoCloseable { Configuration conf, @Nullable PluginManager pluginManager, @Nullable ExecutorService historicalPartitionExecutor, - int serverId, + File firstDataDir, + long lookupVolumeBytes, Ticker ticker, com.github.benmanes.caffeine.cache.Scheduler cacheScheduler) { this.conf = checkNotNull(conf, "conf must not be null."); this.pluginManager = pluginManager; - this.paimonLookupTempDir = resolvePaimonLookupTempDir(conf, serverId); + this.historicalLookupCacheRootDir = + FlussPaths.historicalLookupRootDir( + checkNotNull(firstDataDir, "firstDataDir must not be null.")); + checkArgument(lookupVolumeBytes > 0, "lookupVolumeBytes must be greater than 0."); + this.lookupVolumeBytes = lookupVolumeBytes; this.budgetManager = new HistoricalLookupCacheBudgetManager( - conf.get( + capacityBytes( + conf.get( ConfigOptions - .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE) - .getBytes()); + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO))); this.capacityEvictions = new ThreadSafeSimpleCounter(); this.maxQueuedHistoricalRequests = conf.get(ConfigOptions.NETTY_SERVER_MAX_QUEUED_HISTORICAL_REQUESTS); @@ -216,19 +223,19 @@ private static com.github.benmanes.caffeine.cache.Scheduler createCacheScheduler /** * Attempts to clean lookup cache files left by a previous TabletServer process. * - *

Only this server's directory is removed. It is recreated lazily when the first table - * lookuper is created. + *

The cache root under this server's first data directory is removed. It is recreated lazily + * when the first table lookuper is created. */ synchronized void startup() { if (started) { return; } try { - FileUtils.deleteDirectory(paimonLookupTempDir); + FileUtils.deleteDirectory(historicalLookupCacheRootDir); } catch (IOException e) { LOG.warn( - "Failed to clean Paimon lookup temporary directory {}.", - paimonLookupTempDir, + "Failed to clean historical lookup cache directory {}.", + historicalLookupCacheRootDir, e); } started = true; @@ -348,10 +355,10 @@ void reconfigure(Configuration newConf) { .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS); synchronized (this) { long newMaxBytes = - newConf.get( + capacityBytes( + newConf.get( ConfigOptions - .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE) - .getBytes(); + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO)); if (newMaxBytes != budgetManager.maxBytes()) { budgetManager.updateGlobalLimit(newMaxBytes); } @@ -400,7 +407,7 @@ private LookupResultForBucket lookupInternal( long currentLakeConfigVersion = lakeConfigVersion; Configuration currentConf = conf; long cacheSizeBytes = - tableConfig.getHistoricalPartitionLookupCacheMaxDiskSize().getBytes(); + capacityBytes(tableConfig.getHistoricalPartitionLookupCacheMaxDiskRatio()); cachedLookuper = acquireCachedLookuper( context, @@ -550,7 +557,7 @@ private static boolean matchesLookupConfiguration( try { File tableLookupDir = FlussPaths.historicalLookupTableDir( - getOrCreatePaimonLookupTempDir(), + getOrCreateHistoricalLookupCacheRootDir(), context.tablePath, context.tableId); LakeTableLookuper lookuper = @@ -558,6 +565,7 @@ private static boolean matchesLookupConfiguration( context.tablePath, tableLookupDir.getAbsolutePath(), tableConfig, + cacheSizeBytes, clusterConf); return new CachedLakeTableLookuper( context.tableId, @@ -576,7 +584,9 @@ private static boolean matchesLookupConfiguration( File tableLookupDir = FlussPaths.historicalLookupTableDir( - getOrCreatePaimonLookupTempDir(), context.tablePath, context.tableId); + getOrCreateHistoricalLookupCacheRootDir(), + context.tablePath, + context.tableId); // Build the replacement first so a creation failure leaves the current lookuper and its // reservation unchanged in the cache. LakeTableLookuper lookuper = @@ -584,6 +594,7 @@ private static boolean matchesLookupConfiguration( context.tablePath, tableLookupDir.getAbsolutePath(), tableConfig, + cacheSizeBytes, clusterConf); // Replace the reservation atomically: the old and replacement cache sizes never count // against the global budget at the same time. @@ -704,6 +715,7 @@ LakeTableLookuper createLakeTableLookuper( TablePath tablePath, String ioTmpDir, TableConfig tableConfig, + long cacheSizeBytes, Configuration clusterConf) { DataLakeFormat dataLakeFormat = clusterConf.get(ConfigOptions.DATALAKE_FORMAT); if (dataLakeFormat == null) { @@ -728,7 +740,7 @@ LakeTableLookuper createLakeTableLookuper( LakeStorage lakeStorage = lakeStoragePlugin.createLakeStorage(Configuration.fromMap(lakeProperties)); return lakeStorage.createLakeTableLookuper( - tablePath, new LakeStorage.LookuperContext(ioTmpDir, tableConfig)); + tablePath, new LakeStorage.LookuperContext(ioTmpDir, tableConfig, cacheSizeBytes)); } private static boolean hasLakeConfigChanged(Configuration currentConf, Configuration newConf) { @@ -738,25 +750,26 @@ private static boolean hasLakeConfigChanged(Configuration currentConf, Configura extractLakeProperties(currentConf), extractLakeProperties(newConf)); } - private synchronized File getOrCreatePaimonLookupTempDir() { - if (paimonLookupTempDirCreated) { - return paimonLookupTempDir; + private synchronized File getOrCreateHistoricalLookupCacheRootDir() { + if (historicalLookupCacheRootDirCreated) { + return historicalLookupCacheRootDir; } try { - Files.createDirectories(paimonLookupTempDir.toPath()); - paimonLookupTempDirCreated = true; + Files.createDirectories(historicalLookupCacheRootDir.toPath()); + historicalLookupCacheRootDirCreated = true; } catch (IOException e) { throw new FlussRuntimeException( - "Failed to create Paimon lookup temporary directory: " + paimonLookupTempDir, + "Failed to create historical lookup cache directory: " + + historicalLookupCacheRootDir, e); } - return paimonLookupTempDir; + return historicalLookupCacheRootDir; } - private static File resolvePaimonLookupTempDir(Configuration conf, int serverId) { - return new File( - new File(conf.get(ConfigOptions.SERVER_IO_TMP_DIR), PAIMON_LOOKUP_DIR_NAME), - String.valueOf(serverId)); + private long capacityBytes(double ratio) { + checkArgument(ratio > 0.0 && ratio <= 1.0, "ratio must be within (0.0, 1.0]."); + long bytes = (long) Math.ceil(lookupVolumeBytes * ratio); + return Math.min(lookupVolumeBytes, bytes); } private static void closeLookuper(CachedLakeTableLookuper cachedLookuper) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 118cee03348..d868dc6710e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -360,8 +360,17 @@ public ReplicaManager( this.ioExecutor = ioExecutor; this.minInSyncReplicas = conf.get(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER); this.scannerManager = checkNotNull(scannerManager, "scannerManager"); + // Historical lookup cache capacity currently uses only the first data volume. + File historicalLookupDataDir = localDiskManager.dataDirs().get(0); + long historicalLookupVolumeBytes = + Files.getFileStore(historicalLookupDataDir.toPath()).getTotalSpace(); this.historicalLakeLookupManager = - new HistoricalLakeLookupManager(conf, pluginManager, serverId, scheduler); + new HistoricalLakeLookupManager( + conf, + pluginManager, + historicalLookupDataDir, + historicalLookupVolumeBytes, + scheduler); serverMetricGroup.registerHistoricalPartitionInflightRequests( "lookup", historicalLakeLookupManager::numInflightRequests); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 4687b95128c..31a7ff27bb6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -22,7 +22,6 @@ import org.apache.fluss.config.ConfigOption; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.ReadableConfig; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.InvalidAlterTableException; @@ -94,7 +93,7 @@ public static void validateTableDescriptor( int maxBucketNum, @Nullable DataLakeFormat clusterDataLakeFormat, TablePath tablePath, - MemorySize historicalLookupCacheMaxSize) { + double historicalLookupCacheMaxRatio) { Schema schema = tableDescriptor.getSchema(); boolean hasPrimaryKey = schema.getPrimaryKey().isPresent(); Configuration tableConf = Configuration.fromMap(tableDescriptor.getProperties()); @@ -133,7 +132,7 @@ public static void validateTableDescriptor( checkTieredLog(tableConf); checkHistoricalPartition(tableDescriptor, tableConf); if (tableConf.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED)) { - checkHistoricalLookupCacheSize(tableConf, tablePath, historicalLookupCacheMaxSize); + checkHistoricalLookupCacheRatio(tableConf, tablePath, historicalLookupCacheMaxRatio); } checkPartition(tableConf, tableDescriptor.getPartitionKeys(), schema.getRowType()); checkSystemColumns(schema.getRowType()); @@ -234,33 +233,33 @@ private static void checkHistoricalPartition( } } - private static void checkHistoricalLookupCacheSize( - Configuration tableConf, TablePath tablePath, MemorySize historicalLookupCacheMaxSize) { - MemorySize tableCacheSize = + private static void checkHistoricalLookupCacheRatio( + Configuration tableConf, TablePath tablePath, double historicalLookupCacheMaxRatio) { + double tableCacheRatio = tableConf.get( ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE); - if (tableCacheSize.getBytes() <= 0) { + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); + if (!(tableCacheRatio > 0.0 && tableCacheRatio <= 1.0)) { throw new InvalidConfigException( String.format( - "'%s' for table '%s' must be greater than 0 bytes.", + "'%s' for table '%s' must be within (0.0, 1.0].", ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key(), tablePath)); } - if (tableCacheSize.compareTo(historicalLookupCacheMaxSize) > 0) { + if (Double.compare(tableCacheRatio, historicalLookupCacheMaxRatio) > 0) { throw new InvalidConfigException( String.format( "'%s' (%s) for table '%s' must be less than or equal to '%s' (%s).", ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key(), - tableCacheSize, + tableCacheRatio, tablePath, - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key(), - historicalLookupCacheMaxSize)); + historicalLookupCacheMaxRatio)); } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/TabletManagerBaseTest.java b/fluss-server/src/test/java/org/apache/fluss/server/TabletManagerBaseTest.java index 498c9b95f15..01d43c5562f 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/TabletManagerBaseTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/TabletManagerBaseTest.java @@ -19,9 +19,12 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FlussRuntimeException; +import org.apache.fluss.utils.FlussPaths; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import java.io.File; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -39,6 +42,21 @@ /** Test for {@link TabletManagerBase}. */ final class TabletManagerBaseTest { + @TempDir private File tempDir; + + @Test + void testIgnoresHistoricalLookupCacheDirectoryWhenLoadingTablets() { + File fakeTabletDir = + new File( + new File(FlussPaths.historicalLookupRootDir(tempDir), "database"), + "kv-table-1"); + assertThat(fakeTabletDir.mkdirs()).isTrue(); + + TestingTabletManager tabletManager = new TestingTabletManager(tempDir); + + assertThat(tabletManager.tabletsToLoad(tempDir)).isEmpty(); + } + @Test void testCloseTabletsConcurrentlyWaitsForAllTasksAndShutsDownPoolOnFailure() throws Exception { TestingTabletManager tabletManager = new TestingTabletManager(2); @@ -92,6 +110,14 @@ private TestingTabletManager(int closingThreads) { super(TabletType.KV, Collections.emptyList(), new Configuration(), closingThreads); } + private TestingTabletManager(File dataDir) { + super(TabletType.KV, Collections.singletonList(dataDir), new Configuration(), 1); + } + + private List tabletsToLoad(File dataDir) { + return listTabletsToLoad(dataDir); + } + private CompletableFuture closeTablets( List tablets, Consumer closeAction) { return closeTabletsConcurrently(tablets, "testing-tablet-closing", closeAction); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index 8171465b2c1..0b6f7af3d9e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -35,6 +35,7 @@ import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.server.entity.LookupDataForBucket; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.FlussPaths; import com.github.benmanes.caffeine.cache.Scheduler; import com.github.benmanes.caffeine.cache.Ticker; @@ -70,7 +71,7 @@ /** Tests for {@link HistoricalLakeLookupManager}. */ class HistoricalLakeLookupManagerTest { - private static final int SERVER_ID = 1; + private static final long LOOKUP_VOLUME_BYTES = MemorySize.parse("800gb").getBytes(); private static final TableBucket HISTORICAL_BUCKET = new TableBucket(PARTITION_TABLE_ID, 1L, 0); private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = (lookupTimeNanos, lookupFileMaterialization) -> {}; @@ -187,7 +188,8 @@ void testRejectNonPositiveHistoricalLookupMaxQueuedRequests() { conf, null, executor, - SERVER_ID, + ioTmpDir, + LOOKUP_VOLUME_BYTES, Ticker.systemTicker(), Scheduler.disabledScheduler())) .isInstanceOf(IllegalArgumentException.class) @@ -207,7 +209,8 @@ void testRejectNonPositiveHistoricalPartitionThreadPoolMaxSize(int maxThreadPool conf, null, null, - SERVER_ID, + ioTmpDir, + LOOKUP_VOLUME_BYTES, Ticker.systemTicker(), Scheduler.disabledScheduler())) .isInstanceOf(IllegalArgumentException.class) @@ -216,9 +219,8 @@ void testRejectNonPositiveHistoricalPartitionThreadPoolMaxSize(int maxThreadPool } @Test - void testCleansPaimonLookupTempDirectoryOnStartupAndCreatesItLazily() throws Exception { - File serverLookupDir = - new File(new File(ioTmpDir, "paimon-lookup"), String.valueOf(SERVER_ID)); + void testCleansLookupCacheDirectoryOnStartupAndCreatesItLazily() throws Exception { + File serverLookupDir = FlussPaths.historicalLookupRootDir(ioTmpDir); assertThat(serverLookupDir.mkdirs()).isTrue(); File staleLookupFile = new File(serverLookupDir, "stale-lookup-file"); assertThat(staleLookupFile.createNewFile()).isTrue(); @@ -317,15 +319,11 @@ void testInvalidatesLookuperOnSchemaAndLifecycleChanges() throws Exception { } @Test - void testReplacesLookuperOnlyWhenEffectiveCacheSizeChanges() throws Exception { + void testReplacesLookuperOnlyWhenEffectiveCacheRatioChanges() throws Exception { ManualExecutor executor = new ManualExecutor(); TestingHistoricalLakeLookupManager manager = createTestingManager(executor); - lookupAndRun( - manager, - executor, - PARTITION_TABLE_INFO, - tableConfigWithCacheSize(MemorySize.parse("8gb"))); + lookupAndRun(manager, executor, PARTITION_TABLE_INFO, tableConfigWithCacheRatio(0.01)); TestingLakeTableLookuper initialLookuper = manager.createdLookupers.get(0); Configuration unrelatedChange = new Configuration(); @@ -333,28 +331,16 @@ void testReplacesLookuperOnlyWhenEffectiveCacheSizeChanges() throws Exception { ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS, ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.defaultValue() + 1); lookupAndRun(manager, executor, PARTITION_TABLE_INFO, new TableConfig(unrelatedChange)); - lookupAndRun( - manager, - executor, - PARTITION_TABLE_INFO, - tableConfigWithCacheSize(MemorySize.parse("8gb"))); + lookupAndRun(manager, executor, PARTITION_TABLE_INFO, tableConfigWithCacheRatio(0.01)); assertThat(manager.createdLookupers).hasSize(1); assertThat(initialLookuper.closed).isFalse(); - lookupAndRun( - manager, - executor, - PARTITION_TABLE_INFO, - tableConfigWithCacheSize(MemorySize.parse("4gb"))); + lookupAndRun(manager, executor, PARTITION_TABLE_INFO, tableConfigWithCacheRatio(0.005)); assertThat(initialLookuper.closed).isTrue(); assertThat(manager.createdLookupers).hasSize(2); - assertThat( - manager.createdTableConfigs - .get(1) - .getHistoricalPartitionLookupCacheMaxDiskSize()) - .isEqualTo(MemorySize.parse("4gb")); + assertThat(manager.createdCacheSizes.get(1)).isEqualTo(MemorySize.parse("4gb").getBytes()); } @Test @@ -399,9 +385,7 @@ void testEvictsLeastRecentlyUsedLookuperWhenCapacityIsFull() throws Exception { ManualExecutor executor = new ManualExecutor(); AtomicLong tickerNanos = new AtomicLong(); Configuration conf = conf(1); - conf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.parse("16gb")); + conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.02); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager( conf, executor, tickerNanos::get, Scheduler.disabledScheduler()); @@ -431,9 +415,7 @@ void testEvictsLeastRecentlyUsedLookuperWhenCapacityIsFull() throws Exception { void testReconfiguresGlobalCapacityLazily() throws Exception { ManualExecutor executor = new ManualExecutor(); Configuration conf = conf(1); - conf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.parse("96gb")); + conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.12); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(conf, executor); manager.startup(); @@ -452,8 +434,7 @@ void testReconfiguresGlobalCapacityLazily() throws Exception { // Changing only the global limit must not recreate an existing lookuper. Configuration increasedConf = new Configuration(conf); increasedConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.parse("104gb")); + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.13); manager.reconfigure(increasedConf); lookupAndRun( manager, @@ -467,8 +448,7 @@ void testReconfiguresGlobalCapacityLazily() throws Exception { // A reduction is lazy: cached lookupers remain until another admission needs capacity. Configuration reducedConf = new Configuration(increasedConf); reducedConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.parse("88gb")); + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.11); manager.reconfigure(reducedConf); assertThat(manager.cachedTableCount()).isEqualTo(12); @@ -521,24 +501,24 @@ void testEvictsOutsideConcurrentTableReplacements() throws Exception { thread.setDaemon(true); return thread; }); - MemorySize initialCacheSize = MemorySize.parse("4gb"); - MemorySize replacementCacheSize = MemorySize.parse("8gb"); + double initialCacheRatio = 0.005; + double replacementCacheRatio = 0.01; Configuration conf = conf(2); conf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - replacementCacheSize); + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, + replacementCacheRatio); CoordinatedReplacementManager manager = - new CoordinatedReplacementManager(conf, executor, replacementCacheSize); + new CoordinatedReplacementManager(conf, executor, replacementCacheRatio); manager.startup(); TableInfo first = - tableInfoWithCacheSize( - PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId(), initialCacheSize); + tableInfoWithCacheRatio( + PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId(), initialCacheRatio); TableInfo second = - tableInfoWithCacheSize( + tableInfoWithCacheRatio( PARTITION_TABLE_ID + 1, PARTITION_TABLE_INFO.getSchemaId(), - initialCacheSize); + initialCacheRatio); try { assertThat(lookup(manager, first).get(5, TimeUnit.SECONDS).failed()).isFalse(); assertThat(lookup(manager, second).get(5, TimeUnit.SECONDS).failed()).isFalse(); @@ -546,11 +526,11 @@ void testEvictsOutsideConcurrentTableReplacements() throws Exception { // Both replacements hold their own table's compute lock before admission fails. LRU // eviction must happen after those locks are released to avoid cross-key deadlock. TableInfo firstReplacement = - tableInfoWithCacheSize( - first.getTableId(), first.getSchemaId() + 1, replacementCacheSize); + tableInfoWithCacheRatio( + first.getTableId(), first.getSchemaId() + 1, replacementCacheRatio); TableInfo secondReplacement = - tableInfoWithCacheSize( - second.getTableId(), second.getSchemaId() + 1, replacementCacheSize); + tableInfoWithCacheRatio( + second.getTableId(), second.getSchemaId() + 1, replacementCacheRatio); CompletableFuture firstResult = lookup(manager, firstReplacement); CompletableFuture secondResult = @@ -571,12 +551,10 @@ void testEvictsOutsideConcurrentTableReplacements() throws Exception { } @Test - void testThrottlesWhenTableCacheSizeExceedsRuntimeLimit() throws Exception { + void testThrottlesWhenTableCacheRatioExceedsRuntimeLimit() throws Exception { ManualExecutor executor = new ManualExecutor(); Configuration conf = conf(1); - conf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - MemorySize.parse("8gb")); + conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.01); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(conf, executor); manager.startup(); @@ -585,10 +563,8 @@ void testThrottlesWhenTableCacheSizeExceedsRuntimeLimit() throws Exception { lookupResultAndRun( manager, executor, - tableInfoWithCacheSize( - PARTITION_TABLE_ID, - PARTITION_TABLE_INFO.getSchemaId(), - MemorySize.parse("16gb"))); + tableInfoWithCacheRatio( + PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId(), 0.02)); assertThat(result.getError().error()).isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); assertThat(manager.createdLookupers).isEmpty(); @@ -602,7 +578,8 @@ private HistoricalLakeLookupManager createManager( conf(maxQueuedHistoricalRequests), null, executor, - SERVER_ID, + ioTmpDir, + LOOKUP_VOLUME_BYTES, Ticker.systemTicker(), Scheduler.disabledScheduler()); manager.startup(); @@ -621,7 +598,7 @@ private Configuration conf(int maxQueuedHistoricalRequests) { conf.set( ConfigOptions.NETTY_SERVER_MAX_QUEUED_HISTORICAL_REQUESTS, maxQueuedHistoricalRequests); - conf.set(ConfigOptions.SERVER_IO_TMP_DIR, ioTmpDir.getAbsolutePath()); + conf.set(ConfigOptions.DATA_DIR, ioTmpDir.getAbsolutePath()); return conf; } @@ -664,14 +641,14 @@ private static TableInfo tableInfo(long tableId, int schemaId) { PARTITION_TABLE_INFO.getModifiedTime()); } - private static TableInfo tableInfoWithCacheSize( - long tableId, int schemaId, MemorySize cacheSize) { + private static TableInfo tableInfoWithCacheRatio( + long tableId, int schemaId, double cacheRatio) { TableDescriptor descriptor = TableDescriptor.builder(PARTITION_TABLE_INFO.toTableDescriptor()) .property( ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - cacheSize) + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, + cacheRatio) .build(); return TableInfo.of( PARTITION_TABLE_INFO.getTablePath(), @@ -683,11 +660,11 @@ private static TableInfo tableInfoWithCacheSize( PARTITION_TABLE_INFO.getModifiedTime()); } - private static TableConfig tableConfigWithCacheSize(MemorySize cacheSize) { + private static TableConfig tableConfigWithCacheRatio(double cacheRatio) { Configuration conf = new Configuration(); conf.set( - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - cacheSize); + ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, + cacheRatio); return new TableConfig(conf); } @@ -761,6 +738,7 @@ private static final class TestingHistoricalLakeLookupManager private final List createdLookupers = new ArrayList<>(); private final List createdIoTmpDirs = new ArrayList<>(); private final List createdTableConfigs = new ArrayList<>(); + private final List createdCacheSizes = new ArrayList<>(); private final List createdClusterConfigs = new ArrayList<>(); private TestingHistoricalLakeLookupManager(Configuration conf, ManualExecutor executor) { @@ -768,7 +746,8 @@ private TestingHistoricalLakeLookupManager(Configuration conf, ManualExecutor ex conf, null, executor, - SERVER_ID, + new File(conf.get(ConfigOptions.DATA_DIR)), + LOOKUP_VOLUME_BYTES, Ticker.systemTicker(), Scheduler.disabledScheduler()); } @@ -778,7 +757,14 @@ private TestingHistoricalLakeLookupManager( ManualExecutor executor, Ticker ticker, Scheduler cacheScheduler) { - super(conf, null, executor, SERVER_ID, ticker, cacheScheduler); + super( + conf, + null, + executor, + new File(conf.get(ConfigOptions.DATA_DIR)), + LOOKUP_VOLUME_BYTES, + ticker, + cacheScheduler); } @Override @@ -786,11 +772,13 @@ LakeTableLookuper createLakeTableLookuper( TablePath tablePath, String ioTmpDir, TableConfig tableConfig, + long cacheSizeBytes, Configuration clusterConf) { TestingLakeTableLookuper lookuper = new TestingLakeTableLookuper(); createdLookupers.add(lookuper); createdIoTmpDirs.add(ioTmpDir); createdTableConfigs.add(tableConfig); + createdCacheSizes.add(cacheSizeBytes); createdClusterConfigs.add(clusterConf); return lookuper; } @@ -816,20 +804,21 @@ public void close() { } private static final class CoordinatedReplacementManager extends HistoricalLakeLookupManager { - private final MemorySize replacementCacheSize; + private final double replacementCacheRatio; private final CyclicBarrier replacementBarrier = new CyclicBarrier(2); private final AtomicInteger coordinatedCreations = new AtomicInteger(); private CoordinatedReplacementManager( - Configuration conf, ExecutorService executor, MemorySize replacementCacheSize) { + Configuration conf, ExecutorService executor, double replacementCacheRatio) { super( conf, null, executor, - SERVER_ID, + new File(conf.get(ConfigOptions.DATA_DIR)), + LOOKUP_VOLUME_BYTES, Ticker.systemTicker(), Scheduler.disabledScheduler()); - this.replacementCacheSize = replacementCacheSize; + this.replacementCacheRatio = replacementCacheRatio; } @Override @@ -837,10 +826,12 @@ LakeTableLookuper createLakeTableLookuper( TablePath tablePath, String ioTmpDir, TableConfig tableConfig, + long cacheSizeBytes, Configuration clusterConf) { - if (tableConfig - .getHistoricalPartitionLookupCacheMaxDiskSize() - .equals(replacementCacheSize) + if (Double.compare( + tableConfig.getHistoricalPartitionLookupCacheMaxDiskRatio(), + replacementCacheRatio) + == 0 && coordinatedCreations.getAndIncrement() < 2) { try { replacementBarrier.await(5, TimeUnit.SECONDS); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java index 6b0c45a88fe..400d64460ab 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java @@ -199,7 +199,6 @@ public void setup(TestInfo testInfo) throws Exception { } else { conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath()); } - conf.set(ConfigOptions.SERVER_IO_TMP_DIR, tempDir.getAbsolutePath()); conf.setString(ConfigOptions.COORDINATOR_HOST, "localhost"); conf.set(ConfigOptions.REMOTE_DATA_DIR, tempDir.getAbsolutePath() + "/remote_data_dir"); conf.set(ConfigOptions.SERVER_IO_POOL_SIZE, 2); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java index fcbaed882cc..8a1f6fb41e5 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java @@ -522,7 +522,6 @@ private ReplicaManager createReplicaManager(int serverId, LocalDiskManager local Configuration conf = new Configuration(); conf.set(ConfigOptions.TABLET_SERVER_ID, serverId); conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath() + "/server-" + serverId); - conf.set(ConfigOptions.SERVER_IO_TMP_DIR, tempDir.getAbsolutePath()); conf.set(ConfigOptions.WRITER_ID_EXPIRATION_TIME, Duration.ofHours(12)); Scheduler scheduler = new FlussScheduler(2); scheduler.startup(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index eb8942c534a..4cae527523a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -18,7 +18,6 @@ package org.apache.fluss.server.utils; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.InvalidConfigException; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.Schema; @@ -54,7 +53,7 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { DataLakeFormat.PAIMON, TABLE_PATH, ConfigOptions - .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .defaultValue())) .isInstanceOf(InvalidConfigException.class) .hasMessage( @@ -86,7 +85,7 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { DataLakeFormat.PAIMON, TABLE_PATH, ConfigOptions - .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .defaultValue())) .isInstanceOf(InvalidConfigException.class) .hasMessage( @@ -98,35 +97,35 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { } @Test - void testValidateHistoricalLookupCacheSize() { + void testValidateHistoricalLookupCacheRatio() { TableDescriptor ordinaryTableDescriptor = TableDescriptor.builder() .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) .distributedBy(1) .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) .build(); - assertThatCode(() -> validate(ordinaryTableDescriptor, MemorySize.parse("4gb"))) - .doesNotThrowAnyException(); + assertThatCode(() -> validate(ordinaryTableDescriptor, 0.05)).doesNotThrowAnyException(); - TableDescriptor zeroSizeDescriptor = descriptorWithCacheSize(MemorySize.ZERO); - assertThatThrownBy(() -> validate(zeroSizeDescriptor, MemorySize.parse("80gb"))) + TableDescriptor zeroRatioDescriptor = descriptorWithCacheRatio(0.0); + assertThatThrownBy(() -> validate(zeroRatioDescriptor, 0.1)) .isInstanceOf(InvalidConfigException.class) .hasMessageContaining( - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE + ConfigOptions + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO .key()) .hasMessageContaining(TABLE_PATH.toString()) - .hasMessageContaining("greater than 0 bytes"); + .hasMessageContaining("within (0.0, 1.0]"); - TableDescriptor oversizedDescriptor = descriptorWithCacheSize(MemorySize.parse("16gb")); - assertThatThrownBy(() -> validate(oversizedDescriptor, MemorySize.parse("8gb"))) + TableDescriptor oversizedDescriptor = descriptorWithCacheRatio(0.2); + assertThatThrownBy(() -> validate(oversizedDescriptor, 0.1)) .isInstanceOf(InvalidConfigException.class) - .hasMessageContaining("16 gb") + .hasMessageContaining("0.2") .hasMessageContaining( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE.key()) - .hasMessageContaining("8 gb"); + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO.key()) + .hasMessageContaining("0.1"); } - private static TableDescriptor descriptorWithCacheSize(MemorySize cacheSize) { + private static TableDescriptor descriptorWithCacheRatio(double cacheRatio) { return TableDescriptor.builder() .schema( Schema.newBuilder() @@ -143,13 +142,13 @@ private static TableDescriptor descriptorWithCacheSize(MemorySize cacheSize) { .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) .property( ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_SIZE, - cacheSize) + .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, + cacheRatio) .build(); } - private static void validate(TableDescriptor descriptor, MemorySize globalCacheSize) { + private static void validate(TableDescriptor descriptor, double globalCacheRatio) { TableDescriptorValidation.validateTableDescriptor( - descriptor, 100, DataLakeFormat.PAIMON, TABLE_PATH, globalCacheSize); + descriptor, 100, DataLakeFormat.PAIMON, TABLE_PATH, globalCacheRatio); } } From e4918039ff410cbb4d9bea1097345b3beb53c92d Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Thu, 6 Aug 2026 07:08:49 +0800 Subject: [PATCH 09/15] [test] Exclude lookup IO wrapper from coverage Exclude the delegating TrackingIOManager inner class from the per-class JaCoCo threshold. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 3/3 AI-Contributed/UT: 0/0 --- fluss-test-coverage/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fluss-test-coverage/pom.xml b/fluss-test-coverage/pom.xml index b742c2d8e1e..485dbf0ed51 100644 --- a/fluss-test-coverage/pom.xml +++ b/fluss-test-coverage/pom.xml @@ -500,6 +500,9 @@ org.apache.fluss.flink.DummyClass120 org.apache.fluss.lake.batch.ArrowRecordBatch org.apache.fluss.lake.committer.CommittedLakeSnapshot + + org.apache.fluss.lake.paimon.lookup.PaimonLakeTableLookuper.TrackingIOManager + org.apache.fluss.lake.paimon.utils.FlussDataTypeToPaimonDataType org.apache.paimon.arrow.converter.Arrow2PaimonVectorConverter* From 7fb6ed61e7c86042da39c307fe23ba9d25b940dc Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Thu, 6 Aug 2026 11:55:27 +0800 Subject: [PATCH 10/15] [server] Refine historical lookup metric names and docs Clarify cache metric naming, lookup lifecycle comments, and data-directory parameters. Document the historical lookup metrics exposed to operators. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 66/81 AI-Contributed/UT: 0/0 --- .../org/apache/fluss/metrics/MetricNames.java | 3 +- .../org/apache/fluss/utils/FlussPaths.java | 12 ++--- .../lookup/PaimonLakeTableLookuper.java | 2 + .../metrics/group/TableMetricGroup.java | 4 ++ .../replica/HistoricalLakeLookupManager.java | 10 ++-- .../fluss/server/replica/ReplicaManager.java | 2 +- .../observability/monitor-metrics.md | 48 +++++++++++++++++-- 7 files changed, 64 insertions(+), 17 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java index e80ad687cfe..62d1cab2041 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java +++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java @@ -107,7 +107,8 @@ public class MetricNames { // for historical lookup cache public static final String HISTORICAL_LOOKUP_CACHED_TABLE_COUNT = "historicalLookupCachedTableCount"; - public static final String HISTORICAL_LOOKUP_CACHE_EVICTIONS = "historicalLookupCacheEvictions"; + public static final String HISTORICAL_LOOKUP_CACHED_TABLE_CAPACITY_EVICTIONS = + "historicalLookupCachedTableCapacityEvictions"; // -------------------------------------------------------------------------------------------- // metrics for user diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java index 4554d9a49f4..fef00c6c558 100644 --- a/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java +++ b/fluss-common/src/main/java/org/apache/fluss/utils/FlussPaths.java @@ -53,9 +53,7 @@ public class FlussPaths { /** Prefix of a local kv tablet directory to store kv files for a specific kv tablet. */ public static final String KV_TABLET_DIR_PREFIX = "kv-"; - /** - * The directory name for historical lookup cache files under the first local data directory. - */ + /** The directory name for historical lookup cache files under a local data directory. */ public static final String HISTORICAL_LOOKUP_CACHE_DIR_NAME = ".historical-lookup-cache"; /** Prefix for a partition id to distinguish between table id and partition id. */ @@ -154,12 +152,12 @@ public static File kvTabletDir( } /** - * Returns the historical lookup cache root under the first local data directory. + * Returns the historical lookup cache root under the local data directory. * - * @param firstDataDir the first available local data directory + * @param dataDir the local data directory */ - public static File historicalLookupRootDir(File firstDataDir) { - return new File(firstDataDir, HISTORICAL_LOOKUP_CACHE_DIR_NAME); + public static File historicalLookupRootDir(File dataDir) { + return new File(dataDir, HISTORICAL_LOOKUP_CACHE_DIR_NAME); } /** diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index 5e0a4166466..fdaefd08ec4 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -158,6 +158,8 @@ public PaimonLakeTableLookuper( context.lookupMetricRecorder() .recordLookup( System.nanoTime() - lookupStartNanos, + // An increase means this lookup materialized at least one local lookup + // file through the tracking IO manager. lookupFileMaterializationCount > materializationCountBeforeLookup); } if (paimonRow == null) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java index f78fce63b93..ce6a7e8aa55 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java @@ -614,6 +614,10 @@ protected String getGroupName(CharacterFilter filter) { } } + /** + * Collects historical lookup metrics in the KV table scope and separates lake lookup metrics by + * whether the lookup materialized local files. + */ private static class HistoricalLookupMetricGroup extends AbstractMetricGroup { private final Counter totalLookupRequests; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index cba2eef9800..39db95cbe8d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -134,14 +134,14 @@ class HistoricalLakeLookupManager implements AutoCloseable { HistoricalLakeLookupManager( Configuration conf, @Nullable PluginManager pluginManager, - File firstDataDir, + File dataDir, long lookupVolumeBytes, Scheduler scheduler) { this( conf, pluginManager, null, - firstDataDir, + dataDir, lookupVolumeBytes, Ticker.systemTicker(), createCacheScheduler(scheduler)); @@ -152,7 +152,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { Configuration conf, @Nullable PluginManager pluginManager, @Nullable ExecutorService historicalPartitionExecutor, - File firstDataDir, + File dataDir, long lookupVolumeBytes, Ticker ticker, com.github.benmanes.caffeine.cache.Scheduler cacheScheduler) { @@ -160,7 +160,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { this.pluginManager = pluginManager; this.historicalLookupCacheRootDir = FlussPaths.historicalLookupRootDir( - checkNotNull(firstDataDir, "firstDataDir must not be null.")); + checkNotNull(dataDir, "dataDir must not be null.")); checkArgument(lookupVolumeBytes > 0, "lookupVolumeBytes must be greater than 0."); this.lookupVolumeBytes = lookupVolumeBytes; this.budgetManager = @@ -519,6 +519,8 @@ private CachedLakeTableLookuper acquireCachedLookuper( selectedLookuper.acquire(); return selectedLookuper; }); + // Replacement admission may preserve the old mapping without acquiring it. Return null for + // that stale mapping so the caller can evict another table and retry. return matchesLookupConfiguration( cachedLookuper, context, currentLakeConfigVersion, cacheSizeBytes) ? cachedLookuper diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index d868dc6710e..be113765143 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -462,7 +462,7 @@ private void registerMetrics() { MetricNames.HISTORICAL_LOOKUP_CACHED_TABLE_COUNT, historicalLakeLookupManager::cachedTableCount); serverMetricGroup.counter( - MetricNames.HISTORICAL_LOOKUP_CACHE_EVICTIONS, + MetricNames.HISTORICAL_LOOKUP_CACHED_TABLE_CAPACITY_EVICTIONS, historicalLakeLookupManager.capacityEvictions()); MetricGroup logicalStorage = serverMetricGroup.addGroup("logicalStorage"); diff --git a/website/docs/maintenance/observability/monitor-metrics.md b/website/docs/maintenance/observability/monitor-metrics.md index 18a4edccb2a..74f3c510c63 100644 --- a/website/docs/maintenance/observability/monitor-metrics.md +++ b/website/docs/maintenance/observability/monitor-metrics.md @@ -463,8 +463,8 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM - tabletserver - - + tabletserver + - messagesInPerSecond The number of messages written per second to this server. Meter @@ -589,6 +589,21 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM The number of kv pre-write buffer truncate due to the error happened when writing cdc to log per second. Meter + + historicalPartitionInflightRequests + The number of accepted historical partition requests that have not completed, labeled with operation. Historical lookup requests use operation="lookup". + Gauge + + + historicalLookupCachedTableCount + The number of table lookupers currently retained in the historical lookup cache. + Gauge + + + historicalLookupCachedTableCapacityEvictions + The cumulative number of cached table lookupers evicted to free historical lookup cache capacity. + Counter + logicalStorage logSize @@ -679,6 +694,7 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM request_produceLog request_putKv request_lookup + request_historicalLookup request_prefixLookup request_metadata request_fetchLogClient @@ -767,8 +783,8 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM - tabletserver - table + tabletserver + table messagesInPerSecond The number of messages written per second to this table. Meter @@ -823,6 +839,26 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM The number of failed lookup requests to lookup value by key from this table per second. Meter + + totalHistoricalLookupRequestsPerSecond + The number of historical lookup requests to this table per second. + Meter + + + failedHistoricalLookupRequestsPerSecond + The number of historical lookup requests that failed unexpectedly for this table per second. + Meter + + + lakeLookupsPerSecond + The number of historical lake point lookups performed for this table per second, labeled with lookup_file_materialization. + Meter + + + lakeLookupTimeMs + The time spent on a historical lake point lookup, in milliseconds, labeled with lookup_file_materialization. + Histogram + totalLimitScanRequestsPerSecond The number of limit scan requests to scan records with limit from this table per second. @@ -941,6 +977,10 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM +For lakeLookupsPerSecond and lakeLookupTimeMs, +lookup_file_materialization="true" means that the lookup created at least one local +lookup file; false means that it did not create a local lookup file. + ### RocksDB RocksDB metrics provide insights into the performance and health of the underlying RocksDB storage engine used by Fluss. These metrics are categorized into table-level metrics (aggregated from all buckets of a table) and server-level metrics (aggregated from all tables in a server). From 72345fab66926bfe1170a0b08aedb85cb72a766e Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Sun, 9 Aug 2026 08:38:04 +0800 Subject: [PATCH 11/15] [rpc] Avoid scanning all lookup buckets Determine lookup request type from the first bucket and validate normal lookup conversion so mixed requests are still rejected in either order. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 20/20 AI-Contributed/UT: 0/0 --- .../fluss/rpc/util/CommonRpcMessageUtils.java | 16 ++++++++-------- .../server/utils/ServerRpcMessageUtils.java | 4 ++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 0a7d260db78..a5aa22cb3b8 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -31,7 +31,6 @@ import org.apache.fluss.rpc.messages.PbAclInfo; import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket; import org.apache.fluss.rpc.messages.PbKeyValue; -import org.apache.fluss.rpc.messages.PbLookupReqForBucket; import org.apache.fluss.rpc.messages.PbPartitionSpec; import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; @@ -61,14 +60,15 @@ */ public class CommonRpcMessageUtils { - /** Returns whether the lookup request contains historical partition lookup data. */ + /** + * Returns whether the lookup request is for historical partition lookup. + * + *

Normal and historical lookup buckets cannot be mixed in the same request, so the first + * bucket determines the request type. + */ public static boolean hasHistoricalLookup(LookupRequest lookupRequest) { - for (PbLookupReqForBucket bucketRequest : lookupRequest.getBucketsReqsList()) { - if (bucketRequest.hasOriginalPartitionName()) { - return true; - } - } - return false; + return lookupRequest.getBucketsReqsCount() > 0 + && lookupRequest.getBucketsReqAt(0).hasOriginalPartitionName(); } public static List toPbAclInfos(Collection aclBindings) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 60212736d6c..89ae6f7138e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -1123,6 +1123,10 @@ public static Map> toLookupData(LookupRequest lookupRe long tableId = lookupRequest.getTableId(); Map> lookupEntryData = new HashMap<>(); for (PbLookupReqForBucket lookupReqForBucket : lookupRequest.getBucketsReqsList()) { + if (lookupReqForBucket.hasOriginalPartitionName()) { + throw new IllegalArgumentException( + "Normal and historical lookups cannot be mixed in the same request."); + } TableBucket tb = new TableBucket( tableId, From e9870e3ea103efb7238bcfaf3194828c3df8e6a4 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Sun, 9 Aug 2026 08:47:02 +0800 Subject: [PATCH 12/15] [server] Hide internal historical partition from listings Exclude the internal historical system partition from list partition responses and extend the existing admin integration test. Leave a TODO to return lake-native partitions in the future. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 5/5 AI-Contributed/UT: 16/16 --- .../fluss/client/admin/FlussAdminITCase.java | 16 ++++++++++++++-- .../server/utils/ServerRpcMessageUtils.java | 5 +++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index 150b84e5679..93733b32bb8 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -124,6 +124,7 @@ import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.InternalRowAssert.assertThatRow; import static org.apache.fluss.testutils.common.CommonTestUtils.waitUntil; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -1102,22 +1103,33 @@ void testListPartitionInfos() throws Exception { .column("id", DataTypes.STRING()) .column("name", DataTypes.STRING()) .column("pt", DataTypes.STRING()) + .primaryKey("id", "pt") .build()) .comment("test table") .distributedBy(3, "id") .partitionedBy("pt") .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) + .property(ConfigOptions.TABLE_AUTO_PARTITION_KEY, "pt") .property( ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, AutoPartitionTimeUnit.YEAR) + .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) + .property(ConfigOptions.TABLE_DATALAKE_FORMAT, PAIMON) + .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) .build(); TablePath partitionedTablePath = TablePath.of(dbName, "test_partitioned_table"); admin.createTable(partitionedTablePath, partitionedTable, true).get(); Map partitionIdByNames = - FLUSS_CLUSTER_EXTENSION.waitUntilPartitionAllReady(partitionedTablePath); + FLUSS_CLUSTER_EXTENSION.waitUntilPartitionAllReady( + partitionedTablePath, + ConfigOptions.TABLE_AUTO_PARTITION_NUM_PRECREATE.defaultValue() + 1); + assertThat(partitionIdByNames).containsKey(HISTORICAL_PARTITION_VALUE); List partitionInfos = admin.listPartitionInfos(partitionedTablePath).get(); - assertThat(partitionInfos).hasSize(partitionIdByNames.size()); + assertThat(partitionInfos) + .hasSize(partitionIdByNames.size() - 1) + .extracting(PartitionInfo::getPartitionName) + .doesNotContain(HISTORICAL_PARTITION_VALUE); for (PartitionInfo partitionInfo : partitionInfos) { assertThat(partitionIdByNames.get(partitionInfo.getPartitionName())) .isEqualTo(partitionInfo.getPartitionId()); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 89ae6f7138e..799e4726705 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -232,6 +232,7 @@ import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toByteBuffer; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toPbAclInfo; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** @@ -1780,6 +1781,10 @@ public static ListPartitionInfosResponse toListPartitionInfosResponse( ListPartitionInfosResponse listPartitionsResponse = new ListPartitionInfosResponse(); for (Map.Entry partitionRegistration : partitionRegistrations.entrySet()) { + // TODO: Return the actual lake partitions instead of the internal historical partition. + if (HISTORICAL_PARTITION_VALUE.equals(partitionRegistration.getKey())) { + continue; + } ResolvedPartitionSpec spec = ResolvedPartitionSpec.fromPartitionName( partitionKeys, partitionRegistration.getKey()); From 2466d8ce2544762e4684d0873dab424c76890a5c Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Sun, 9 Aug 2026 13:11:15 +0800 Subject: [PATCH 13/15] [test] Fix historical partition lookup IT case Read the internal historical partition from ZooKeeper now that public partition listings hide it. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 23/23 --- .../HistoricalPartitionLookupITCase.java | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java index e4ab4130db8..091c72bc21b 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/HistoricalPartitionLookupITCase.java @@ -34,6 +34,7 @@ import org.apache.fluss.metadata.TablePath; import org.apache.fluss.row.InternalRow; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.server.zk.data.PartitionRegistration; import org.apache.fluss.types.DataTypes; import org.apache.flink.core.execution.JobClient; @@ -46,6 +47,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import static org.apache.fluss.testutils.DataTestUtils.row; @@ -98,13 +100,12 @@ void testLookupExpiredPartitionFromPaimon(boolean defaultBucketKey) throws Excep false) .get(); // The ALTER RPC must not complete until the required system partition is persisted. - assertThat( - FLUSS_CLUSTER_EXTENSION - .getZooKeeperClient() - .getPartition(tablePath, HISTORICAL_PARTITION_VALUE)) - .isPresent(); - waitUntilPartitionCreated(tablePath, HISTORICAL_PARTITION_VALUE); - long historicalPartitionId = getPartitionId(tablePath, HISTORICAL_PARTITION_VALUE); + Optional historicalPartition = + FLUSS_CLUSTER_EXTENSION + .getZooKeeperClient() + .getPartition(tablePath, HISTORICAL_PARTITION_VALUE); + assertThat(historicalPartition).isPresent(); + long historicalPartitionId = historicalPartition.get().getPartitionId(); FLUSS_CLUSTER_EXTENSION.waitUntilTablePartitionReady(tableId, historicalPartitionId); // Keep the initial retention wide enough so this old partition can be created and written @@ -347,12 +348,4 @@ private static void waitUntilPartitionDropped(TablePath tablePath, String partit assertThat(admin.listPartitionInfos(tablePath).get()) .noneMatch(p -> partitionName.equals(p.getPartitionName()))); } - - private static void waitUntilPartitionCreated(TablePath tablePath, String partitionName) { - retry( - Duration.ofMinutes(1), - () -> - assertThat(admin.listPartitionInfos(tablePath).get()) - .anyMatch(p -> partitionName.equals(p.getPartitionName()))); - } } From 46b1319d24f33d61e35f33ae73089eee033db747 Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 12 Aug 2026 14:19:09 +0800 Subject: [PATCH 14/15] [server] Address historical lookup review comments Simplify historical lookup cache capacity to a fixed table limit, integrate disk-write protection and historical metrics, and clean up configuration, lifecycle, naming, and cache observability. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 862/1279 AI-Contributed/UT: 456/636 --- .../apache/fluss/config/ConfigOptions.java | 13 +- .../apache/fluss/config/FlussConfigUtils.java | 3 - .../org/apache/fluss/config/TableConfig.java | 6 - .../lake/lakestorage/LakeTableLookuper.java | 4 +- .../org/apache/fluss/metrics/MetricNames.java | 15 +- .../fluss/config/FlussConfigUtilsTest.java | 7 - .../apache/fluss/config/TableConfigTest.java | 13 - .../lookup/PaimonLakeTableLookuper.java | 12 +- .../lookup/PaimonLakeTableLookuperTest.java | 10 +- .../rpc/netty/server/NettyServerHandler.java | 6 +- .../rpc/netty/server/RequestsMetrics.java | 28 +- .../apache/fluss/server/RpcServiceBase.java | 3 + .../fluss/server/TabletManagerBase.java | 4 +- .../server/coordinator/CoordinatorServer.java | 2 +- .../coordinator/CoordinatorService.java | 2 +- ...HistoricalLookupCacheConfigValidator.java} | 18 +- .../server/coordinator/MetadataManager.java | 15 +- .../metrics/group/TableMetricGroup.java | 131 +++-- .../group/TabletServerMetricGroup.java | 36 -- .../replica/HistoricalLakeLookupManager.java | 471 +++++++----------- .../HistoricalLookupCacheBudgetManager.java | 175 ------- .../apache/fluss/server/replica/Replica.java | 11 +- .../fluss/server/replica/ReplicaManager.java | 102 ++-- .../server/utils/ServerRpcMessageUtils.java | 5 - .../utils/TableDescriptorValidation.java | 38 +- .../HistoricalLakeLookupManagerTest.java | 439 ++++++---------- ...istoricalLookupCacheBudgetManagerTest.java | 95 ---- ...istoricalPartitionTableValidationTest.java | 72 +-- .../observability/monitor-metrics.md | 67 +-- 29 files changed, 534 insertions(+), 1269 deletions(-) rename fluss-server/src/main/java/org/apache/fluss/server/coordinator/{HistoricalLookupCacheConfigUpdater.java => HistoricalLookupCacheConfigValidator.java} (76%) delete mode 100644 fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java delete mode 100644 fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index a798a44c424..455096713a7 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -434,8 +434,8 @@ public class ConfigOptions { .doubleType() .defaultValue(0.10) .withDescription( - "The maximum fraction of the total capacity of the volume containing the first available data directory that historical partition lookup caches may reserve on a TabletServer. " - + "Historical lookup cache files are stored under that data directory; additional data volumes are not used. " + "The maximum fraction of the total capacity of the volume containing the first available data directory allocated to historical partition lookup caches on a TabletServer. " + + "Up to ten table lookupers are cached, and each receives one tenth of this capacity. Historical lookup cache files are stored under that data directory; additional data volumes are not used. " + "The valid range is (0.0, 1.0]."); public static final ConfigOption @@ -1876,15 +1876,6 @@ public class ConfigOptions { + "to look up historical partition data so that their clients load the " + "updated table configuration."); - public static final ConfigOption - TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO = - key("table.datalake.historical-partition.lookup-cache.max-disk-ratio") - .doubleType() - .defaultValue(0.01) - .withDescription( - "The maximum fraction of the total capacity of the volume containing the first available TabletServer data directory reserved for this table's historical partition lookup cache. " - + "When the table is created or altered, the value must be within (0.0, 1.0] and no greater than the current TabletServer historical lookup cache ratio."); - public static final ConfigOption TABLE_DATALAKE_FORMAT = key("table.datalake.format") .enumType(DataLakeFormat.class) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index f4751af1da0..5f1489baac3 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -49,9 +49,6 @@ public class FlussConfigUtils { Arrays.asList( ConfigOptions.TABLE_DATALAKE_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED.key(), - ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO - .key(), ConfigOptions.TABLE_DATALAKE_FRESHNESS.key(), ConfigOptions.TABLE_DATALAKE_AUTO_COMPACTION.key(), ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.key(), diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index e7d53ae1b04..931c33e9a75 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -105,12 +105,6 @@ public boolean isHistoricalPartitionEnabled() { return config.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED); } - /** Gets the maximum local disk ratio of the historical partition lookup cache. */ - public double getHistoricalPartitionLookupCacheMaxDiskRatio() { - return config.get( - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); - } - /** * Return the data lake format of the table. It'll be the datalake format configured in Fluss * whiling creating the table. Return empty if no datalake format configured while creating. diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java index ab50acdb8f2..d5d22888157 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuper.java @@ -41,9 +41,9 @@ interface LookupMetricRecorder { * Records a completed lake table point lookup. * * @param lookupTimeNanos time spent on the lake table point lookup, in nanoseconds - * @param lookupFileMaterialization whether the lookup triggered lookup file materialization + * @param lookupFileDownloaded whether the lookup downloaded a lookup file */ - void recordLookup(long lookupTimeNanos, boolean lookupFileMaterialization); + void recordLookup(long lookupTimeNanos, boolean lookupFileDownloaded); } /** diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java index 62d1cab2041..98f93ba8ad1 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java +++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java @@ -92,8 +92,7 @@ public class MetricNames { "delayedFetchFromFollowerExpiresPerSecond"; public static final String DELAYED_FETCH_FROM_CLIENT_EXPIRES_RATE = "delayedFetchFromClientExpiresPerSecond"; - public static final String HISTORICAL_PARTITION_INFLIGHT_REQUESTS = - "historicalPartitionInflightRequests"; + public static final String HISTORICAL_INFLIGHT_REQUESTS = "inflightRequests"; public static final String SERVER_LOGICAL_STORAGE_LOG_SIZE = "logSize"; public static final String SERVER_LOGICAL_STORAGE_KV_SIZE = "kvSize"; @@ -105,10 +104,10 @@ public class MetricNames { public static final String DISK_WRITE_LOCKED = "diskWriteLocked"; // for historical lookup cache - public static final String HISTORICAL_LOOKUP_CACHED_TABLE_COUNT = - "historicalLookupCachedTableCount"; - public static final String HISTORICAL_LOOKUP_CACHED_TABLE_CAPACITY_EVICTIONS = - "historicalLookupCachedTableCapacityEvictions"; + public static final String HISTORICAL_LOOKUP_CACHE_DISK_SIZE = "lookupCacheDiskSize"; + public static final String HISTORICAL_LOOKUP_CACHE_TABLE_COUNT = "lookupCacheTableCount"; + public static final String HISTORICAL_LOOKUP_CACHE_CAPACITY_EVICTIONS = + "lookupCacheCapacityEvictions"; // -------------------------------------------------------------------------------------------- // metrics for user @@ -138,10 +137,6 @@ public class MetricNames { public static final String TOTAL_LOOKUP_REQUESTS_RATE = "totalLookupRequestsPerSecond"; public static final String FAILED_LOOKUP_REQUESTS_RATE = "failedLookupRequestsPerSecond"; - public static final String TOTAL_HISTORICAL_LOOKUP_REQUESTS_RATE = - "totalHistoricalLookupRequestsPerSecond"; - public static final String FAILED_HISTORICAL_LOOKUP_REQUESTS_RATE = - "failedHistoricalLookupRequestsPerSecond"; public static final String LAKE_LOOKUPS_RATE = "lakeLookupsPerSecond"; public static final String LAKE_LOOKUP_TIME_MS = "lakeLookupTimeMs"; public static final String TOTAL_PUT_KV_REQUESTS_RATE = "totalPutKvRequestsPerSecond"; diff --git a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java index 17ec9fd8492..61a00c01d2e 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java @@ -241,13 +241,6 @@ void testValidateHistoricalLookupCacheConfigs() { .hasMessageContaining( ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS .key()); - - assertThat( - FlussConfigUtils.isAlterableTableOption( - ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO - .key())) - .isTrue(); } @Test diff --git a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java index 8470cb3f3ad..5d18fcd1c97 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java @@ -44,17 +44,4 @@ void testDeleteBehavior() { TableConfig tableConfig3 = new TableConfig(conf); assertThat(tableConfig3.getDeleteBehavior()).hasValue(DeleteBehavior.IGNORE); } - - @Test - void testHistoricalPartitionLookupCacheMaxDiskRatio() { - Configuration conf = new Configuration(); - TableConfig tableConfig = new TableConfig(conf); - assertThat(tableConfig.getHistoricalPartitionLookupCacheMaxDiskRatio()).isEqualTo(0.01); - - conf.set( - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, - 0.02); - assertThat(new TableConfig(conf).getHistoricalPartitionLookupCacheMaxDiskRatio()) - .isEqualTo(0.02); - } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index fdaefd08ec4..1d64eed809a 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -104,7 +104,7 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private @Nullable LocalTableQuery localTableQuery; private @Nullable RowPartitionKeyExtractor partitionKeyExtractor; private int primaryKeyFieldCount; - private long lookupFileMaterializationCount; + private long lookupFileDownloadCount; // Both encoders are initialized only for a kv-format-v2 table whose bucket key differs from // its physical primary key. They remain null when the incoming Fluss key already uses Paimon's @@ -147,7 +147,7 @@ public PaimonLakeTableLookuper( org.apache.paimon.data.BinaryRow keyRow = toPaimonLookupKey(key); initializeFilesIfNeeded(partition, context.bucketId()); - long materializationCountBeforeLookup = lookupFileMaterializationCount; + long downloadCountBeforeLookup = lookupFileDownloadCount; long lookupStartNanos = System.nanoTime(); org.apache.paimon.data.InternalRow paimonRow; try { @@ -158,9 +158,9 @@ public PaimonLakeTableLookuper( context.lookupMetricRecorder() .recordLookup( System.nanoTime() - lookupStartNanos, - // An increase means this lookup materialized at least one local lookup - // file through the tracking IO manager. - lookupFileMaterializationCount > materializationCountBeforeLookup); + // An increase means this lookup downloaded at least one lookup file + // through the tracking IO manager. + lookupFileDownloadCount > downloadCountBeforeLookup); } if (paimonRow == null) { return null; @@ -463,7 +463,7 @@ public FileIOChannel.ID createChannel() { @Override public FileIOChannel.ID createChannel(String prefix) { - lookupFileMaterializationCount++; + lookupFileDownloadCount++; return delegate.createChannel(prefix); } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index 9f871140b3c..367ec01112f 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -80,7 +80,7 @@ class PaimonLakeTableLookuperTest { private static final short EVOLVED_SCHEMA_ID = 2; private static final long LOOKUP_CACHE_MAX_DISK_BYTES = MemorySize.parse("8gb").getBytes(); private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = - (lookupTimeNanos, lookupFileMaterialization) -> {}; + (lookupTimeNanos, lookupFileDownloaded) -> {}; @TempDir private File tempWarehouseDir; @@ -126,15 +126,15 @@ void testLookupPartitionedPrimaryKeyTable() throws Exception { tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED), LOOKUP_CACHE_MAX_DISK_BYTES)) { - List lookupFileMaterializations = new ArrayList<>(); + List lookupFileDownloads = new ArrayList<>(); LakeTableLookuper.LookupContext context = lookupContext( schema, "20240101", 0, SCHEMA_ID, - (lookupTimeNanos, lookupFileMaterialization) -> - lookupFileMaterializations.add(lookupFileMaterialization)); + (lookupTimeNanos, lookupFileDownloaded) -> + lookupFileDownloads.add(lookupFileDownloaded)); byte[] value = lookuper.lookup(paimonKey(schema, 1, "20240101"), context); BinaryValue decodedValue = decodeValue(value, SCHEMA_ID, schema); @@ -150,7 +150,7 @@ void testLookupPartitionedPrimaryKeyTable() throws Exception { assertThat(lookuper.lookup(compactedKey(schema, 1, "20240101"), context)).isNull(); // The first lookup creates the local lookup file, while subsequent lookups reuse it. - assertThat(lookupFileMaterializations).containsExactly(true, false, false); + assertThat(lookupFileDownloads).containsExactly(true, false, false); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java index deb3367ec7d..25328fe44e6 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java @@ -300,16 +300,16 @@ private void updateRequestMetrics(FlussRequest request, long requestEndTimeMs) { private Optional getMetrics(FlussRequest request) { boolean isFromFollower = false; - boolean isHistoricalLookup = false; + boolean isHistorical = false; ApiMessage requestMessage = request.getMessage(); if (request.getApiKey() == ApiKeys.FETCH_LOG.id) { // for fetch, we need to identify it's from client or follower FetchLogRequest fetchLogRequest = (FetchLogRequest) requestMessage; isFromFollower = fetchLogRequest.getFollowerServerId() >= 0; } else if (request.getApiKey() == ApiKeys.LOOKUP.id) { - isHistoricalLookup = hasHistoricalLookup((LookupRequest) requestMessage); + isHistorical = hasHistoricalLookup((LookupRequest) requestMessage); } - return requestsMetrics.getMetrics(request.getApiKey(), isFromFollower, isHistoricalLookup); + return requestsMetrics.getMetrics(request.getApiKey(), isFromFollower, isHistorical); } @VisibleForTesting diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/RequestsMetrics.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/RequestsMetrics.java index aebaa473f7e..3fff71f70a6 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/RequestsMetrics.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/RequestsMetrics.java @@ -41,8 +41,6 @@ */ public class RequestsMetrics { - private static final String HISTORICAL_LOOKUP_METRICS_KEY = "historicalLookup"; - // a map from request name to the metrics registered for the request name private final Map metricsByRequest = new HashMap<>(); @@ -53,16 +51,14 @@ private RequestsMetrics(MetricGroup serverMetricsGroup, Collection apiK for (ApiKeys apiKey : apiKeys) { // we create a metrics group for each type of request, with the request type // as variable + addMetrics(serverMetricsGroup, toRequestName(apiKey, false, false)); if (apiKey == ApiKeys.FETCH_LOG) { - // if it's fetch, we need two metrics group, one for client, one for follower - addMetrics(serverMetricsGroup, toRequestName(apiKey, true)); - addMetrics(serverMetricsGroup, toRequestName(apiKey, false)); - } else { - addMetrics(serverMetricsGroup, toRequestName(apiKey, false)); + // For fetch, register separate metric groups for clients and followers. + addMetrics(serverMetricsGroup, toRequestName(apiKey, true, false)); + } + if (apiKey == ApiKeys.LOOKUP) { + addMetrics(serverMetricsGroup, toRequestName(apiKey, false, true)); } - } - if (apiKeys.contains(ApiKeys.LOOKUP)) { - addMetrics(serverMetricsGroup, HISTORICAL_LOOKUP_METRICS_KEY); } this.requestMetricGroup = serverMetricsGroup.addGroup("request"); } @@ -101,14 +97,15 @@ private void addMetrics(MetricGroup parentMetricGroup, String requestName) { requestName, new Metrics(parentMetricGroup.addGroup("request", requestName))); } - private static String toRequestName(ApiKeys apiKeys, boolean isFromFollower) { + private static String toRequestName( + ApiKeys apiKeys, boolean isFromFollower, boolean isHistorical) { switch (apiKeys) { case PRODUCE_LOG: return "produceLog"; case PUT_KV: return "putKv"; case LOOKUP: - return "lookup"; + return isHistorical ? "historicalLookup" : "lookup"; case PREFIX_LOOKUP: return "prefixLookup"; case FETCH_LOG: @@ -121,11 +118,8 @@ private static String toRequestName(ApiKeys apiKeys, boolean isFromFollower) { } public Optional getMetrics( - short apiKey, boolean isFromFollower, boolean isHistoricalLookup) { - if (apiKey == ApiKeys.LOOKUP.id && isHistoricalLookup) { - return Optional.ofNullable(metricsByRequest.get(HISTORICAL_LOOKUP_METRICS_KEY)); - } - String requestName = toRequestName(ApiKeys.forId(apiKey), isFromFollower); + short apiKey, boolean isFromFollower, boolean isHistorical) { + String requestName = toRequestName(ApiKeys.forId(apiKey), isFromFollower, isHistorical); return Optional.ofNullable(metricsByRequest.get(requestName)); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java index 4ae6b09a331..a534a35516d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java @@ -127,6 +127,7 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPbConfigEntries; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPbDatabaseSummary; import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toTablePath; +import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkState; /** @@ -486,6 +487,8 @@ public CompletableFuture listPartitionInfos( } else { partitionRegistrations = metadataManager.listPartitions(tablePath); } + // TODO: Return the actual lake partitions instead of the internal historical partition. + partitionRegistrations.remove(HISTORICAL_PARTITION_VALUE); TableInfo tableInfo = metadataManager.getTable(tablePath); List partitionKeys = tableInfo.getPartitionKeys(); return CompletableFuture.completedFuture( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/TabletManagerBase.java b/fluss-server/src/main/java/org/apache/fluss/server/TabletManagerBase.java index 0c476b10902..389c85918a1 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/TabletManagerBase.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/TabletManagerBase.java @@ -59,6 +59,7 @@ import static org.apache.fluss.utils.FlussPaths.HISTORICAL_LOOKUP_CACHE_DIR_NAME; import static org.apache.fluss.utils.FlussPaths.KV_TABLET_DIR_PREFIX; import static org.apache.fluss.utils.FlussPaths.LOG_TABLET_DIR_PREFIX; +import static org.apache.fluss.utils.FlussPaths.REMOTE_LOG_INDEX_LOCAL_CACHE; import static org.apache.fluss.utils.FlussPaths.isPartitionDir; /** @@ -116,7 +117,8 @@ protected List listTabletsToLoad(File dataDir) { // Get all database directory. File[] dbDirs = FileUtils.listDirectories(dataDir); for (File dbDir : dbDirs) { - if (dbDir.getName().equals(HISTORICAL_LOOKUP_CACHE_DIR_NAME)) { + if (dbDir.getName().equals(HISTORICAL_LOOKUP_CACHE_DIR_NAME) + || dbDir.getName().equals(REMOTE_LOG_INDEX_LOCAL_CACHE)) { continue; } // Get all table path directory. diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index 5b73fbc8aa2..a2f9b4fad97 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -305,8 +305,8 @@ protected void initCoordinatorStandby() throws Exception { dynamicConfigManager.register(lakeCatalogDynamicLoader); dynamicConfigManager.register(remoteDirDynamicLoader); dynamicConfigManager.register(replicaCapacityController); - dynamicConfigManager.register(new HistoricalLookupCacheConfigUpdater(metadataManager)); // Register stateless validators for coordinator-side upfront validation + dynamicConfigManager.register(new HistoricalLookupCacheConfigValidator()); dynamicConfigManager.register(new DiskWriteLimitConfigValidator()); rpcServer.getServerReconfigurables().forEach(dynamicConfigManager::register); dynamicConfigManager.startup(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index e35d03f20b8..04119bb0fd9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -486,7 +486,7 @@ public CompletableFuture createTable(CreateTableRequest req // validate table descriptor before creating table in lake or fluss metadata, // to avoid orphaned lake tables when validation fails - metadataManager.validateTableDescriptor(tablePath, tableDescriptor); + metadataManager.validateTableDescriptor(tableDescriptor); // the distribution and bucket count must be set now //noinspection OptionalGetWithoutIsPresent diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java similarity index 76% rename from fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java rename to fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java index 5b9e5c55e25..c9d763f410a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigUpdater.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/HistoricalLookupCacheConfigValidator.java @@ -24,16 +24,8 @@ import java.time.Duration; -import static org.apache.fluss.utils.Preconditions.checkNotNull; - -/** Validates dynamic historical lookup cache settings and updates the metadata manager. */ -final class HistoricalLookupCacheConfigUpdater implements ServerReconfigurable { - - private final MetadataManager metadataManager; - - HistoricalLookupCacheConfigUpdater(MetadataManager metadataManager) { - this.metadataManager = checkNotNull(metadataManager, "metadataManager must not be null."); - } +/** Validates dynamic historical lookup cache settings. */ +final class HistoricalLookupCacheConfigValidator implements ServerReconfigurable { @Override public void validate(Configuration newConfig) throws ConfigException { @@ -63,9 +55,5 @@ public void validate(Configuration newConfig) throws ConfigException { } @Override - public void reconfigure(Configuration newConfig) { - metadataManager.updateHistoricalLookupCacheMaxRatio( - newConfig.get( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO)); - } + public void reconfigure(Configuration newConfig) {} } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index b197054581e..43d98434252 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -87,7 +87,6 @@ public class MetadataManager { private final ZooKeeperClient zookeeperClient; private final int maxPartitionNum; private final int maxBucketNum; - private volatile double historicalLookupCacheMaxRatio; private final LakeCatalogDynamicLoader lakeCatalogDynamicLoader; public static final Set SENSITIVE_TABLE_OPTIONS = new HashSet<>(); @@ -111,23 +110,15 @@ public MetadataManager( this.zookeeperClient = zookeeperClient; this.maxPartitionNum = conf.get(ConfigOptions.MAX_PARTITION_NUM); this.maxBucketNum = conf.get(ConfigOptions.MAX_BUCKET_NUM); - this.historicalLookupCacheMaxRatio = - conf.get(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); this.lakeCatalogDynamicLoader = lakeCatalogDynamicLoader; } /** Validates the table descriptor. */ - public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDescriptor) { + public void validateTableDescriptor(TableDescriptor tableDescriptor) { TableDescriptorValidation.validateTableDescriptor( tableDescriptor, maxBucketNum, - lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat(), - tablePath, - historicalLookupCacheMaxRatio); - } - - void updateHistoricalLookupCacheMaxRatio(double newMaxRatio) { - historicalLookupCacheMaxRatio = newMaxRatio; + lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat()); } public void createDatabase( @@ -560,7 +551,7 @@ public void alterTableProperties( } // reuse the same validate logic with the createTable() method - validateTableDescriptor(tablePath, newDescriptor); + validateTableDescriptor(newDescriptor); beforeUpdate.accept(tableInfo, newDescriptor); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java index ce6a7e8aa55..303ed580514 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java @@ -28,6 +28,7 @@ import org.apache.fluss.metrics.NoOpCounter; import org.apache.fluss.metrics.ThreadSafeSimpleCounter; import org.apache.fluss.metrics.groups.AbstractMetricGroup; +import org.apache.fluss.metrics.groups.MetricGroup; import org.apache.fluss.metrics.registry.MetricRegistry; import org.apache.fluss.server.kv.rocksdb.RocksDBStatistics; @@ -107,6 +108,16 @@ protected String getGroupName(CharacterFilter filter) { return "table"; } + /** Closes this table metric group and its directly created tablet metric groups. */ + @Override + public void close() { + if (kvMetrics != null) { + kvMetrics.close(); + } + logMetrics.close(); + super.close(); + } + public void incLogMessageIn(long n) { logMetrics.messagesIn.inc(n); serverMetrics.messageIn().inc(n); @@ -197,7 +208,7 @@ public Counter totalHistoricalLookupRequests() { if (kvMetrics == null) { return NoOpCounter.INSTANCE; } else { - return kvMetrics.historicalLookupMetrics.totalLookupRequests(); + return kvMetrics.totalHistoricalLookupRequests; } } @@ -206,7 +217,7 @@ public Counter failedHistoricalLookupRequests() { if (kvMetrics == null) { return NoOpCounter.INSTANCE; } else { - return kvMetrics.historicalLookupMetrics.failedLookupRequests(); + return kvMetrics.failedHistoricalLookupRequests; } } @@ -214,13 +225,11 @@ public Counter failedHistoricalLookupRequests() { * Records a historical lake table point lookup. * * @param lookupTimeNanos time spent on the lake table point lookup, in nanoseconds - * @param lookupFileMaterialization whether the lookup triggered lookup file materialization + * @param lookupFileDownloaded whether the lookup downloaded a lookup file */ - public void recordHistoricalLakeLookup( - long lookupTimeNanos, boolean lookupFileMaterialization) { + public void recordHistoricalLakeLookup(long lookupTimeNanos, boolean lookupFileDownloaded) { if (kvMetrics != null) { - kvMetrics.historicalLookupMetrics.recordLakeLookup( - lookupTimeNanos, lookupFileMaterialization); + kvMetrics.recordHistoricalLakeLookup(lookupTimeNanos, lookupFileDownloaded); } } @@ -563,9 +572,14 @@ protected String getGroupName(CharacterFilter filter) { private static class KvMetricGroup extends TabletMetricGroup { + private static final String LOOKUP_FILE_DOWNLOADED = "lookup_file_downloaded"; + private final Counter totalLookupRequests; private final Counter failedLookupRequests; - private final HistoricalLookupMetricGroup historicalLookupMetrics; + private final Counter totalHistoricalLookupRequests; + private final Counter failedHistoricalLookupRequests; + private final LookupFileDownloadedMetricGroup downloadedHistoricalLookupMetrics; + private final LookupFileDownloadedMetricGroup nonDownloadedHistoricalLookupMetrics; private final Counter totalPutKvRequests; private final Counter failedPutKvRequests; private final Counter totalLimitScanRequests; @@ -581,7 +595,22 @@ public KvMetricGroup(TableMetricGroup tableMetricGroup) { meter(MetricNames.TOTAL_LOOKUP_REQUESTS_RATE, new MeterView(totalLookupRequests)); failedLookupRequests = new ThreadSafeSimpleCounter(); meter(MetricNames.FAILED_LOOKUP_REQUESTS_RATE, new MeterView(failedLookupRequests)); - historicalLookupMetrics = new HistoricalLookupMetricGroup(registry, this); + // for historical lookup request + MetricGroup historicalLookupMetrics = addGroup("historical"); + totalHistoricalLookupRequests = new ThreadSafeSimpleCounter(); + historicalLookupMetrics.meter( + MetricNames.TOTAL_LOOKUP_REQUESTS_RATE, + new MeterView(totalHistoricalLookupRequests)); + failedHistoricalLookupRequests = new ThreadSafeSimpleCounter(); + historicalLookupMetrics.meter( + MetricNames.FAILED_LOOKUP_REQUESTS_RATE, + new MeterView(failedHistoricalLookupRequests)); + // Separate groups expose the same metric names with different downloaded-file labels + // without adding the label key to the logical metric scope. + downloadedHistoricalLookupMetrics = + new LookupFileDownloadedMetricGroup(registry, this, true); + nonDownloadedHistoricalLookupMetrics = + new LookupFileDownloadedMetricGroup(registry, this, false); // for put kv request totalPutKvRequests = new ThreadSafeSimpleCounter(); meter(MetricNames.TOTAL_PUT_KV_REQUESTS_RATE, new MeterView(totalPutKvRequests)); @@ -609,76 +638,39 @@ public KvMetricGroup(TableMetricGroup tableMetricGroup) { } @Override - protected String getGroupName(CharacterFilter filter) { - return super.getGroupName(filter); - } - } - - /** - * Collects historical lookup metrics in the KV table scope and separates lake lookup metrics by - * whether the lookup materialized local files. - */ - private static class HistoricalLookupMetricGroup extends AbstractMetricGroup { - - private final Counter totalLookupRequests; - private final Counter failedLookupRequests; - private final LookupFileMaterializationMetricGroup materializedLookupMetrics; - private final LookupFileMaterializationMetricGroup nonMaterializedLookupMetrics; - - private HistoricalLookupMetricGroup(MetricRegistry registry, KvMetricGroup parent) { - super(registry, parent.getScopeComponents(), parent); - - totalLookupRequests = new ThreadSafeSimpleCounter(); - meter( - MetricNames.TOTAL_HISTORICAL_LOOKUP_REQUESTS_RATE, - new MeterView(totalLookupRequests)); - failedLookupRequests = new ThreadSafeSimpleCounter(); - meter( - MetricNames.FAILED_HISTORICAL_LOOKUP_REQUESTS_RATE, - new MeterView(failedLookupRequests)); - materializedLookupMetrics = - new LookupFileMaterializationMetricGroup(registry, this, true); - nonMaterializedLookupMetrics = - new LookupFileMaterializationMetricGroup(registry, this, false); - } - - final Counter totalLookupRequests() { - return totalLookupRequests; - } - - final Counter failedLookupRequests() { - return failedLookupRequests; + public void close() { + downloadedHistoricalLookupMetrics.close(); + nonDownloadedHistoricalLookupMetrics.close(); + super.close(); + } + + private void recordHistoricalLakeLookup( + long lookupTimeNanos, boolean lookupFileDownloaded) { + LookupFileDownloadedMetricGroup metricGroup = + lookupFileDownloaded + ? downloadedHistoricalLookupMetrics + : nonDownloadedHistoricalLookupMetrics; + metricGroup.recordLookup(lookupTimeNanos); } @Override protected String getGroupName(CharacterFilter filter) { - return ""; - } - - private void recordLakeLookup(long lookupTimeNanos, boolean lookupFileMaterialization) { - LookupFileMaterializationMetricGroup metricGroup = - lookupFileMaterialization - ? materializedLookupMetrics - : nonMaterializedLookupMetrics; - metricGroup.recordLookup(lookupTimeNanos); + return super.getGroupName(filter); } } - private static class LookupFileMaterializationMetricGroup extends AbstractMetricGroup { + private static final class LookupFileDownloadedMetricGroup extends AbstractMetricGroup { - private static final int WINDOW_SIZE = 1024; + private static final int WINDOW_SIZE = 64; - private final boolean lookupFileMaterialization; + private final boolean lookupFileDownloaded; private final Counter lakeLookups; private final Histogram lakeLookupTimeMs; - private LookupFileMaterializationMetricGroup( - MetricRegistry registry, - HistoricalLookupMetricGroup parent, - boolean lookupFileMaterialization) { - super(registry, parent.getScopeComponents(), parent); - this.lookupFileMaterialization = lookupFileMaterialization; - + private LookupFileDownloadedMetricGroup( + MetricRegistry registry, KvMetricGroup parent, boolean lookupFileDownloaded) { + super(registry, makeScope(parent, "historical"), parent); + this.lookupFileDownloaded = lookupFileDownloaded; lakeLookups = new ThreadSafeSimpleCounter(); meter(MetricNames.LAKE_LOOKUPS_RATE, new MeterView(lakeLookups)); lakeLookupTimeMs = @@ -694,12 +686,13 @@ private void recordLookup(long lookupTimeNanos) { @Override protected void putVariables(Map variables) { - variables.put("lookup_file_materialization", String.valueOf(lookupFileMaterialization)); + variables.put( + KvMetricGroup.LOOKUP_FILE_DOWNLOADED, String.valueOf(lookupFileDownloaded)); } @Override protected String getGroupName(CharacterFilter filter) { - return ""; + return "historical"; } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java index d176ecac64d..22215bc6de9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java @@ -23,7 +23,6 @@ import org.apache.fluss.metrics.CharacterFilter; import org.apache.fluss.metrics.Counter; import org.apache.fluss.metrics.DescriptiveStatisticsHistogram; -import org.apache.fluss.metrics.Gauge; import org.apache.fluss.metrics.Histogram; import org.apache.fluss.metrics.MeterView; import org.apache.fluss.metrics.MetricNames; @@ -240,20 +239,6 @@ public Counter failedIsrUpdates() { return failedIsrUpdates; } - /** - * Registers the number of in-flight historical partition requests for an operation. - * - * @param operation historical partition operation - * @param inflightRequests gauge for accepted requests that have not completed - */ - public void registerHistoricalPartitionInflightRequests( - String operation, Gauge inflightRequests) { - HistoricalPartitionOperationMetricGroup operationMetricGroup = - new HistoricalPartitionOperationMetricGroup(registry, this, operation); - operationMetricGroup.gauge( - MetricNames.HISTORICAL_PARTITION_INFLIGHT_REQUESTS, inflightRequests); - } - // ------------------------------------------------------------------------ // table buckets groups // ------------------------------------------------------------------------ @@ -282,25 +267,4 @@ public void removeTableBucketMetricGroup(TablePath tablePath, TableBucket bucket } } } - - private static class HistoricalPartitionOperationMetricGroup extends AbstractMetricGroup { - - private final String operation; - - private HistoricalPartitionOperationMetricGroup( - MetricRegistry registry, TabletServerMetricGroup parent, String operation) { - super(registry, parent.getScopeComponents(), parent); - this.operation = operation; - } - - @Override - protected void putVariables(Map variables) { - variables.put("operation", operation); - } - - @Override - protected String getGroupName(CharacterFilter filter) { - return ""; - } - } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index 39db95cbe8d..9a993eda045 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -21,6 +21,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.DiskWriteLockedException; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; @@ -41,7 +42,7 @@ import org.apache.fluss.rpc.entity.LookupResultForBucket; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.server.entity.LookupDataForBucket; -import org.apache.fluss.server.replica.HistoricalLookupCacheBudgetManager.Reservation; +import org.apache.fluss.server.storage.LocalDiskManager; import org.apache.fluss.utils.ExecutorUtils; import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; @@ -61,7 +62,9 @@ import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -75,6 +78,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import static org.apache.fluss.server.utils.LakeStorageUtils.extractLakeProperties; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -95,11 +99,17 @@ * the current request. Active lookups can finish on the old lookuper, which is closed after its * last lookup releases it. * + *

Up to ten table lookupers are cached. Each lookuper receives one tenth of the server-level + * disk budget, and Caffeine evicts lookupers when the table limit is exceeded. + * + *

Historical lookup cache I/O participates in TabletServer disk write protection. A lookup is + * rejected when the data disk is write-locked, and cached lookupers are invalidated so their local + * files are released. Lookups can create fresh cache files again after the disk recovers. + * *

A lookuper is closed when replaced, explicitly invalidated by a replica lifecycle event, - * evicted to admit another table within the configured disk budget, the manager shuts down, or - * after the configured idle expiration. Caffeine expiration is scheduled on the shared TabletServer - * scheduler, allowing idle resources to be released even if no subsequent lookup accesses the - * cache. + * evicted when the table limit is exceeded, the manager shuts down, or after the configured idle + * expiration. Caffeine expiration is scheduled on the shared TabletServer scheduler, allowing idle + * resources to be released even if no subsequent lookup accesses the cache. */ class HistoricalLakeLookupManager implements AutoCloseable { @@ -107,15 +117,21 @@ class HistoricalLakeLookupManager implements AutoCloseable { private static final String LOOKUPER_CACHE_EXPIRATION_TASK_NAME = "historical-lookuper-cache-expiration"; + private static final String LOOKUP_CACHE_DISK_SIZE_TASK_NAME = + "historical-lookup-cache-disk-size"; + private static final Duration LOOKUP_CACHE_DISK_SIZE_CHECK_INTERVAL = Duration.ofMinutes(3); private static final Duration HISTORICAL_PARTITION_THREAD_KEEP_ALIVE = Duration.ofMinutes(10); private static final Duration HISTORICAL_PARTITION_EXECUTOR_SHUTDOWN_TIMEOUT = Duration.ofSeconds(10); private static final String HISTORICAL_PARTITION_THREAD_NAME_PREFIX = "historical-partition-io"; + // TODO: Share one Paimon IOManager disk budget across all table lookupers and evict cached + // entries by data file instead of reserving fixed per-table capacity. See + // https://github.com/apache/fluss/issues/3955. + private static final int MAX_CACHED_TABLES = 10; private volatile Configuration conf; private volatile long lakeConfigVersion; private final @Nullable PluginManager pluginManager; - private final HistoricalLookupCacheBudgetManager budgetManager; private final Counter capacityEvictions; private final int maxQueuedHistoricalRequests; private final Semaphore lookupPermits; @@ -124,27 +140,35 @@ class HistoricalLakeLookupManager implements AutoCloseable { private final Cache lakeTableLookupers; private final ExecutorService historicalPartitionExecutor; private final File historicalLookupCacheRootDir; - private final long lookupVolumeBytes; + private final long dataDirVolumeBytes; + private final Runnable diskWriteGuard; + + private volatile long lookupCacheMaxDiskBytesPerTable; + private volatile long lookupCacheDiskSize; @GuardedBy("this") private boolean historicalLookupCacheRootDirCreated; private volatile boolean started; + /** Creates a historical lake lookup manager. */ HistoricalLakeLookupManager( Configuration conf, @Nullable PluginManager pluginManager, + LocalDiskManager localDiskManager, File dataDir, - long lookupVolumeBytes, + long dataDirVolumeBytes, Scheduler scheduler) { this( conf, pluginManager, null, dataDir, - lookupVolumeBytes, + dataDirVolumeBytes, Ticker.systemTicker(), - createCacheScheduler(scheduler)); + createCacheScheduler(scheduler), + checkNotNull(localDiskManager, "localDiskManager must not be null.") + ::ensureWritable); } @VisibleForTesting @@ -153,22 +177,23 @@ class HistoricalLakeLookupManager implements AutoCloseable { @Nullable PluginManager pluginManager, @Nullable ExecutorService historicalPartitionExecutor, File dataDir, - long lookupVolumeBytes, + long dataDirVolumeBytes, Ticker ticker, - com.github.benmanes.caffeine.cache.Scheduler cacheScheduler) { + com.github.benmanes.caffeine.cache.Scheduler cacheScheduler, + Runnable diskWriteGuard) { this.conf = checkNotNull(conf, "conf must not be null."); this.pluginManager = pluginManager; this.historicalLookupCacheRootDir = FlussPaths.historicalLookupRootDir( checkNotNull(dataDir, "dataDir must not be null.")); - checkArgument(lookupVolumeBytes > 0, "lookupVolumeBytes must be greater than 0."); - this.lookupVolumeBytes = lookupVolumeBytes; - this.budgetManager = - new HistoricalLookupCacheBudgetManager( - capacityBytes( - conf.get( - ConfigOptions - .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO))); + checkArgument(dataDirVolumeBytes > 0, "dataDirVolumeBytes must be greater than 0."); + this.dataDirVolumeBytes = dataDirVolumeBytes; + this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); + this.lookupCacheMaxDiskBytesPerTable = + cacheBytesPerTable( + conf.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO)); this.capacityEvictions = new ThreadSafeSimpleCounter(); this.maxQueuedHistoricalRequests = conf.get(ConfigOptions.NETTY_SERVER_MAX_QUEUED_HISTORICAL_REQUESTS); @@ -188,6 +213,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { : historicalPartitionExecutor; this.lakeTableLookupers = Caffeine.newBuilder() + .maximumSize(MAX_CACHED_TABLES) .expireAfterAccess( conf.get( ConfigOptions @@ -198,8 +224,16 @@ class HistoricalLakeLookupManager implements AutoCloseable { .removalListener( (Long ignored, CachedLakeTableLookuper cachedLookuper, - RemovalCause ignoredCause) -> { + RemovalCause cause) -> { if (cachedLookuper != null) { + if (cause == RemovalCause.SIZE) { + capacityEvictions.inc(); + LOG.info( + "Evicted historical lookup cache for table {} (table ID {}) because the cache retains at most {} tables.", + cachedLookuper.tablePath, + cachedLookuper.tableId, + MAX_CACHED_TABLES); + } onLookuperRemoved(cachedLookuper); } }) @@ -241,14 +275,34 @@ synchronized void startup() { started = true; } + /** Starts periodic sampling of the historical lookup cache footprint. */ + void startLookupCacheDiskSizeMonitor(Scheduler scheduler) { + checkNotNull(scheduler, "scheduler must not be null."); + scheduler.schedule( + LOOKUP_CACHE_DISK_SIZE_TASK_NAME, + this::updateLookupCacheDiskSize, + 0L, + LOOKUP_CACHE_DISK_SIZE_CHECK_INTERVAL.toMillis()); + } + + /** Looks up a batch of keys from one historical lake partition. */ CompletableFuture lookup( LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo, - TableConfig tableConfig, LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { checkState(started, "Historical lake lookup manager has not been started."); TableBucket tableBucket = lookupData.tableBucket(); + try { + ensureDiskWritable(); + } catch (DiskWriteLockedException e) { + return CompletableFuture.completedFuture( + new LookupResultForBucket( + tableBucket, + null, + lookupData.originalPartitionName(), + ApiError.fromThrowable(e))); + } if (!lookupPermits.tryAcquire()) { return CompletableFuture.completedFuture( new LookupResultForBucket( @@ -269,7 +323,6 @@ CompletableFuture lookup( lookupData, tableInfo, schemaInfo, - tableConfig, checkNotNull( lookupMetricRecorder, "lookupMetricRecorder must not be null.")); @@ -300,17 +353,12 @@ private CompletableFuture submitLookup( LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo, - TableConfig tableConfig, LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { CompletableFuture future = CompletableFuture.supplyAsync( () -> lookupInternal( - lookupData, - tableInfo, - schemaInfo, - tableConfig, - lookupMetricRecorder), + lookupData, tableInfo, schemaInfo, lookupMetricRecorder), historicalPartitionExecutor); pendingLookups.add(future); return future; @@ -329,39 +377,44 @@ private ExecutorService createHistoricalPartitionExecutor(int maxThreadPoolSize) return executor; } + /** Invalidates the cached lake lookuper for the given table. */ void invalidateTableLookuper(long tableId) { lakeTableLookupers.invalidate(tableId); } + /** Returns the number of table lookupers currently cached. */ int cachedTableCount() { return lakeTableLookupers.asMap().size(); } + /** Returns the counter for table lookuper evictions caused by the cached table limit. */ Counter capacityEvictions() { return capacityEvictions; } + /** Returns the number of accepted historical lookup requests that have not completed. */ int numInflightRequests() { return maxQueuedHistoricalRequests - lookupPermits.availablePermits(); } + /** Applies dynamic historical lookup configuration changes. */ void reconfigure(Configuration newConf) { checkNotNull(newConf, "newConf must not be null."); boolean lakeConfigChanged; + boolean cacheLimitChanged; boolean expirationChanged; Duration newExpiration = newConf.get( ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS); synchronized (this) { - long newMaxBytes = - capacityBytes( + long newMaxBytesPerTable = + cacheBytesPerTable( newConf.get( ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO)); - if (newMaxBytes != budgetManager.maxBytes()) { - budgetManager.updateGlobalLimit(newMaxBytes); - } + cacheLimitChanged = newMaxBytesPerTable != lookupCacheMaxDiskBytesPerTable; + lookupCacheMaxDiskBytesPerTable = newMaxBytesPerTable; lakeConfigChanged = hasLakeConfigChanged(conf, newConf); expirationChanged = @@ -383,11 +436,12 @@ void reconfigure(Configuration newConf) { .get() .setExpiresAfter(newExpiration.toMillis(), TimeUnit.MILLISECONDS); } - if (lakeConfigChanged) { + if (lakeConfigChanged || cacheLimitChanged) { // Do not invalidate while holding this monitor: lookuper creation holds a cache key - // lock before preparing the lookup directory under the same monitor. Inactive - // lookupers close now, while active lookupers close after their last lookup releases - // them. + // lock before preparing the lookup directory under the same monitor. Invalidation + // closes inactive lookupers immediately and active lookupers after their last lookup + // releases them. After a cache limit change, the next lookup creates a Paimon lookuper + // with the updated per-table limit. lakeTableLookupers.invalidateAll(); lakeTableLookupers.cleanUp(); } @@ -397,7 +451,6 @@ private LookupResultForBucket lookupInternal( LookupDataForBucket lookupData, TableInfo tableInfo, SchemaInfo schemaInfo, - TableConfig tableConfig, LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { TableBucket tableBucket = lookupData.tableBucket(); CachedLakeTableLookuper cachedLookuper = null; @@ -406,15 +459,52 @@ private LookupResultForBucket lookupInternal( createLookupContext(lookupData, tableInfo, schemaInfo, lookupMetricRecorder); long currentLakeConfigVersion = lakeConfigVersion; Configuration currentConf = conf; - long cacheSizeBytes = - capacityBytes(tableConfig.getHistoricalPartitionLookupCacheMaxDiskRatio()); + long cacheSizeBytes = lookupCacheMaxDiskBytesPerTable; cachedLookuper = - acquireCachedLookuper( - context, - tableConfig, - currentConf, - currentLakeConfigVersion, - cacheSizeBytes); + lakeTableLookupers + .asMap() + .compute( + context.tableId, + (ignored, currentLookuper) -> { + CachedLakeTableLookuper selectedLookuper = currentLookuper; + // Create the lookuper lazily, and recreate it after schema, + // lake configuration, or server cache size changes so it + // reloads lake table/query state and uses the current + // settings. + if (selectedLookuper == null + || selectedLookuper.schemaId != context.schemaId + || selectedLookuper.lakeConfigVersion + != currentLakeConfigVersion + || selectedLookuper.cacheSizeBytes + != cacheSizeBytes) { + File tableLookupDir = + FlussPaths.historicalLookupTableDir( + getOrCreateHistoricalLookupCacheRootDir(), + context.tablePath, + context.tableId); + LakeTableLookuper lookuper = + createLakeTableLookuper( + context.tablePath, + tableLookupDir.getAbsolutePath(), + tableInfo.getTableConfig(), + cacheSizeBytes, + currentConf); + selectedLookuper = + new CachedLakeTableLookuper( + context.tableId, + context.tablePath, + context.schemaId, + currentLakeConfigVersion, + cacheSizeBytes, + tableLookupDir, + lookuper); + } + // Pin the lookuper before leaving the atomic cache update. + // Eviction or invalidation can then defer closing it until + // this lookup releases it. + selectedLookuper.acquire(); + return selectedLookuper; + }); List values = new ArrayList<>(lookupData.keys().size()); for (byte[] key : lookupData.keys()) { values.add(cachedLookuper.lookuper.lookup(key, context.lookupContext)); @@ -434,245 +524,20 @@ private LookupResultForBucket lookupInternal( } } - private CachedLakeTableLookuper acquireCachedLookuper( - LookupContext context, - TableConfig tableConfig, - Configuration clusterConf, - long currentLakeConfigVersion, - long cacheSizeBytes) { - CachedLakeTableLookuper cachedLookuper = - tryAcquireCachedLookuper( - context, - tableConfig, - clusterConf, - currentLakeConfigVersion, - cacheSizeBytes); - if (cachedLookuper != null) { - return cachedLookuper; - } - - int maxEvictions = lakeTableLookupers.asMap().size(); - for (int evictions = 0; evictions < maxEvictions; evictions++) { - // Evict only after compute releases the target table's cache lock. Updating a - // different table mapping from inside compute can deadlock with a concurrent - // replacement performing the inverse update. - if (!evictLeastRecentlyUsed(context.tableId)) { - break; - } - cachedLookuper = - tryAcquireCachedLookuper( - context, - tableConfig, - clusterConf, - currentLakeConfigVersion, - cacheSizeBytes); - if (cachedLookuper != null) { - return cachedLookuper; - } - } - throw capacityThrottledException(context, cacheSizeBytes); - } - - /** - * Makes one atomic attempt to acquire a matching cached lookuper without evicting other tables. - * - * @return the acquired lookuper, or {@code null} if its capacity cannot be reserved - */ - private @Nullable CachedLakeTableLookuper tryAcquireCachedLookuper( - LookupContext context, - TableConfig tableConfig, - Configuration clusterConf, - long currentLakeConfigVersion, - long cacheSizeBytes) { - CachedLakeTableLookuper cachedLookuper = - lakeTableLookupers - .asMap() - .compute( - context.tableId, - (ignored, currentLookuper) -> { - CachedLakeTableLookuper selectedLookuper = currentLookuper; - // Create the lookuper lazily, and recreate it after schema, - // lake configuration, or effective cache size changes so it - // reloads lake table/query state and uses the current settings. - if (!matchesLookupConfiguration( - selectedLookuper, - context, - currentLakeConfigVersion, - cacheSizeBytes)) { - selectedLookuper = - tryCreateCachedLookuper( - context, - tableConfig, - clusterConf, - currentLakeConfigVersion, - cacheSizeBytes, - currentLookuper); - if (selectedLookuper == null) { - // Preserve the current mapping and leave compute before - // attempting to evict another table. - return currentLookuper; - } - } - // Pin the lookuper before leaving the atomic cache update. - // Eviction or invalidation can then defer closing it until this - // lookup releases it. - selectedLookuper.acquire(); - return selectedLookuper; - }); - // Replacement admission may preserve the old mapping without acquiring it. Return null for - // that stale mapping so the caller can evict another table and retry. - return matchesLookupConfiguration( - cachedLookuper, context, currentLakeConfigVersion, cacheSizeBytes) - ? cachedLookuper - : null; - } - - private static boolean matchesLookupConfiguration( - @Nullable CachedLakeTableLookuper cachedLookuper, - LookupContext context, - long currentLakeConfigVersion, - long cacheSizeBytes) { - return cachedLookuper != null - && cachedLookuper.schemaId == context.schemaId - && cachedLookuper.lakeConfigVersion == currentLakeConfigVersion - && cachedLookuper.cacheSizeBytes == cacheSizeBytes; - } - - /** - * Creates a lookuper after atomically reserving its configured cache capacity. - * - * @return the new cached lookuper, or {@code null} if its capacity cannot be reserved - */ - private @Nullable CachedLakeTableLookuper tryCreateCachedLookuper( - LookupContext context, - TableConfig tableConfig, - Configuration clusterConf, - long currentLakeConfigVersion, - long cacheSizeBytes, - @Nullable CachedLakeTableLookuper currentLookuper) { - if (currentLookuper == null) { - // A cache miss must obtain capacity before creating any local lookup resources. - Reservation reservation = budgetManager.tryReserve(context.tableId, cacheSizeBytes); - if (reservation == null) { - return null; - } - try { - File tableLookupDir = - FlussPaths.historicalLookupTableDir( - getOrCreateHistoricalLookupCacheRootDir(), - context.tablePath, - context.tableId); - LakeTableLookuper lookuper = - createLakeTableLookuper( - context.tablePath, - tableLookupDir.getAbsolutePath(), - tableConfig, - cacheSizeBytes, - clusterConf); - return new CachedLakeTableLookuper( - context.tableId, - context.tablePath, - context.schemaId, - currentLakeConfigVersion, - cacheSizeBytes, - tableLookupDir, - reservation, - lookuper); - } catch (Throwable throwable) { - budgetManager.release(reservation); - throw throwable; - } - } - - File tableLookupDir = - FlussPaths.historicalLookupTableDir( - getOrCreateHistoricalLookupCacheRootDir(), - context.tablePath, - context.tableId); - // Build the replacement first so a creation failure leaves the current lookuper and its - // reservation unchanged in the cache. - LakeTableLookuper lookuper = - createLakeTableLookuper( - context.tablePath, - tableLookupDir.getAbsolutePath(), - tableConfig, - cacheSizeBytes, - clusterConf); - // Replace the reservation atomically: the old and replacement cache sizes never count - // against the global budget at the same time. - Reservation reservation = - budgetManager.tryReplace(currentLookuper.reservation, cacheSizeBytes); - if (reservation == null) { - // The candidate was never published, while the current lookuper remains usable. - closeLookuper(lookuper, tableLookupDir); - return null; - } - return new CachedLakeTableLookuper( - context.tableId, - context.tablePath, - context.schemaId, - currentLakeConfigVersion, - cacheSizeBytes, - tableLookupDir, - reservation, - lookuper); - } - - /** - * Evicts one eligible cached lookuper using best-effort LRU order. - * - *

Candidates use Caffeine's expire-after-access order. A candidate accessed after the - * snapshot is taken may therefore still be evicted. - */ - private boolean evictLeastRecentlyUsed(long excludedTableId) { - Map candidates = - lakeTableLookupers - .policy() - .expireAfterAccess() - .get() - .oldest(lakeTableLookupers.asMap().size()); - for (CachedLakeTableLookuper candidate : candidates.values()) { - if (candidate.tableId == excludedTableId) { - continue; - } - // The snapshot may be stale after expiration or replacement. Compare-and-remove - // prevents this eviction from removing a newer lookuper for the same table. - boolean removed = lakeTableLookupers.asMap().remove(candidate.tableId, candidate); - if (!removed) { - continue; - } - - // The direct Caffeine executor normally invokes the listener inline. Repeat the - // transition explicitly so admission does not depend on listener scheduling. - onLookuperRemoved(candidate); - capacityEvictions.inc(); - LOG.info( - "Evicted historical lookup cache for table {} (table ID {}, cache size {} bytes, reserved {} of {} bytes).", - candidate.tablePath, - candidate.tableId, - candidate.cacheSizeBytes, - budgetManager.reservedBytes(), - budgetManager.maxBytes()); - return true; + private void ensureDiskWritable() { + try { + diskWriteGuard.run(); + } catch (DiskWriteLockedException e) { + // Release local cache files after the disk is write-locked. Idle lookupers close + // immediately, while active lookupers close after their final lookup releases them. + // Run pending cache maintenance now so removal callbacks are processed promptly. + lakeTableLookupers.invalidateAll(); + lakeTableLookupers.cleanUp(); + throw e; } - return false; - } - - private HistoricalPartitionThrottledException capacityThrottledException( - LookupContext context, long cacheSizeBytes) { - return new HistoricalPartitionThrottledException( - String.format( - "Historical lookup cache capacity is unavailable for table %s (table ID %s): requested %s bytes, reserved %s of %s bytes across %s cached tables.", - context.tablePath, - context.tableId, - cacheSizeBytes, - budgetManager.reservedBytes(), - budgetManager.maxBytes(), - cachedTableCount())); } private void onLookuperRemoved(CachedLakeTableLookuper cachedLookuper) { - budgetManager.release(cachedLookuper.reservation); cachedLookuper.invalidate(); } @@ -768,10 +633,43 @@ private synchronized File getOrCreateHistoricalLookupCacheRootDir() { return historicalLookupCacheRootDir; } - private long capacityBytes(double ratio) { + private long cacheBytesPerTable(double ratio) { checkArgument(ratio > 0.0 && ratio <= 1.0, "ratio must be within (0.0, 1.0]."); - long bytes = (long) Math.ceil(lookupVolumeBytes * ratio); - return Math.min(lookupVolumeBytes, bytes); + long totalCacheBytes = + Math.min(dataDirVolumeBytes, (long) Math.ceil(dataDirVolumeBytes * ratio)); + return Math.max(1L, totalCacheBytes / MAX_CACHED_TABLES); + } + + /** Returns the most recently sampled historical lookup cache footprint, in bytes. */ + long lookupCacheDiskSize() { + return lookupCacheDiskSize; + } + + private void updateLookupCacheDiskSize() { + if (!historicalLookupCacheRootDir.exists()) { + lookupCacheDiskSize = 0L; + return; + } + try (Stream paths = Files.walk(historicalLookupCacheRootDir.toPath())) { + lookupCacheDiskSize = + paths.filter(Files::isRegularFile) + .mapToLong(HistoricalLakeLookupManager::fileSize) + .sum(); + } catch (IOException | UncheckedIOException e) { + LOG.warn( + "Failed to calculate historical lookup cache usage under {}. Keeping the last sampled value of {} bytes.", + historicalLookupCacheRootDir, + lookupCacheDiskSize, + e); + } + } + + private static long fileSize(Path path) { + try { + return Files.size(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } private static void closeLookuper(CachedLakeTableLookuper cachedLookuper) { @@ -824,7 +722,6 @@ private static final class CachedLakeTableLookuper { private final long lakeConfigVersion; private final long cacheSizeBytes; private final File tableLookupDir; - private final Reservation reservation; private final LakeTableLookuper lookuper; private int activeLookups; private boolean invalidated; @@ -837,7 +734,6 @@ private CachedLakeTableLookuper( long lakeConfigVersion, long cacheSizeBytes, File tableLookupDir, - Reservation reservation, LakeTableLookuper lookuper) { this.tableId = tableId; this.tablePath = tablePath; @@ -845,7 +741,6 @@ private CachedLakeTableLookuper( this.lakeConfigVersion = lakeConfigVersion; this.cacheSizeBytes = cacheSizeBytes; this.tableLookupDir = tableLookupDir; - this.reservation = reservation; this.lookuper = lookuper; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java deleted file mode 100644 index 6bf27301503..00000000000 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManager.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.server.replica; - -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.ThreadSafe; - -import java.util.HashMap; -import java.util.Map; - -import static org.apache.fluss.utils.Preconditions.checkArgument; - -/** - * Tracks the configured disk capacity reserved by historical lookupers. - * - *

This manager accounts for configured cache capacity, not the bytes currently present on disk. - * A reservation belongs to a table's current or creating lookuper. Once that lookuper is removed - * from the cache mapping, its reservation is released immediately even if active requests keep the - * retired lookuper alive for a short time. - * - *

All mutable state is protected by this instance's monitor. The following invariants therefore - * hold after every operation: - * - *

    - *
  • Each table ID has at most one current reservation. - *
  • {@code reservedBytes} is the sum of the reservations in {@code reservationsByTableId}. - *
  • {@code reservedBytes >= 0}. After a dynamic limit reduction, existing reservations may - * exceed {@code maxBytes}; the caller may evict cached lookupers before retrying the next - * admission. - *
- */ -@ThreadSafe -final class HistoricalLookupCacheBudgetManager { - - @GuardedBy("this") - private long maxBytes; - - // Contains only reservations that currently count against the budget. Retired lookupers are - // deliberately absent even when they are still serving an already acquired lookup. - @GuardedBy("this") - private final Map reservationsByTableId = new HashMap<>(); - - @GuardedBy("this") - private long reservedBytes; - - /** Creates a budget manager with the given positive capacity limit. */ - HistoricalLookupCacheBudgetManager(long maxBytes) { - checkArgument(maxBytes > 0, "maxBytes must be greater than 0."); - this.maxBytes = maxBytes; - } - - /** - * Tries to reserve capacity for a new table lookuper. - * - *

The check and reservation insertion are one atomic operation. A request fails when the - * table already owns a reservation or the remaining budget is too small. Subtraction is used - * for the capacity check to avoid overflowing {@code reservedBytes + bytes}. - * - * @return the new reservation, or {@code null} if the table already has a reservation or there - * is insufficient remaining capacity - */ - synchronized @Nullable Reservation tryReserve(long tableId, long bytes) { - checkArgument(bytes > 0, "bytes must be greater than 0."); - if (reservationsByTableId.containsKey(tableId) || bytes > maxBytes - reservedBytes) { - return null; - } - - Reservation reservation = new Reservation(tableId, bytes); - reservationsByTableId.put(tableId, reservation); - reservedBytes = Math.addExact(reservedBytes, bytes); - return reservation; - } - - /** - * Atomically replaces a table's current reservation for a replacement lookuper. - * - *

The supplied reservation must still be the table's current reservation. A stale object can - * be observed after expiration, LRU eviction, or another replacement and must not overwrite the - * newer reservation. If the identity or capacity check fails, the old reservation remains - * unchanged. - * - * @return the replacement reservation, or {@code null} if the supplied reservation is no longer - * current or the replacement does not fit within the capacity limit - */ - synchronized @Nullable Reservation tryReplace(Reservation oldReservation, long newBytes) { - checkArgument(newBytes > 0, "newBytes must be greater than 0."); - Reservation currentReservation = reservationsByTableId.get(oldReservation.getTableId()); - if (currentReservation != oldReservation) { - return null; - } - - long reservedWithoutOld = Math.subtractExact(reservedBytes, oldReservation.getBytes()); - if (newBytes > maxBytes - reservedWithoutOld) { - return null; - } - - Reservation newReservation = new Reservation(oldReservation.getTableId(), newBytes); - reservationsByTableId.put(oldReservation.getTableId(), newReservation); - reservedBytes = Math.addExact(reservedWithoutOld, newBytes); - return newReservation; - } - - /** Updates the limit used by subsequent attempts without modifying existing reservations. */ - synchronized void updateGlobalLimit(long newMaxBytes) { - checkArgument(newMaxBytes > 0, "newMaxBytes must be greater than 0."); - maxBytes = newMaxBytes; - } - - /** - * Releases a reservation if it is still the table's current reservation. - * - *

Removal listeners, creation cleanup, and delayed retired-lookuper callbacks can all - * attempt a release. Comparing the reservation object identity makes those calls idempotent and - * prevents an old lookuper from releasing its replacement's capacity. - */ - synchronized void release(Reservation reservation) { - Reservation currentReservation = reservationsByTableId.get(reservation.getTableId()); - if (currentReservation != reservation) { - return; - } - - reservationsByTableId.remove(reservation.getTableId()); - reservedBytes = Math.subtractExact(reservedBytes, reservation.getBytes()); - } - - /** Returns the capacity currently reserved by current and creating lookupers. */ - synchronized long reservedBytes() { - return reservedBytes; - } - - /** Returns the configured capacity limit. */ - synchronized long maxBytes() { - return maxBytes; - } - - /** - * A capacity reservation for one cached lookuper. - * - *

Each reserve or replace operation creates an immutable instance. The manager compares - * object identity so delayed callbacks carrying an older instance are harmless. - */ - static final class Reservation { - private final long tableId; - private final long bytes; - - private Reservation(long tableId, long bytes) { - this.tableId = tableId; - this.bytes = bytes; - } - - long getTableId() { - return tableId; - } - - long getBytes() { - return bytes; - } - } -} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index 29d30546eea..5192cdeb0b6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -186,8 +186,7 @@ public final class Replica { private final SchemaGetter schemaGetter; private final TableInfo tableInfo; - // Metadata updates replace this snapshot after applying configuration-specific side effects. - private volatile TableConfig tableConfig; + private final TableConfig tableConfig; // logFormat and arrowCompressionInfo are used in hot-path, so cache them here. private final LogFormat logFormat; private final ArrowCompressionInfo arrowCompressionInfo; @@ -347,14 +346,6 @@ public TableInfo getTableInfo() { return tableInfo; } - TableConfig getTableConfig() { - return tableConfig; - } - - void updateTableConfig(TableConfig tableConfig) { - this.tableConfig = checkNotNull(tableConfig, "tableConfig"); - } - public @Nullable Integer getLeaderId() { return leaderReplicaIdOpt.get(); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index be113765143..ff3d1952ad0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -361,24 +361,23 @@ public ReplicaManager( this.minInSyncReplicas = conf.get(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER); this.scannerManager = checkNotNull(scannerManager, "scannerManager"); // Historical lookup cache capacity currently uses only the first data volume. - File historicalLookupDataDir = localDiskManager.dataDirs().get(0); - long historicalLookupVolumeBytes = - Files.getFileStore(historicalLookupDataDir.toPath()).getTotalSpace(); + File dataDir = localDiskManager.dataDirs().get(0); + long dataDirVolumeBytes = Files.getFileStore(dataDir.toPath()).getTotalSpace(); this.historicalLakeLookupManager = new HistoricalLakeLookupManager( conf, pluginManager, - historicalLookupDataDir, - historicalLookupVolumeBytes, + localDiskManager, + dataDir, + dataDirVolumeBytes, scheduler); - serverMetricGroup.registerHistoricalPartitionInflightRequests( - "lookup", historicalLakeLookupManager::numInflightRequests); registerMetrics(); } public void startup() { historicalLakeLookupManager.startup(); + historicalLakeLookupManager.startLookupCacheDiskSizeMonitor(scheduler); // start up ISR expiration thread. // A follower can log behind leader for up tp configOptions#LOG_REPLICA_MAX_LAG_TIME x 1.5 @@ -446,6 +445,21 @@ public void reconfigure(Configuration newConfig) { } private void registerMetrics() { + // for historical lookup metrics + MetricGroup historicalMetrics = serverMetricGroup.addGroup("historical"); + historicalMetrics.gauge( + MetricNames.HISTORICAL_INFLIGHT_REQUESTS, + historicalLakeLookupManager::numInflightRequests); + historicalMetrics.gauge( + MetricNames.HISTORICAL_LOOKUP_CACHE_DISK_SIZE, + historicalLakeLookupManager::lookupCacheDiskSize); + historicalMetrics.gauge( + MetricNames.HISTORICAL_LOOKUP_CACHE_TABLE_COUNT, + historicalLakeLookupManager::cachedTableCount); + historicalMetrics.counter( + MetricNames.HISTORICAL_LOOKUP_CACHE_CAPACITY_EVICTIONS, + historicalLakeLookupManager.capacityEvictions()); + serverMetricGroup.gauge( MetricNames.REPLICA_LEADER_COUNT, () -> onlineReplicas().filter(Replica::isLeader).count()); @@ -458,12 +472,6 @@ private void registerMetrics() { serverMetricGroup.gauge(MetricNames.UNDER_REPLICATED, this::underReplicatedCount); serverMetricGroup.gauge(MetricNames.UNDER_MIN_ISR, this::underMinIsrCount); serverMetricGroup.gauge(MetricNames.AT_MIN_ISR, this::atMinIsrCount); - serverMetricGroup.gauge( - MetricNames.HISTORICAL_LOOKUP_CACHED_TABLE_COUNT, - historicalLakeLookupManager::cachedTableCount); - serverMetricGroup.counter( - MetricNames.HISTORICAL_LOOKUP_CACHED_TABLE_CAPACITY_EVICTIONS, - historicalLakeLookupManager.capacityEvictions()); MetricGroup logicalStorage = serverMetricGroup.addGroup("logicalStorage"); logicalStorage.gauge( @@ -619,18 +627,11 @@ public void maybeUpdateMetadataCache(int coordinatorEpoch, ClusterMetadata clust private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { Map tableIdToLakeFlag = new HashMap<>(); Map tableIdToTieredLogLocalSegments = new HashMap<>(); - Map tableIdToTableConfig = new HashMap<>(); for (TableMetadata tableMetadata : clusterMetadata.getTableMetadataList()) { TableInfo tableInfo = tableMetadata.getTableInfo(); long tableId = tableInfo.getTableId(); - // Deleted-table markers do not carry authoritative table configuration. - if (tableId != TableMetadata.DELETED_TABLE_ID - && !tableInfo.getTablePath().equals(TableMetadata.DELETED_TABLE_PATH)) { - tableIdToTableConfig.put(tableId, tableInfo.getTableConfig()); - } - // Collect datalake enabled configuration if (tableInfo.getTableConfig().getDataLakeFormat().isPresent()) { boolean dataLakeEnabled = tableInfo.getTableConfig().isDataLakeEnabled(); @@ -642,9 +643,7 @@ private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { tableIdToTieredLogLocalSegments.put(tableId, tieredLogLocalSegments); } - if (tableIdToLakeFlag.isEmpty() - && tableIdToTieredLogLocalSegments.isEmpty() - && tableIdToTableConfig.isEmpty()) { + if (tableIdToLakeFlag.isEmpty() && tableIdToTieredLogLocalSegments.isEmpty()) { return; } @@ -664,11 +663,6 @@ private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { replica.updateTieredLogLocalSegments( tableIdToTieredLogLocalSegments.get(tableId)); } - - // Publish the new snapshot after applying configuration-specific side effects. - if (tableIdToTableConfig.containsKey(tableId)) { - replica.updateTableConfig(tableIdToTableConfig.get(tableId)); - } } } } @@ -856,12 +850,11 @@ public void historicalLookups( Collections.synchronizedList(new ArrayList<>(lookupData.size())); AtomicInteger remainingLookups = new AtomicInteger(lookupData.size()); for (LookupDataForBucket data : lookupData) { + Replica replica; CompletableFuture lookupFuture; - TableMetricGroup tableMetrics = null; try { - Replica replica = getReplicaOrException(data.tableBucket()); - tableMetrics = replica.tableMetrics(); - tableMetrics.totalHistoricalLookupRequests().inc(); + replica = getReplicaOrException(data.tableBucket()); + replica.tableMetrics().totalHistoricalLookupRequests().inc(); if (!replica.isKvTable()) { throw new NonPrimaryKeyTableException( "Historical lookup is only supported for primary key tables, but " @@ -878,36 +871,33 @@ public void historicalLookups( data, replica.getTableInfo(), latestSchemaInfo, - replica.getTableConfig(), - tableMetrics::recordHistoricalLakeLookup); + replica.tableMetrics()::recordHistoricalLakeLookup); } catch (Exception e) { - lookupFuture = - CompletableFuture.completedFuture( - new LookupResultForBucket( - data.tableBucket(), - null, - data.originalPartitionName(), - ApiError.fromThrowable(e))); + result.add( + new LookupResultForBucket( + data.tableBucket(), + null, + data.originalPartitionName(), + ApiError.fromThrowable(e))); + if (remainingLookups.decrementAndGet() == 0) { + responseCallback.accept(result); + } + continue; } - TableMetricGroup historicalLookupMetrics = tableMetrics; lookupFuture.whenComplete( (bucketResult, error) -> { - LookupResultForBucket completedResult; - if (error == null) { - completedResult = bucketResult; - } else { - completedResult = - new LookupResultForBucket( - data.tableBucket(), - null, - data.originalPartitionName(), - ApiError.fromThrowable(error)); - } - if (historicalLookupMetrics != null - && completedResult.failed() + LookupResultForBucket completedResult = + error == null + ? bucketResult + : new LookupResultForBucket( + data.tableBucket(), + null, + data.originalPartitionName(), + ApiError.fromThrowable(error)); + if (completedResult.failed() && isUnexpectedHistoricalLookupException( completedResult.getError().exception())) { - historicalLookupMetrics.failedHistoricalLookupRequests().inc(); + replica.tableMetrics().failedHistoricalLookupRequests().inc(); } result.add(completedResult); if (remainingLookups.decrementAndGet() == 0) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 799e4726705..89ae6f7138e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -232,7 +232,6 @@ import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toByteBuffer; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.toPbAclInfo; -import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** @@ -1781,10 +1780,6 @@ public static ListPartitionInfosResponse toListPartitionInfosResponse( ListPartitionInfosResponse listPartitionsResponse = new ListPartitionInfosResponse(); for (Map.Entry partitionRegistration : partitionRegistrations.entrySet()) { - // TODO: Return the actual lake partitions instead of the internal historical partition. - if (HISTORICAL_PARTITION_VALUE.equals(partitionRegistration.getKey())) { - continue; - } ResolvedPartitionSpec spec = ResolvedPartitionSpec.fromPartitionName( partitionKeys, partitionRegistration.getKey()); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 31a7ff27bb6..e8ba9714dd6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -38,7 +38,6 @@ import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; -import org.apache.fluss.metadata.TablePath; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypeRoot; import org.apache.fluss.types.RowType; @@ -91,9 +90,7 @@ public class TableDescriptorValidation { public static void validateTableDescriptor( TableDescriptor tableDescriptor, int maxBucketNum, - @Nullable DataLakeFormat clusterDataLakeFormat, - TablePath tablePath, - double historicalLookupCacheMaxRatio) { + @Nullable DataLakeFormat clusterDataLakeFormat) { Schema schema = tableDescriptor.getSchema(); boolean hasPrimaryKey = schema.getPrimaryKey().isPresent(); Configuration tableConf = Configuration.fromMap(tableDescriptor.getProperties()); @@ -131,9 +128,6 @@ public static void validateTableDescriptor( checkDeleteBehavior(tableConf, hasPrimaryKey); checkTieredLog(tableConf); checkHistoricalPartition(tableDescriptor, tableConf); - if (tableConf.get(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED)) { - checkHistoricalLookupCacheRatio(tableConf, tablePath, historicalLookupCacheMaxRatio); - } checkPartition(tableConf, tableDescriptor.getPartitionKeys(), schema.getRowType()); checkSystemColumns(schema.getRowType()); validateStatisticsConfig(tableDescriptor); @@ -233,36 +227,6 @@ private static void checkHistoricalPartition( } } - private static void checkHistoricalLookupCacheRatio( - Configuration tableConf, TablePath tablePath, double historicalLookupCacheMaxRatio) { - double tableCacheRatio = - tableConf.get( - ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO); - if (!(tableCacheRatio > 0.0 && tableCacheRatio <= 1.0)) { - throw new InvalidConfigException( - String.format( - "'%s' for table '%s' must be within (0.0, 1.0].", - ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO - .key(), - tablePath)); - } - if (Double.compare(tableCacheRatio, historicalLookupCacheMaxRatio) > 0) { - throw new InvalidConfigException( - String.format( - "'%s' (%s) for table '%s' must be less than or equal to '%s' (%s).", - ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO - .key(), - tableCacheRatio, - tablePath, - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO - .key(), - historicalLookupCacheMaxRatio)); - } - } - public static void validateAlterTableProperties( TableInfo currentTable, Set tableKeysToChange) { TableConfig currentConfig = currentTable.getTableConfig(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index 0b6f7af3d9e..3c78834f8b2 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -21,6 +21,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.DiskWriteLockedException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.metadata.DataLakeFormat; @@ -45,6 +46,7 @@ import org.junit.jupiter.params.provider.ValueSource; import java.io.File; +import java.io.RandomAccessFile; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; @@ -52,14 +54,11 @@ import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -71,10 +70,11 @@ /** Tests for {@link HistoricalLakeLookupManager}. */ class HistoricalLakeLookupManagerTest { - private static final long LOOKUP_VOLUME_BYTES = MemorySize.parse("800gb").getBytes(); + private static final long DATA_DIR_VOLUME_BYTES = MemorySize.parse("800gb").getBytes(); private static final TableBucket HISTORICAL_BUCKET = new TableBucket(PARTITION_TABLE_ID, 1L, 0); private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = - (lookupTimeNanos, lookupFileMaterialization) -> {}; + (lookupTimeNanos, lookupFileDownloaded) -> {}; + private static final Runnable NO_OP_DISK_WRITE_GUARD = () -> {}; @TempDir private File ioTmpDir; @@ -89,7 +89,6 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { lookupData(HISTORICAL_BUCKET), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig(), NO_OP_LOOKUP_METRIC_RECORDER); assertThat(first).isNotDone(); assertThat(executor.numQueuedTasks()).isEqualTo(1); @@ -101,7 +100,6 @@ void testHistoricalLookupThrottledWhenPermitsExhausted() throws Exception { lookupData(secondBucket), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig(), NO_OP_LOOKUP_METRIC_RECORDER) .get(1, TimeUnit.SECONDS); @@ -123,7 +121,6 @@ void testHistoricalLookupReleasesPermitOnFailure() throws Exception { lookupData(HISTORICAL_BUCKET), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig(), NO_OP_LOOKUP_METRIC_RECORDER); executor.runNext(); LookupResultForBucket firstResult = first.get(1, TimeUnit.SECONDS); @@ -137,7 +134,6 @@ void testHistoricalLookupReleasesPermitOnFailure() throws Exception { lookupData(HISTORICAL_BUCKET), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig(), NO_OP_LOOKUP_METRIC_RECORDER); assertThat(second).isNotDone(); assertThat(executor.numQueuedTasks()).isEqualTo(1); @@ -153,21 +149,18 @@ void testHistoricalLookupMaxQueuedRequestsUsesExplicitConfig() throws Exception lookupData(new TableBucket(PARTITION_TABLE_ID, 1L, 0)), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig(), NO_OP_LOOKUP_METRIC_RECORDER); CompletableFuture second = manager.lookup( lookupData(new TableBucket(PARTITION_TABLE_ID, 2L, 0)), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig(), NO_OP_LOOKUP_METRIC_RECORDER); LookupResultForBucket third = manager.lookup( lookupData(new TableBucket(PARTITION_TABLE_ID, 3L, 0)), PARTITION_TABLE_INFO, PARTITION_TABLE_INFO.getSchemaInfo(), - PARTITION_TABLE_INFO.getTableConfig(), NO_OP_LOOKUP_METRIC_RECORDER) .get(1, TimeUnit.SECONDS); @@ -189,9 +182,10 @@ void testRejectNonPositiveHistoricalLookupMaxQueuedRequests() { null, executor, ioTmpDir, - LOOKUP_VOLUME_BYTES, + DATA_DIR_VOLUME_BYTES, Ticker.systemTicker(), - Scheduler.disabledScheduler())) + Scheduler.disabledScheduler(), + NO_OP_DISK_WRITE_GUARD)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( ConfigOptions.NETTY_SERVER_MAX_QUEUED_HISTORICAL_REQUESTS.key()); @@ -210,9 +204,10 @@ void testRejectNonPositiveHistoricalPartitionThreadPoolMaxSize(int maxThreadPool null, null, ioTmpDir, - LOOKUP_VOLUME_BYTES, + DATA_DIR_VOLUME_BYTES, Ticker.systemTicker(), - Scheduler.disabledScheduler())) + Scheduler.disabledScheduler(), + NO_OP_DISK_WRITE_GUARD)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( ConfigOptions.SERVER_HISTORICAL_PARTITION_THREAD_POOL_MAX_SIZE.key()); @@ -319,28 +314,32 @@ void testInvalidatesLookuperOnSchemaAndLifecycleChanges() throws Exception { } @Test - void testReplacesLookuperOnlyWhenEffectiveCacheRatioChanges() throws Exception { + void testDoesNotReplaceLookuperForUnrelatedTableConfigChange() throws Exception { ManualExecutor executor = new ManualExecutor(); TestingHistoricalLakeLookupManager manager = createTestingManager(executor); - lookupAndRun(manager, executor, PARTITION_TABLE_INFO, tableConfigWithCacheRatio(0.01)); + lookupAndRun(manager, executor, PARTITION_TABLE_INFO); TestingLakeTableLookuper initialLookuper = manager.createdLookupers.get(0); - Configuration unrelatedChange = new Configuration(); - unrelatedChange.set( - ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS, - ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.defaultValue() + 1); - lookupAndRun(manager, executor, PARTITION_TABLE_INFO, new TableConfig(unrelatedChange)); - lookupAndRun(manager, executor, PARTITION_TABLE_INFO, tableConfigWithCacheRatio(0.01)); + TableDescriptor changedDescriptor = + TableDescriptor.builder(PARTITION_TABLE_INFO.toTableDescriptor()) + .property( + ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS, + ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.defaultValue() + 1) + .build(); + TableInfo changedTableInfo = + TableInfo.of( + PARTITION_TABLE_INFO.getTablePath(), + PARTITION_TABLE_INFO.getTableId(), + PARTITION_TABLE_INFO.getSchemaId(), + changedDescriptor, + PARTITION_TABLE_INFO.getRemoteDataDir(), + PARTITION_TABLE_INFO.getCreatedTime(), + PARTITION_TABLE_INFO.getModifiedTime()); + lookupAndRun(manager, executor, changedTableInfo); assertThat(manager.createdLookupers).hasSize(1); assertThat(initialLookuper.closed).isFalse(); - - lookupAndRun(manager, executor, PARTITION_TABLE_INFO, tableConfigWithCacheRatio(0.005)); - - assertThat(initialLookuper.closed).isTrue(); - assertThat(manager.createdLookupers).hasSize(2); - assertThat(manager.createdCacheSizes.get(1)).isEqualTo(MemorySize.parse("4gb").getBytes()); } @Test @@ -381,88 +380,74 @@ void testDynamicallyUpdatesExpirationAndExpiresIdleLookuper() throws Exception { } @Test - void testEvictsLeastRecentlyUsedLookuperWhenCapacityIsFull() throws Exception { + void testEvictsLookuperWhenCachedTableLimitIsExceeded() throws Exception { ManualExecutor executor = new ManualExecutor(); - AtomicLong tickerNanos = new AtomicLong(); Configuration conf = conf(1); - conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.02); + conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.20); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager( - conf, executor, tickerNanos::get, Scheduler.disabledScheduler()); + conf, + executor, + Ticker.systemTicker(), + Scheduler.disabledScheduler(), + 100, + 0); manager.startup(); - TableInfo first = tableInfo(PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId()); - TableInfo second = tableInfo(PARTITION_TABLE_ID + 1, PARTITION_TABLE_INFO.getSchemaId()); - TableInfo third = tableInfo(PARTITION_TABLE_ID + 2, PARTITION_TABLE_INFO.getSchemaId()); - - lookupAndRun(manager, executor, first); - tickerNanos.incrementAndGet(); - lookupAndRun(manager, executor, second); - tickerNanos.incrementAndGet(); - lookupAndRun(manager, executor, first); - tickerNanos.incrementAndGet(); - lookupAndRun(manager, executor, third); + for (int i = 0; i < 11; i++) { + lookupAndRun( + manager, + executor, + tableInfo(PARTITION_TABLE_ID + i, PARTITION_TABLE_INFO.getSchemaId())); + } - assertThat(manager.createdLookupers).hasSize(3); - assertThat(manager.createdLookupers.get(0).closed).isFalse(); - assertThat(manager.createdLookupers.get(1).closed).isTrue(); - assertThat(manager.createdLookupers.get(2).closed).isFalse(); - assertThat(manager.cachedTableCount()).isEqualTo(2); + assertThat(manager.createdLookupers).hasSize(11); + assertThat(manager.createdLookupers).filteredOn(lookuper -> lookuper.closed).hasSize(1); + assertThat(manager.createdCacheSizes).containsOnly(2L); + assertThat(manager.cachedTableCount()).isEqualTo(10); assertThat(manager.capacityEvictions().getCount()).isEqualTo(1); } @Test - void testReconfiguresGlobalCapacityLazily() throws Exception { + void testRejectsLookupsAndClearsCacheWhenDiskWriteLocked() throws Exception { ManualExecutor executor = new ManualExecutor(); - Configuration conf = conf(1); - conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.12); + AtomicBoolean diskWriteLocked = new AtomicBoolean(); + Runnable diskWriteGuard = + () -> { + if (diskWriteLocked.get()) { + throw new DiskWriteLockedException("Data disk is write-locked."); + } + }; TestingHistoricalLakeLookupManager manager = - new TestingHistoricalLakeLookupManager(conf, executor); + new TestingHistoricalLakeLookupManager( + conf(1), + executor, + Ticker.systemTicker(), + Scheduler.disabledScheduler(), + 100, + 6, + diskWriteGuard); manager.startup(); - for (int i = 0; i < 12; i++) { - lookupAndRun( - manager, - executor, - tableInfo(PARTITION_TABLE_ID + i, PARTITION_TABLE_INFO.getSchemaId())); - } + lookupAndRun(manager, executor, PARTITION_TABLE_INFO); + TestingLakeTableLookuper cachedLookuper = manager.createdLookupers.get(0); + assertThat(cachedLookuper.cacheFile).exists().hasSize(6); + + // A write lock rejects the read before it reaches the executor and releases the existing + // disk cache. Once the disk recovers, a later lookup can create a fresh lookuper. + diskWriteLocked.set(true); + LookupResultForBucket rejected = + lookup(manager, PARTITION_TABLE_INFO).get(1, TimeUnit.SECONDS); + assertThat(rejected.getError().error()).isEqualTo(Errors.DISK_WRITE_LOCKED); + assertThat(executor.numQueuedTasks()).isZero(); + assertThat(manager.numInflightRequests()).isZero(); + assertThat(manager.cachedTableCount()).isZero(); + assertThat(cachedLookuper.cacheFile).doesNotExist(); + assertThat(cachedLookuper.closed).isTrue(); - assertThat(manager.createdLookupers).hasSize(12); - assertThat(manager.cachedTableCount()).isEqualTo(12); - assertThat(manager.capacityEvictions().getCount()).isZero(); - - // Changing only the global limit must not recreate an existing lookuper. - Configuration increasedConf = new Configuration(conf); - increasedConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.13); - manager.reconfigure(increasedConf); - lookupAndRun( - manager, - executor, - tableInfo(PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId())); - - assertThat(manager.createdLookupers).hasSize(12); - assertThat(manager.cachedTableCount()).isEqualTo(12); - assertThat(manager.capacityEvictions().getCount()).isZero(); - - // A reduction is lazy: cached lookupers remain until another admission needs capacity. - Configuration reducedConf = new Configuration(increasedConf); - reducedConf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.11); - manager.reconfigure(reducedConf); - - assertThat(manager.cachedTableCount()).isEqualTo(12); - assertThat(manager.capacityEvictions().getCount()).isZero(); - - lookupAndRun( - manager, - executor, - tableInfo(PARTITION_TABLE_ID + 12, PARTITION_TABLE_INFO.getSchemaId())); - - assertThat(manager.createdLookupers).hasSize(13); - assertThat(manager.cachedTableCount()).isEqualTo(11); - assertThat(manager.createdLookupers).filteredOn(lookuper -> lookuper.closed).hasSize(2); - assertThat(manager.capacityEvictions().getCount()).isEqualTo(2); + diskWriteLocked.set(false); + lookupAndRun(manager, executor, PARTITION_TABLE_INFO); + assertThat(manager.createdLookupers).hasSize(2); } @Test @@ -490,87 +475,6 @@ void testReconfiguresLakePropertiesAndInvalidatesLookuper() throws Exception { .containsEntry("datalake.paimon.warehouse", "new-warehouse"); } - @Test - void testEvictsOutsideConcurrentTableReplacements() throws Exception { - ExecutorService executor = - Executors.newFixedThreadPool( - 2, - runnable -> { - Thread thread = - new Thread(runnable, "historical-lookup-replacement-test"); - thread.setDaemon(true); - return thread; - }); - double initialCacheRatio = 0.005; - double replacementCacheRatio = 0.01; - Configuration conf = conf(2); - conf.set( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, - replacementCacheRatio); - CoordinatedReplacementManager manager = - new CoordinatedReplacementManager(conf, executor, replacementCacheRatio); - manager.startup(); - - TableInfo first = - tableInfoWithCacheRatio( - PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId(), initialCacheRatio); - TableInfo second = - tableInfoWithCacheRatio( - PARTITION_TABLE_ID + 1, - PARTITION_TABLE_INFO.getSchemaId(), - initialCacheRatio); - try { - assertThat(lookup(manager, first).get(5, TimeUnit.SECONDS).failed()).isFalse(); - assertThat(lookup(manager, second).get(5, TimeUnit.SECONDS).failed()).isFalse(); - - // Both replacements hold their own table's compute lock before admission fails. LRU - // eviction must happen after those locks are released to avoid cross-key deadlock. - TableInfo firstReplacement = - tableInfoWithCacheRatio( - first.getTableId(), first.getSchemaId() + 1, replacementCacheRatio); - TableInfo secondReplacement = - tableInfoWithCacheRatio( - second.getTableId(), second.getSchemaId() + 1, replacementCacheRatio); - CompletableFuture firstResult = - lookup(manager, firstReplacement); - CompletableFuture secondResult = - lookup(manager, secondReplacement); - - LookupResultForBucket firstReplacementResult = firstResult.get(5, TimeUnit.SECONDS); - LookupResultForBucket secondReplacementResult = secondResult.get(5, TimeUnit.SECONDS); - assertThat(firstReplacementResult.getError().error()) - .isIn(Errors.NONE, Errors.HISTORICAL_PARTITION_THROTTLED); - assertThat(secondReplacementResult.getError().error()) - .isIn(Errors.NONE, Errors.HISTORICAL_PARTITION_THROTTLED); - assertThat(firstReplacementResult.failed() && secondReplacementResult.failed()) - .isFalse(); - manager.close(); - } finally { - executor.shutdownNow(); - } - } - - @Test - void testThrottlesWhenTableCacheRatioExceedsRuntimeLimit() throws Exception { - ManualExecutor executor = new ManualExecutor(); - Configuration conf = conf(1); - conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.01); - TestingHistoricalLakeLookupManager manager = - new TestingHistoricalLakeLookupManager(conf, executor); - manager.startup(); - - LookupResultForBucket result = - lookupResultAndRun( - manager, - executor, - tableInfoWithCacheRatio( - PARTITION_TABLE_ID, PARTITION_TABLE_INFO.getSchemaId(), 0.02)); - - assertThat(result.getError().error()).isEqualTo(Errors.HISTORICAL_PARTITION_THROTTLED); - assertThat(manager.createdLookupers).isEmpty(); - assertThat(manager.cachedTableCount()).isZero(); - } - private HistoricalLakeLookupManager createManager( int maxQueuedHistoricalRequests, ManualExecutor executor) { HistoricalLakeLookupManager manager = @@ -579,9 +483,10 @@ private HistoricalLakeLookupManager createManager( null, executor, ioTmpDir, - LOOKUP_VOLUME_BYTES, + DATA_DIR_VOLUME_BYTES, Ticker.systemTicker(), - Scheduler.disabledScheduler()); + Scheduler.disabledScheduler(), + NO_OP_DISK_WRITE_GUARD); manager.startup(); return manager; } @@ -617,16 +522,10 @@ private static LookupDataForBucket lookupData(TableBucket tableBucket) { private static CompletableFuture lookup( HistoricalLakeLookupManager manager, TableInfo tableInfo) { - return lookup(manager, tableInfo, tableInfo.getTableConfig()); - } - - private static CompletableFuture lookup( - HistoricalLakeLookupManager manager, TableInfo tableInfo, TableConfig tableConfig) { return manager.lookup( lookupData(new TableBucket(tableInfo.getTableId(), 1L, 0)), tableInfo, tableInfo.getSchemaInfo(), - tableConfig, NO_OP_LOOKUP_METRIC_RECORDER); } @@ -641,52 +540,12 @@ private static TableInfo tableInfo(long tableId, int schemaId) { PARTITION_TABLE_INFO.getModifiedTime()); } - private static TableInfo tableInfoWithCacheRatio( - long tableId, int schemaId, double cacheRatio) { - TableDescriptor descriptor = - TableDescriptor.builder(PARTITION_TABLE_INFO.toTableDescriptor()) - .property( - ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, - cacheRatio) - .build(); - return TableInfo.of( - PARTITION_TABLE_INFO.getTablePath(), - tableId, - schemaId, - descriptor, - PARTITION_TABLE_INFO.getRemoteDataDir(), - PARTITION_TABLE_INFO.getCreatedTime(), - PARTITION_TABLE_INFO.getModifiedTime()); - } - - private static TableConfig tableConfigWithCacheRatio(double cacheRatio) { - Configuration conf = new Configuration(); - conf.set( - ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, - cacheRatio); - return new TableConfig(conf); - } - private static void lookupAndRun( HistoricalLakeLookupManager manager, ManualExecutor executor, TableInfo tableInfo) throws Exception { lookupAndRun(manager, executor, tableInfo, tableInfo.getSchemaInfo()); } - private static void lookupAndRun( - HistoricalLakeLookupManager manager, - ManualExecutor executor, - TableInfo tableInfo, - TableConfig tableConfig) - throws Exception { - LookupResultForBucket result = - lookupResultAndRun( - manager, executor, tableInfo, tableInfo.getSchemaInfo(), tableConfig); - assertThat(result.failed()).isFalse(); - assertThat(result.originalPartitionName()).isEqualTo("2024"); - } - private static void lookupAndRun( HistoricalLakeLookupManager manager, ManualExecutor executor, @@ -710,24 +569,12 @@ private static LookupResultForBucket lookupResultAndRun( TableInfo tableInfo, SchemaInfo schemaInfo) throws Exception { - return lookupResultAndRun( - manager, executor, tableInfo, schemaInfo, tableInfo.getTableConfig()); - } - - private static LookupResultForBucket lookupResultAndRun( - HistoricalLakeLookupManager manager, - ManualExecutor executor, - TableInfo tableInfo, - SchemaInfo schemaInfo, - TableConfig tableConfig) - throws Exception { TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 1L, 0); CompletableFuture future = manager.lookup( lookupData(tableBucket), tableInfo, schemaInfo, - tableConfig, NO_OP_LOOKUP_METRIC_RECORDER); executor.runNext(); return future.get(1, TimeUnit.SECONDS); @@ -740,6 +587,7 @@ private static final class TestingHistoricalLakeLookupManager private final List createdTableConfigs = new ArrayList<>(); private final List createdCacheSizes = new ArrayList<>(); private final List createdClusterConfigs = new ArrayList<>(); + private final long lookupCacheFileBytes; private TestingHistoricalLakeLookupManager(Configuration conf, ManualExecutor executor) { super( @@ -747,9 +595,11 @@ private TestingHistoricalLakeLookupManager(Configuration conf, ManualExecutor ex null, executor, new File(conf.get(ConfigOptions.DATA_DIR)), - LOOKUP_VOLUME_BYTES, + DATA_DIR_VOLUME_BYTES, Ticker.systemTicker(), - Scheduler.disabledScheduler()); + Scheduler.disabledScheduler(), + NO_OP_DISK_WRITE_GUARD); + this.lookupCacheFileBytes = 0L; } private TestingHistoricalLakeLookupManager( @@ -762,9 +612,48 @@ private TestingHistoricalLakeLookupManager( null, executor, new File(conf.get(ConfigOptions.DATA_DIR)), - LOOKUP_VOLUME_BYTES, + DATA_DIR_VOLUME_BYTES, + ticker, + cacheScheduler, + NO_OP_DISK_WRITE_GUARD); + this.lookupCacheFileBytes = 0L; + } + + private TestingHistoricalLakeLookupManager( + Configuration conf, + ManualExecutor executor, + Ticker ticker, + Scheduler cacheScheduler, + long dataDirVolumeBytes, + long lookupCacheFileBytes) { + this( + conf, + executor, + ticker, + cacheScheduler, + dataDirVolumeBytes, + lookupCacheFileBytes, + () -> {}); + } + + private TestingHistoricalLakeLookupManager( + Configuration conf, + ManualExecutor executor, + Ticker ticker, + Scheduler cacheScheduler, + long dataDirVolumeBytes, + long lookupCacheFileBytes, + Runnable diskWriteGuard) { + super( + conf, + null, + executor, + new File(conf.get(ConfigOptions.DATA_DIR)), + dataDirVolumeBytes, ticker, - cacheScheduler); + cacheScheduler, + diskWriteGuard); + this.lookupCacheFileBytes = lookupCacheFileBytes; } @Override @@ -774,7 +663,8 @@ LakeTableLookuper createLakeTableLookuper( TableConfig tableConfig, long cacheSizeBytes, Configuration clusterConf) { - TestingLakeTableLookuper lookuper = new TestingLakeTableLookuper(); + TestingLakeTableLookuper lookuper = + new TestingLakeTableLookuper(new File(ioTmpDir), lookupCacheFileBytes); createdLookupers.add(lookuper); createdIoTmpDirs.add(ioTmpDir); createdTableConfigs.add(tableConfig); @@ -785,61 +675,40 @@ LakeTableLookuper createLakeTableLookuper( } private static final class TestingLakeTableLookuper implements LakeTableLookuper { + private final File cacheFile; + private final long cacheFileBytes; private boolean closed; + private boolean cacheFileDownloaded; private final List lookupContexts = new ArrayList<>(); + private TestingLakeTableLookuper(File lookupDir, long cacheFileBytes) { + this.cacheFile = new File(lookupDir, "cache-file"); + this.cacheFileBytes = cacheFileBytes; + } + @Override - public byte[] lookup(byte[] key, LookupContext context) { + public byte[] lookup(byte[] key, LookupContext context) throws Exception { if (closed) { throw new IllegalStateException("Lookuper is already closed."); } lookupContexts.add(context); + boolean downloaded = false; + if (!cacheFileDownloaded && cacheFileBytes > 0) { + java.nio.file.Files.createDirectories(cacheFile.getParentFile().toPath()); + try (RandomAccessFile file = new RandomAccessFile(cacheFile, "rw")) { + file.setLength(cacheFileBytes); + } + cacheFileDownloaded = true; + downloaded = true; + } + context.lookupMetricRecorder().recordLookup(1L, downloaded); return key; } @Override - public void close() { + public void close() throws Exception { closed = true; - } - } - - private static final class CoordinatedReplacementManager extends HistoricalLakeLookupManager { - private final double replacementCacheRatio; - private final CyclicBarrier replacementBarrier = new CyclicBarrier(2); - private final AtomicInteger coordinatedCreations = new AtomicInteger(); - - private CoordinatedReplacementManager( - Configuration conf, ExecutorService executor, double replacementCacheRatio) { - super( - conf, - null, - executor, - new File(conf.get(ConfigOptions.DATA_DIR)), - LOOKUP_VOLUME_BYTES, - Ticker.systemTicker(), - Scheduler.disabledScheduler()); - this.replacementCacheRatio = replacementCacheRatio; - } - - @Override - LakeTableLookuper createLakeTableLookuper( - TablePath tablePath, - String ioTmpDir, - TableConfig tableConfig, - long cacheSizeBytes, - Configuration clusterConf) { - if (Double.compare( - tableConfig.getHistoricalPartitionLookupCacheMaxDiskRatio(), - replacementCacheRatio) - == 0 - && coordinatedCreations.getAndIncrement() < 2) { - try { - replacementBarrier.await(5, TimeUnit.SECONDS); - } catch (Exception e) { - throw new RuntimeException("Failed to coordinate lookuper replacements.", e); - } - } - return new TestingLakeTableLookuper(); + java.nio.file.Files.deleteIfExists(cacheFile.toPath()); } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java deleted file mode 100644 index 6ef7fefa9e1..00000000000 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLookupCacheBudgetManagerTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.server.replica; - -import org.apache.fluss.server.replica.HistoricalLookupCacheBudgetManager.Reservation; - -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -/** Tests for {@link HistoricalLookupCacheBudgetManager}. */ -class HistoricalLookupCacheBudgetManagerTest { - - @Test - void testReserveAndReleaseWithinLimit() { - HistoricalLookupCacheBudgetManager manager = new HistoricalLookupCacheBudgetManager(10); - - Reservation first = manager.tryReserve(1, 4); - Reservation second = manager.tryReserve(2, 6); - assertThat(first).isNotNull(); - assertThat(second).isNotNull(); - assertThat(manager.reservedBytes()).isEqualTo(10); - assertThat(manager.tryReserve(3, 1)).isNull(); - assertThat(manager.tryReserve(1, 1)).isNull(); - - manager.release(first); - manager.release(first); - assertThat(manager.reservedBytes()).isEqualTo(6); - Reservation third = manager.tryReserve(3, 4); - assertThat(third).isNotNull(); - assertThat(manager.reservedBytes()).isEqualTo(10); - } - - @Test - void testReplaceReservationAtomically() { - HistoricalLookupCacheBudgetManager manager = new HistoricalLookupCacheBudgetManager(12); - Reservation oldReservation = manager.tryReserve(1, 4); - Reservation otherReservation = manager.tryReserve(2, 6); - assertThat(oldReservation).isNotNull(); - assertThat(otherReservation).isNotNull(); - - Reservation replacement = manager.tryReplace(oldReservation, 5); - assertThat(replacement).isNotNull(); - assertThat(replacement.getTableId()).isEqualTo(1); - assertThat(replacement.getBytes()).isEqualTo(5); - assertThat(manager.reservedBytes()).isEqualTo(11); - - // Releasing the retired reservation must not affect the replacement. - manager.release(oldReservation); - assertThat(manager.reservedBytes()).isEqualTo(11); - assertThat(manager.tryReplace(oldReservation, 1)).isNull(); - - // A failed replacement leaves the current reservation unchanged. - assertThat(manager.tryReplace(replacement, 7)).isNull(); - assertThat(manager.reservedBytes()).isEqualTo(11); - manager.release(replacement); - assertThat(manager.reservedBytes()).isEqualTo(6); - } - - @Test - void testReducedLimitAppliesToSubsequentReservations() { - HistoricalLookupCacheBudgetManager manager = new HistoricalLookupCacheBudgetManager(10); - Reservation first = manager.tryReserve(1, 6); - Reservation second = manager.tryReserve(2, 4); - assertThat(first).isNotNull(); - assertThat(second).isNotNull(); - - // Shrinking is lazy: existing reservations remain even though their total exceeds the new - // limit. - manager.updateGlobalLimit(7); - assertThat(manager.maxBytes()).isEqualTo(7); - assertThat(manager.reservedBytes()).isEqualTo(10); - - // Subsequent reservations use the reduced limit and succeed only after capacity is freed. - assertThat(manager.tryReserve(3, 1)).isNull(); - - manager.release(second); - assertThat(manager.tryReserve(3, 1)).isNotNull(); - } -} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java index 4cae527523a..cb427c23ec8 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/HistoricalPartitionTableValidationTest.java @@ -22,18 +22,14 @@ import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.metadata.TablePath; import org.apache.fluss.types.DataTypes; import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; class HistoricalPartitionTableValidationTest { - private static final TablePath TABLE_PATH = TablePath.of("test_db", "test_table"); - @Test void testReportsAllUnmetHistoricalPartitionRequirements() { // Case 1: Report disabled options, a missing format, and missing table keys together. @@ -50,11 +46,7 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { TableDescriptorValidation.validateTableDescriptor( allRequirementsMissingDescriptor, 100, - DataLakeFormat.PAIMON, - TABLE_PATH, - ConfigOptions - .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO - .defaultValue())) + DataLakeFormat.PAIMON)) .isInstanceOf(InvalidConfigException.class) .hasMessage( "'table.datalake.historical-partition.enabled' has unmet requirements: " @@ -82,11 +74,7 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { TableDescriptorValidation.validateTableDescriptor( relatedValidationFailuresDescriptor, 100, - DataLakeFormat.PAIMON, - TABLE_PATH, - ConfigOptions - .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO - .defaultValue())) + DataLakeFormat.PAIMON)) .isInstanceOf(InvalidConfigException.class) .hasMessage( "'table.datalake.historical-partition.enabled' has unmet requirements: " @@ -95,60 +83,4 @@ void testReportsAllUnmetHistoricalPartitionRequirements() { + "the table must define a primary key; " + "the table must define exactly one partition key (found 0)."); } - - @Test - void testValidateHistoricalLookupCacheRatio() { - TableDescriptor ordinaryTableDescriptor = - TableDescriptor.builder() - .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) - .distributedBy(1) - .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) - .build(); - assertThatCode(() -> validate(ordinaryTableDescriptor, 0.05)).doesNotThrowAnyException(); - - TableDescriptor zeroRatioDescriptor = descriptorWithCacheRatio(0.0); - assertThatThrownBy(() -> validate(zeroRatioDescriptor, 0.1)) - .isInstanceOf(InvalidConfigException.class) - .hasMessageContaining( - ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO - .key()) - .hasMessageContaining(TABLE_PATH.toString()) - .hasMessageContaining("within (0.0, 1.0]"); - - TableDescriptor oversizedDescriptor = descriptorWithCacheRatio(0.2); - assertThatThrownBy(() -> validate(oversizedDescriptor, 0.1)) - .isInstanceOf(InvalidConfigException.class) - .hasMessageContaining("0.2") - .hasMessageContaining( - ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO.key()) - .hasMessageContaining("0.1"); - } - - private static TableDescriptor descriptorWithCacheRatio(double cacheRatio) { - return TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("dt", DataTypes.STRING()) - .primaryKey("id", "dt") - .build()) - .partitionedBy("dt") - .distributedBy(1) - .property(ConfigOptions.TABLE_REPLICATION_FACTOR, 1) - .property(ConfigOptions.TABLE_AUTO_PARTITION_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_ENABLED, true) - .property(ConfigOptions.TABLE_DATALAKE_FORMAT, DataLakeFormat.PAIMON) - .property(ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED, true) - .property( - ConfigOptions - .TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, - cacheRatio) - .build(); - } - - private static void validate(TableDescriptor descriptor, double globalCacheRatio) { - TableDescriptorValidation.validateTableDescriptor( - descriptor, 100, DataLakeFormat.PAIMON, TABLE_PATH, globalCacheRatio); - } } diff --git a/website/docs/maintenance/observability/monitor-metrics.md b/website/docs/maintenance/observability/monitor-metrics.md index 74f3c510c63..4bb313e61bc 100644 --- a/website/docs/maintenance/observability/monitor-metrics.md +++ b/website/docs/maintenance/observability/monitor-metrics.md @@ -463,8 +463,8 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM - tabletserver - - + tabletserver + - messagesInPerSecond The number of messages written per second to this server. Meter @@ -590,18 +590,24 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM Meter - historicalPartitionInflightRequests - The number of accepted historical partition requests that have not completed, labeled with operation. Historical lookup requests use operation="lookup". + historical + inflightRequests + The number of accepted historical requests that have not completed. Gauge - historicalLookupCachedTableCount + lookupCacheDiskSize + The current historical lookup cache footprint on local disk, in bytes. + Gauge + + + lookupCacheTableCount The number of table lookupers currently retained in the historical lookup cache. Gauge - historicalLookupCachedTableCapacityEvictions - The cumulative number of cached table lookupers evicted to free historical lookup cache capacity. + lookupCacheCapacityEvictions + The cumulative number of cached table lookupers evicted because the cache retains at most ten tables. Counter @@ -784,7 +790,7 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM tabletserver - table + table messagesInPerSecond The number of messages written per second to this table. Meter @@ -839,26 +845,6 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM The number of failed lookup requests to lookup value by key from this table per second. Meter - - totalHistoricalLookupRequestsPerSecond - The number of historical lookup requests to this table per second. - Meter - - - failedHistoricalLookupRequestsPerSecond - The number of historical lookup requests that failed unexpectedly for this table per second. - Meter - - - lakeLookupsPerSecond - The number of historical lake point lookups performed for this table per second, labeled with lookup_file_materialization. - Meter - - - lakeLookupTimeMs - The time spent on a historical lake point lookup, in milliseconds, labeled with lookup_file_materialization. - Histogram - totalLimitScanRequestsPerSecond The number of limit scan requests to scan records with limit from this table per second. @@ -903,6 +889,27 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM remoteLogDeleteErrorPerSecond The number of failed delete remote log requests to delete remote log after log ttl per second. Meter + + + table_historical + totalLookupRequestsPerSecond + The number of historical lookup requests to this table per second. + Meter + + + failedLookupRequestsPerSecond + The number of historical lookup requests that failed unexpectedly for this table per second. + Meter + + + lakeLookupsPerSecond + The number of historical lake point lookups performed for this table per second, labeled with lookup_file_downloaded. + Meter + + + lakeLookupTimeMs + The time spent on a historical lake point lookup, in milliseconds, labeled with lookup_file_downloaded. + Histogram table_bucket_log @@ -978,8 +985,8 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM For lakeLookupsPerSecond and lakeLookupTimeMs, -lookup_file_materialization="true" means that the lookup created at least one local -lookup file; false means that it did not create a local lookup file. +lookup_file_downloaded="true" means that the lookup downloaded at least one local +lookup file; false means that it did not download a local lookup file. ### RocksDB From 47858146300059e733b5e78376f382b351e3e36f Mon Sep 17 00:00:00 2001 From: luoyuxia Date: Wed, 12 Aug 2026 19:13:32 +0800 Subject: [PATCH 15/15] [server] Refine historical lookup disk protection Check disk write protection only when Paimon downloads lookup cache files, preserve cached lookups under write lock, and simplify cache startup and removal lifecycle. Co-Authored-By: Codex AI-Model: gpt-5 AI-Contributed/Feature: 157/157 AI-Contributed/UT: 188/188 --- .../fluss/lake/lakestorage/LakeStorage.java | 13 ++- .../fluss/lake/paimon/PaimonLakeStorage.java | 3 +- .../lookup/PaimonLakeTableLookuper.java | 22 +++- .../lookup/PaimonLakeTableLookuperTest.java | 82 +++++++++++-- .../replica/HistoricalLakeLookupManager.java | 110 ++++++------------ .../fluss/server/replica/ReplicaManager.java | 3 +- .../server/storage/LocalDiskManager.java | 6 +- .../HistoricalLakeLookupManagerTest.java | 106 ++++++----------- 8 files changed, 181 insertions(+), 164 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java index 7087dd0e36a..446aa1184c4 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java @@ -73,6 +73,7 @@ final class LookuperContext { private final String ioTmpDir; private final TableConfig tableConfig; private final long lookupCacheMaxDiskBytes; + private final Runnable diskWriteGuard; /** * Creates a lookuper context. @@ -80,14 +81,19 @@ final class LookuperContext { * @param ioTmpDir local directory for temporary files used by the lookuper * @param tableConfig configuration of the Fluss table * @param lookupCacheMaxDiskBytes maximum local lookup cache size in bytes + * @param diskWriteGuard guard invoked before creating a local lookup cache file */ public LookuperContext( - String ioTmpDir, TableConfig tableConfig, long lookupCacheMaxDiskBytes) { + String ioTmpDir, + TableConfig tableConfig, + long lookupCacheMaxDiskBytes, + Runnable diskWriteGuard) { this.ioTmpDir = checkNotNull(ioTmpDir, "ioTmpDir must not be null."); this.tableConfig = checkNotNull(tableConfig, "tableConfig must not be null."); checkArgument( lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); this.lookupCacheMaxDiskBytes = lookupCacheMaxDiskBytes; + this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); } /** Returns the local directory for temporary files used by the lookuper. */ @@ -104,5 +110,10 @@ public TableConfig tableConfig() { public long lookupCacheMaxDiskBytes() { return lookupCacheMaxDiskBytes; } + + /** Returns the guard invoked before creating a local lookup cache file. */ + public Runnable diskWriteGuard() { + return diskWriteGuard; + } } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java index 0b7131a6555..80e66398985 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java @@ -61,6 +61,7 @@ public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, LookuperCo tablePath, context.ioTmpDir(), context.tableConfig(), - context.lookupCacheMaxDiskBytes()); + context.lookupCacheMaxDiskBytes(), + context.diskWriteGuard()); } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index 1d64eed809a..18afe2d982e 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -20,6 +20,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.DiskWriteLockedException; import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.paimon.utils.PaimonPartitionBucket; @@ -33,6 +34,7 @@ import org.apache.fluss.row.encode.ValueEncoder; import org.apache.fluss.row.encode.paimon.PaimonKeyEncoder; import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.ExceptionUtils; import org.apache.fluss.utils.IOUtils; import org.apache.paimon.CoreOptions; @@ -57,6 +59,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -95,6 +98,7 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private final String ioTmpDir; private final TableConfig tableConfig; private final long lookupCacheMaxDiskBytes; + private final Runnable diskWriteGuard; private final Set initializedBuckets; @@ -123,7 +127,8 @@ public PaimonLakeTableLookuper( TablePath tablePath, String ioTmpDir, TableConfig tableConfig, - long lookupCacheMaxDiskBytes) { + long lookupCacheMaxDiskBytes, + Runnable diskWriteGuard) { this.paimonConfig = checkNotNull(paimonConfig, "paimonConfig must not be null."); this.tablePath = checkNotNull(tablePath, "tablePath must not be null."); this.ioTmpDir = checkNotNull(ioTmpDir, "ioTmpDir must not be null."); @@ -131,6 +136,7 @@ public PaimonLakeTableLookuper( checkArgument( lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); this.lookupCacheMaxDiskBytes = lookupCacheMaxDiskBytes; + this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); this.initializedBuckets = new HashSet<>(); } @@ -154,6 +160,13 @@ public PaimonLakeTableLookuper( paimonRow = lookupWithFileRefresh( partition, context.bucketId(), keyRow, context.valueRowType()); + } catch (Exception e) { + DiskWriteLockedException diskWriteLockedException = + ExceptionUtils.findThrowable(e, DiskWriteLockedException.class).orElse(null); + if (diskWriteLockedException != null) { + throw diskWriteLockedException; + } + throw e; } finally { context.lookupMetricRecorder() .recordLookup( @@ -463,6 +476,13 @@ public FileIOChannel.ID createChannel() { @Override public FileIOChannel.ID createChannel(String prefix) { + try { + diskWriteGuard.run(); + } catch (DiskWriteLockedException e) { + // IOManager does not allow createChannel to declare IOException. Preserve the + // I/O boundary here and unwrap the retriable Fluss exception in lookup(). + throw new UncheckedIOException(new IOException(e)); + } lookupFileDownloadCount++; return delegate.createChannel(prefix); } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index 367ec01112f..1bf0ec3a16f 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -21,6 +21,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.DiskWriteLockedException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.lakestorage.TestingLakeCatalogContext; import org.apache.fluss.lake.paimon.PaimonLakeCatalog; @@ -63,6 +64,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; @@ -81,6 +83,7 @@ class PaimonLakeTableLookuperTest { private static final long LOOKUP_CACHE_MAX_DISK_BYTES = MemorySize.parse("8gb").getBytes(); private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = (lookupTimeNanos, lookupFileDownloaded) -> {}; + private static final Runnable NO_OP_DISK_WRITE_GUARD = () -> {}; @TempDir private File tempWarehouseDir; @@ -125,7 +128,8 @@ void testLookupPartitionedPrimaryKeyTable() throws Exception { tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED), - LOOKUP_CACHE_MAX_DISK_BYTES)) { + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { List lookupFileDownloads = new ArrayList<>(); LakeTableLookuper.LookupContext context = lookupContext( @@ -154,6 +158,58 @@ void testLookupPartitionedPrimaryKeyTable() throws Exception { } } + @Test + void testDiskWriteLockBlocksOnlyLookupFileDownloads() throws Exception { + TablePath tablePath = TablePath.of(DB, "disk_write_lock"); + Schema schema = pkSchema(); + FileStoreTable table = createPaimonTable(tablePath, partitionedPkDescriptor(schema)); + writeAndCommitData( + table, + Collections.singletonMap( + 0, + Arrays.asList( + paimonRow(1, "20240101", "Alice"), + paimonRow(2, "20240102", "Bob")))); + AtomicBoolean diskWriteLocked = new AtomicBoolean(); + Runnable diskWriteGuard = + () -> { + if (diskWriteLocked.get()) { + throw new DiskWriteLockedException("Data disk is write-locked."); + } + }; + + try (LakeTableLookuper lookuper = + new PaimonLakeTableLookuper( + paimonConfig, + tablePath, + tempWarehouseDir.getAbsolutePath(), + tableConfig(KvFormat.COMPACTED), + LOOKUP_CACHE_MAX_DISK_BYTES, + diskWriteGuard)) { + LakeTableLookuper.LookupContext cachedPartition = + lookupContext(schema, "20240101", 0, SCHEMA_ID); + LakeTableLookuper.LookupContext uncachedPartition = + lookupContext(schema, "20240102", 0, SCHEMA_ID); + + assertThat(lookuper.lookup(paimonKey(schema, 1, "20240101"), cachedPartition)) + .isNotNull(); + diskWriteLocked.set(true); + + // Cache hits remain available, while a lookup that needs a new local file is rejected. + assertThat(lookuper.lookup(paimonKey(schema, 1, "20240101"), cachedPartition)) + .isNotNull(); + assertThatThrownBy( + () -> + lookuper.lookup( + paimonKey(schema, 2, "20240102"), uncachedPartition)) + .isInstanceOf(DiskWriteLockedException.class); + + diskWriteLocked.set(false); + assertThat(lookuper.lookup(paimonKey(schema, 2, "20240102"), uncachedPartition)) + .isNotNull(); + } + } + @Test void testLookupPartitionsWithSameHashCode() throws Exception { // These distinct partition values produce the same BinaryRow hash code, reproducing the @@ -185,7 +241,8 @@ void testLookupPartitionsWithSameHashCode() throws Exception { tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED), - LOOKUP_CACHE_MAX_DISK_BYTES)) { + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { BinaryValue firstValue = decodeValue( lookuper.lookup( @@ -226,7 +283,8 @@ void testLookupWithIndexedKvFormat() throws Exception { tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.INDEXED), - LOOKUP_CACHE_MAX_DISK_BYTES)) { + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { LakeTableLookuper.LookupContext context = lookupContext(schema, "20240101", 0, SCHEMA_ID); @@ -271,7 +329,8 @@ void testLookupKvFormatV2WithNonDefaultBucketKey() throws Exception { tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED, KV_FORMAT_VERSION_2), - LOOKUP_CACHE_MAX_DISK_BYTES)) { + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { LakeTableLookuper.LookupContext context = lookupContext(schema, "20240101", 0, SCHEMA_ID); byte[] compactedKey = @@ -327,7 +386,8 @@ void testRetriesInitializationAfterLookupKeyConverterFailure() throws Exception tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED, KV_FORMAT_VERSION_2), - LOOKUP_CACHE_MAX_DISK_BYTES)) { + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { // Inject a late initialization failure: the Paimon table requires sub_id in its // lookup key, but the first lookup's value row type deliberately omits that field. assertThatThrownBy( @@ -370,7 +430,8 @@ void testRefreshFilesAfterCompactionAndSnapshotExpiration() throws Exception { tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED), - LOOKUP_CACHE_MAX_DISK_BYTES)) { + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { LakeTableLookuper.LookupContext context = lookupContext(schema, "20240101", 0, SCHEMA_ID); assertThat(lookuper.lookup(paimonKey(schema, 5, "20240101"), context)).isNotNull(); @@ -437,7 +498,8 @@ void testLookupWithNonStringPartitionKey() throws Exception { tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED), - LOOKUP_CACHE_MAX_DISK_BYTES)) { + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { LakeTableLookuper.LookupContext context = new LakeTableLookuper.LookupContext( ResolvedPartitionSpec.fromPartitionName( @@ -474,7 +536,8 @@ void testRejectAppendOnlyTable() throws Exception { tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED), - LOOKUP_CACHE_MAX_DISK_BYTES)) { + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { LakeTableLookuper.LookupContext context = new LakeTableLookuper.LookupContext( new ResolvedPartitionSpec( @@ -532,7 +595,8 @@ void testLookupAfterSchemaEvolutionPadsNewColumnsWithNull() throws Exception { tablePath, tempWarehouseDir.getAbsolutePath(), tableConfig(KvFormat.COMPACTED), - LOOKUP_CACHE_MAX_DISK_BYTES)) { + LOOKUP_CACHE_MAX_DISK_BYTES, + NO_OP_DISK_WRITE_GUARD)) { BinaryValue oldSchemaValue = decodeValue( lookuper.lookup( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java index 9a993eda045..977b608402b 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/HistoricalLakeLookupManager.java @@ -21,7 +21,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; -import org.apache.fluss.exception.DiskWriteLockedException; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.exception.InvalidPartitionException; @@ -58,7 +57,6 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; import java.io.File; import java.io.IOException; @@ -102,9 +100,9 @@ *

Up to ten table lookupers are cached. Each lookuper receives one tenth of the server-level * disk budget, and Caffeine evicts lookupers when the table limit is exceeded. * - *

Historical lookup cache I/O participates in TabletServer disk write protection. A lookup is - * rejected when the data disk is write-locked, and cached lookupers are invalidated so their local - * files are released. Lookups can create fresh cache files again after the disk recovers. + *

Historical lookup cache I/O participates in TabletServer disk write protection. Existing cache + * hits remain available when the data disk is write-locked, while lookups that need to download new + * cache files are rejected until the disk recovers. * *

A lookuper is closed when replaced, explicitly invalidated by a replica lifecycle event, * evicted when the table limit is exceeded, the manager shuts down, or after the configured idle @@ -141,14 +139,14 @@ class HistoricalLakeLookupManager implements AutoCloseable { private final ExecutorService historicalPartitionExecutor; private final File historicalLookupCacheRootDir; private final long dataDirVolumeBytes; + // TODO: Introduce a minimum lookup cache disk ratio (default 0.01). When disk usage is high, + // evict cached entries down to the minimum ratio instead of clearing the entire cache; allow + // the cache to grow back to the maximum ratio after disk usage recovers. private final Runnable diskWriteGuard; private volatile long lookupCacheMaxDiskBytesPerTable; private volatile long lookupCacheDiskSize; - @GuardedBy("this") - private boolean historicalLookupCacheRootDirCreated; - private volatile boolean started; /** Creates a historical lake lookup manager. */ @@ -221,22 +219,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { .ticker(checkNotNull(ticker, "ticker must not be null.")) .scheduler(checkNotNull(cacheScheduler, "cacheScheduler must not be null.")) .executor(Runnable::run) - .removalListener( - (Long ignored, - CachedLakeTableLookuper cachedLookuper, - RemovalCause cause) -> { - if (cachedLookuper != null) { - if (cause == RemovalCause.SIZE) { - capacityEvictions.inc(); - LOG.info( - "Evicted historical lookup cache for table {} (table ID {}) because the cache retains at most {} tables.", - cachedLookuper.tablePath, - cachedLookuper.tableId, - MAX_CACHED_TABLES); - } - onLookuperRemoved(cachedLookuper); - } - }) + .removalListener(this::onLookuperRemoved) .build(); this.lookupPermits = new Semaphore(maxQueuedHistoricalRequests); this.pendingLookups = ConcurrentHashMap.newKeySet(); @@ -257,10 +240,11 @@ private static com.github.benmanes.caffeine.cache.Scheduler createCacheScheduler /** * Attempts to clean lookup cache files left by a previous TabletServer process. * - *

The cache root under this server's first data directory is removed. It is recreated lazily - * when the first table lookuper is created. + *

The cache root under this server's first data directory is removed and recreated before + * lookups are accepted. */ - synchronized void startup() { + synchronized void startup(Scheduler scheduler) { + checkNotNull(scheduler, "scheduler must not be null."); if (started) { return; } @@ -272,17 +256,20 @@ synchronized void startup() { historicalLookupCacheRootDir, e); } - started = true; - } - - /** Starts periodic sampling of the historical lookup cache footprint. */ - void startLookupCacheDiskSizeMonitor(Scheduler scheduler) { - checkNotNull(scheduler, "scheduler must not be null."); + try { + Files.createDirectories(historicalLookupCacheRootDir.toPath()); + } catch (IOException e) { + throw new FlussRuntimeException( + "Failed to create historical lookup cache directory: " + + historicalLookupCacheRootDir, + e); + } scheduler.schedule( LOOKUP_CACHE_DISK_SIZE_TASK_NAME, this::updateLookupCacheDiskSize, 0L, LOOKUP_CACHE_DISK_SIZE_CHECK_INTERVAL.toMillis()); + started = true; } /** Looks up a batch of keys from one historical lake partition. */ @@ -293,16 +280,6 @@ CompletableFuture lookup( LakeTableLookuper.LookupMetricRecorder lookupMetricRecorder) { checkState(started, "Historical lake lookup manager has not been started."); TableBucket tableBucket = lookupData.tableBucket(); - try { - ensureDiskWritable(); - } catch (DiskWriteLockedException e) { - return CompletableFuture.completedFuture( - new LookupResultForBucket( - tableBucket, - null, - lookupData.originalPartitionName(), - ApiError.fromThrowable(e))); - } if (!lookupPermits.tryAcquire()) { return CompletableFuture.completedFuture( new LookupResultForBucket( @@ -479,7 +456,7 @@ private LookupResultForBucket lookupInternal( != cacheSizeBytes) { File tableLookupDir = FlussPaths.historicalLookupTableDir( - getOrCreateHistoricalLookupCacheRootDir(), + historicalLookupCacheRootDir, context.tablePath, context.tableId); LakeTableLookuper lookuper = @@ -524,20 +501,19 @@ private LookupResultForBucket lookupInternal( } } - private void ensureDiskWritable() { - try { - diskWriteGuard.run(); - } catch (DiskWriteLockedException e) { - // Release local cache files after the disk is write-locked. Idle lookupers close - // immediately, while active lookupers close after their final lookup releases them. - // Run pending cache maintenance now so removal callbacks are processed promptly. - lakeTableLookupers.invalidateAll(); - lakeTableLookupers.cleanUp(); - throw e; + private void onLookuperRemoved( + Long ignored, @Nullable CachedLakeTableLookuper cachedLookuper, RemovalCause cause) { + if (cachedLookuper == null) { + return; + } + if (cause == RemovalCause.SIZE) { + capacityEvictions.inc(); + LOG.info( + "Evicted historical lookup cache for table {} (table ID {}) because the cache retains at most {} tables.", + cachedLookuper.tablePath, + cachedLookuper.tableId, + MAX_CACHED_TABLES); } - } - - private void onLookuperRemoved(CachedLakeTableLookuper cachedLookuper) { cachedLookuper.invalidate(); } @@ -607,7 +583,9 @@ LakeTableLookuper createLakeTableLookuper( LakeStorage lakeStorage = lakeStoragePlugin.createLakeStorage(Configuration.fromMap(lakeProperties)); return lakeStorage.createLakeTableLookuper( - tablePath, new LakeStorage.LookuperContext(ioTmpDir, tableConfig, cacheSizeBytes)); + tablePath, + new LakeStorage.LookuperContext( + ioTmpDir, tableConfig, cacheSizeBytes, diskWriteGuard)); } private static boolean hasLakeConfigChanged(Configuration currentConf, Configuration newConf) { @@ -617,22 +595,6 @@ private static boolean hasLakeConfigChanged(Configuration currentConf, Configura extractLakeProperties(currentConf), extractLakeProperties(newConf)); } - private synchronized File getOrCreateHistoricalLookupCacheRootDir() { - if (historicalLookupCacheRootDirCreated) { - return historicalLookupCacheRootDir; - } - try { - Files.createDirectories(historicalLookupCacheRootDir.toPath()); - historicalLookupCacheRootDirCreated = true; - } catch (IOException e) { - throw new FlussRuntimeException( - "Failed to create historical lookup cache directory: " - + historicalLookupCacheRootDir, - e); - } - return historicalLookupCacheRootDir; - } - private long cacheBytesPerTable(double ratio) { checkArgument(ratio > 0.0 && ratio <= 1.0, "ratio must be within (0.0, 1.0]."); long totalCacheBytes = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index ff3d1952ad0..75b0f2ed1c3 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -376,8 +376,7 @@ public ReplicaManager( } public void startup() { - historicalLakeLookupManager.startup(); - historicalLakeLookupManager.startLookupCacheDiskSizeMonitor(scheduler); + historicalLakeLookupManager.startup(scheduler); // start up ISR expiration thread. // A follower can log behind leader for up tp configOptions#LOG_REPLICA_MAX_LAG_TIME x 1.5 diff --git a/fluss-server/src/main/java/org/apache/fluss/server/storage/LocalDiskManager.java b/fluss-server/src/main/java/org/apache/fluss/server/storage/LocalDiskManager.java index b12e897ab62..e34283e4548 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/storage/LocalDiskManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/storage/LocalDiskManager.java @@ -512,9 +512,9 @@ public double getDiskWriteRecoverRatio() { /** * Throws {@link DiskWriteLockedException} when the local data disk usage has crossed the - * configured write-limit ratio. Only client-driven writes ({@code appendLog} / {@code putKv}) - * should call this; follower replication paths must bypass this check to preserve replica - * consistency. + * configured write-limit ratio. Client-driven writes ({@code appendLog} / {@code putKv}) and + * lookup cache file download paths should call this; follower replication paths must bypass + * this check to preserve replica consistency. */ public void ensureWritable() { if (diskWriteLocked) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java index 3c78834f8b2..afa53e5516b 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HistoricalLakeLookupManagerTest.java @@ -21,7 +21,6 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; -import org.apache.fluss.exception.DiskWriteLockedException; import org.apache.fluss.exception.HistoricalPartitionThrottledException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.metadata.DataLakeFormat; @@ -57,8 +56,8 @@ import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -75,6 +74,8 @@ class HistoricalLakeLookupManagerTest { private static final LakeTableLookuper.LookupMetricRecorder NO_OP_LOOKUP_METRIC_RECORDER = (lookupTimeNanos, lookupFileDownloaded) -> {}; private static final Runnable NO_OP_DISK_WRITE_GUARD = () -> {}; + private static final org.apache.fluss.utils.concurrent.Scheduler NO_OP_SCHEDULER = + new NoOpScheduler(); @TempDir private File ioTmpDir; @@ -214,7 +215,7 @@ void testRejectNonPositiveHistoricalPartitionThreadPoolMaxSize(int maxThreadPool } @Test - void testCleansLookupCacheDirectoryOnStartupAndCreatesItLazily() throws Exception { + void testCleansAndCreatesLookupCacheDirectoryOnStartup() throws Exception { File serverLookupDir = FlussPaths.historicalLookupRootDir(ioTmpDir); assertThat(serverLookupDir.mkdirs()).isTrue(); File staleLookupFile = new File(serverLookupDir, "stale-lookup-file"); @@ -225,16 +226,15 @@ void testCleansLookupCacheDirectoryOnStartupAndCreatesItLazily() throws Exceptio new TestingHistoricalLakeLookupManager(conf(1), executor); assertThat(staleLookupFile).exists(); - manager.startup(); + manager.startup(NO_OP_SCHEDULER); assertThat(staleLookupFile).doesNotExist(); - assertThat(serverLookupDir).doesNotExist(); - lookupAndRun(manager, executor, PARTITION_TABLE_INFO); assertThat(serverLookupDir).isDirectory(); + lookupAndRun(manager, executor, PARTITION_TABLE_INFO); assertThat(manager.createdIoTmpDirs.get(0)).startsWith(serverLookupDir.getAbsolutePath()); File liveLookupFile = new File(serverLookupDir, "live-lookup-file"); assertThat(liveLookupFile.createNewFile()).isTrue(); - manager.startup(); + manager.startup(NO_OP_SCHEDULER); assertThat(liveLookupFile).exists(); } @@ -364,7 +364,7 @@ void testDynamicallyUpdatesExpirationAndExpiresIdleLookuper() throws Exception { executor, tickerNanos::get, cacheScheduler); - manager.startup(); + manager.startup(NO_OP_SCHEDULER); lookupAndRun(manager, executor, PARTITION_TABLE_INFO); TestingLakeTableLookuper expiredLookuper = manager.createdLookupers.get(0); @@ -392,7 +392,7 @@ void testEvictsLookuperWhenCachedTableLimitIsExceeded() throws Exception { Scheduler.disabledScheduler(), 100, 0); - manager.startup(); + manager.startup(NO_OP_SCHEDULER); for (int i = 0; i < 11; i++) { lookupAndRun( @@ -408,48 +408,6 @@ void testEvictsLookuperWhenCachedTableLimitIsExceeded() throws Exception { assertThat(manager.capacityEvictions().getCount()).isEqualTo(1); } - @Test - void testRejectsLookupsAndClearsCacheWhenDiskWriteLocked() throws Exception { - ManualExecutor executor = new ManualExecutor(); - AtomicBoolean diskWriteLocked = new AtomicBoolean(); - Runnable diskWriteGuard = - () -> { - if (diskWriteLocked.get()) { - throw new DiskWriteLockedException("Data disk is write-locked."); - } - }; - TestingHistoricalLakeLookupManager manager = - new TestingHistoricalLakeLookupManager( - conf(1), - executor, - Ticker.systemTicker(), - Scheduler.disabledScheduler(), - 100, - 6, - diskWriteGuard); - manager.startup(); - - lookupAndRun(manager, executor, PARTITION_TABLE_INFO); - TestingLakeTableLookuper cachedLookuper = manager.createdLookupers.get(0); - assertThat(cachedLookuper.cacheFile).exists().hasSize(6); - - // A write lock rejects the read before it reaches the executor and releases the existing - // disk cache. Once the disk recovers, a later lookup can create a fresh lookuper. - diskWriteLocked.set(true); - LookupResultForBucket rejected = - lookup(manager, PARTITION_TABLE_INFO).get(1, TimeUnit.SECONDS); - assertThat(rejected.getError().error()).isEqualTo(Errors.DISK_WRITE_LOCKED); - assertThat(executor.numQueuedTasks()).isZero(); - assertThat(manager.numInflightRequests()).isZero(); - assertThat(manager.cachedTableCount()).isZero(); - assertThat(cachedLookuper.cacheFile).doesNotExist(); - assertThat(cachedLookuper.closed).isTrue(); - - diskWriteLocked.set(false); - lookupAndRun(manager, executor, PARTITION_TABLE_INFO); - assertThat(manager.createdLookupers).hasSize(2); - } - @Test void testReconfiguresLakePropertiesAndInvalidatesLookuper() throws Exception { Configuration initialConf = conf(1); @@ -458,7 +416,7 @@ void testReconfiguresLakePropertiesAndInvalidatesLookuper() throws Exception { ManualExecutor executor = new ManualExecutor(); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(initialConf, executor); - manager.startup(); + manager.startup(NO_OP_SCHEDULER); lookupAndRun(manager, executor, PARTITION_TABLE_INFO); TestingLakeTableLookuper initialLookuper = manager.createdLookupers.get(0); @@ -487,14 +445,14 @@ private HistoricalLakeLookupManager createManager( Ticker.systemTicker(), Scheduler.disabledScheduler(), NO_OP_DISK_WRITE_GUARD); - manager.startup(); + manager.startup(NO_OP_SCHEDULER); return manager; } private TestingHistoricalLakeLookupManager createTestingManager(ManualExecutor executor) { TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(conf(1), executor); - manager.startup(); + manager.startup(NO_OP_SCHEDULER); return manager; } @@ -626,24 +584,6 @@ private TestingHistoricalLakeLookupManager( Scheduler cacheScheduler, long dataDirVolumeBytes, long lookupCacheFileBytes) { - this( - conf, - executor, - ticker, - cacheScheduler, - dataDirVolumeBytes, - lookupCacheFileBytes, - () -> {}); - } - - private TestingHistoricalLakeLookupManager( - Configuration conf, - ManualExecutor executor, - Ticker ticker, - Scheduler cacheScheduler, - long dataDirVolumeBytes, - long lookupCacheFileBytes, - Runnable diskWriteGuard) { super( conf, null, @@ -652,7 +592,7 @@ private TestingHistoricalLakeLookupManager( dataDirVolumeBytes, ticker, cacheScheduler, - diskWriteGuard); + NO_OP_DISK_WRITE_GUARD); this.lookupCacheFileBytes = lookupCacheFileBytes; } @@ -712,6 +652,26 @@ public void close() throws Exception { } } + private static final class NoOpScheduler + implements org.apache.fluss.utils.concurrent.Scheduler { + + @Override + public void startup() { + // no-op + } + + @Override + public void shutdown() { + // no-op + } + + @Override + public ScheduledFuture schedule( + String name, Runnable task, long delayMs, long periodMs) { + return null; + } + } + private static final class ManualExecutor extends AbstractExecutorService { private final BlockingQueue tasks = new LinkedBlockingQueue<>(); private volatile boolean shutdown;