diff --git a/src/common/settings.json b/src/common/settings.json index 945acff346e3..e9e1f0cf73b8 100644 --- a/src/common/settings.json +++ b/src/common/settings.json @@ -182,6 +182,13 @@ "#if defined(DUCKDB_EXTENSION_AUTOLOAD_DEFAULT) && DUCKDB_EXTENSION_AUTOLOAD_DEFAULT": "true" } }, + { + "name": "background_queue_purge", + "description": "Run eviction queue purge in a background thread instead of inline.", + "type": "BOOLEAN", + "scope": "global", + "default_value": "false" + }, { "name": "block_allocator_memory", "description": "Physical memory that the block allocator is allowed to use (this memory is never freed and cannot be reduced).", diff --git a/src/include/duckdb/main/settings.hpp b/src/include/duckdb/main/settings.hpp index eaac31ed9888..2e6175dd280e 100644 --- a/src/include/duckdb/main/settings.hpp +++ b/src/include/duckdb/main/settings.hpp @@ -349,6 +349,16 @@ struct AutoloadKnownExtensionsSetting { static constexpr idx_t SettingIndex = 14; }; +struct BackgroundQueuePurgeSetting { + using RETURN_TYPE = bool; + static constexpr const char *Name = "background_queue_purge"; + static constexpr const char *Description = "Run eviction queue purge in a background thread instead of inline."; + static constexpr const char *InputType = "BOOLEAN"; + static constexpr const char *DefaultValue = "false"; + static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; + static constexpr idx_t SettingIndex = 15; +}; + struct BlockAllocatorMemorySetting { using RETURN_TYPE = string; static constexpr const char *Name = "block_allocator_memory"; @@ -368,7 +378,7 @@ struct CatalogErrorMaxSchemasSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "100"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 15; + static constexpr idx_t SettingIndex = 16; }; struct CheckpointOnDetachSetting { @@ -380,7 +390,7 @@ struct CheckpointOnDetachSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "DEFAULT"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 16; + static constexpr idx_t SettingIndex = 17; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -403,7 +413,7 @@ struct CurrentTransactionInvalidationPolicySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "STANDARD_POLICY"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 17; + static constexpr idx_t SettingIndex = 18; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -414,7 +424,7 @@ struct CustomExtensionRepositorySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 18; + static constexpr idx_t SettingIndex = 19; }; struct CustomProfilingSettingsSetting { @@ -444,7 +454,7 @@ struct DebugAsofIejoinSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 19; + static constexpr idx_t SettingIndex = 20; }; struct DebugCheckpointAbortSetting { @@ -455,7 +465,7 @@ struct DebugCheckpointAbortSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "NONE"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 20; + static constexpr idx_t SettingIndex = 21; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -466,7 +476,7 @@ struct DebugCheckpointSleepMsSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "0"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 21; + static constexpr idx_t SettingIndex = 22; }; struct DebugEvictionQueueSleepMicroSecondsSetting { @@ -477,7 +487,7 @@ struct DebugEvictionQueueSleepMicroSecondsSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "0"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 22; + static constexpr idx_t SettingIndex = 23; }; struct DebugForceExternalSetting { @@ -499,7 +509,7 @@ struct DebugForceNoCrossProductSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 23; + static constexpr idx_t SettingIndex = 24; }; struct DebugPhysicalTableScanExecutionStrategySetting { @@ -510,7 +520,7 @@ struct DebugPhysicalTableScanExecutionStrategySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "DEFAULT"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 24; + static constexpr idx_t SettingIndex = 25; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -521,7 +531,7 @@ struct DebugSkipCheckpointOnCommitSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 25; + static constexpr idx_t SettingIndex = 26; }; struct DebugVerifyBlocksSetting { @@ -531,7 +541,7 @@ struct DebugVerifyBlocksSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 26; + static constexpr idx_t SettingIndex = 27; }; struct DebugVerifyVectorSetting { @@ -541,7 +551,7 @@ struct DebugVerifyVectorSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "NONE"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 27; + static constexpr idx_t SettingIndex = 28; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -552,7 +562,7 @@ struct DebugWindowModeSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "WINDOW"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 28; + static constexpr idx_t SettingIndex = 29; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -564,7 +574,7 @@ struct DefaultBlockSizeSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "262144"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 29; + static constexpr idx_t SettingIndex = 30; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -575,7 +585,7 @@ struct DefaultCollationSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 30; + static constexpr idx_t SettingIndex = 31; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -586,7 +596,7 @@ struct DefaultNullOrderSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "NULLS_LAST"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 31; + static constexpr idx_t SettingIndex = 32; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -597,7 +607,7 @@ struct DefaultOrderSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "ASCENDING"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 32; + static constexpr idx_t SettingIndex = 33; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -618,7 +628,7 @@ struct DeprecatedUsingKeySyntaxSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "DEFAULT"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 33; + static constexpr idx_t SettingIndex = 34; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -631,7 +641,7 @@ struct DisableDatabaseInvalidationSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 34; + static constexpr idx_t SettingIndex = 35; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -642,7 +652,7 @@ struct DisableTimestamptzCastsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 35; + static constexpr idx_t SettingIndex = 36; }; struct DisabledCompressionMethodsSetting { @@ -692,7 +702,7 @@ struct DuckDBAPISetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 36; + static constexpr idx_t SettingIndex = 37; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -704,7 +714,7 @@ struct DynamicOrFilterThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "50"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 37; + static constexpr idx_t SettingIndex = 38; }; struct EnableCachingOperatorsSetting { @@ -726,7 +736,7 @@ struct EnableExternalAccessSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 38; + static constexpr idx_t SettingIndex = 39; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -737,7 +747,7 @@ struct EnableExternalFileCacheSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 39; + static constexpr idx_t SettingIndex = 40; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -749,7 +759,7 @@ struct EnableFSSTVectorsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 40; + static constexpr idx_t SettingIndex = 41; }; struct EnableHTTPLoggingSetting { @@ -769,7 +779,7 @@ struct EnableHTTPMetadataCacheSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 41; + static constexpr idx_t SettingIndex = 42; }; struct EnableLogging { @@ -790,7 +800,7 @@ struct EnableMacroDependenciesSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 42; + static constexpr idx_t SettingIndex = 43; }; struct EnableObjectCacheSetting { @@ -800,7 +810,7 @@ struct EnableObjectCacheSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 43; + static constexpr idx_t SettingIndex = 44; }; struct EnableProfilingSetting { @@ -846,7 +856,7 @@ struct EnableViewDependenciesSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 44; + static constexpr idx_t SettingIndex = 45; }; struct EnabledLogTypes { @@ -866,7 +876,7 @@ struct ErrorsAsJSONSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 45; + static constexpr idx_t SettingIndex = 46; }; struct ExperimentalMetadataReuseSetting { @@ -876,7 +886,7 @@ struct ExperimentalMetadataReuseSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 46; + static constexpr idx_t SettingIndex = 47; }; struct ExplainOutputSetting { @@ -886,7 +896,7 @@ struct ExplainOutputSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "PHYSICAL_ONLY"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 47; + static constexpr idx_t SettingIndex = 48; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -907,7 +917,7 @@ struct ExtensionDirectorySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 48; + static constexpr idx_t SettingIndex = 49; }; struct ExternalThreadsSetting { @@ -917,7 +927,7 @@ struct ExternalThreadsSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "1"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 49; + static constexpr idx_t SettingIndex = 50; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -928,7 +938,7 @@ struct FileSearchPathSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 50; + static constexpr idx_t SettingIndex = 51; }; struct ForceBitpackingModeSetting { @@ -938,7 +948,7 @@ struct ForceBitpackingModeSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "AUTO"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 51; + static constexpr idx_t SettingIndex = 52; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -949,7 +959,7 @@ struct ForceCompressionSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "auto"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 52; + static constexpr idx_t SettingIndex = 53; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -982,7 +992,7 @@ struct GeometryMinimumShreddingSize { static constexpr const char *InputType = "BIGINT"; static constexpr const char *DefaultValue = "30000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 53; + static constexpr idx_t SettingIndex = 54; }; struct HomeDirectorySetting { @@ -992,7 +1002,7 @@ struct HomeDirectorySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 54; + static constexpr idx_t SettingIndex = 55; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1014,7 +1024,7 @@ struct HTTPProxySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 55; + static constexpr idx_t SettingIndex = 56; }; struct HTTPProxyPasswordSetting { @@ -1024,7 +1034,7 @@ struct HTTPProxyPasswordSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 56; + static constexpr idx_t SettingIndex = 57; }; struct HTTPProxyUsernameSetting { @@ -1034,7 +1044,7 @@ struct HTTPProxyUsernameSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 57; + static constexpr idx_t SettingIndex = 58; }; struct IeeeFloatingPointOpsSetting { @@ -1045,7 +1055,7 @@ struct IeeeFloatingPointOpsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 58; + static constexpr idx_t SettingIndex = 59; }; struct IgnoreUnknownCrsSetting { @@ -1056,7 +1066,7 @@ struct IgnoreUnknownCrsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 59; + static constexpr idx_t SettingIndex = 60; }; struct ImmediateTransactionModeSetting { @@ -1067,7 +1077,7 @@ struct ImmediateTransactionModeSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 60; + static constexpr idx_t SettingIndex = 61; }; struct IndexScanMaxCountSetting { @@ -1079,7 +1089,7 @@ struct IndexScanMaxCountSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "2048"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 61; + static constexpr idx_t SettingIndex = 62; }; struct IndexScanPercentageSetting { @@ -1091,7 +1101,7 @@ struct IndexScanPercentageSetting { static constexpr const char *InputType = "DOUBLE"; static constexpr const char *DefaultValue = "0.001"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 62; + static constexpr idx_t SettingIndex = 63; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1105,7 +1115,7 @@ struct InitialColumnSegmentSizeSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "2048"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 63; + static constexpr idx_t SettingIndex = 64; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1117,7 +1127,7 @@ struct IntegerDivisionSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 64; + static constexpr idx_t SettingIndex = 65; }; struct LambdaSyntaxSetting { @@ -1128,7 +1138,7 @@ struct LambdaSyntaxSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "DEFAULT"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 65; + static constexpr idx_t SettingIndex = 66; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1140,7 +1150,7 @@ struct LateMaterializationMaxRowsSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "50"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 66; + static constexpr idx_t SettingIndex = 67; }; struct LockConfigurationSetting { @@ -1150,7 +1160,7 @@ struct LockConfigurationSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 67; + static constexpr idx_t SettingIndex = 68; }; struct LogQueryPathSetting { @@ -1161,7 +1171,7 @@ struct LogQueryPathSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 68; + static constexpr idx_t SettingIndex = 69; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1204,7 +1214,7 @@ struct MaxExpressionDepthSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "1000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 69; + static constexpr idx_t SettingIndex = 70; }; struct MaxMemorySetting { @@ -1235,7 +1245,7 @@ struct MaxVacuumTasksSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "100"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 70; + static constexpr idx_t SettingIndex = 71; }; struct MergeJoinThresholdSetting { @@ -1245,7 +1255,7 @@ struct MergeJoinThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "1000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 71; + static constexpr idx_t SettingIndex = 72; }; struct NestedLoopJoinThresholdSetting { @@ -1256,7 +1266,7 @@ struct NestedLoopJoinThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "5"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 72; + static constexpr idx_t SettingIndex = 73; }; struct OldImplicitCastingSetting { @@ -1266,7 +1276,7 @@ struct OldImplicitCastingSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 73; + static constexpr idx_t SettingIndex = 74; }; struct OrderByNonIntegerLiteralSetting { @@ -1277,7 +1287,7 @@ struct OrderByNonIntegerLiteralSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 74; + static constexpr idx_t SettingIndex = 75; }; struct OrderedAggregateThresholdSetting { @@ -1287,7 +1297,7 @@ struct OrderedAggregateThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "262144"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 75; + static constexpr idx_t SettingIndex = 76; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1299,7 +1309,7 @@ struct PartitionedWriteFlushThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "524288"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 76; + static constexpr idx_t SettingIndex = 77; }; struct PartitionedWriteMaxOpenFilesSetting { @@ -1310,7 +1320,7 @@ struct PartitionedWriteMaxOpenFilesSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "100"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 77; + static constexpr idx_t SettingIndex = 78; }; struct PasswordSetting { @@ -1320,7 +1330,7 @@ struct PasswordSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 78; + static constexpr idx_t SettingIndex = 79; }; struct PerfectHtThresholdSetting { @@ -1330,7 +1340,7 @@ struct PerfectHtThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "12"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 79; + static constexpr idx_t SettingIndex = 80; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1342,7 +1352,7 @@ struct PinThreadsSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "auto"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 80; + static constexpr idx_t SettingIndex = 81; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1354,7 +1364,7 @@ struct PivotFilterThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "20"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 81; + static constexpr idx_t SettingIndex = 82; }; struct PivotLimitSetting { @@ -1364,7 +1374,7 @@ struct PivotLimitSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "100000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 82; + static constexpr idx_t SettingIndex = 83; }; struct PreferRangeJoinsSetting { @@ -1374,7 +1384,7 @@ struct PreferRangeJoinsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 83; + static constexpr idx_t SettingIndex = 84; }; struct PreserveIdentifierCaseSetting { @@ -1385,7 +1395,7 @@ struct PreserveIdentifierCaseSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 84; + static constexpr idx_t SettingIndex = 85; }; struct PreserveInsertionOrderSetting { @@ -1397,7 +1407,7 @@ struct PreserveInsertionOrderSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 85; + static constexpr idx_t SettingIndex = 86; }; struct ProduceArrowStringViewSetting { @@ -1408,7 +1418,7 @@ struct ProduceArrowStringViewSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 86; + static constexpr idx_t SettingIndex = 87; }; struct ProfileOutputSetting { @@ -1461,7 +1471,7 @@ struct ScalarSubqueryErrorOnMultipleRowsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 87; + static constexpr idx_t SettingIndex = 88; }; struct SchedulerProcessPartialSetting { @@ -1476,7 +1486,7 @@ struct SchedulerProcessPartialSetting { static constexpr const char *DefaultValue = "false"; #endif static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 88; + static constexpr idx_t SettingIndex = 89; }; struct SchemaSetting { @@ -1518,7 +1528,7 @@ struct StorageBlockPrefetchSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "REMOTE_ONLY"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 89; + static constexpr idx_t SettingIndex = 90; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1560,7 +1570,7 @@ struct TempFileEncryptionSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 90; + static constexpr idx_t SettingIndex = 91; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1581,7 +1591,7 @@ struct UsernameSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 91; + static constexpr idx_t SettingIndex = 92; }; struct VacuumRebuildIndexesSetting { @@ -1593,7 +1603,7 @@ struct VacuumRebuildIndexesSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "0"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 92; + static constexpr idx_t SettingIndex = 93; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1606,7 +1616,7 @@ struct ValidateExternalFileCacheSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "VALIDATE_ALL"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 93; + static constexpr idx_t SettingIndex = 94; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1618,7 +1628,7 @@ struct VariantMinimumShreddingSizeSetting { static constexpr const char *InputType = "BIGINT"; static constexpr const char *DefaultValue = "30000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 94; + static constexpr idx_t SettingIndex = 95; }; struct WalAutocheckpointEntriesSetting { @@ -1629,7 +1639,7 @@ struct WalAutocheckpointEntriesSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "0"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 95; + static constexpr idx_t SettingIndex = 96; }; struct WarningsAsErrorsSetting { @@ -1639,7 +1649,7 @@ struct WarningsAsErrorsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 96; + static constexpr idx_t SettingIndex = 97; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1651,7 +1661,7 @@ struct WriteBufferRowGroupCountSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "5"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 97; + static constexpr idx_t SettingIndex = 98; }; struct ZstdMinStringLengthSetting { @@ -1662,11 +1672,11 @@ struct ZstdMinStringLengthSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "4096"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 98; + static constexpr idx_t SettingIndex = 99; }; struct GeneratedSettingInfo { - static constexpr idx_t MaxSettingIndex = 99; + static constexpr idx_t MaxSettingIndex = 100; }; //===----------------------------------------------------------------------===// diff --git a/src/include/duckdb/storage/buffer/buffer_pool.hpp b/src/include/duckdb/storage/buffer/buffer_pool.hpp index f407f9b592bb..8c174d3d91fc 100644 --- a/src/include/duckdb/storage/buffer/buffer_pool.hpp +++ b/src/include/duckdb/storage/buffer/buffer_pool.hpp @@ -34,6 +34,7 @@ struct BufferEvictionNode { bool CanUnload(BlockMemory &memory); shared_ptr TryGetBlockMemory(); + bool IsDeadNode(idx_t debug_sleep_micros = 0); }; //! The BufferPool is in charge of handling memory management for one or more databases. It defines memory limits diff --git a/src/include/duckdb/storage/table/update_segment.hpp b/src/include/duckdb/storage/table/update_segment.hpp index 295b67c8670b..f6fa86c92867 100644 --- a/src/include/duckdb/storage/table/update_segment.hpp +++ b/src/include/duckdb/storage/table/update_segment.hpp @@ -100,6 +100,7 @@ class UpdateSegment { void InitializeUpdateInfo(idx_t vector_idx); void InitializeUpdateInfo(UpdateInfo &info, row_t *ids, const SelectionVector &sel, idx_t count, idx_t vector_index, idx_t vector_offset); + void ReallocateRootInfoIfNeeded(UpdateInfo ¤t_info, idx_t update_count, idx_t vector_index); }; struct UpdateNode { diff --git a/src/include/duckdb/transaction/update_info.hpp b/src/include/duckdb/transaction/update_info.hpp index b62c066f7d23..5d2eaad9011a 100644 --- a/src/include/duckdb/transaction/update_info.hpp +++ b/src/include/duckdb/transaction/update_info.hpp @@ -93,11 +93,18 @@ struct UpdateInfo { bool HasPrev() const; bool HasNext() const; static UpdateInfo &Get(UndoBufferReference &entry); - //! Returns the total allocation size for an UpdateInfo entry, together with space for the tuple data + //! Returns the total allocation size for an UpdateInfo entry with max capacity (STANDARD_VECTOR_SIZE) static idx_t GetAllocSize(idx_t type_size); + //! Returns the total allocation size for an UpdateInfo entry with a specific capacity + static idx_t GetAllocSize(idx_t type_size, idx_t capacity); + //! Computes a compact capacity for a given count (rounds up with growth headroom) + static idx_t GetCompactCapacity(idx_t count); //! Initialize an UpdateInfo struct that has been allocated using GetAllocSize (i.e. has extra space after it) static void Initialize(UpdateInfo &info, DataTable &data_table, transaction_t transaction_id, idx_t row_group_start); + //! Initialize with a specific capacity (for compact allocations) + static void Initialize(UpdateInfo &info, DataTable &data_table, transaction_t transaction_id, + idx_t row_group_start, idx_t capacity); }; } // namespace duckdb diff --git a/src/main/config.cpp b/src/main/config.cpp index 08d6585dd2f5..8168beea7d7c 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -88,6 +88,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(AutoinstallExtensionRepositorySetting), DUCKDB_SETTING(AutoinstallKnownExtensionsSetting), DUCKDB_SETTING(AutoloadKnownExtensionsSetting), + DUCKDB_SETTING(BackgroundQueuePurgeSetting), DUCKDB_GLOBAL(BlockAllocatorMemorySetting), DUCKDB_SETTING(CatalogErrorMaxSchemasSetting), DUCKDB_SETTING_CALLBACK(CheckpointOnDetachSetting), @@ -211,12 +212,12 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(ZstdMinStringLengthSetting), FINAL_SETTING}; -static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 102), - DUCKDB_SETTING_ALIAS("null_order", 44), - DUCKDB_SETTING_ALIAS("profiling_output", 121), - DUCKDB_SETTING_ALIAS("user", 136), - DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 26), - DUCKDB_SETTING_ALIAS("worker_threads", 135), +static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 103), + DUCKDB_SETTING_ALIAS("null_order", 45), + DUCKDB_SETTING_ALIAS("profiling_output", 122), + DUCKDB_SETTING_ALIAS("user", 137), + DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 27), + DUCKDB_SETTING_ALIAS("worker_threads", 136), FINAL_ALIAS}; vector DBConfig::GetOptions() { diff --git a/src/storage/buffer/block_handle.cpp b/src/storage/buffer/block_handle.cpp index a652940a65dc..af10a5481330 100644 --- a/src/storage/buffer/block_handle.cpp +++ b/src/storage/buffer/block_handle.cpp @@ -34,8 +34,10 @@ BlockMemory::~BlockMemory() { // NOLINT: allow internal exceptions // The block memory is being destroyed, meaning that any unswizzled pointers are now binary junk. SetSwizzling(nullptr); D_ASSERT(!GetBuffer() || GetBuffer()->GetBufferType() == GetBufferType()); - if (GetBuffer() && GetBufferType() != FileBufferType::TINY_BUFFER) { - // Kill the latest version in the eviction queue. + if (GetEvictionSequenceNumber() > 0 && GetBufferType() != FileBufferType::TINY_BUFFER) { + // The eviction_seq_num is >0 only when there is an active entry in the queue for this + // block (it's reset to 0 on Unload/eviction). When this BlockMemory is destroyed with + // seq_num >0, that queue entry's weak_ptr will expire — mark it dead. GetBufferManager().GetBufferPool().IncrementDeadNodes(*this); } diff --git a/src/storage/buffer/buffer_pool.cpp b/src/storage/buffer/buffer_pool.cpp index d2a6fab47d70..a29c09bc68f6 100644 --- a/src/storage/buffer/buffer_pool.cpp +++ b/src/storage/buffer/buffer_pool.cpp @@ -66,13 +66,27 @@ shared_ptr BufferEvictionNode::TryGetBlockMemory() { return shared_memory_p; } +bool BufferEvictionNode::IsDeadNode(idx_t debug_sleep_micros) { + auto shared_memory_p = memory_p.lock(); + if (debug_sleep_micros > 0) { + ThreadUtil::SleepMicroSeconds(debug_sleep_micros); + } + if (!shared_memory_p) { + return true; + } + if (handle_sequence_number != shared_memory_p->GetEvictionSequenceNumber()) { + return true; + } + return false; +} + typedef duckdb_moodycamel::ConcurrentQueue eviction_queue_t; struct EvictionQueue { public: explicit EvictionQueue(const vector &file_buffer_types_p) : file_buffer_types(file_buffer_types_p), debug_eviction_queue_sleep(0), evict_queue_insertions(0), - total_dead_nodes(0) { + total_dead_nodes(0), purge_consumer_token(q), purge_producer_token(q) { } public: @@ -82,7 +96,8 @@ struct EvictionQueue { //! Tries to dequeue an element from the eviction queue, but only after acquiring the purge queue lock. bool TryDequeueWithLock(BufferEvictionNode &node); //! Garbage collect dead nodes in the eviction queue. - void Purge(); + //! When full_purge is true, skip early-out conditions and process the entire queue. + void Purge(bool full_purge = false); template void IterateUnloadableBlocks(FN fn); @@ -108,8 +123,9 @@ struct EvictionQueue { } private: - //! Bulk purge dead nodes from the eviction queue. Then, enqueue those that are still alive. - void PurgeIteration(const idx_t purge_size); + //! Dequeue a batch via consumer token, drop dead nodes, re-enqueue alive nodes + //! via producer token. + void PurgeIterationWithTokens(const idx_t purge_size); public: //! The type of the buffers in this queue and helper function (both for verification only) @@ -143,6 +159,15 @@ struct EvictionQueue { mutex purge_lock; //! A pre-allocated vector of eviction nodes. We reuse this to keep the allocation overhead of purges small. vector purge_nodes; + //! Consumer token for full_purge: ensures forward progress through sub-queues. + eviction_queue_t::consumer_token_t purge_consumer_token; + //! Producer token for full_purge: alive nodes go into a dedicated sub-queue + //! that the consumer has already passed, avoiding re-processing. + eviction_queue_t::producer_token_t purge_producer_token; + +public: + //! Whether a background purge thread is currently running for this queue. + atomic purge_in_flight {false}; }; bool EvictionQueue::AddToEvictionQueue(BufferEvictionNode &&node) { @@ -155,7 +180,7 @@ bool EvictionQueue::TryDequeueWithLock(BufferEvictionNode &node) { return q.try_dequeue(node); } -void EvictionQueue::Purge() { +void EvictionQueue::Purge(bool full_purge) { // only one thread purges the queue, all other threads early-out unique_lock guard(purge_lock, std::try_to_lock); if (!guard.owns_lock()) { @@ -174,78 +199,72 @@ void EvictionQueue::Purge() { return; } - // There are two types of situations. - - // For most scenarios, purging INSERT_INTERVAL * PURGE_SIZE_MULTIPLIER nodes is enough. - // Purging more nodes than we insert also counters oscillation for scenarios where most nodes are dead. - // If we always purge slightly more, we trigger a purge less often, as we purge below the trigger. - - // However, if the pressure on the queue becomes too contested, we need to purge more aggressively, - // i.e., we actively seek a specific number of dead nodes to purge. We use the total number of existing dead nodes. - // We detect this situation by observing the queue's ratio between alive vs. dead nodes. If the ratio of alive vs. - // dead nodes grows faster than we can purge, we keep purging until we hit one of the following conditions. - - // 2.1. We're back at an approximate queue size less than purge_size * EARLY_OUT_MULTIPLIER. - // 2.2. We're back at a ratio of 1*alive_node:ALIVE_NODE_MULTIPLIER*dead_nodes. - // 2.3. We've purged the entire queue: max_purges is zero. This is a worst-case scenario, - // guaranteeing that we always exit the loop. - idx_t max_purges = approx_q_size / purge_size; + + // Use consumer/producer tokens to avoid re-processing alive nodes. + // The consumer token progresses through sub-queues sequentially; alive nodes + // are re-enqueued via a dedicated producer token whose sub-queue is "behind" + // the consumer position, so they won't be revisited until a full wrap-around. while (max_purges != 0) { - PurgeIteration(purge_size); + PurgeIterationWithTokens(purge_size); - // update relevant sizes and potentially early-out - approx_q_size = q.size_approx(); + if (!full_purge) { + approx_q_size = q.size_approx(); - // early-out according to (2.1) - if (approx_q_size < purge_size * EARLY_OUT_MULTIPLIER) { - break; - } + // early-out according to (2.1) + if (approx_q_size < purge_size * EARLY_OUT_MULTIPLIER) { + break; + } - idx_t approx_dead_nodes = total_dead_nodes; - approx_dead_nodes = approx_dead_nodes > approx_q_size ? approx_q_size : approx_dead_nodes; - idx_t approx_alive_nodes = approx_q_size - approx_dead_nodes; + idx_t approx_dead_nodes = total_dead_nodes; + approx_dead_nodes = approx_dead_nodes > approx_q_size ? approx_q_size : approx_dead_nodes; + idx_t approx_alive_nodes = approx_q_size - approx_dead_nodes; - // early-out according to (2.2) - if (approx_alive_nodes * (ALIVE_NODE_MULTIPLIER - 1) > approx_dead_nodes) { - break; + // early-out according to (2.2) + if (approx_alive_nodes * (ALIVE_NODE_MULTIPLIER - 1) > approx_dead_nodes) { + break; + } } max_purges--; } } -void EvictionQueue::PurgeIteration(const idx_t purge_size) { - // if this purge is significantly smaller or bigger than the previous purge, then - // we need to resize the purge_nodes vector. Note that this barely happens, as we - // purge queue_insertions * PURGE_SIZE_MULTIPLIER nodes +void EvictionQueue::PurgeIterationWithTokens(const idx_t purge_size) { idx_t previous_purge_size = purge_nodes.size(); if (purge_size < previous_purge_size / 2 || purge_size > previous_purge_size) { purge_nodes.resize(purge_size); } - // bulk purge - const idx_t actually_dequeued = q.try_dequeue_bulk(purge_nodes.begin(), purge_size); + // Dequeue using consumer token — progresses through sub-queues sequentially + const idx_t actually_dequeued = q.try_dequeue_bulk(purge_consumer_token, purge_nodes.begin(), purge_size); + if (actually_dequeued == 0) { + return; + } - // retrieve all alive nodes that have been wrongly dequeued - idx_t alive_nodes = 0; + idx_t dead_count = 0; + idx_t alive_count = 0; auto debug_sleep_micros = debug_eviction_queue_sleep.load(std::memory_order_relaxed); for (idx_t i = 0; i < actually_dequeued; i++) { auto &node = purge_nodes[i]; - auto handle = node.TryGetBlockMemory(); - if (debug_sleep_micros > 0) { - // Debug race conditions regarding the ownership of the BlockMemory. - ThreadUtil::SleepMicroSeconds(debug_sleep_micros); - } - if (handle) { - purge_nodes[alive_nodes++] = std::move(node); + if (node.IsDeadNode(debug_sleep_micros)) { + dead_count++; + } else { + // Move alive nodes to the front for bulk re-enqueue + if (alive_count != i) { + purge_nodes[alive_count] = std::move(node); + } + alive_count++; } } - // bulk re-add (TODO order them by timestamp to better retain the LRU behavior) - q.enqueue_bulk(purge_nodes.begin(), alive_nodes); + // Re-enqueue alive nodes via producer token — goes into a dedicated sub-queue + // that the consumer token has already passed + if (alive_count > 0) { + q.enqueue_bulk(purge_producer_token, purge_nodes.begin(), alive_count); + } - total_dead_nodes -= actually_dequeued - alive_nodes; + total_dead_nodes -= dead_count; } BufferPool::BufferPool(BlockAllocator &block_allocator, idx_t maximum_memory, bool track_eviction_timestamps, @@ -500,7 +519,18 @@ void BufferPool::PurgeQueue(const BlockHandle &block) { const auto queue_sleep_micros = Settings::Get(buffer_manager.GetDatabase()); eviction_queue.debug_eviction_queue_sleep = queue_sleep_micros; - eviction_queue.Purge(); + if (Settings::Get(buffer_manager.GetDatabase())) { + bool expected = false; + if (eviction_queue.purge_in_flight.compare_exchange_strong(expected, true)) { + thread purge_thread([&eviction_queue]() { + eviction_queue.Purge(true); + eviction_queue.purge_in_flight.store(false); + }); + purge_thread.detach(); + } + } else { + eviction_queue.Purge(); + } } void BufferPool::SetLimit(idx_t limit, const char *exception_postscript) { diff --git a/src/storage/buffer/buffer_pool_reservation.cpp b/src/storage/buffer/buffer_pool_reservation.cpp index f9fbe209ac00..756ebb4baf04 100644 --- a/src/storage/buffer/buffer_pool_reservation.cpp +++ b/src/storage/buffer/buffer_pool_reservation.cpp @@ -13,6 +13,7 @@ BufferPoolReservation::BufferPoolReservation(BufferPoolReservation &&src) noexce } BufferPoolReservation &BufferPoolReservation::operator=(BufferPoolReservation &&src) noexcept { + pool.UpdateUsedMemory(tag, -UnsafeNumericCast(size)); tag = src.tag; size = src.size; src.size = 0; diff --git a/src/storage/table/column_data.cpp b/src/storage/table/column_data.cpp index 9c8460cd971c..b7db54b65e26 100644 --- a/src/storage/table/column_data.cpp +++ b/src/storage/table/column_data.cpp @@ -475,7 +475,11 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) { !last_segment.GetCompressionFunction().init_append) { // we cannot append to this segment - append a new segment auto total_rows = segment->GetRowStart() + last_segment.count; - AppendTransientSegment(l, total_rows, last_segment); + + // Persistent segments are resized to block_size during checkpoint, wo we pass nullptr to start fresh + const bool is_persistent = last_segment.segment_type == ColumnSegmentType::PERSISTENT; + AppendTransientSegment(l, total_rows, is_persistent ? nullptr : &last_segment); + state.current = data.GetLastSegment(l); } else { state.current = segment; diff --git a/src/storage/table/update_segment.cpp b/src/storage/table/update_segment.cpp index 7f0fefd2c495..a993984f1066 100644 --- a/src/storage/table/update_segment.cpp +++ b/src/storage/table/update_segment.cpp @@ -102,6 +102,30 @@ idx_t UpdateInfo::GetAllocSize(idx_t type_size) { return AlignValue(sizeof(UpdateInfo) + (sizeof(sel_t) + type_size) * STANDARD_VECTOR_SIZE); } +idx_t UpdateInfo::GetAllocSize(idx_t type_size, idx_t capacity) { + return AlignValue(sizeof(UpdateInfo) + (sizeof(sel_t) + type_size) * capacity); +} + +idx_t UpdateInfo::GetCompactCapacity(idx_t count) { + // Round up to next power of two, with a minimum of 8, capped at STANDARD_VECTOR_SIZE + if (count <= 8) { + return 8; + } + idx_t capacity = NextPowerOfTwo(count); + return MinValue(capacity, STANDARD_VECTOR_SIZE); +} + +void UpdateInfo::Initialize(UpdateInfo &info, DataTable &data_table, transaction_t transaction_id, + idx_t row_group_start, idx_t capacity) { + info.max = UnsafeNumericCast(capacity); + info.row_group_start = row_group_start; + info.version_number = transaction_id; + info.table = &data_table; + info.segment = nullptr; + info.prev.entry = nullptr; + info.next.entry = nullptr; +} + void UpdateInfo::Initialize(UpdateInfo &info, DataTable &data_table, transaction_t transaction_id, idx_t row_group_start) { info.max = STANDARD_VECTOR_SIZE; @@ -1277,6 +1301,39 @@ void UpdateSegment::InitializeUpdateInfo(idx_t vector_idx) { } } +void UpdateSegment::ReallocateRootInfoIfNeeded(UpdateInfo ¤t_info, idx_t update_count, idx_t vector_index) { + idx_t required_capacity = MinValue(idx_t(current_info.N) + update_count, STANDARD_VECTOR_SIZE); + if (required_capacity <= idx_t(current_info.max)) { + return; + } + idx_t new_capacity = UpdateInfo::GetCompactCapacity(required_capacity); + idx_t new_alloc_size = UpdateInfo::GetAllocSize(type_size, new_capacity); + auto new_handle = root->allocator.Allocate(new_alloc_size); + auto &new_info = UpdateInfo::Get(new_handle); + + new_info.segment = current_info.segment; + new_info.table = current_info.table; + new_info.column_index = current_info.column_index; + new_info.row_group_start = current_info.row_group_start; + new_info.version_number.store(current_info.version_number.load()); + new_info.vector_index = current_info.vector_index; + new_info.N = current_info.N; + new_info.max = UnsafeNumericCast(new_capacity); + new_info.prev = current_info.prev; + new_info.next = current_info.next; + + memcpy(new_info.GetTuples(), current_info.GetTuples(), sizeof(sel_t) * current_info.N); + memcpy(new_info.GetValues(), current_info.GetValues(), type_size * current_info.N); + + if (new_info.next.IsSet()) { + auto next_pin = new_info.next.Pin(); + auto &next_info = UpdateInfo::Get(next_pin); + next_info.prev = new_handle.GetBufferPointer(); + } + + root->info[vector_index] = new_handle.GetBufferPointer(); +} + void UpdateSegment::Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_p, row_t *ids, idx_t count, Vector &base_data, idx_t row_group_start) { // obtain an exclusive lock @@ -1328,11 +1385,15 @@ void UpdateSegment::Update(TransactionData transaction, DataTable &data_table, i // this transaction in the version chain auto root_pointer = root->info[vector_index]; auto root_pin = root_pointer.Pin(); - auto &base_info = UpdateInfo::Get(root_pin); UndoBufferReference node_ref; - CheckForConflicts(base_info.next, transaction, ids, sel, count, UnsafeNumericCast(vector_offset), - node_ref); + CheckForConflicts(UpdateInfo::Get(root_pin).next, transaction, ids, sel, count, + UnsafeNumericCast(vector_offset), node_ref); + + ReallocateRootInfoIfNeeded(UpdateInfo::Get(root_pin), count, vector_index); + root_pointer = root->info[vector_index]; + root_pin = root_pointer.Pin(); + auto &base_info = UpdateInfo::Get(root_pin); // there are no conflicts - continue with the update unsafe_unique_array update_info_data; @@ -1375,11 +1436,12 @@ void UpdateSegment::Update(TransactionData transaction, DataTable &data_table, i node->Verify(); } else { // there is no version info yet: create the top level update info and fill it with the updates - // allocate space for the UpdateInfo in the allocator - idx_t alloc_size = UpdateInfo::GetAllocSize(type_size); + // allocate space for the UpdateInfo in the allocator (compact: sized to actual count) + idx_t compact_capacity = UpdateInfo::GetCompactCapacity(count); + idx_t alloc_size = UpdateInfo::GetAllocSize(type_size, compact_capacity); auto handle = root->allocator.Allocate(alloc_size); auto &update_info = UpdateInfo::Get(handle); - UpdateInfo::Initialize(update_info, data_table, TRANSACTION_ID_START - 1, row_group_start); + UpdateInfo::Initialize(update_info, data_table, TRANSACTION_ID_START - 1, row_group_start, compact_capacity); update_info.column_index = column_index; InitializeUpdateInfo(update_info, ids, sel, count, vector_index, vector_offset); diff --git a/test/api/test_buffer_pool_eviction.cpp b/test/api/test_buffer_pool_eviction.cpp index 2966179684b8..0632a108ce4d 100644 --- a/test/api/test_buffer_pool_eviction.cpp +++ b/test/api/test_buffer_pool_eviction.cpp @@ -262,3 +262,138 @@ TEST_CASE("Test buffer pool eviction: failed to allocate space if every page and const auto final_memory_usage = buffer_manager.GetUsedMemory(); REQUIRE(final_memory_usage == total_memory_limit); } + +namespace { + +idx_t SumDeadNodes(const vector &info) { + idx_t total = 0; + for (const auto &q : info) { + total += q.dead_nodes; + } + return total; +} + +idx_t SumApproxSize(const vector &info) { + idx_t total = 0; + for (const auto &q : info) { + total += q.approximate_size; + } + return total; +} + +} // namespace + +// Regression test for an eviction-queue dead-node accounting bug. +// +// Each non-tiny BlockMemory placed in the eviction queue stays referenced from there via a +// weak_ptr + sequence number. When the BlockMemory is destroyed, the latest queue +// entry pointing at it becomes dead and must be reflected in the per-queue dead_nodes counter. +// +// The destructor uses eviction_seq_num > 0 to determine whether an entry exists in the queue: +// after eviction (Unload), seq_num is reset to 0, meaning the entry was already consumed. +// Only blocks that still have a live queue entry (seq_num > 0) should increment dead_nodes. +TEST_CASE("Test eviction queue: dead_nodes is incremented on BlockMemory destruction even after eviction", + "[storage][buffer_pool]") { + DuckDB db; + Connection con(db); + auto &context = *con.context; + auto &buffer_manager = BufferManager::GetBufferManager(context); + auto &buffer_pool = DatabaseInstance::GetDatabase(context).GetBufferPool(); + const idx_t initial_memory = buffer_pool.GetUsedMemory(); + + constexpr idx_t buffer_size = 1024 * 1024; // 1 MiB + constexpr idx_t total_buffers = 6; + constexpr idx_t held_buffers = 2; + const idx_t actual_alloc_size = BufferManager::GetAllocSize(buffer_size + Storage::DEFAULT_BLOCK_HEADER_SIZE); + + // Memory limit only large enough to hold `held_buffers` blocks at once, so subsequent + // allocations evict older destroyable blocks (their buffer becomes nullptr). + const idx_t memory_limit = initial_memory + held_buffers * actual_alloc_size; + buffer_pool.SetLimit(memory_limit, EXCEPTION_POSTSCRIPT); + + vector> handles; + handles.reserve(total_buffers); + for (idx_t i = 0; i < total_buffers; ++i) { + auto pin = buffer_manager.Allocate(MemoryTag::EXTENSION, buffer_size, /*can_destroy=*/true); + handles.emplace_back(pin.GetBlockHandle()); + // Pin destroyed at scope exit -> block enters the eviction queue. + } + + // At this point only the most recent `held_buffers` BlockHandles still own a buffer; the + // older `total_buffers - held_buffers` have been evicted (buffer == nullptr) but the + // BlockMemory is still alive because we hold the shared_ptr. + idx_t evicted_observed = 0; + idx_t loaded_observed = 0; + for (auto &handle : handles) { + if (handle->GetMemory().GetState() == BlockState::BLOCK_UNLOADED) { + evicted_observed++; + } else { + loaded_observed++; + } + } + REQUIRE(evicted_observed == total_buffers - held_buffers); + REQUIRE(loaded_observed == held_buffers); + + const idx_t dead_before = SumDeadNodes(buffer_pool.GetEvictionQueueInfo()); + + // Drop all BlockHandles. Only the `held_buffers` blocks that are still loaded (seq_num > 0, + // meaning their queue entry is still present) will increment dead_nodes. The evicted blocks + // had their entries consumed during eviction (seq_num reset to 0) — no queue entry remains. + handles.clear(); + + const idx_t dead_after = SumDeadNodes(buffer_pool.GetEvictionQueueInfo()); + + REQUIRE(dead_after - dead_before == held_buffers); +} + +// Sanity check the dead_nodes counter never exceeds the queue size and never decrements past +// zero across a destroyable-block churn workload. This indirectly exercises PurgeIteration: +// before the fix, pinned-but-current entries dequeued during purge would be both dropped from +// the queue AND wrongly subtracted from the dead-node counter, leading to underflow on the +// unsigned counter (observable as an absurdly large dead_nodes value). +TEST_CASE("Test eviction queue: dead_nodes invariants hold under destroyable block churn", "[storage][buffer_pool]") { + DuckDB db; + Connection con(db); + auto &context = *con.context; + auto &buffer_manager = BufferManager::GetBufferManager(context); + auto &buffer_pool = DatabaseInstance::GetDatabase(context).GetBufferPool(); + const idx_t initial_memory = buffer_pool.GetUsedMemory(); + + constexpr idx_t buffer_size = 64 * 1024; // 64 KiB - smaller buffers so we can churn many of them + constexpr idx_t resident_buffers = 32; + constexpr idx_t churn_iterations = 20000; // > INSERT_INTERVAL (4096) to trigger purges + const idx_t actual_alloc_size = BufferManager::GetAllocSize(buffer_size + Storage::DEFAULT_BLOCK_HEADER_SIZE); + + const idx_t memory_limit = initial_memory + resident_buffers * actual_alloc_size; + buffer_pool.SetLimit(memory_limit, EXCEPTION_POSTSCRIPT); + + // Keep a small set of pinned buffers that stay resident throughout the test. + // Their queue entries (after each unpin/repin cycle below) are alive, latest-version, + // and currently unevictable - exactly the case the old PurgeIteration mishandled. + vector pinned_resident; + pinned_resident.reserve(resident_buffers / 4); + for (idx_t i = 0; i < resident_buffers / 4; ++i) { + pinned_resident.emplace_back(buffer_manager.Allocate(MemoryTag::EXTENSION, buffer_size, /*can_destroy=*/true)); + } + + // Drive churn: allocate, briefly hold, release. Each unpin enqueues a node; many of + // them go stale immediately when the BlockHandle is dropped, generating dead nodes. + for (idx_t i = 0; i < churn_iterations; ++i) { + auto pin = buffer_manager.Allocate(MemoryTag::EXTENSION, buffer_size, /*can_destroy=*/true); + // pin destroyed -> enqueue; BlockHandle dropped -> BlockMemory destroyed -> dead++ + } + + const auto info = buffer_pool.GetEvictionQueueInfo(); + const idx_t dead = SumDeadNodes(info); + const idx_t approx_size = SumApproxSize(info); + + // Underflow check: total_dead_nodes is an unsigned atomic. If the old PurgeIteration + // over-decremented (treating pinned-but-current entries as dead), the counter would wrap + // to a value far larger than any plausible queue size. + REQUIRE(dead < approx_size + churn_iterations); + + // Every queue must individually satisfy dead_nodes <= total_insertions. + for (const auto &q : info) { + REQUIRE(q.dead_nodes <= q.total_insertions); + } +} diff --git a/test/sql/storage/test_eviction_queue_no_underflow.test_slow b/test/sql/storage/test_eviction_queue_no_underflow.test_slow new file mode 100644 index 000000000000..f06ad3802247 --- /dev/null +++ b/test/sql/storage/test_eviction_queue_no_underflow.test_slow @@ -0,0 +1,48 @@ +# name: test/sql/storage/test_eviction_queue_no_underflow.test_slow +# description: Concurrent high-load test ensuring eviction queue dead_nodes never underflows +# group: [storage] + +statement ok +SET threads=8; + +statement ok +PRAGMA memory_limit='100MB'; + +statement ok +CREATE MACRO eviction_queues_sane() AS ( + SELECT COUNT(*) = 0 FROM duckdb_eviction_queues() + WHERE dead_nodes < 0 + OR dead_nodes > total_insertions +); + +concurrentloop x 0 8 + +loop i 0 200 + +statement ok +CREATE OR REPLACE TABLE t_{x}_{i} AS SELECT range AS id, repeat('x', 1000) AS payload FROM range(10000); + +statement ok +SELECT COUNT(*) FROM t_{x}_{i} WHERE length(payload) > 0; + +statement ok +DROP TABLE IF EXISTS t_{x}_{i}; + +endloop + +endloop + +query I +SELECT eviction_queues_sane(); +---- +1 + +query I +SELECT COUNT(*) = 0 FROM duckdb_eviction_queues() WHERE dead_nodes < 0; +---- +1 + +query I +SELECT COUNT(*) = 0 FROM duckdb_eviction_queues() WHERE dead_nodes > total_insertions; +---- +1 diff --git a/test/sql/update/compact_update_allocation.test b/test/sql/update/compact_update_allocation.test new file mode 100644 index 000000000000..cdc941f9aa3b --- /dev/null +++ b/test/sql/update/compact_update_allocation.test @@ -0,0 +1,321 @@ +# name: test/sql/update/compact_update_allocation.test +# description: Compact UpdateInfo allocation - memory footprint and correctness +# group: [update] + +# disable auto-checkpoint so updates stay in-memory +statement ok +SET wal_autocheckpoint='1TB' + +statement ok +SET threads=1 + +# --- Memory footprint test --- +# Create a table with multiple row groups (each ~122K rows with vectors of 2048) +statement ok +CREATE TABLE mem_test (a INTEGER, b INTEGER, c INTEGER, d VARCHAR) + +statement ok +INSERT INTO mem_test SELECT i, i%500, i%200, 'val_' || (i%100)::VARCHAR FROM range(500000) t(i) + +# Record memory before updates +statement ok +CREATE TABLE mem_before AS SELECT memory_usage_bytes FROM duckdb_memory() WHERE tag = 'TRANSACTION' + +# Perform sparse updates: 1 row per vector across many vectors +# This is the worst case for old allocation (12KB per vector, regardless of update count) +statement ok +UPDATE mem_test SET a = a + 1 WHERE a % 2048 = 0 + +# Check memory after sparse updates +# With compact allocation, updating ~244 rows (one per vector) should use much less than +# 244 * 12KB = ~2.9MB that the old code would use for root UpdateInfo alone +# We allow up to 1.5MB due to bump allocator block overhead +statement ok +CREATE TABLE mem_after AS SELECT memory_usage_bytes FROM duckdb_memory() WHERE tag = 'TRANSACTION' + +query I +SELECT CASE WHEN (a.memory_usage_bytes - b.memory_usage_bytes) < 1500000 + THEN 'ok' + ELSE error(concat('expected < 1.5MB increase, got ', + (a.memory_usage_bytes - b.memory_usage_bytes), ' bytes')) + END +FROM mem_after a, mem_before b +---- +ok + +statement ok +DROP TABLE mem_before + +statement ok +DROP TABLE mem_after + +# --- Correctness: single row update --- +statement ok +CREATE TABLE single_update (i INTEGER NOT NULL, v VARCHAR) + +statement ok +INSERT INTO single_update SELECT range, 'hello' FROM range(10000) + +statement ok +UPDATE single_update SET v = 'updated' WHERE i = 5000 + +query II +SELECT i, v FROM single_update WHERE i = 5000 +---- +5000 updated + +query I +SELECT count(*) FROM single_update WHERE v = 'hello' +---- +9999 + +# --- Correctness: multiple updates to same vector (triggers re-allocation) --- +statement ok +UPDATE single_update SET v = 'second' WHERE i = 5001 + +statement ok +UPDATE single_update SET v = 'third' WHERE i = 5002 + +statement ok +UPDATE single_update SET v = 'fourth' WHERE i = 5003 + +statement ok +UPDATE single_update SET v = 'fifth' WHERE i = 5004 + +statement ok +UPDATE single_update SET v = 'sixth' WHERE i = 5005 + +statement ok +UPDATE single_update SET v = 'seventh' WHERE i = 5006 + +statement ok +UPDATE single_update SET v = 'eighth' WHERE i = 5007 + +statement ok +UPDATE single_update SET v = 'ninth' WHERE i = 5008 + +# Now we have 9 updates in the same vector (5000-5008 are all in one 2048-row vector) +# This should have triggered re-allocation from capacity 8 to 16 +query I +SELECT count(*) FROM single_update WHERE v NOT IN ('hello') +---- +9 + +query II +SELECT i, v FROM single_update WHERE i BETWEEN 5000 AND 5008 ORDER BY i +---- +5000 updated +5001 second +5002 third +5003 fourth +5004 fifth +5005 sixth +5006 seventh +5007 eighth +5008 ninth + +# --- Correctness: update all rows in a vector (capacity grows to 2048) --- +statement ok +CREATE TABLE full_vector_update (i INTEGER NOT NULL) + +statement ok +INSERT INTO full_vector_update SELECT range FROM range(4096) + +# Update all rows in the first vector (rows 0-2047) +statement ok +UPDATE full_vector_update SET i = i + 10000 WHERE i < 2048 + +query I +SELECT MIN(i) FROM full_vector_update +---- +2048 + +query I +SELECT MAX(i) FROM full_vector_update WHERE i >= 10000 +---- +12047 + +query I +SELECT count(*) FROM full_vector_update WHERE i >= 10000 +---- +2048 + +# --- Correctness: updates survive checkpoint --- +statement ok +CREATE TABLE checkpoint_test (a INTEGER, b VARCHAR) + +statement ok +INSERT INTO checkpoint_test SELECT range, 'original' FROM range(10000) + +statement ok +UPDATE checkpoint_test SET b = 'modified' WHERE a % 1000 = 0 + +query I +SELECT count(*) FROM checkpoint_test WHERE b = 'modified' +---- +10 + +statement ok +CHECKPOINT + +query I +SELECT count(*) FROM checkpoint_test WHERE b = 'modified' +---- +10 + +query I +SELECT count(*) FROM checkpoint_test WHERE b = 'original' +---- +9990 + +# --- Correctness: rollback with compact allocation --- +statement ok +CREATE TABLE rollback_test (i INTEGER NOT NULL) + +statement ok +INSERT INTO rollback_test SELECT range FROM range(5000) + +statement ok +BEGIN TRANSACTION + +statement ok +UPDATE rollback_test SET i = i + 99999 WHERE i = 2500 + +query I +SELECT i FROM rollback_test WHERE i > 99000 +---- +102499 + +statement ok +ROLLBACK + +query I +SELECT i FROM rollback_test WHERE i = 2500 +---- +2500 + +query I +SELECT count(*) FROM rollback_test WHERE i > 99000 +---- +0 + +# --- Correctness: concurrent transactions with compact allocation --- +statement ok +CREATE TABLE concurrent_test (i INTEGER NOT NULL) + +statement ok +INSERT INTO concurrent_test SELECT range FROM range(5000) + +statement ok +BEGIN TRANSACTION + +statement ok +UPDATE concurrent_test SET i = i + 100000 WHERE i = 1000 + +# In the same transaction, read the updated value +query I +SELECT i FROM concurrent_test WHERE i > 99000 +---- +101000 + +statement ok +COMMIT + +query I +SELECT i FROM concurrent_test WHERE i > 99000 +---- +101000 + +# --- Correctness: many scattered updates across different vectors --- +statement ok +CREATE TABLE scatter_test (a BIGINT NOT NULL, b DOUBLE, c VARCHAR) + +statement ok +INSERT INTO scatter_test SELECT range, range * 1.5, 'str_' || range::VARCHAR FROM range(100000) + +# Update 1 row in each of ~49 different vectors (every 2048th row) +statement ok +UPDATE scatter_test SET b = -1.0, c = 'UPDATED' WHERE a % 2048 = 1024 + +query I +SELECT count(*) FROM scatter_test WHERE c = 'UPDATED' +---- +49 + +query I +SELECT count(*) FROM scatter_test WHERE b = -1.0 +---- +49 + +# All non-updated rows should be intact +query I +SELECT count(*) FROM scatter_test WHERE b != -1.0 AND b = a * 1.5 +---- +99951 + +# --- Correctness: NULL handling with compact updates --- +statement ok +CREATE TABLE null_test (i INTEGER) + +statement ok +INSERT INTO null_test SELECT range FROM range(5000) + +# Update some rows to NULL +statement ok +UPDATE null_test SET i = NULL WHERE i % 500 = 0 + +query I +SELECT count(*) FROM null_test WHERE i IS NULL +---- +10 + +# Update NULL rows back to a value +statement ok +UPDATE null_test SET i = -1 WHERE i IS NULL + +query I +SELECT count(*) FROM null_test WHERE i IS NULL +---- +0 + +query I +SELECT count(*) FROM null_test WHERE i = -1 +---- +10 + +# --- Correctness: different types --- +statement ok +CREATE TABLE types_test ( + ti TINYINT, + si SMALLINT, + bi BIGINT, + f FLOAT, + d DOUBLE, + v VARCHAR, + bl BOOLEAN +) + +statement ok +INSERT INTO types_test +SELECT (range % 127)::TINYINT, + (range % 32000)::SMALLINT, + range::BIGINT, + range::FLOAT, + range::DOUBLE, + range::VARCHAR, + (range % 2 = 0) +FROM range(5000) + +statement ok +UPDATE types_test SET ti = -1, si = -1, bi = -1, f = -1.0, d = -1.0, v = 'X', bl = true WHERE bi = 2500 + +query IIIIITI +SELECT ti, si, bi, f, d, v, bl FROM types_test WHERE bi = -1 +---- +-1 -1 -1 -1.0 -1.0 X true + +# Verify other rows are not affected +query I +SELECT count(*) FROM types_test WHERE bi >= 0 +---- +4999