diff --git a/.github/workflows/Main.yml b/.github/workflows/Main.yml index 854a962cd53f..814170a5238f 100644 --- a/.github/workflows/Main.yml +++ b/.github/workflows/Main.yml @@ -385,6 +385,12 @@ jobs: run: | ./build/release/test/unittest --test-config test/configs/block_size_16kB.json + - name: test/configs/adaptive_segment_allocation.json + if: success() || failure() + shell: bash + run: | + python3 scripts/ci/run_tests.py --test-config=test/configs/adaptive_segment_allocation.json ./build/release/test/unittest + - name: test/configs/latest_storage_block_size_16kB.json if: (success() || failure()) && steps.build.conclusion == 'success' shell: bash diff --git a/src/common/settings.json b/src/common/settings.json index a45efc890ae8..8743400720f3 100644 --- a/src/common/settings.json +++ b/src/common/settings.json @@ -15,6 +15,13 @@ "set" ] }, + { + "name": "adaptive_segment_allocation", + "description": "When enabled, transient column segments start small and double in size with each overflow, up to the block size", + "type": "BOOLEAN", + "default_scope": "global", + "default_value": "false" + }, { "name": "allocator_background_threads", "description": "Whether to enable the allocator background thread.", @@ -735,6 +742,13 @@ "set" ] }, + { + "name": "initial_segment_rows", + "description": "Number of rows allocated for the first transient column segment when adaptive_segment_allocation is enabled. Subsequent segments double this until reaching the block size.", + "type": "UBIGINT", + "default_scope": "global", + "default_value": "2048" + }, { "name": "integer_division", "description": "Whether or not the / operator defaults to integer division, or to floating point division", diff --git a/src/include/duckdb/main/settings.hpp b/src/include/duckdb/main/settings.hpp index 3f4a08c94f32..e20dfe53e967 100644 --- a/src/include/duckdb/main/settings.hpp +++ b/src/include/duckdb/main/settings.hpp @@ -108,6 +108,17 @@ struct AccessModeSetting { static Value GetSetting(const ClientContext &context); }; +struct AdaptiveSegmentAllocationSetting { + using RETURN_TYPE = bool; + static constexpr const char *Name = "adaptive_segment_allocation"; + static constexpr const char *Description = "When enabled, transient column segments start small and double in size " + "with each overflow, up to the block size"; + static constexpr const char *InputType = "BOOLEAN"; + static constexpr const char *DefaultValue = "false"; + static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; + static constexpr idx_t SettingIndex = 0; +}; + struct AllocatorBackgroundThreadsSetting { using RETURN_TYPE = bool; static constexpr const char *Name = "allocator_background_threads"; @@ -115,7 +126,7 @@ struct AllocatorBackgroundThreadsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 0; + static constexpr idx_t SettingIndex = 1; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -148,7 +159,7 @@ struct AllowCommunityExtensionsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 1; + static constexpr idx_t SettingIndex = 2; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -159,7 +170,7 @@ struct AllowExtensionsMetadataMismatchSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 2; + static constexpr idx_t SettingIndex = 3; }; struct AllowParserOverrideExtensionSetting { @@ -169,7 +180,7 @@ struct AllowParserOverrideExtensionSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "DEFAULT"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 3; + static constexpr idx_t SettingIndex = 4; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -191,7 +202,7 @@ struct AllowUnredactedSecretsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 4; + static constexpr idx_t SettingIndex = 5; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -202,7 +213,7 @@ struct AllowUnsignedExtensionsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 5; + static constexpr idx_t SettingIndex = 6; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -247,7 +258,7 @@ struct ArrowLargeBufferSizeSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 6; + static constexpr idx_t SettingIndex = 7; }; struct ArrowLosslessConversionSetting { @@ -259,7 +270,7 @@ struct ArrowLosslessConversionSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 7; + static constexpr idx_t SettingIndex = 8; }; struct ArrowOutputListViewSetting { @@ -270,7 +281,7 @@ struct ArrowOutputListViewSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 8; + static constexpr idx_t SettingIndex = 9; }; struct ArrowOutputVersionSetting { @@ -281,7 +292,7 @@ struct ArrowOutputVersionSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "1.0"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 9; + static constexpr idx_t SettingIndex = 10; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -293,7 +304,7 @@ struct AsofLoopJoinThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "64"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 10; + static constexpr idx_t SettingIndex = 11; }; struct AutoCheckpointSkipWalThresholdSetting { @@ -305,7 +316,7 @@ struct AutoCheckpointSkipWalThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "100000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 11; + static constexpr idx_t SettingIndex = 12; }; struct AutoinstallExtensionRepositorySetting { @@ -316,7 +327,7 @@ struct AutoinstallExtensionRepositorySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 12; + static constexpr idx_t SettingIndex = 13; }; struct AutoinstallKnownExtensionsSetting { @@ -331,7 +342,7 @@ struct AutoinstallKnownExtensionsSetting { static constexpr const char *DefaultValue = "false"; #endif static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 13; + static constexpr idx_t SettingIndex = 14; }; struct AutoloadKnownExtensionsSetting { @@ -346,7 +357,7 @@ struct AutoloadKnownExtensionsSetting { static constexpr const char *DefaultValue = "false"; #endif static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 14; + static constexpr idx_t SettingIndex = 15; }; struct BlockAllocatorMemorySetting { @@ -368,7 +379,20 @@ struct CatalogErrorMaxSchemasSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "100"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 15; + static constexpr idx_t SettingIndex = 16; +}; + +struct CheckpointOnDetachSetting { + using RETURN_TYPE = CheckpointOnDetach; + static constexpr const char *Name = "checkpoint_on_detach"; + static constexpr const char *Description = + "Override checkpoint behavior when detaching a database. ENABLED always checkpoints, DISABLED never " + "checkpoints, DEFAULT defers to the global checkpoint_on_shutdown setting."; + static constexpr const char *InputType = "VARCHAR"; + static constexpr const char *DefaultValue = "DEFAULT"; + static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; + static constexpr idx_t SettingIndex = 17; + static void OnSet(SettingCallbackInfo &info, Value &input); }; struct CheckpointThresholdSetting { @@ -390,7 +414,7 @@ struct CurrentTransactionInvalidationPolicySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "STANDARD_POLICY"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 16; + static constexpr idx_t SettingIndex = 18; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -401,7 +425,7 @@ struct CustomExtensionRepositorySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 17; + static constexpr idx_t SettingIndex = 19; }; struct CustomProfilingSettingsSetting { @@ -431,7 +455,7 @@ struct DebugAsofIejoinSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 18; + static constexpr idx_t SettingIndex = 20; }; struct DebugCheckpointAbortSetting { @@ -442,7 +466,7 @@ struct DebugCheckpointAbortSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "NONE"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 19; + static constexpr idx_t SettingIndex = 21; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -453,7 +477,7 @@ struct DebugCheckpointSleepMsSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "0"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 20; + static constexpr idx_t SettingIndex = 22; }; struct DebugEvictionQueueSleepMicroSecondsSetting { @@ -464,7 +488,7 @@ struct DebugEvictionQueueSleepMicroSecondsSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "0"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 21; + static constexpr idx_t SettingIndex = 23; }; struct DebugForceExternalSetting { @@ -486,7 +510,7 @@ struct DebugForceNoCrossProductSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 22; + static constexpr idx_t SettingIndex = 24; }; struct DebugPhysicalTableScanExecutionStrategySetting { @@ -497,7 +521,7 @@ struct DebugPhysicalTableScanExecutionStrategySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "DEFAULT"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 23; + static constexpr idx_t SettingIndex = 25; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -508,7 +532,7 @@ struct DebugSkipCheckpointOnCommitSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 24; + static constexpr idx_t SettingIndex = 26; }; struct DebugVerifyBlocksSetting { @@ -518,7 +542,7 @@ struct DebugVerifyBlocksSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 25; + static constexpr idx_t SettingIndex = 27; }; struct DebugVerifyVectorSetting { @@ -528,7 +552,7 @@ struct DebugVerifyVectorSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "NONE"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 26; + static constexpr idx_t SettingIndex = 28; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -539,7 +563,7 @@ struct DebugWindowModeSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "WINDOW"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 27; + static constexpr idx_t SettingIndex = 29; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -551,7 +575,7 @@ struct DefaultBlockSizeSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "262144"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 28; + static constexpr idx_t SettingIndex = 30; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -562,7 +586,7 @@ struct DefaultCollationSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 29; + static constexpr idx_t SettingIndex = 31; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -573,7 +597,7 @@ struct DefaultNullOrderSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "NULLS_LAST"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 30; + static constexpr idx_t SettingIndex = 32; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -584,7 +608,7 @@ struct DefaultOrderSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "ASCENDING"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 31; + static constexpr idx_t SettingIndex = 33; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -605,7 +629,7 @@ struct DeprecatedUsingKeySyntaxSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "DEFAULT"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 32; + static constexpr idx_t SettingIndex = 34; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -618,7 +642,7 @@ struct DisableDatabaseInvalidationSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 33; + static constexpr idx_t SettingIndex = 35; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -629,7 +653,7 @@ struct DisableTimestamptzCastsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 34; + static constexpr idx_t SettingIndex = 36; }; struct DisabledCompressionMethodsSetting { @@ -679,7 +703,7 @@ struct DuckDBAPISetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 35; + static constexpr idx_t SettingIndex = 37; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -691,7 +715,7 @@ struct DynamicOrFilterThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "50"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 36; + static constexpr idx_t SettingIndex = 38; }; struct EnableCachingOperatorsSetting { @@ -713,7 +737,7 @@ struct EnableExternalAccessSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 37; + static constexpr idx_t SettingIndex = 39; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -724,7 +748,7 @@ struct EnableExternalFileCacheSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 38; + static constexpr idx_t SettingIndex = 40; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -736,7 +760,7 @@ struct EnableFSSTVectorsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 39; + static constexpr idx_t SettingIndex = 41; }; struct EnableHTTPLoggingSetting { @@ -756,7 +780,7 @@ struct EnableHTTPMetadataCacheSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 40; + static constexpr idx_t SettingIndex = 42; }; struct EnableLogging { @@ -777,7 +801,7 @@ struct EnableMacroDependenciesSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 41; + static constexpr idx_t SettingIndex = 43; }; struct EnableObjectCacheSetting { @@ -787,7 +811,7 @@ struct EnableObjectCacheSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 42; + static constexpr idx_t SettingIndex = 44; }; struct EnableProfilingSetting { @@ -833,7 +857,7 @@ struct EnableViewDependenciesSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 43; + static constexpr idx_t SettingIndex = 45; }; struct EnabledLogTypes { @@ -853,7 +877,7 @@ struct ErrorsAsJSONSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 44; + static constexpr idx_t SettingIndex = 46; }; struct ExperimentalMetadataReuseSetting { @@ -863,7 +887,7 @@ struct ExperimentalMetadataReuseSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 45; + static constexpr idx_t SettingIndex = 47; }; struct ExplainOutputSetting { @@ -873,7 +897,7 @@ struct ExplainOutputSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "PHYSICAL_ONLY"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 46; + static constexpr idx_t SettingIndex = 48; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -894,7 +918,7 @@ struct ExtensionDirectorySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 47; + static constexpr idx_t SettingIndex = 49; }; struct ExternalThreadsSetting { @@ -904,7 +928,7 @@ struct ExternalThreadsSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "1"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 48; + static constexpr idx_t SettingIndex = 50; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -915,7 +939,7 @@ struct FileSearchPathSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 49; + static constexpr idx_t SettingIndex = 51; }; struct ForceBitpackingModeSetting { @@ -925,7 +949,7 @@ struct ForceBitpackingModeSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "AUTO"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 50; + static constexpr idx_t SettingIndex = 52; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -936,7 +960,7 @@ struct ForceCompressionSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "auto"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 51; + static constexpr idx_t SettingIndex = 53; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -969,7 +993,7 @@ struct GeometryMinimumShreddingSize { static constexpr const char *InputType = "BIGINT"; static constexpr const char *DefaultValue = "30000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 52; + static constexpr idx_t SettingIndex = 54; }; struct HomeDirectorySetting { @@ -979,7 +1003,7 @@ struct HomeDirectorySetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 53; + static constexpr idx_t SettingIndex = 55; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -997,12 +1021,11 @@ struct HTTPLoggingOutputSetting { struct HTTPProxySetting { using RETURN_TYPE = string; static constexpr const char *Name = "http_proxy"; - static constexpr const char *Description = - "HTTP proxy host (defaults to the HTTP_PROXY environment variable when unset)"; + static constexpr const char *Description = "HTTP proxy host"; static constexpr const char *InputType = "VARCHAR"; - static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value ¶meter); - static void ResetGlobal(DatabaseInstance *db, DBConfig &config); - static Value GetSetting(const ClientContext &context); + static constexpr const char *DefaultValue = ""; + static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; + static constexpr idx_t SettingIndex = 56; }; struct HTTPProxyPasswordSetting { @@ -1012,7 +1035,7 @@ struct HTTPProxyPasswordSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 54; + static constexpr idx_t SettingIndex = 57; }; struct HTTPProxyUsernameSetting { @@ -1022,7 +1045,7 @@ struct HTTPProxyUsernameSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 55; + static constexpr idx_t SettingIndex = 58; }; struct IeeeFloatingPointOpsSetting { @@ -1033,7 +1056,7 @@ struct IeeeFloatingPointOpsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 56; + static constexpr idx_t SettingIndex = 59; }; struct IgnoreUnknownCrsSetting { @@ -1044,7 +1067,7 @@ struct IgnoreUnknownCrsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 57; + static constexpr idx_t SettingIndex = 60; }; struct ImmediateTransactionModeSetting { @@ -1055,7 +1078,7 @@ struct ImmediateTransactionModeSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 58; + static constexpr idx_t SettingIndex = 61; }; struct IndexScanMaxCountSetting { @@ -1067,7 +1090,7 @@ struct IndexScanMaxCountSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "2048"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 59; + static constexpr idx_t SettingIndex = 62; }; struct IndexScanPercentageSetting { @@ -1079,10 +1102,22 @@ struct IndexScanPercentageSetting { static constexpr const char *InputType = "DOUBLE"; static constexpr const char *DefaultValue = "0.001"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 60; + static constexpr idx_t SettingIndex = 63; static void OnSet(SettingCallbackInfo &info, Value &input); }; +struct InitialSegmentRowsSetting { + using RETURN_TYPE = idx_t; + static constexpr const char *Name = "initial_segment_rows"; + static constexpr const char *Description = + "Number of rows allocated for the first transient column segment when adaptive_segment_allocation is enabled. " + "Subsequent segments double this until reaching the block size."; + static constexpr const char *InputType = "UBIGINT"; + static constexpr const char *DefaultValue = "2048"; + static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; + static constexpr idx_t SettingIndex = 64; +}; + struct IntegerDivisionSetting { using RETURN_TYPE = bool; static constexpr const char *Name = "integer_division"; @@ -1091,7 +1126,7 @@ struct IntegerDivisionSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 61; + static constexpr idx_t SettingIndex = 65; }; struct LambdaSyntaxSetting { @@ -1102,7 +1137,7 @@ struct LambdaSyntaxSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "DEFAULT"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 62; + static constexpr idx_t SettingIndex = 66; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1114,7 +1149,7 @@ struct LateMaterializationMaxRowsSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "50"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 63; + static constexpr idx_t SettingIndex = 67; }; struct LockConfigurationSetting { @@ -1124,7 +1159,7 @@ struct LockConfigurationSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 64; + static constexpr idx_t SettingIndex = 68; }; struct LogQueryPathSetting { @@ -1135,7 +1170,7 @@ struct LogQueryPathSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 65; + static constexpr idx_t SettingIndex = 69; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1178,7 +1213,7 @@ struct MaxExpressionDepthSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "1000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 66; + static constexpr idx_t SettingIndex = 70; }; struct MaxMemorySetting { @@ -1209,7 +1244,7 @@ struct MaxVacuumTasksSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "100"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 67; + static constexpr idx_t SettingIndex = 71; }; struct MergeJoinThresholdSetting { @@ -1219,7 +1254,7 @@ struct MergeJoinThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "1000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 68; + static constexpr idx_t SettingIndex = 72; }; struct NestedLoopJoinThresholdSetting { @@ -1230,7 +1265,7 @@ struct NestedLoopJoinThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "5"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 69; + static constexpr idx_t SettingIndex = 73; }; struct OldImplicitCastingSetting { @@ -1240,7 +1275,7 @@ struct OldImplicitCastingSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 70; + static constexpr idx_t SettingIndex = 74; }; struct OrderByNonIntegerLiteralSetting { @@ -1251,7 +1286,7 @@ struct OrderByNonIntegerLiteralSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 71; + static constexpr idx_t SettingIndex = 75; }; struct OrderedAggregateThresholdSetting { @@ -1261,7 +1296,7 @@ struct OrderedAggregateThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "262144"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 72; + static constexpr idx_t SettingIndex = 76; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1273,7 +1308,7 @@ struct PartitionedWriteFlushThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "524288"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 73; + static constexpr idx_t SettingIndex = 77; }; struct PartitionedWriteMaxOpenFilesSetting { @@ -1284,7 +1319,7 @@ struct PartitionedWriteMaxOpenFilesSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "100"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 74; + static constexpr idx_t SettingIndex = 78; }; struct PasswordSetting { @@ -1294,7 +1329,7 @@ struct PasswordSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 75; + static constexpr idx_t SettingIndex = 79; }; struct PerfectHtThresholdSetting { @@ -1304,7 +1339,7 @@ struct PerfectHtThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "12"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 76; + static constexpr idx_t SettingIndex = 80; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1316,7 +1351,7 @@ struct PinThreadsSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "auto"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 77; + static constexpr idx_t SettingIndex = 81; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1328,7 +1363,7 @@ struct PivotFilterThresholdSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "20"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 78; + static constexpr idx_t SettingIndex = 82; }; struct PivotLimitSetting { @@ -1338,7 +1373,7 @@ struct PivotLimitSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "100000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 79; + static constexpr idx_t SettingIndex = 83; }; struct PreferRangeJoinsSetting { @@ -1348,7 +1383,7 @@ struct PreferRangeJoinsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 80; + static constexpr idx_t SettingIndex = 84; }; struct PreserveIdentifierCaseSetting { @@ -1359,7 +1394,7 @@ struct PreserveIdentifierCaseSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 81; + static constexpr idx_t SettingIndex = 85; }; struct PreserveInsertionOrderSetting { @@ -1371,7 +1406,7 @@ struct PreserveInsertionOrderSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 82; + static constexpr idx_t SettingIndex = 86; }; struct ProduceArrowStringViewSetting { @@ -1382,7 +1417,7 @@ struct ProduceArrowStringViewSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 83; + static constexpr idx_t SettingIndex = 87; }; struct ProfileOutputSetting { @@ -1435,7 +1470,7 @@ struct ScalarSubqueryErrorOnMultipleRowsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "true"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::LOCAL_DEFAULT; - static constexpr idx_t SettingIndex = 84; + static constexpr idx_t SettingIndex = 88; }; struct SchedulerProcessPartialSetting { @@ -1450,7 +1485,7 @@ struct SchedulerProcessPartialSetting { static constexpr const char *DefaultValue = "false"; #endif static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 85; + static constexpr idx_t SettingIndex = 89; }; struct SchemaSetting { @@ -1492,7 +1527,7 @@ struct StorageBlockPrefetchSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "REMOTE_ONLY"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 86; + static constexpr idx_t SettingIndex = 90; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1534,7 +1569,7 @@ struct TempFileEncryptionSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 87; + static constexpr idx_t SettingIndex = 91; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1555,7 +1590,7 @@ struct UsernameSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = ""; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 88; + static constexpr idx_t SettingIndex = 92; }; struct VacuumRebuildIndexesSetting { @@ -1567,7 +1602,7 @@ struct VacuumRebuildIndexesSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "0"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 89; + static constexpr idx_t SettingIndex = 93; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1580,7 +1615,7 @@ struct ValidateExternalFileCacheSetting { static constexpr const char *InputType = "VARCHAR"; static constexpr const char *DefaultValue = "VALIDATE_ALL"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 90; + static constexpr idx_t SettingIndex = 94; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1592,7 +1627,7 @@ struct VariantMinimumShreddingSizeSetting { static constexpr const char *InputType = "BIGINT"; static constexpr const char *DefaultValue = "30000"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 91; + static constexpr idx_t SettingIndex = 95; }; struct WalAutocheckpointEntriesSetting { @@ -1603,7 +1638,7 @@ struct WalAutocheckpointEntriesSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "0"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 92; + static constexpr idx_t SettingIndex = 96; }; struct WarningsAsErrorsSetting { @@ -1613,7 +1648,7 @@ struct WarningsAsErrorsSetting { static constexpr const char *InputType = "BOOLEAN"; static constexpr const char *DefaultValue = "false"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 93; + static constexpr idx_t SettingIndex = 97; static void OnSet(SettingCallbackInfo &info, Value &input); }; @@ -1625,7 +1660,7 @@ struct WriteBufferRowGroupCountSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "5"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_DEFAULT; - static constexpr idx_t SettingIndex = 94; + static constexpr idx_t SettingIndex = 98; }; struct ZstdMinStringLengthSetting { @@ -1636,11 +1671,11 @@ struct ZstdMinStringLengthSetting { static constexpr const char *InputType = "UBIGINT"; static constexpr const char *DefaultValue = "4096"; static constexpr SettingScopeTarget Scope = SettingScopeTarget::GLOBAL_ONLY; - static constexpr idx_t SettingIndex = 95; + static constexpr idx_t SettingIndex = 99; }; struct GeneratedSettingInfo { - static constexpr idx_t MaxSettingIndex = 96; + static constexpr idx_t MaxSettingIndex = 100; }; //===----------------------------------------------------------------------===// diff --git a/src/include/duckdb/storage/table/column_data.hpp b/src/include/duckdb/storage/table/column_data.hpp index 0660e778e539..99653c511fbf 100644 --- a/src/include/duckdb/storage/table/column_data.hpp +++ b/src/include/duckdb/storage/table/column_data.hpp @@ -253,6 +253,8 @@ class ColumnData : public enable_shared_from_this { mutable mutex stats_lock; //! Total transient allocation size atomic allocation_size; + //! Size of the last transient segment - drives the adaptive doubling in AppendTransientSegment + idx_t last_transient_segment_size = 0; //! The stats of the root segment unique_ptr stats; diff --git a/src/main/config.cpp b/src/main/config.cpp index 52b913f32cfb..5a2d0e4fa0c9 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -67,6 +67,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_GLOBAL(DeltaOnlyVariantEncodingEnabledSetting), DUCKDB_GLOBAL(AccessModeSetting), + DUCKDB_SETTING(AdaptiveSegmentAllocationSetting), DUCKDB_SETTING_CALLBACK(AllocatorBackgroundThreadsSetting), DUCKDB_GLOBAL(AllocatorBulkDeallocationFlushThresholdSetting), DUCKDB_GLOBAL(AllocatorFlushThresholdSetting), @@ -90,6 +91,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(AutoloadKnownExtensionsSetting), DUCKDB_GLOBAL(BlockAllocatorMemorySetting), DUCKDB_SETTING(CatalogErrorMaxSchemasSetting), + DUCKDB_SETTING_CALLBACK(CheckpointOnDetachSetting), DUCKDB_GLOBAL(CheckpointThresholdSetting), DUCKDB_SETTING_CALLBACK(CurrentTransactionInvalidationPolicySetting), DUCKDB_SETTING(CustomExtensionRepositorySetting), @@ -148,7 +150,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(GeometryMinimumShreddingSize), DUCKDB_SETTING_CALLBACK(HomeDirectorySetting), DUCKDB_LOCAL(HTTPLoggingOutputSetting), - DUCKDB_GLOBAL(HTTPProxySetting), + DUCKDB_SETTING(HTTPProxySetting), DUCKDB_SETTING(HTTPProxyPasswordSetting), DUCKDB_SETTING(HTTPProxyUsernameSetting), DUCKDB_SETTING(IeeeFloatingPointOpsSetting), @@ -156,6 +158,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(ImmediateTransactionModeSetting), DUCKDB_SETTING(IndexScanMaxCountSetting), DUCKDB_SETTING_CALLBACK(IndexScanPercentageSetting), + DUCKDB_SETTING(InitialSegmentRowsSetting), DUCKDB_SETTING(IntegerDivisionSetting), DUCKDB_SETTING_CALLBACK(LambdaSyntaxSetting), DUCKDB_SETTING(LateMaterializationMaxRowsSetting), @@ -209,12 +212,12 @@ static const ConfigurationOption internal_options[] = { DUCKDB_SETTING(ZstdMinStringLengthSetting), FINAL_SETTING}; -static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 100), - DUCKDB_SETTING_ALIAS("null_order", 43), - DUCKDB_SETTING_ALIAS("profiling_output", 119), - DUCKDB_SETTING_ALIAS("user", 134), - DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 25), - DUCKDB_SETTING_ALIAS("worker_threads", 133), +static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 103), + DUCKDB_SETTING_ALIAS("null_order", 45), + DUCKDB_SETTING_ALIAS("profiling_output", 122), + DUCKDB_SETTING_ALIAS("user", 137), + DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 27), + DUCKDB_SETTING_ALIAS("worker_threads", 136), FINAL_ALIAS}; vector DBConfig::GetOptions() { diff --git a/src/storage/buffer/buffer_pool.cpp b/src/storage/buffer/buffer_pool.cpp index 729bb89f7fd5..8285d6d028f6 100644 --- a/src/storage/buffer/buffer_pool.cpp +++ b/src/storage/buffer/buffer_pool.cpp @@ -3,6 +3,8 @@ #include "duckdb/common/exception.hpp" #include "duckdb/common/thread.hpp" #include "duckdb/common/typedefs.hpp" +#include "duckdb/logging/logger.hpp" +#include #include "duckdb/main/settings.hpp" #include "duckdb/parallel/concurrentqueue.hpp" #include "duckdb/parallel/task_scheduler.hpp" @@ -82,7 +84,7 @@ struct EvictionQueue { //! Tries to dequeue an element from the eviction queue, but only after acquiring the purge queue lock. bool TryDequeueWithLock(BufferEvictionNode &node); //! Garbage collect dead nodes in the eviction queue. - void Purge(); + void Purge(const DatabaseInstance &db); template void IterateUnloadableBlocks(FN fn); @@ -146,13 +148,15 @@ bool EvictionQueue::TryDequeueWithLock(BufferEvictionNode &node) { return q.try_dequeue(node); } -void EvictionQueue::Purge() { +void EvictionQueue::Purge(const DatabaseInstance &db) { // only one thread purges the queue, all other threads early-out unique_lock guard(purge_lock, std::try_to_lock); if (!guard.owns_lock()) { return; } + auto purge_start = std::chrono::steady_clock::now(); + // we purge INSERT_INTERVAL * PURGE_SIZE_MULTIPLIER nodes idx_t purge_size = INSERT_INTERVAL * PURGE_SIZE_MULTIPLIER; @@ -182,6 +186,7 @@ void EvictionQueue::Purge() { // guaranteeing that we always exit the loop. idx_t max_purges = approx_q_size / purge_size; + idx_t initial_max_purges = max_purges; while (max_purges != 0) { PurgeIteration(purge_size); @@ -204,6 +209,15 @@ void EvictionQueue::Purge() { max_purges--; } + + idx_t iterations = initial_max_purges - max_purges; + auto elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - purge_start) + .count(); + if (iterations > 10 || elapsed_ms > 1000) { + DUCKDB_LOG_WARNING(db, "EvictionQueue::Purge took %lldms with %llu iterations, queue_size=%llu, dead_nodes=%llu", + elapsed_ms, iterations, q.size_approx(), (idx_t)total_dead_nodes); + } } void EvictionQueue::PurgeIteration(const idx_t purge_size) { @@ -491,7 +505,7 @@ void BufferPool::PurgeQueue(const BlockHandle &block) { const auto queue_sleep_micros = Settings::Get(buffer_manager.GetDatabase()); eviction_queue.debug_eviction_queue_sleep = queue_sleep_micros; - eviction_queue.Purge(); + eviction_queue.Purge(buffer_manager.GetDatabase()); } void BufferPool::SetLimit(idx_t limit, const char *exception_postscript) { diff --git a/src/storage/table/column_data.cpp b/src/storage/table/column_data.cpp index d32d2d2f4bcd..818b9953f19f 100644 --- a/src/storage/table/column_data.cpp +++ b/src/storage/table/column_data.cpp @@ -1,5 +1,7 @@ #include "duckdb/storage/table/column_data.hpp" #include "duckdb/common/exception/transaction_exception.hpp" +#include "duckdb/common/types/validity_mask.hpp" +#include "duckdb/main/settings.hpp" #include "duckdb/function/compression_function.hpp" #include "duckdb/planner/table_filter.hpp" #include "duckdb/storage/data_pointer.hpp" @@ -606,24 +608,41 @@ void ColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) { const auto block_size = block_manager.GetBlockSize(); const auto type_size = GetTypeIdSize(type.InternalType()); - auto vector_segment_size = block_size; - if (data_type == ColumnDataType::INITIAL_TRANSACTION_LOCAL && start_row == 0) { + auto &db = GetDatabase(); + auto &config = DBConfig::GetConfig(db); + + idx_t segment_size; + if (Settings::Get(config)) { + if (last_transient_segment_size == 0) { + const auto initial_rows = Settings::Get(config); + segment_size = MinValue(block_size, MaxValue(type_size, initial_rows * type_size)); + } else { + segment_size = MinValue(block_size, last_transient_segment_size * 2); + } + // BIT (validity) segments can only hold rows in multiples of STANDARD_VECTOR_SIZE; + // any segment below STANDARD_MASK_SIZE triggering a dead-segment overflow chain + if (type.InternalType() == PhysicalType::BIT) { + segment_size = MaxValue(segment_size, ValidityMask::STANDARD_MASK_SIZE); + } + last_transient_segment_size = segment_size; + } else { + auto vector_segment_size = block_size; + + if (data_type == ColumnDataType::INITIAL_TRANSACTION_LOCAL && start_row == 0) { #if STANDARD_VECTOR_SIZE < 1024 - vector_segment_size = 1024 * type_size; + vector_segment_size = 1024 * type_size; #else - vector_segment_size = STANDARD_VECTOR_SIZE * type_size; + vector_segment_size = STANDARD_VECTOR_SIZE * type_size; #endif - } + } - // The segment size is bound by the block size, but can be smaller. - idx_t segment_size = block_size < vector_segment_size ? block_size : vector_segment_size; + // The segment size is bound by the block size, but can be smaller. + segment_size = block_size < vector_segment_size ? block_size : vector_segment_size; + } allocation_size += segment_size; - auto &db = GetDatabase(); - auto &config = DBConfig::GetConfig(db); auto function = config.GetCompressionFunction(CompressionType::COMPRESSION_UNCOMPRESSED, type.InternalType()); - auto new_segment = ColumnSegment::CreateTransientSegment(db, function, type, segment_size, block_manager); AppendSegment(l, std::move(new_segment)); } diff --git a/test/configs/adaptive_segment_allocation.json b/test/configs/adaptive_segment_allocation.json new file mode 100644 index 000000000000..dd62a68e902c --- /dev/null +++ b/test/configs/adaptive_segment_allocation.json @@ -0,0 +1,5 @@ +{ + "description": "Run with adaptive transient segment allocation enabled (initial_segment_rows=2).", + "on_init": "SET adaptive_segment_allocation=true; SET initial_segment_rows=2;", + "skip_compiled": "true" +} diff --git a/test/configs/force_storage_restart.json b/test/configs/force_storage_restart.json index 1fb1ff0199b9..a3edccdec40b 100644 --- a/test/configs/force_storage_restart.json +++ b/test/configs/force_storage_restart.json @@ -46,6 +46,12 @@ "test/sql/catalog/dependencies/test_drop_column_sequence_dependency.test", "test/sql/catalog/dependencies/test_set_default_sequence_dependency.test" ] + }, + { + "reason": "Tests transient segment structure before checkpoint. force_restart checkpoints after every statement, merging segments.", + "paths": [ + "test/sql/storage/adaptive_segment_allocation.test" + ] } ] } diff --git a/test/sql/storage/adaptive_segment_allocation.test b/test/sql/storage/adaptive_segment_allocation.test new file mode 100644 index 000000000000..4ca48cc0e24d --- /dev/null +++ b/test/sql/storage/adaptive_segment_allocation.test @@ -0,0 +1,219 @@ +# name: test/sql/storage/adaptive_segment_allocation.test +# description: Adaptive transient segment allocation - memory footprint and correctness +# group: [storage] + +require vector_size 2048 + +statement ok +SET adaptive_segment_allocation = true + +statement ok +SET initial_segment_rows = 2 + +# disable auto-checkpoint so the transient segment structure is visible in pragma_storage_info +statement ok +SET wal_autocheckpoint='1TB' + +# with initial_segment_rows=2 and 5 rows, the doubling chain produces exactly 2 segments +statement ok +CREATE TABLE step_test (i INT NOT NULL) + +statement ok +INSERT INTO step_test VALUES (1), (2), (3), (4), (5) + +# segment 0 holds the first 2 rows (= initial_segment_rows), segment 1 holds the remaining 3 +query II +SELECT segment_id, count +FROM pragma_storage_info('step_test') +WHERE column_name = 'i' AND segment_type = 'INTEGER' +ORDER BY segment_id +---- +0 2 +1 3 + +statement ok +DROP TABLE step_test + +statement ok +CHECKPOINT + +statement ok +CREATE TABLE mem_adaptive (a INT NOT NULL, b INT NOT NULL, c INT NOT NULL, d INT NOT NULL, e INT NOT NULL) + +statement ok +INSERT INTO mem_adaptive VALUES (1, 2, 3, 4, 5) + +# adaptive: 5 cols * (8 bytes data + 256 bytes validity) = ~1320 bytes +# non-adaptive allocates ~51200 bytes for the same insert +query I +SELECT CASE WHEN memory_usage_bytes < 2000 + THEN 'ok' + ELSE error(concat('expected < 2000 bytes, got ', memory_usage_bytes)) + END +FROM duckdb_memory() +WHERE tag = 'IN_MEMORY_TABLE' +---- +ok + +statement ok +CHECKPOINT + +statement ok +DROP TABLE mem_adaptive + +statement ok +SET adaptive_segment_allocation = false + +statement ok +CREATE TABLE mem_default (a INT NOT NULL, b INT NOT NULL, c INT NOT NULL, d INT NOT NULL, e INT NOT NULL) + +statement ok +INSERT INTO mem_default VALUES (1, 2, 3, 4, 5) + +# non-adaptive: 5 cols * (STANDARD_VECTOR_SIZE * 4 + STANDARD_VECTOR_SIZE / 8) = 51200 bytes +query I +SELECT CASE WHEN memory_usage_bytes = 51200 + THEN 'ok' + ELSE error(concat('expected 51200 bytes, got ', memory_usage_bytes)) + END +FROM duckdb_memory() +WHERE tag = 'IN_MEMORY_TABLE' +---- +ok + +statement ok +DROP TABLE mem_default + +statement ok +SET adaptive_segment_allocation = true + +statement ok +SET initial_segment_rows = 2 + +# correctness: 100 rows split across doubling segments (2, 4, 8, 16, 32, 38) +statement ok +CREATE TABLE overflow_test (i BIGINT NOT NULL) + +statement ok +INSERT INTO overflow_test SELECT range FROM range(100) + +query I +SELECT COUNT(*) FROM overflow_test +---- +100 + +query I +SELECT SUM(i) FROM overflow_test +---- +4950 + +query I +SELECT MIN(i) = 0 AND MAX(i) = 99 FROM overflow_test +---- +true + +# data must survive checkpoint (transient segments converted to persistent) +statement ok +CHECKPOINT + +query I +SELECT COUNT(*) FROM overflow_test +---- +100 + +query I +SELECT SUM(i) FROM overflow_test +---- +4950 + +statement ok +DROP TABLE overflow_test + +statement ok +CREATE TABLE validity_test (i INTEGER) + +# every 7th row is NULL, crossing every segment boundary +statement ok +INSERT INTO validity_test + SELECT CASE WHEN range % 7 = 6 THEN NULL ELSE range END + FROM range(100) + +query I +SELECT COUNT(*) FROM validity_test +---- +100 + +# 14 NULLs at positions 6, 13, 20, ..., 97 +query I +SELECT COUNT(*) FILTER (WHERE i IS NULL) FROM validity_test +---- +14 + +query I +SELECT COUNT(*) FILTER (WHERE i IS NOT NULL) FROM validity_test +---- +86 + +# if bit positions were misaligned across segments, this would return non-zero +query I +SELECT COUNT(*) FROM ( + SELECT ROW_NUMBER() OVER () - 1 AS rn, i FROM validity_test +) WHERE (rn % 7 = 6) != (i IS NULL) +---- +0 + +statement ok +SET adaptive_segment_allocation = true + +statement ok +SET initial_segment_rows = 2 + +statement ok +CREATE TABLE string_test (s VARCHAR) + +# even rows: short strings (stored inline) +# odd rows: 50-char prefix + number (stored in overflow pages) +statement ok +INSERT INTO string_test + SELECT CASE WHEN range % 2 = 0 + THEN range::VARCHAR + ELSE repeat('x', 50) || range::VARCHAR + END + FROM range(100) + +query I +SELECT COUNT(*) FROM string_test +---- +100 + +# 50 short strings (inline) and 50 long strings (overflow) +query I +SELECT COUNT(*) FILTER (WHERE len(s) <= 12) FROM string_test +---- +50 + +query I +SELECT COUNT(*) FILTER (WHERE len(s) > 50) FROM string_test +---- +50 + +# data must survive checkpoint +statement ok +CHECKPOINT + +query I +SELECT COUNT(*) FROM string_test +---- +100 + +# verify a specific short string is intact +query I +SELECT s FROM string_test WHERE s = '42' +---- +42 + +# verify a specific long string is intact and not truncated +query I +SELECT len(s) FROM string_test WHERE s = repeat('x', 50) || '1' +---- +51