From cc53525c9df66e3ba46eab234ed2df56f122ce54 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Mon, 25 May 2026 16:26:29 +0530 Subject: [PATCH 1/7] [Storage] Change default parallelism to CPU-based value across azblob, azfile, azdatalake Update default concurrency for uploads and downloads from a fixed value of 5 (or 1 for stream uploads) to Clamp(NumCPU, 8, 96), matching the Rust SDK implementation. This improves throughput on modern multi-core VMs while clamping prevents resource exhaustion on small or very large machines. Adds AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true env var opt-out to revert to the previous default of 5. --- sdk/storage/azblob/blob/constants.go | 9 +++- sdk/storage/azblob/blob/models.go | 9 ++-- .../azblob/blockblob/chunkwriting_test.go | 2 +- sdk/storage/azblob/blockblob/models.go | 8 +-- .../azblob/internal/shared/batch_transfer.go | 17 +++++- .../azblob/internal/shared/shared_test.go | 43 +++++++++++++++ sdk/storage/azdatalake/file/models.go | 14 +++-- .../internal/shared/batch_transfer.go | 21 +++++++- .../internal/shared/batch_transfer_test.go | 53 +++++++++++++++++++ sdk/storage/azfile/file/models.go | 17 +++--- .../azfile/internal/shared/batch_transfer.go | 21 +++++++- .../internal/shared/batch_transfer_test.go | 53 +++++++++++++++++++ 12 files changed, 245 insertions(+), 22 deletions(-) create mode 100644 sdk/storage/azdatalake/internal/shared/batch_transfer_test.go create mode 100644 sdk/storage/azfile/internal/shared/batch_transfer_test.go diff --git a/sdk/storage/azblob/blob/constants.go b/sdk/storage/azblob/blob/constants.go index ef47450b5ea6..6a3f45bcdec6 100644 --- a/sdk/storage/azblob/blob/constants.go +++ b/sdk/storage/azblob/blob/constants.go @@ -17,10 +17,17 @@ const ( // DefaultDownloadBlockSize is default block size DefaultDownloadBlockSize = int64(4 * 1024 * 1024) // 4MB - // DefaultConcurrency is the default number of blocks downloaded or uploaded in parallel + // DefaultConcurrency is the legacy default number of blocks downloaded or uploaded in parallel. + // Deprecated: Use DefaultConcurrencyValue() instead, which returns a value based on CPU core count. DefaultConcurrency = shared.DefaultConcurrency ) +// DefaultConcurrencyValue returns the default concurrency for parallel uploads/downloads. +// The value is based on CPU core count, clamped between 8 and 96. +func DefaultConcurrencyValue() uint16 { + return shared.DefaultConcurrencyValue() +} + // BlobType defines values for BlobType type BlobType = generated.BlobType diff --git a/sdk/storage/azblob/blob/models.go b/sdk/storage/azblob/blob/models.go index 9acec4594fb3..3ecac8bff373 100644 --- a/sdk/storage/azblob/blob/models.go +++ b/sdk/storage/azblob/blob/models.go @@ -122,7 +122,8 @@ type downloadOptions struct { CPKInfo *CPKInfo CPKScopeInfo *CPKScopeInfo - // Concurrency indicates the maximum number of blocks to download in parallel (0=default). + // Concurrency indicates the maximum number of blocks to download in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // RetryReaderOptionsPerBlock is used when downloading each block. @@ -176,7 +177,8 @@ type DownloadBufferOptions struct { // CPKScopeInfo contains a group of parameters for client provided encryption scope. CPKScopeInfo *CPKScopeInfo - // Concurrency indicates the maximum number of blocks to download in parallel (0=default). + // Concurrency indicates the maximum number of blocks to download in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // RetryReaderOptionsPerBlock is used when downloading each block. @@ -204,7 +206,8 @@ type DownloadFileOptions struct { CPKInfo *CPKInfo CPKScopeInfo *CPKScopeInfo - // Concurrency indicates the maximum number of blocks to download in parallel. The default value is 5. + // Concurrency indicates the maximum number of blocks to download in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // RetryReaderOptionsPerBlock is used when downloading each block. diff --git a/sdk/storage/azblob/blockblob/chunkwriting_test.go b/sdk/storage/azblob/blockblob/chunkwriting_test.go index d8f5473ebde1..ed6ee6e90961 100644 --- a/sdk/storage/azblob/blockblob/chunkwriting_test.go +++ b/sdk/storage/azblob/blockblob/chunkwriting_test.go @@ -177,7 +177,7 @@ func TestSlowDestCopyFrom(t *testing.T) { require.ErrorIs(t, err, io.ErrNoProgress) } - require.Equal(t, 1, tracker.Count) + require.GreaterOrEqual(t, tracker.Count, 1) require.True(t, tracker.Freed) } diff --git a/sdk/storage/azblob/blockblob/models.go b/sdk/storage/azblob/blockblob/models.go index 868637ffc481..09799c5b8446 100644 --- a/sdk/storage/azblob/blockblob/models.go +++ b/sdk/storage/azblob/blockblob/models.go @@ -273,7 +273,8 @@ type uploadFromReaderOptions struct { CPKInfo *blob.CPKInfo CPKScopeInfo *blob.CPKScopeInfo - // Concurrency indicates the maximum number of blocks to upload in parallel (0=default) + // Concurrency indicates the maximum number of blocks to upload in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 TransactionalValidation blob.TransferValidationType @@ -334,7 +335,8 @@ type UploadStreamOptions struct { BlockSize int64 // Concurrency defines the max number of concurrent uploads to be performed to upload the file. - // Each concurrent upload will create a buffer of size BlockSize. The default value is one. + // Each concurrent upload will create a buffer of size BlockSize. The default is based on + // CPU core count (min 8, max 96). Concurrency int TransactionalValidation blob.TransferValidationType @@ -350,7 +352,7 @@ type UploadStreamOptions struct { func (u *UploadStreamOptions) setDefaults() { if u.Concurrency == 0 { - u.Concurrency = 1 + u.Concurrency = int(shared.DefaultConcurrencyValue()) } if u.BlockSize < _1MiB { diff --git a/sdk/storage/azblob/internal/shared/batch_transfer.go b/sdk/storage/azblob/internal/shared/batch_transfer.go index 7fc26f543d06..95e38fc7ba44 100644 --- a/sdk/storage/azblob/internal/shared/batch_transfer.go +++ b/sdk/storage/azblob/internal/shared/batch_transfer.go @@ -6,12 +6,27 @@ package shared import ( "context" "errors" + "os" + "runtime" + "strings" ) const ( + // DefaultConcurrency is the legacy default concurrency value. + // Deprecated: Use DefaultConcurrencyValue() instead, which returns a value based on CPU core count. DefaultConcurrency = 5 ) +// DefaultConcurrencyValue returns the default concurrency based on CPU count, +// clamped between 8 and 96. Set the AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY +// environment variable to "true" to revert to the previous default of 5. +func DefaultConcurrencyValue() uint16 { + if strings.EqualFold(os.Getenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY"), "true") { + return DefaultConcurrency + } + return uint16(max(8, min(96, runtime.NumCPU()))) +} + // BatchTransferOptions identifies options used by doBatchTransfer. type BatchTransferOptions struct { TransferSize int64 @@ -30,7 +45,7 @@ func DoBatchTransfer(ctx context.Context, o *BatchTransferOptions) error { } if o.Concurrency == 0 { - o.Concurrency = DefaultConcurrency // default concurrency + o.Concurrency = DefaultConcurrencyValue() } // Prepare and do parallel operations. diff --git a/sdk/storage/azblob/internal/shared/shared_test.go b/sdk/storage/azblob/internal/shared/shared_test.go index d4cd2c7b2e7a..70be7bf26263 100644 --- a/sdk/storage/azblob/internal/shared/shared_test.go +++ b/sdk/storage/azblob/internal/shared/shared_test.go @@ -4,6 +4,7 @@ package shared import ( + "runtime" "strings" "testing" @@ -207,3 +208,45 @@ func TestIsIPEndpointStyle(t *testing.T) { require.True(t, IsIPEndpointStyle("127.0.0.1")) require.True(t, IsIPEndpointStyle("127.0.0.1:80")) } + +func TestDefaultConcurrencyValue_InBounds(t *testing.T) { + val := DefaultConcurrencyValue() + require.GreaterOrEqual(t, val, uint16(8)) + require.LessOrEqual(t, val, uint16(96)) +} + +func TestDefaultConcurrencyValue_Deterministic(t *testing.T) { + val1 := DefaultConcurrencyValue() + val2 := DefaultConcurrencyValue() + require.Equal(t, val1, val2) +} + +func TestDefaultConcurrencyValue_MatchesCPU(t *testing.T) { + cpus := runtime.NumCPU() + val := DefaultConcurrencyValue() + if cpus < 8 { + require.Equal(t, uint16(8), val) + } else if cpus > 96 { + require.Equal(t, uint16(96), val) + } else { + require.Equal(t, uint16(cpus), val) + } +} + +func TestDefaultConcurrencyValue_LegacyEnvVar(t *testing.T) { + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "true") + require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) + + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "TRUE") + require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) + + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "false") + val := DefaultConcurrencyValue() + require.GreaterOrEqual(t, val, uint16(8)) + require.LessOrEqual(t, val, uint16(96)) + + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "") + val = DefaultConcurrencyValue() + require.GreaterOrEqual(t, val, uint16(8)) + require.LessOrEqual(t, val, uint16(96)) +} diff --git a/sdk/storage/azdatalake/file/models.go b/sdk/storage/azdatalake/file/models.go index 4ff0c58b897a..550eb248f743 100644 --- a/sdk/storage/azdatalake/file/models.go +++ b/sdk/storage/azdatalake/file/models.go @@ -141,7 +141,8 @@ type uploadFromReaderOptions struct { // Progress is a function that is invoked periodically as bytes are sent to the FileClient. // Note that the progress reporting is not always increasing; it can go down when retrying a request. Progress func(bytesTransferred int64) - // Concurrency indicates the maximum number of chunks to upload in parallel (default is 5) + // Concurrency indicates the maximum number of chunks to upload in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // AccessConditions contains optional parameters to access leased entity. AccessConditions *AccessConditions @@ -157,7 +158,8 @@ type uploadFromReaderOptions struct { type UploadStreamOptions struct { // ChunkSize specifies the chunk size to use in bytes; the default (and maximum size) is MaxAppendBytes. ChunkSize int64 - // Concurrency indicates the maximum number of chunks to upload in parallel (default is 5) + // Concurrency indicates the maximum number of chunks to upload in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // AccessConditions contains optional parameters to access leased entity. AccessConditions *AccessConditions @@ -324,7 +326,7 @@ func (o *AppendDataOptions) format(offset int64, body io.ReadSeekCloser) (*gener func (u *UploadStreamOptions) setDefaults() { if u.Concurrency == 0 { - u.Concurrency = 1 + u.Concurrency = shared.DefaultConcurrencyValue() } if u.ChunkSize < _1MiB { @@ -428,7 +430,8 @@ type DownloadBufferOptions struct { CPKInfo *CPKInfo // CPKScopeInfo contains a group of parameters for client provided encryption scope. CPKScopeInfo *CPKScopeInfo - // Concurrency indicates the maximum number of chunks to download in parallel (0=default). + // Concurrency indicates the maximum number of chunks to download in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. RetryReaderOptionsPerChunk *RetryReaderOptions @@ -482,7 +485,8 @@ type DownloadFileOptions struct { CPKInfo *CPKInfo // CPKScopeInfo contains a group of parameters for client provided encryption scope. CPKScopeInfo *CPKScopeInfo - // Concurrency indicates the maximum number of chunks to download in parallel. The default value is 5. + // Concurrency indicates the maximum number of chunks to download in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. RetryReaderOptionsPerChunk *RetryReaderOptions diff --git a/sdk/storage/azdatalake/internal/shared/batch_transfer.go b/sdk/storage/azdatalake/internal/shared/batch_transfer.go index 2c5cf54c07c9..1aa30a8868bf 100644 --- a/sdk/storage/azdatalake/internal/shared/batch_transfer.go +++ b/sdk/storage/azdatalake/internal/shared/batch_transfer.go @@ -6,8 +6,27 @@ package shared import ( "context" "errors" + "os" + "runtime" + "strings" ) +const ( + // DefaultConcurrency is the legacy default concurrency value. + // Deprecated: Use DefaultConcurrencyValue() instead, which returns a value based on CPU core count. + DefaultConcurrency = 5 +) + +// DefaultConcurrencyValue returns the default concurrency based on CPU count, +// clamped between 8 and 96. Set the AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY +// environment variable to "true" to revert to the previous default of 5. +func DefaultConcurrencyValue() uint16 { + if strings.EqualFold(os.Getenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY"), "true") { + return DefaultConcurrency + } + return uint16(max(8, min(96, runtime.NumCPU()))) +} + // BatchTransferOptions identifies options used by doBatchTransfer. type BatchTransferOptions struct { TransferSize int64 @@ -25,7 +44,7 @@ func DoBatchTransfer(ctx context.Context, o *BatchTransferOptions) error { } if o.Concurrency == 0 { - o.Concurrency = 5 // default concurrency + o.Concurrency = DefaultConcurrencyValue() } // Prepare and do parallel operations. diff --git a/sdk/storage/azdatalake/internal/shared/batch_transfer_test.go b/sdk/storage/azdatalake/internal/shared/batch_transfer_test.go new file mode 100644 index 000000000000..20bbfbefbc71 --- /dev/null +++ b/sdk/storage/azdatalake/internal/shared/batch_transfer_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package shared + +import ( + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDefaultConcurrencyValue_InBounds(t *testing.T) { + val := DefaultConcurrencyValue() + require.GreaterOrEqual(t, val, uint16(8)) + require.LessOrEqual(t, val, uint16(96)) +} + +func TestDefaultConcurrencyValue_Deterministic(t *testing.T) { + val1 := DefaultConcurrencyValue() + val2 := DefaultConcurrencyValue() + require.Equal(t, val1, val2) +} + +func TestDefaultConcurrencyValue_MatchesCPU(t *testing.T) { + cpus := runtime.NumCPU() + val := DefaultConcurrencyValue() + if cpus < 8 { + require.Equal(t, uint16(8), val) + } else if cpus > 96 { + require.Equal(t, uint16(96), val) + } else { + require.Equal(t, uint16(cpus), val) + } +} + +func TestDefaultConcurrencyValue_LegacyEnvVar(t *testing.T) { + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "true") + require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) + + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "TRUE") + require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) + + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "false") + val := DefaultConcurrencyValue() + require.GreaterOrEqual(t, val, uint16(8)) + require.LessOrEqual(t, val, uint16(96)) + + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "") + val = DefaultConcurrencyValue() + require.GreaterOrEqual(t, val, uint16(8)) + require.LessOrEqual(t, val, uint16(96)) +} diff --git a/sdk/storage/azfile/file/models.go b/sdk/storage/azfile/file/models.go index 0dd5122c1556..1cb8c8496cd7 100644 --- a/sdk/storage/azfile/file/models.go +++ b/sdk/storage/azfile/file/models.go @@ -606,7 +606,8 @@ type downloadOptions struct { // LeaseAccessConditions contains optional parameters to access leased entity. LeaseAccessConditions *LeaseAccessConditions - // Concurrency indicates the maximum number of chunks to download in parallel (0=default). + // Concurrency indicates the maximum number of chunks to download in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. @@ -646,7 +647,8 @@ type DownloadBufferOptions struct { // LeaseAccessConditions contains optional parameters to access leased entity. LeaseAccessConditions *LeaseAccessConditions - // Concurrency indicates the maximum number of chunks to download in parallel (0=default). + // Concurrency indicates the maximum number of chunks to download in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. @@ -669,7 +671,8 @@ type DownloadFileOptions struct { // LeaseAccessConditions contains optional parameters to access leased entity. LeaseAccessConditions *LeaseAccessConditions - // Concurrency indicates the maximum number of chunks to download in parallel (0=default). + // Concurrency indicates the maximum number of chunks to download in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. @@ -1050,7 +1053,8 @@ type uploadFromReaderOptions struct { // Note that the progress reporting is not always increasing; it can go down when retrying a request. Progress func(bytesTransferred int64) - // Concurrency indicates the maximum number of chunks to upload in parallel (default is 5) + // Concurrency indicates the maximum number of chunks to upload in parallel. + // The default is based on CPU core count (min 8, max 96). Concurrency uint16 // LeaseAccessConditions contains optional parameters to access leased entity. @@ -1078,7 +1082,8 @@ type UploadStreamOptions struct { ChunkSize int64 // Concurrency defines the max number of concurrent uploads to be performed to upload the file. - // Each concurrent upload will create a buffer of size ChunkSize. The default value is one. + // Each concurrent upload will create a buffer of size ChunkSize. The default is based on + // CPU core count (min 8, max 96). Concurrency int // LeaseAccessConditions contains optional parameters to access leased entity. @@ -1087,7 +1092,7 @@ type UploadStreamOptions struct { func (u *UploadStreamOptions) setDefaults() { if u.Concurrency == 0 { - u.Concurrency = 1 + u.Concurrency = int(shared.DefaultConcurrencyValue()) } if u.ChunkSize < _1MiB { diff --git a/sdk/storage/azfile/internal/shared/batch_transfer.go b/sdk/storage/azfile/internal/shared/batch_transfer.go index 2c5cf54c07c9..1aa30a8868bf 100644 --- a/sdk/storage/azfile/internal/shared/batch_transfer.go +++ b/sdk/storage/azfile/internal/shared/batch_transfer.go @@ -6,8 +6,27 @@ package shared import ( "context" "errors" + "os" + "runtime" + "strings" ) +const ( + // DefaultConcurrency is the legacy default concurrency value. + // Deprecated: Use DefaultConcurrencyValue() instead, which returns a value based on CPU core count. + DefaultConcurrency = 5 +) + +// DefaultConcurrencyValue returns the default concurrency based on CPU count, +// clamped between 8 and 96. Set the AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY +// environment variable to "true" to revert to the previous default of 5. +func DefaultConcurrencyValue() uint16 { + if strings.EqualFold(os.Getenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY"), "true") { + return DefaultConcurrency + } + return uint16(max(8, min(96, runtime.NumCPU()))) +} + // BatchTransferOptions identifies options used by doBatchTransfer. type BatchTransferOptions struct { TransferSize int64 @@ -25,7 +44,7 @@ func DoBatchTransfer(ctx context.Context, o *BatchTransferOptions) error { } if o.Concurrency == 0 { - o.Concurrency = 5 // default concurrency + o.Concurrency = DefaultConcurrencyValue() } // Prepare and do parallel operations. diff --git a/sdk/storage/azfile/internal/shared/batch_transfer_test.go b/sdk/storage/azfile/internal/shared/batch_transfer_test.go new file mode 100644 index 000000000000..20bbfbefbc71 --- /dev/null +++ b/sdk/storage/azfile/internal/shared/batch_transfer_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package shared + +import ( + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDefaultConcurrencyValue_InBounds(t *testing.T) { + val := DefaultConcurrencyValue() + require.GreaterOrEqual(t, val, uint16(8)) + require.LessOrEqual(t, val, uint16(96)) +} + +func TestDefaultConcurrencyValue_Deterministic(t *testing.T) { + val1 := DefaultConcurrencyValue() + val2 := DefaultConcurrencyValue() + require.Equal(t, val1, val2) +} + +func TestDefaultConcurrencyValue_MatchesCPU(t *testing.T) { + cpus := runtime.NumCPU() + val := DefaultConcurrencyValue() + if cpus < 8 { + require.Equal(t, uint16(8), val) + } else if cpus > 96 { + require.Equal(t, uint16(96), val) + } else { + require.Equal(t, uint16(cpus), val) + } +} + +func TestDefaultConcurrencyValue_LegacyEnvVar(t *testing.T) { + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "true") + require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) + + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "TRUE") + require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) + + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "false") + val := DefaultConcurrencyValue() + require.GreaterOrEqual(t, val, uint16(8)) + require.LessOrEqual(t, val, uint16(96)) + + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "") + val = DefaultConcurrencyValue() + require.GreaterOrEqual(t, val, uint16(8)) + require.LessOrEqual(t, val, uint16(96)) +} From 43cd16f985d097477a09975796b4b242b038991f Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Mon, 25 May 2026 17:55:15 +0530 Subject: [PATCH 2/7] Add GiB-scale parallelism benchmarks for upload/download buffer and stream --- .../blockblob/parallelism_bench_test.go | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 sdk/storage/azblob/blockblob/parallelism_bench_test.go diff --git a/sdk/storage/azblob/blockblob/parallelism_bench_test.go b/sdk/storage/azblob/blockblob/parallelism_bench_test.go new file mode 100644 index 000000000000..780300cb6a67 --- /dev/null +++ b/sdk/storage/azblob/blockblob/parallelism_bench_test.go @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package blockblob_test + +import ( + "bytes" + "context" + "crypto/rand" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" +) + +// GiB-scale parallelism benchmarks to measure the impact of default concurrency changes. +// These use UploadBuffer/DownloadBuffer which perform parallel chunked transfers, +// where concurrency directly affects throughput. +// +// Run with: +// AZURE_STORAGE_ACCOUNT_NAME=... AZURE_STORAGE_ACCOUNT_KEY=... \ +// go test ./blockblob/... -bench 'BenchmarkParallelism' -benchtime=3x -run '^$' -timeout 1800s +// +// To compare old vs new defaults: +// # Old default (concurrency=5): +// AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true go test ... +// # New default (concurrency=NumCPU clamped 8-96): +// go test ... + +func BenchmarkParallelismUploadBuffer(b *testing.B) { + sizes := []struct { + name string + size int + }{ + {"256MB", 256 * 1024 * 1024}, + {"512MB", 512 * 1024 * 1024}, + {"1GiB", 1024 * 1024 * 1024}, + {"2GiB", 2 * 1024 * 1024 * 1024}, + {"4GiB", 4 * 1024 * 1024 * 1024}, + } + + for _, sz := range sizes { + b.Run(sz.name, func(b *testing.B) { + data := make([]byte, sz.size) + _, _ = rand.Read(data) + + env := setupBenchEnv(b) + b.SetBytes(int64(sz.size)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + blobName := fmt.Sprintf("par-upbuf-%s-%d", sz.name, i) + bbClient := env.containerClient.NewBlockBlobClient(blobName) + _, err := bbClient.UploadBuffer(context.Background(), data, &blockblob.UploadBufferOptions{}) + if err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkParallelismUploadStream(b *testing.B) { + sizes := []struct { + name string + size int + }{ + {"256MB", 256 * 1024 * 1024}, + {"512MB", 512 * 1024 * 1024}, + {"1GiB", 1024 * 1024 * 1024}, + {"2GiB", 2 * 1024 * 1024 * 1024}, + {"4GiB", 4 * 1024 * 1024 * 1024}, + } + + for _, sz := range sizes { + b.Run(sz.name, func(b *testing.B) { + data := make([]byte, sz.size) + _, _ = rand.Read(data) + + env := setupBenchEnv(b) + b.SetBytes(int64(sz.size)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + blobName := fmt.Sprintf("par-upstr-%s-%d", sz.name, i) + bbClient := env.containerClient.NewBlockBlobClient(blobName) + _, err := bbClient.UploadStream(context.Background(), + bytes.NewReader(data), + &blockblob.UploadStreamOptions{}) + if err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkParallelismDownloadBuffer(b *testing.B) { + sizes := []struct { + name string + size int + }{ + {"256MB", 256 * 1024 * 1024}, + {"512MB", 512 * 1024 * 1024}, + {"1GiB", 1024 * 1024 * 1024}, + {"2GiB", 2 * 1024 * 1024 * 1024}, + {"4GiB", 4 * 1024 * 1024 * 1024}, + } + + for _, sz := range sizes { + b.Run(sz.name, func(b *testing.B) { + data := make([]byte, sz.size) + _, _ = rand.Read(data) + + env := setupBenchEnv(b) + + // Upload the blob once before benchmarking download + blobName := fmt.Sprintf("par-dlbuf-%s", sz.name) + bbClient := env.containerClient.NewBlockBlobClient(blobName) + _, err := bbClient.UploadBuffer(context.Background(), data, nil) + if err != nil { + b.Fatal(err) + } + + // Get blob properties to know the size for buffer allocation + props, err := bbClient.GetProperties(context.Background(), nil) + if err != nil { + b.Fatal(err) + } + blobSize := *props.ContentLength + + b.SetBytes(blobSize) + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf := make([]byte, blobSize) + _, err := bbClient.BlobClient().DownloadBuffer(context.Background(), buf, &blob.DownloadBufferOptions{}) + if err != nil { + b.Fatal(err) + } + } + }) + } +} From f35f22ea371ecdd2a11b7666c64051b68275893f Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Mon, 25 May 2026 18:03:50 +0530 Subject: [PATCH 3/7] Remove duplicate test files from azfile and azdatalake --- .../internal/shared/batch_transfer_test.go | 53 ------------------- .../internal/shared/batch_transfer_test.go | 53 ------------------- 2 files changed, 106 deletions(-) delete mode 100644 sdk/storage/azdatalake/internal/shared/batch_transfer_test.go delete mode 100644 sdk/storage/azfile/internal/shared/batch_transfer_test.go diff --git a/sdk/storage/azdatalake/internal/shared/batch_transfer_test.go b/sdk/storage/azdatalake/internal/shared/batch_transfer_test.go deleted file mode 100644 index 20bbfbefbc71..000000000000 --- a/sdk/storage/azdatalake/internal/shared/batch_transfer_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -package shared - -import ( - "runtime" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestDefaultConcurrencyValue_InBounds(t *testing.T) { - val := DefaultConcurrencyValue() - require.GreaterOrEqual(t, val, uint16(8)) - require.LessOrEqual(t, val, uint16(96)) -} - -func TestDefaultConcurrencyValue_Deterministic(t *testing.T) { - val1 := DefaultConcurrencyValue() - val2 := DefaultConcurrencyValue() - require.Equal(t, val1, val2) -} - -func TestDefaultConcurrencyValue_MatchesCPU(t *testing.T) { - cpus := runtime.NumCPU() - val := DefaultConcurrencyValue() - if cpus < 8 { - require.Equal(t, uint16(8), val) - } else if cpus > 96 { - require.Equal(t, uint16(96), val) - } else { - require.Equal(t, uint16(cpus), val) - } -} - -func TestDefaultConcurrencyValue_LegacyEnvVar(t *testing.T) { - t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "true") - require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) - - t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "TRUE") - require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) - - t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "false") - val := DefaultConcurrencyValue() - require.GreaterOrEqual(t, val, uint16(8)) - require.LessOrEqual(t, val, uint16(96)) - - t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "") - val = DefaultConcurrencyValue() - require.GreaterOrEqual(t, val, uint16(8)) - require.LessOrEqual(t, val, uint16(96)) -} diff --git a/sdk/storage/azfile/internal/shared/batch_transfer_test.go b/sdk/storage/azfile/internal/shared/batch_transfer_test.go deleted file mode 100644 index 20bbfbefbc71..000000000000 --- a/sdk/storage/azfile/internal/shared/batch_transfer_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -package shared - -import ( - "runtime" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestDefaultConcurrencyValue_InBounds(t *testing.T) { - val := DefaultConcurrencyValue() - require.GreaterOrEqual(t, val, uint16(8)) - require.LessOrEqual(t, val, uint16(96)) -} - -func TestDefaultConcurrencyValue_Deterministic(t *testing.T) { - val1 := DefaultConcurrencyValue() - val2 := DefaultConcurrencyValue() - require.Equal(t, val1, val2) -} - -func TestDefaultConcurrencyValue_MatchesCPU(t *testing.T) { - cpus := runtime.NumCPU() - val := DefaultConcurrencyValue() - if cpus < 8 { - require.Equal(t, uint16(8), val) - } else if cpus > 96 { - require.Equal(t, uint16(96), val) - } else { - require.Equal(t, uint16(cpus), val) - } -} - -func TestDefaultConcurrencyValue_LegacyEnvVar(t *testing.T) { - t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "true") - require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) - - t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "TRUE") - require.Equal(t, uint16(DefaultConcurrency), DefaultConcurrencyValue()) - - t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "false") - val := DefaultConcurrencyValue() - require.GreaterOrEqual(t, val, uint16(8)) - require.LessOrEqual(t, val, uint16(96)) - - t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "") - val = DefaultConcurrencyValue() - require.GreaterOrEqual(t, val, uint16(8)) - require.LessOrEqual(t, val, uint16(96)) -} From 0ac90732b8af1d32ae7e6b367300b38f5815ae45 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Mon, 25 May 2026 18:04:28 +0530 Subject: [PATCH 4/7] Remove benchmark file from PR --- .../blockblob/parallelism_bench_test.go | 142 ------------------ 1 file changed, 142 deletions(-) delete mode 100644 sdk/storage/azblob/blockblob/parallelism_bench_test.go diff --git a/sdk/storage/azblob/blockblob/parallelism_bench_test.go b/sdk/storage/azblob/blockblob/parallelism_bench_test.go deleted file mode 100644 index 780300cb6a67..000000000000 --- a/sdk/storage/azblob/blockblob/parallelism_bench_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -package blockblob_test - -import ( - "bytes" - "context" - "crypto/rand" - "fmt" - "testing" - - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" -) - -// GiB-scale parallelism benchmarks to measure the impact of default concurrency changes. -// These use UploadBuffer/DownloadBuffer which perform parallel chunked transfers, -// where concurrency directly affects throughput. -// -// Run with: -// AZURE_STORAGE_ACCOUNT_NAME=... AZURE_STORAGE_ACCOUNT_KEY=... \ -// go test ./blockblob/... -bench 'BenchmarkParallelism' -benchtime=3x -run '^$' -timeout 1800s -// -// To compare old vs new defaults: -// # Old default (concurrency=5): -// AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true go test ... -// # New default (concurrency=NumCPU clamped 8-96): -// go test ... - -func BenchmarkParallelismUploadBuffer(b *testing.B) { - sizes := []struct { - name string - size int - }{ - {"256MB", 256 * 1024 * 1024}, - {"512MB", 512 * 1024 * 1024}, - {"1GiB", 1024 * 1024 * 1024}, - {"2GiB", 2 * 1024 * 1024 * 1024}, - {"4GiB", 4 * 1024 * 1024 * 1024}, - } - - for _, sz := range sizes { - b.Run(sz.name, func(b *testing.B) { - data := make([]byte, sz.size) - _, _ = rand.Read(data) - - env := setupBenchEnv(b) - b.SetBytes(int64(sz.size)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - blobName := fmt.Sprintf("par-upbuf-%s-%d", sz.name, i) - bbClient := env.containerClient.NewBlockBlobClient(blobName) - _, err := bbClient.UploadBuffer(context.Background(), data, &blockblob.UploadBufferOptions{}) - if err != nil { - b.Fatal(err) - } - } - }) - } -} - -func BenchmarkParallelismUploadStream(b *testing.B) { - sizes := []struct { - name string - size int - }{ - {"256MB", 256 * 1024 * 1024}, - {"512MB", 512 * 1024 * 1024}, - {"1GiB", 1024 * 1024 * 1024}, - {"2GiB", 2 * 1024 * 1024 * 1024}, - {"4GiB", 4 * 1024 * 1024 * 1024}, - } - - for _, sz := range sizes { - b.Run(sz.name, func(b *testing.B) { - data := make([]byte, sz.size) - _, _ = rand.Read(data) - - env := setupBenchEnv(b) - b.SetBytes(int64(sz.size)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - blobName := fmt.Sprintf("par-upstr-%s-%d", sz.name, i) - bbClient := env.containerClient.NewBlockBlobClient(blobName) - _, err := bbClient.UploadStream(context.Background(), - bytes.NewReader(data), - &blockblob.UploadStreamOptions{}) - if err != nil { - b.Fatal(err) - } - } - }) - } -} - -func BenchmarkParallelismDownloadBuffer(b *testing.B) { - sizes := []struct { - name string - size int - }{ - {"256MB", 256 * 1024 * 1024}, - {"512MB", 512 * 1024 * 1024}, - {"1GiB", 1024 * 1024 * 1024}, - {"2GiB", 2 * 1024 * 1024 * 1024}, - {"4GiB", 4 * 1024 * 1024 * 1024}, - } - - for _, sz := range sizes { - b.Run(sz.name, func(b *testing.B) { - data := make([]byte, sz.size) - _, _ = rand.Read(data) - - env := setupBenchEnv(b) - - // Upload the blob once before benchmarking download - blobName := fmt.Sprintf("par-dlbuf-%s", sz.name) - bbClient := env.containerClient.NewBlockBlobClient(blobName) - _, err := bbClient.UploadBuffer(context.Background(), data, nil) - if err != nil { - b.Fatal(err) - } - - // Get blob properties to know the size for buffer allocation - props, err := bbClient.GetProperties(context.Background(), nil) - if err != nil { - b.Fatal(err) - } - blobSize := *props.ContentLength - - b.SetBytes(blobSize) - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf := make([]byte, blobSize) - _, err := bbClient.BlobClient().DownloadBuffer(context.Background(), buf, &blob.DownloadBufferOptions{}) - if err != nil { - b.Fatal(err) - } - } - }) - } -} From 625689624345af7367c151c60f3dab52b4216b31 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Mon, 25 May 2026 18:15:37 +0530 Subject: [PATCH 5/7] Address review comments: hermetic tests and env var in doc comments --- sdk/storage/azblob/blob/constants.go | 1 + sdk/storage/azblob/blob/models.go | 6 +++--- sdk/storage/azblob/blockblob/models.go | 4 ++-- sdk/storage/azblob/internal/shared/shared_test.go | 3 +++ sdk/storage/azdatalake/file/models.go | 8 ++++---- sdk/storage/azfile/file/models.go | 10 +++++----- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/sdk/storage/azblob/blob/constants.go b/sdk/storage/azblob/blob/constants.go index 6a3f45bcdec6..918d587b38c8 100644 --- a/sdk/storage/azblob/blob/constants.go +++ b/sdk/storage/azblob/blob/constants.go @@ -24,6 +24,7 @@ const ( // DefaultConcurrencyValue returns the default concurrency for parallel uploads/downloads. // The value is based on CPU core count, clamped between 8 and 96. +// Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default of 5. func DefaultConcurrencyValue() uint16 { return shared.DefaultConcurrencyValue() } diff --git a/sdk/storage/azblob/blob/models.go b/sdk/storage/azblob/blob/models.go index 3ecac8bff373..a23329c19f70 100644 --- a/sdk/storage/azblob/blob/models.go +++ b/sdk/storage/azblob/blob/models.go @@ -123,7 +123,7 @@ type downloadOptions struct { CPKScopeInfo *CPKScopeInfo // Concurrency indicates the maximum number of blocks to download in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // RetryReaderOptionsPerBlock is used when downloading each block. @@ -178,7 +178,7 @@ type DownloadBufferOptions struct { CPKScopeInfo *CPKScopeInfo // Concurrency indicates the maximum number of blocks to download in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // RetryReaderOptionsPerBlock is used when downloading each block. @@ -207,7 +207,7 @@ type DownloadFileOptions struct { CPKScopeInfo *CPKScopeInfo // Concurrency indicates the maximum number of blocks to download in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // RetryReaderOptionsPerBlock is used when downloading each block. diff --git a/sdk/storage/azblob/blockblob/models.go b/sdk/storage/azblob/blockblob/models.go index 09799c5b8446..4e3bac2863f8 100644 --- a/sdk/storage/azblob/blockblob/models.go +++ b/sdk/storage/azblob/blockblob/models.go @@ -274,7 +274,7 @@ type uploadFromReaderOptions struct { CPKScopeInfo *blob.CPKScopeInfo // Concurrency indicates the maximum number of blocks to upload in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 TransactionalValidation blob.TransferValidationType @@ -336,7 +336,7 @@ type UploadStreamOptions struct { // Concurrency defines the max number of concurrent uploads to be performed to upload the file. // Each concurrent upload will create a buffer of size BlockSize. The default is based on - // CPU core count (min 8, max 96). + // CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency int TransactionalValidation blob.TransferValidationType diff --git a/sdk/storage/azblob/internal/shared/shared_test.go b/sdk/storage/azblob/internal/shared/shared_test.go index 70be7bf26263..e9eafee8699e 100644 --- a/sdk/storage/azblob/internal/shared/shared_test.go +++ b/sdk/storage/azblob/internal/shared/shared_test.go @@ -210,18 +210,21 @@ func TestIsIPEndpointStyle(t *testing.T) { } func TestDefaultConcurrencyValue_InBounds(t *testing.T) { + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "") val := DefaultConcurrencyValue() require.GreaterOrEqual(t, val, uint16(8)) require.LessOrEqual(t, val, uint16(96)) } func TestDefaultConcurrencyValue_Deterministic(t *testing.T) { + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "") val1 := DefaultConcurrencyValue() val2 := DefaultConcurrencyValue() require.Equal(t, val1, val2) } func TestDefaultConcurrencyValue_MatchesCPU(t *testing.T) { + t.Setenv("AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY", "") cpus := runtime.NumCPU() val := DefaultConcurrencyValue() if cpus < 8 { diff --git a/sdk/storage/azdatalake/file/models.go b/sdk/storage/azdatalake/file/models.go index 550eb248f743..5d8857888af3 100644 --- a/sdk/storage/azdatalake/file/models.go +++ b/sdk/storage/azdatalake/file/models.go @@ -142,7 +142,7 @@ type uploadFromReaderOptions struct { // Note that the progress reporting is not always increasing; it can go down when retrying a request. Progress func(bytesTransferred int64) // Concurrency indicates the maximum number of chunks to upload in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // AccessConditions contains optional parameters to access leased entity. AccessConditions *AccessConditions @@ -159,7 +159,7 @@ type UploadStreamOptions struct { // ChunkSize specifies the chunk size to use in bytes; the default (and maximum size) is MaxAppendBytes. ChunkSize int64 // Concurrency indicates the maximum number of chunks to upload in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // AccessConditions contains optional parameters to access leased entity. AccessConditions *AccessConditions @@ -431,7 +431,7 @@ type DownloadBufferOptions struct { // CPKScopeInfo contains a group of parameters for client provided encryption scope. CPKScopeInfo *CPKScopeInfo // Concurrency indicates the maximum number of chunks to download in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. RetryReaderOptionsPerChunk *RetryReaderOptions @@ -486,7 +486,7 @@ type DownloadFileOptions struct { // CPKScopeInfo contains a group of parameters for client provided encryption scope. CPKScopeInfo *CPKScopeInfo // Concurrency indicates the maximum number of chunks to download in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. RetryReaderOptionsPerChunk *RetryReaderOptions diff --git a/sdk/storage/azfile/file/models.go b/sdk/storage/azfile/file/models.go index 1cb8c8496cd7..bfdce6de2de1 100644 --- a/sdk/storage/azfile/file/models.go +++ b/sdk/storage/azfile/file/models.go @@ -607,7 +607,7 @@ type downloadOptions struct { LeaseAccessConditions *LeaseAccessConditions // Concurrency indicates the maximum number of chunks to download in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. @@ -648,7 +648,7 @@ type DownloadBufferOptions struct { LeaseAccessConditions *LeaseAccessConditions // Concurrency indicates the maximum number of chunks to download in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. @@ -672,7 +672,7 @@ type DownloadFileOptions struct { LeaseAccessConditions *LeaseAccessConditions // Concurrency indicates the maximum number of chunks to download in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // RetryReaderOptionsPerChunk is used when downloading each chunk. @@ -1054,7 +1054,7 @@ type uploadFromReaderOptions struct { Progress func(bytesTransferred int64) // Concurrency indicates the maximum number of chunks to upload in parallel. - // The default is based on CPU core count (min 8, max 96). + // The default is based on CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency uint16 // LeaseAccessConditions contains optional parameters to access leased entity. @@ -1083,7 +1083,7 @@ type UploadStreamOptions struct { // Concurrency defines the max number of concurrent uploads to be performed to upload the file. // Each concurrent upload will create a buffer of size ChunkSize. The default is based on - // CPU core count (min 8, max 96). + // CPU core count (min 8, max 96). Set AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY=true to revert to the previous default. Concurrency int // LeaseAccessConditions contains optional parameters to access leased entity. From 236bfd899e43726b714cf84faa3964a204727886 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Mon, 25 May 2026 18:28:22 +0530 Subject: [PATCH 6/7] Fix gocritic deprecatedComment: separate Deprecated notice into its own paragraph --- sdk/storage/azblob/blob/constants.go | 1 + sdk/storage/azblob/internal/shared/batch_transfer.go | 1 + sdk/storage/azdatalake/internal/shared/batch_transfer.go | 1 + sdk/storage/azfile/internal/shared/batch_transfer.go | 1 + 4 files changed, 4 insertions(+) diff --git a/sdk/storage/azblob/blob/constants.go b/sdk/storage/azblob/blob/constants.go index 918d587b38c8..c00a2019467f 100644 --- a/sdk/storage/azblob/blob/constants.go +++ b/sdk/storage/azblob/blob/constants.go @@ -18,6 +18,7 @@ const ( DefaultDownloadBlockSize = int64(4 * 1024 * 1024) // 4MB // DefaultConcurrency is the legacy default number of blocks downloaded or uploaded in parallel. + // // Deprecated: Use DefaultConcurrencyValue() instead, which returns a value based on CPU core count. DefaultConcurrency = shared.DefaultConcurrency ) diff --git a/sdk/storage/azblob/internal/shared/batch_transfer.go b/sdk/storage/azblob/internal/shared/batch_transfer.go index 95e38fc7ba44..e6cc0a03039e 100644 --- a/sdk/storage/azblob/internal/shared/batch_transfer.go +++ b/sdk/storage/azblob/internal/shared/batch_transfer.go @@ -13,6 +13,7 @@ import ( const ( // DefaultConcurrency is the legacy default concurrency value. + // // Deprecated: Use DefaultConcurrencyValue() instead, which returns a value based on CPU core count. DefaultConcurrency = 5 ) diff --git a/sdk/storage/azdatalake/internal/shared/batch_transfer.go b/sdk/storage/azdatalake/internal/shared/batch_transfer.go index 1aa30a8868bf..98f0239d2005 100644 --- a/sdk/storage/azdatalake/internal/shared/batch_transfer.go +++ b/sdk/storage/azdatalake/internal/shared/batch_transfer.go @@ -13,6 +13,7 @@ import ( const ( // DefaultConcurrency is the legacy default concurrency value. + // // Deprecated: Use DefaultConcurrencyValue() instead, which returns a value based on CPU core count. DefaultConcurrency = 5 ) diff --git a/sdk/storage/azfile/internal/shared/batch_transfer.go b/sdk/storage/azfile/internal/shared/batch_transfer.go index 1aa30a8868bf..98f0239d2005 100644 --- a/sdk/storage/azfile/internal/shared/batch_transfer.go +++ b/sdk/storage/azfile/internal/shared/batch_transfer.go @@ -13,6 +13,7 @@ import ( const ( // DefaultConcurrency is the legacy default concurrency value. + // // Deprecated: Use DefaultConcurrencyValue() instead, which returns a value based on CPU core count. DefaultConcurrency = 5 ) From 18eb0ba8b4c79edfe95860fbe1b45cfef599873d Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Mon, 25 May 2026 18:38:47 +0530 Subject: [PATCH 7/7] Fix staticcheck SA1019 for deprecated const re-export --- sdk/storage/azblob/blob/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azblob/blob/constants.go b/sdk/storage/azblob/blob/constants.go index c00a2019467f..956e5163dc09 100644 --- a/sdk/storage/azblob/blob/constants.go +++ b/sdk/storage/azblob/blob/constants.go @@ -20,7 +20,7 @@ const ( // DefaultConcurrency is the legacy default number of blocks downloaded or uploaded in parallel. // // Deprecated: Use DefaultConcurrencyValue() instead, which returns a value based on CPU core count. - DefaultConcurrency = shared.DefaultConcurrency + DefaultConcurrency = shared.DefaultConcurrency //nolint:staticcheck // intentional re-export of deprecated const for backward compat ) // DefaultConcurrencyValue returns the default concurrency for parallel uploads/downloads.