diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationKeyVaultOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationKeyVaultOptions.cs index cca16df80..70a4d75e5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationKeyVaultOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationKeyVaultOptions.cs @@ -31,6 +31,13 @@ public class AzureAppConfigurationKeyVaultOptions internal TimeSpan? DefaultSecretRefreshInterval = null; internal bool IsKeyVaultRefreshConfigured = false; + /// + /// Specifies whether Key Vault references should be resolved in parallel. + /// Default value is false. Enabling this can reduce the time required to resolve Key Vault references + /// when many references are loaded from Azure App Configuration. + /// + public bool ParallelSecretResolutionEnabled { get; set; } + /// /// Sets the credentials used to authenticate to key vaults that have no registered . /// diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index e5b6e2585..68862f518 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -137,12 +137,17 @@ internal IEnumerable Adapters /// /// Flag to indicate whether Key Vault options have been configured. /// - internal bool IsKeyVaultConfigured { get; private set; } = false; + internal bool IsKeyVaultConfigured { get; private set; } /// /// Flag to indicate whether Key Vault secret values will be refreshed automatically. /// - internal bool IsKeyVaultRefreshConfigured { get; private set; } = false; + internal bool IsKeyVaultRefreshConfigured { get; private set; } + + /// + /// Flag to indicate whether Key Vault references should be resolved in parallel. + /// + internal bool IsParallelSecretResolutionEnabled { get; private set; } /// /// Indicates all feature flag features used by the application. @@ -520,6 +525,7 @@ public AzureAppConfigurationOptions ConfigureKeyVault(Action> PrepareData(Dictionary + adapter.PreloadAsync(data.Values, _logger, cancellationToken))) + .ConfigureAwait(false); + } + foreach (KeyValuePair kvp in data) { - IEnumerable> keyValuePairs = null; - if (_requestTracingEnabled && _requestTracingOptions != null) { _requestTracingOptions.UpdateAiConfigurationTracing(kvp.Value.ContentType); } - keyValuePairs = await ProcessAdapters(kvp.Value, cancellationToken).ConfigureAwait(false); + IEnumerable> keyValuePairs = await ProcessAdapters(kvp.Value, cancellationToken).ConfigureAwait(false); foreach (KeyValuePair kv in keyValuePairs) { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureKeyVaultReference/AzureKeyVaultKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureKeyVaultReference/AzureKeyVaultKeyValueAdapter.cs index 0498ba09e..5e7118ab5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureKeyVaultReference/AzureKeyVaultKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureKeyVaultReference/AzureKeyVaultKeyValueAdapter.cs @@ -9,7 +9,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net.Mime; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -75,15 +74,12 @@ KeyVaultReferenceException CreateKeyVaultReferenceException(string message, Conf public bool CanProcess(ConfigurationSetting setting) { - if (setting == null || - string.IsNullOrWhiteSpace(setting.Value) || - string.IsNullOrWhiteSpace(setting.ContentType)) + if (setting == null || string.IsNullOrWhiteSpace(setting.Value)) { return false; } - return setting.ContentType.TryParseContentType(out ContentType contentType) - && contentType.IsKeyVaultReference(); + return setting.IsKeyVaultReference(); } public void OnChangeDetected(ConfigurationSetting setting = null) @@ -116,6 +112,79 @@ public bool NeedsRefresh() return _secretProvider.ShouldRefreshKeyVaultSecrets(); } + public async Task PreloadAsync(IEnumerable settings, Logger logger, CancellationToken cancellationToken) + { + if (settings == null) + { + return; + } + + HashSet seen = null; + List<(KeyVaultSecretIdentifier Identifier, string Key, string Label)> toFetch = null; + + foreach (ConfigurationSetting setting in settings) + { + if (!CanProcess(setting)) + { + continue; + } + + string secretRefUri = ParseSecretReferenceUri(setting); + + if (string.IsNullOrEmpty(secretRefUri) || + !Uri.TryCreate(secretRefUri, UriKind.Absolute, out Uri secretUri) || + !KeyVaultSecretIdentifier.TryCreate(secretUri, out KeyVaultSecretIdentifier secretIdentifier)) + { + // Invalid references are surfaced from ProcessKeyValue with full exception context. + continue; + } + + seen = seen ?? new HashSet(); + + if (!seen.Add(secretIdentifier.SourceId)) + { + continue; + } + + toFetch = toFetch ?? new List<(KeyVaultSecretIdentifier, string, string)>(); + toFetch.Add((secretIdentifier, setting.Key, setting.Label)); + } + + if (toFetch == null) + { + return; + } + + var tasks = new Task[toFetch.Count]; + + for (int i = 0; i < toFetch.Count; i++) + { + (KeyVaultSecretIdentifier identifier, string key, string label) = toFetch[i]; + tasks[i] = PreloadSecretAsync(identifier, key, label, logger, cancellationToken); + } + + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + private async Task PreloadSecretAsync(KeyVaultSecretIdentifier identifier, string key, string label, Logger logger, CancellationToken cancellationToken) + { + try + { + await _secretProvider.GetSecretValue(identifier, key, label, logger, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch + { + // Per-secret failures are deferred so ProcessKeyValue can throw a properly populated + // KeyVaultReferenceException. Evict the negative cache entry written by GetSecretValue + // so the retry actually re-fetches instead of returning a cached null within the backoff. + _secretProvider.RemoveSecretFromCache(identifier.SourceId); + } + } + private string ParseSecretReferenceUri(ConfigurationSetting setting) { string secretRefUri = null; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureKeyVaultReference/AzureKeyVaultSecretProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureKeyVaultReference/AzureKeyVaultSecretProvider.cs index 57505ff95..145cee7aa 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureKeyVaultReference/AzureKeyVaultSecretProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureKeyVaultReference/AzureKeyVaultSecretProvider.cs @@ -4,8 +4,8 @@ using Azure.Security.KeyVault.Secrets; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using System; +using System.Collections.Concurrent; using System.Collections.Generic; -using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -14,16 +14,14 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault internal class AzureKeyVaultSecretProvider { private readonly AzureAppConfigurationKeyVaultOptions _keyVaultOptions; - private readonly IDictionary _secretClients; - private readonly Dictionary _cachedKeyVaultSecrets; - private Uri _nextRefreshSourceId; - private DateTimeOffset? _nextRefreshTime; + private readonly ConcurrentDictionary _secretClients; + private readonly ConcurrentDictionary _cachedKeyVaultSecrets; public AzureKeyVaultSecretProvider(AzureAppConfigurationKeyVaultOptions keyVaultOptions = null) { _keyVaultOptions = keyVaultOptions ?? new AzureAppConfigurationKeyVaultOptions(); - _cachedKeyVaultSecrets = new Dictionary(); - _secretClients = new Dictionary(StringComparer.OrdinalIgnoreCase); + _cachedKeyVaultSecrets = new ConcurrentDictionary(); + _secretClients = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); if (_keyVaultOptions.SecretClients != null) { @@ -52,6 +50,7 @@ public async Task GetSecretValue(KeyVaultSecretIdentifier secretIdentifi throw new UnauthorizedAccessException("No key vault credential or secret resolver callback configured, and no matching secret client could be found."); } + CachedKeyVaultSecret updatedCachedSecret = null; bool success = false; try @@ -68,12 +67,12 @@ public async Task GetSecretValue(KeyVaultSecretIdentifier secretIdentifi secretValue = await _keyVaultOptions.SecretResolver(secretIdentifier.SourceId).ConfigureAwait(false); } - cachedSecret = new CachedKeyVaultSecret(secretValue, secretIdentifier.SourceId); + updatedCachedSecret = new CachedKeyVaultSecret(secretValue, secretIdentifier.SourceId); success = true; } finally { - SetSecretInCache(secretIdentifier.SourceId, key, cachedSecret, success); + SetSecretInCache(secretIdentifier.SourceId, key, updatedCachedSecret, success); } return secretValue; @@ -81,42 +80,31 @@ public async Task GetSecretValue(KeyVaultSecretIdentifier secretIdentifi public bool ShouldRefreshKeyVaultSecrets() { - return _nextRefreshTime.HasValue && _nextRefreshTime.Value < DateTimeOffset.UtcNow; - } - - public void ClearCache() - { - var sourceIdsToRemove = new List(); - - var utcNow = DateTimeOffset.UtcNow; - foreach (KeyValuePair secret in _cachedKeyVaultSecrets) { - if (secret.Value.LastRefreshTime + RefreshConstants.MinimumSecretRefreshInterval < utcNow) + if (secret.Value.RefreshAt.HasValue && secret.Value.RefreshAt.Value < DateTimeOffset.UtcNow) { - sourceIdsToRemove.Add(secret.Key); + return true; } } - foreach (Uri sourceId in sourceIdsToRemove) - { - _cachedKeyVaultSecrets.Remove(sourceId); - } + return false; + } - if (_cachedKeyVaultSecrets.Any()) + public void ClearCache() + { + foreach (KeyValuePair secret in _cachedKeyVaultSecrets) { - UpdateNextRefreshableSecretFromCache(); + if (secret.Value.LastRefreshTime + RefreshConstants.MinimumSecretRefreshInterval < DateTimeOffset.UtcNow) + { + _cachedKeyVaultSecrets.TryRemove(secret.Key, out _); + } } } public void RemoveSecretFromCache(Uri sourceId) { - _cachedKeyVaultSecrets.Remove(sourceId); - - if (sourceId == _nextRefreshSourceId) - { - UpdateNextRefreshableSecretFromCache(); - } + _cachedKeyVaultSecrets.TryRemove(sourceId, out _); } private SecretClient GetSecretClient(Uri secretUri) @@ -133,14 +121,12 @@ private SecretClient GetSecretClient(Uri secretUri) return null; } - client = new SecretClient( - new Uri(secretUri.GetLeftPart(UriPartial.Authority)), - _keyVaultOptions.Credential, - _keyVaultOptions.ClientOptions); - - _secretClients.Add(keyVaultId, client); - - return client; + return _secretClients.GetOrAdd( + keyVaultId, + _ => new SecretClient( + new Uri(secretUri.GetLeftPart(UriPartial.Authority)), + _keyVaultOptions.Credential, + _keyVaultOptions.ClientOptions)); } private void SetSecretInCache(Uri sourceId, string key, CachedKeyVaultSecret cachedSecret, bool success = true) @@ -152,37 +138,6 @@ private void SetSecretInCache(Uri sourceId, string key, CachedKeyVaultSecret cac UpdateCacheExpirationTimeForSecret(key, cachedSecret, success); _cachedKeyVaultSecrets[sourceId] = cachedSecret; - - if (sourceId == _nextRefreshSourceId) - { - UpdateNextRefreshableSecretFromCache(); - } - else if ((cachedSecret.RefreshAt.HasValue && _nextRefreshTime.HasValue && cachedSecret.RefreshAt.Value < _nextRefreshTime.Value) - || (cachedSecret.RefreshAt.HasValue && !_nextRefreshTime.HasValue)) - { - _nextRefreshSourceId = sourceId; - _nextRefreshTime = cachedSecret.RefreshAt.Value; - } - } - - private void UpdateNextRefreshableSecretFromCache() - { - _nextRefreshSourceId = null; - _nextRefreshTime = DateTimeOffset.MaxValue; - - foreach (KeyValuePair secret in _cachedKeyVaultSecrets) - { - if (secret.Value.RefreshAt.HasValue && secret.Value.RefreshAt.Value < _nextRefreshTime) - { - _nextRefreshTime = secret.Value.RefreshAt; - _nextRefreshSourceId = secret.Key; - } - } - - if (_nextRefreshTime == DateTimeOffset.MaxValue) - { - _nextRefreshTime = null; - } } private void UpdateCacheExpirationTimeForSecret(string key, CachedKeyVaultSecret cachedSecret, bool success) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationSettingExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationSettingExtensions.cs new file mode 100644 index 000000000..28639337c --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationSettingExtensions.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; +using System.Net.Mime; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions +{ + internal static class ConfigurationSettingExtensions + { + public static bool IsKeyVaultReference(this ConfigurationSetting setting) + { + return setting != null + && setting.ContentType.TryParseContentType(out ContentType contentType) + && contentType.IsKeyVaultReference(); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index fdd7f2fdf..a3431cd7b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -87,6 +87,11 @@ public void OnConfigUpdated() return; } + public Task PreloadAsync(IEnumerable settings, Logger logger, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + private List> ProcessDotnetSchemaFeatureFlag(FeatureFlag featureFlag, ConfigurationSetting setting, Uri endpoint) { var keyValues = new List>(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IKeyValueAdapter.cs index de13314e1..9dddc0e02 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IKeyValueAdapter.cs @@ -13,6 +13,10 @@ internal interface IKeyValueAdapter { Task>> ProcessKeyValue(ConfigurationSetting setting, Uri endpoint, Logger logger, CancellationToken cancellationToken); + // Pre-warm any per-setting state (e.g. Key Vault secret cache) before ProcessKeyValue is invoked + // on each setting. Adapters with no pre-fetchable state can return a completed task. + Task PreloadAsync(IEnumerable settings, Logger logger, CancellationToken cancellationToken); + bool CanProcess(ConfigurationSetting setting); void OnChangeDetected(ConfigurationSetting setting = null); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/JsonKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/JsonKeyValueAdapter.cs index d353439fd..9d8826e78 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/JsonKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/JsonKeyValueAdapter.cs @@ -80,5 +80,10 @@ public bool NeedsRefresh() { return false; } + + public Task PreloadAsync(IEnumerable settings, Logger logger, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } } } diff --git a/tests/Tests.AzureAppConfiguration/Unit/KeyVaultReferenceTests.cs b/tests/Tests.AzureAppConfiguration/Unit/KeyVaultReferenceTests.cs index 3e856a1b0..f1214f456 100644 --- a/tests/Tests.AzureAppConfiguration/Unit/KeyVaultReferenceTests.cs +++ b/tests/Tests.AzureAppConfiguration/Unit/KeyVaultReferenceTests.cs @@ -1050,5 +1050,188 @@ MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) Assert.Equal(_secretValue, config[setting.Key]); } } + + [Fact] + public void ParallelSecretResolution_ResolvesAllReferences() + { + // Build a collection of distinct Key Vault references. + const int referenceCount = 20; + var settings = new List(); + + for (int i = 0; i < referenceCount; i++) + { + settings.Add(ConfigurationModelFactory.ConfigurationSetting( + key: $"Key{i}", + value: $@"{{""uri"":""https://keyvault-theclassics.vault.azure.net/secrets/Secret{i}""}}", + eTag: new ETag($"etag-{i}"), + contentType: KeyVaultConstants.ContentType + "; charset=utf-8")); + } + + var mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns(new MockAsyncPageable(settings)); + + var mockSecretClient = new Mock(MockBehavior.Strict); + mockSecretClient.SetupGet(client => client.VaultUri).Returns(new Uri("https://keyvault-theclassics.vault.azure.net")); + mockSecretClient.Setup(client => client.GetSecretAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((string name, string version, CancellationToken cancellationToken) => + Task.FromResult((Response)new MockResponse(new KeyVaultSecret(name, $"value-of-{name}")))); + + var configuration = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigureKeyVault(kv => + { + kv.Register(mockSecretClient.Object); + kv.ParallelSecretResolutionEnabled = true; + }); + }) + .Build(); + + for (int i = 0; i < referenceCount; i++) + { + Assert.Equal($"value-of-Secret{i}", configuration[$"Key{i}"]); + } + } + + [Fact] + public void ParallelSecretResolution_RunsConcurrently() + { + // Use a gated mock secret client to detect concurrent in-flight calls. + const int referenceCount = 10; + var settings = new List(); + + for (int i = 0; i < referenceCount; i++) + { + settings.Add(ConfigurationModelFactory.ConfigurationSetting( + key: $"Key{i}", + value: $@"{{""uri"":""https://keyvault-theclassics.vault.azure.net/secrets/Secret{i}""}}", + eTag: new ETag($"etag-{i}"), + contentType: KeyVaultConstants.ContentType + "; charset=utf-8")); + } + + int inFlight = 0; + int maxInFlight = 0; + var inFlightLock = new object(); + + var mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns(new MockAsyncPageable(settings)); + + var mockSecretClient = new Mock(MockBehavior.Strict); + mockSecretClient.SetupGet(client => client.VaultUri).Returns(new Uri("https://keyvault-theclassics.vault.azure.net")); + mockSecretClient.Setup(client => client.GetSecretAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (string name, string version, CancellationToken cancellationToken) => + { + lock (inFlightLock) + { + inFlight++; + if (inFlight > maxInFlight) + { + maxInFlight = inFlight; + } + } + + try + { + await Task.Delay(50, cancellationToken).ConfigureAwait(false); + } + finally + { + lock (inFlightLock) + { + inFlight--; + } + } + + return (Response)new MockResponse(new KeyVaultSecret(name, $"value-of-{name}")); + }); + + var configuration = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigureKeyVault(kv => + { + kv.Register(mockSecretClient.Object); + kv.ParallelSecretResolutionEnabled = true; + }); + }) + .Build(); + + // Verify all references resolved. + for (int i = 0; i < referenceCount; i++) + { + Assert.Equal($"value-of-Secret{i}", configuration[$"Key{i}"]); + } + + // When run in parallel, more than one secret request must have been in flight at the same time. + Assert.True(maxInFlight > 1, $"Expected concurrent Key Vault requests, but observed max in-flight = {maxInFlight}."); + } + + [Fact] + public void ParallelSecretResolution_DisabledByDefault_RunsSequentially() + { + const int referenceCount = 5; + var settings = new List(); + + for (int i = 0; i < referenceCount; i++) + { + settings.Add(ConfigurationModelFactory.ConfigurationSetting( + key: $"Key{i}", + value: $@"{{""uri"":""https://keyvault-theclassics.vault.azure.net/secrets/Secret{i}""}}", + eTag: new ETag($"etag-{i}"), + contentType: KeyVaultConstants.ContentType + "; charset=utf-8")); + } + + int inFlight = 0; + int maxInFlight = 0; + var inFlightLock = new object(); + + var mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns(new MockAsyncPageable(settings)); + + var mockSecretClient = new Mock(MockBehavior.Strict); + mockSecretClient.SetupGet(client => client.VaultUri).Returns(new Uri("https://keyvault-theclassics.vault.azure.net")); + mockSecretClient.Setup(client => client.GetSecretAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (string name, string version, CancellationToken cancellationToken) => + { + lock (inFlightLock) + { + inFlight++; + if (inFlight > maxInFlight) + { + maxInFlight = inFlight; + } + } + + try + { + await Task.Delay(20, cancellationToken).ConfigureAwait(false); + } + finally + { + lock (inFlightLock) + { + inFlight--; + } + } + + return (Response)new MockResponse(new KeyVaultSecret(name, $"value-of-{name}")); + }); + + new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigureKeyVault(kv => kv.Register(mockSecretClient.Object)); + }) + .Build(); + + // Default (sequential) path should never have more than one in-flight Key Vault request. + Assert.Equal(1, maxInFlight); + } } }