From c2e18d7b2c2c458e6951fc88f17e1ec0ebc13884 Mon Sep 17 00:00:00 2001 From: Ross Grambo Date: Fri, 18 Oct 2024 13:08:23 -0700 Subject: [PATCH 01/16] Adding allocation id --- .../Extensions/StringExtensions.cs | 9 + .../FeatureManagementConstants.cs | 1 + .../FeatureManagementKeyValueAdapter.cs | 86 +++++++++ .../FeatureManagementTests.cs | 164 ++++++++++++++++++ 4 files changed, 260 insertions(+) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs index 7bcf7212c..8b2c488d0 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // +using System; + namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { internal static class LabelFilters @@ -16,5 +18,12 @@ public static string NormalizeNull(this string s) { return s == LabelFilters.Null ? null : s; } + + public static string ToBase64String(this string s) + { + byte[] bytes = System.Text.Encoding.UTF8.GetBytes(s); + + return Convert.ToBase64String(bytes); + } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs index c6d86d849..aa573a1e2 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs @@ -44,6 +44,7 @@ internal class FeatureManagementConstants public const string ETag = "ETag"; public const string FeatureFlagId = "FeatureFlagId"; public const string FeatureFlagReference = "FeatureFlagReference"; + public const string AllocationId = "AllocationId"; // Dotnet schema keys public const string DotnetSchemaSectionName = "FeatureManagement"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index 2b0e08331..9e3678844 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -319,12 +319,98 @@ private List> ProcessMicrosoftSchemaFeatureFlag(Fea keyValues.Add(new KeyValuePair($"{telemetryPath}:{FeatureManagementConstants.Metadata}:{FeatureManagementConstants.ETag}", setting.ETag.ToString())); keyValues.Add(new KeyValuePair($"{telemetryPath}:{FeatureManagementConstants.Enabled}", telemetry.Enabled.ToString())); + + string allocationId = CalculateAllocationId(featureFlag); + + if (allocationId != null) + { + keyValues.Add(new KeyValuePair($"{telemetryPath}:{FeatureManagementConstants.Metadata}:{FeatureManagementConstants.AllocationId}", allocationId)); + } } } return keyValues; } + private string CalculateAllocationId(FeatureFlag flag) + { + if (flag.Allocation == null) + { + return null; + } + + StringBuilder inputBuilder = new StringBuilder(); + + // Seed + inputBuilder.Append($"seed={flag.Allocation.Seed ?? ""}"); + + var allocatedVariants = new HashSet(); + + // DefaultWhenEnabled + if (flag.Allocation.DefaultWhenEnabled != null) + { + allocatedVariants.Add(flag.Allocation.DefaultWhenEnabled); + } + + inputBuilder.Append($"\ndefault_when_enabled={flag.Allocation.DefaultWhenEnabled ?? ""}"); + + // Percentiles + inputBuilder.Append("\npercentiles="); + + if (flag.Allocation.Percentile != null && flag.Allocation.Percentile.Any()) + { + var sortedPercentiles = flag.Allocation.Percentile + .Where(p => p.From != p.To) + .OrderBy(p => p.From) + .ToList(); + + allocatedVariants.UnionWith(sortedPercentiles.Select(p => p.Variant)); + + inputBuilder.Append(string.Join(";", sortedPercentiles.Select(p => $"{p.From},{p.Variant.ToBase64String()},{p.To}"))); + } + + // If there's no custom seed and no variants allocated, stop now and return null + if (flag.Allocation.Seed == null && + !allocatedVariants.Any()) + { + return null; + } + + // Variants + inputBuilder.Append("\nvariants="); + + if (allocatedVariants.Any() && flag.Variants != null && flag.Variants.Any()) + { + var sortedVariants = flag.Variants + .Where(variant => allocatedVariants.Contains(variant.Name)) + .OrderBy(variant => variant.Name) + .ToList(); + + inputBuilder.Append(string.Join(";", sortedVariants.Select(v => + { + var variantValue = ""; + + if (v.ConfigurationValue.ValueKind != JsonValueKind.Null && v.ConfigurationValue.ValueKind != JsonValueKind.Undefined) + { + variantValue = JsonSerializer.Serialize(v.ConfigurationValue); + } + + return $"{v.Name.ToBase64String()},{(variantValue)}"; + }))); + } + + // Example input string + // input == "seed=123abc\ndefault_when_enabled=Control\npercentiles=0,Blshdk,20;20,Test,100\nvariants=TdLa,standard;Qfcd,special" + string input = inputBuilder.ToString(); + + using (SHA256 sha256 = SHA256.Create()) + { + byte[] truncatedHash = new byte[15]; + Array.Copy(sha256.ComputeHash(Encoding.UTF8.GetBytes(input)), truncatedHash, 15); + return truncatedHash.ToBase64Url(); + } + } + private FormatException CreateFeatureFlagFormatException(string jsonPropertyName, string settingKey, string foundJsonValueKind, string expectedJsonValueKind) { return new FormatException(string.Format( diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 44c20b4d0..2bfc415e5 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -622,6 +622,111 @@ public class FeatureManagementTests eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")) }; + List _allocationIdFeatureFlagCollection = new List + { + ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "TelemetryVariant", + value: @" + { + ""id"": ""TelemetryVariant"", + ""enabled"": true, + ""variants"": [ + { + ""name"": ""True_Override"", + ""configuration_value"": ""default"", + ""status_override"": ""Disabled"" + } + ], + ""allocation"": { + ""default_when_enabled"": ""True_Override"" + }, + ""telemetry"": { + ""enabled"": ""true"" + } + } + ", + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("cmwBRcIAq1jUyKL3Kj8bvf9jtxBrFg-R-ayExStMC90")), + + ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "TelemetryVariantPercentile", + value: @" + { + ""id"": ""TelemetryVariant"", + ""enabled"": true, + ""variants"": [ + { + ""name"": ""True_Override"", + ""configuration_value"": { + ""someKey"": ""someValue"", + ""someOtherKey"": { + ""someSubKey"": ""someSubValue"" + } + } + } + ], + ""allocation"": { + ""default_when_enabled"": ""True_Override"", + ""percentile"": [ + { + ""variant"": ""True_Override"", + ""from"": 0, + ""to"": 100 + } + ] + }, + ""telemetry"": { + ""enabled"": ""true"" + } + } + ", + label: "label", + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("cmwBRcIAq1jUyKL3Kj8bvf9jtxBrFg-R-ayExStMC90")), + + // Quote of the day test + ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "Greeting", + value: @" + { + ""id"": ""Greeting"", + ""description"": """", + ""enabled"": true, + ""variants"": [ + { + ""name"": ""On"", + ""configuration_value"": true + }, + { + ""name"": ""Off"", + ""configuration_value"": false + } + ], + ""allocation"": { + ""percentile"": [ + { + ""variant"": ""On"", + ""from"": 0, + ""to"": 50 + }, + { + ""variant"": ""Off"", + ""from"": 50, + ""to"": 100 + } + ], + ""default_when_enabled"": ""Off"", + ""default_when_disabled"": ""Off"" + }, + ""telemetry"": { + ""enabled"": true + } + } + ", + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("8kS3pc_cQmWnfLY9LQ1cd-RfR6_nQqH6sgdlL9eCgek")), + }; + TimeSpan RefreshInterval = TimeSpan.FromSeconds(1); [Fact] @@ -1972,6 +2077,65 @@ public void WithTelemetry() Assert.Equal("Tag2Value", config["feature_management:feature_flags:1:telemetry:metadata:Tags.Tag1"]); } + [Fact] + public void WithAllocationId() + { + var mockResponse = new Mock(); + var mockClient = new Mock(MockBehavior.Strict); + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns(new MockAsyncPageable(_allocationIdFeatureFlagCollection)); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Connect(TestHelpers.PrimaryConfigStoreEndpoint, new DefaultAzureCredential()); + options.UseFeatureFlags(); + }) + .Build(); + + byte[] featureFlagIdHash; + + using (HashAlgorithm hashAlgorithm = SHA256.Create()) + { + featureFlagIdHash = hashAlgorithm.ComputeHash(Encoding.UTF8.GetBytes($"{FeatureManagementConstants.FeatureFlagMarker}TelemetryVariant\n")); + } + + string featureFlagId = Convert.ToBase64String(featureFlagIdHash) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + + // Validate TelemetryVariant + Assert.Equal("True", config["feature_management:feature_flags:0:telemetry:enabled"]); + Assert.Equal("TelemetryVariant", config["feature_management:feature_flags:0:id"]); + + Assert.Equal(featureFlagId, config["feature_management:feature_flags:0:telemetry:metadata:FeatureFlagId"]); + + Assert.Equal($"{TestHelpers.PrimaryConfigStoreEndpoint}kv/{FeatureManagementConstants.FeatureFlagMarker}TelemetryVariant", config["feature_management:feature_flags:0:telemetry:metadata:FeatureFlagReference"]); + + Assert.Equal("MExY1waco2tqen4EcJKK", config["feature_management:feature_flags:0:telemetry:metadata:AllocationId"]); + + // Validate TelemetryVariantPercentile + Assert.Equal("True", config["feature_management:feature_flags:1:telemetry:enabled"]); + Assert.Equal("TelemetryVariant", config["feature_management:feature_flags:1:id"]); + + Assert.Equal($"{TestHelpers.PrimaryConfigStoreEndpoint}kv/{FeatureManagementConstants.FeatureFlagMarker}TelemetryVariantPercentile?label=label", config["feature_management:feature_flags:1:telemetry:metadata:FeatureFlagReference"]); + + Assert.Equal("SCq5T7Bb1k6tWfWjI3qz", config["feature_management:feature_flags:1:telemetry:metadata:AllocationId"]); + + // Validate Greeting + Assert.Equal("True", config["feature_management:feature_flags:2:telemetry:enabled"]); + Assert.Equal("Greeting", config["feature_management:feature_flags:2:id"]); + + Assert.Equal("63pHsrNKDSi5Zfe_FvZPSegwbsEo5TS96hf4k7cc4Zw", config["feature_management:feature_flags:2:telemetry:metadata:FeatureFlagId"]); + + Assert.Equal($"{TestHelpers.PrimaryConfigStoreEndpoint}kv/{FeatureManagementConstants.FeatureFlagMarker}Greeting", config["feature_management:feature_flags:2:telemetry:metadata:FeatureFlagReference"]); + + Assert.Equal("L0m7_ulkdsaQmz6dSw4r", config["feature_management:feature_flags:2:telemetry:metadata:AllocationId"]); + } + [Fact] public void WithRequirementType() { From 1c2c525eee10bf6cb1c7695e1df8bee1504e45fb Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Mon, 21 Oct 2024 14:59:24 -0700 Subject: [PATCH 02/16] serialize with sorted keys --- .../FeatureManagementKeyValueAdapter.cs | 6 +- .../JsonElementExtensions.cs | 87 +++++++++++++++++++ .../FeatureManagementTests.cs | 4 +- 3 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index 9e3678844..3076cd996 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -359,7 +359,7 @@ private string CalculateAllocationId(FeatureFlag flag) if (flag.Allocation.Percentile != null && flag.Allocation.Percentile.Any()) { - var sortedPercentiles = flag.Allocation.Percentile + IEnumerable sortedPercentiles = flag.Allocation.Percentile .Where(p => p.From != p.To) .OrderBy(p => p.From) .ToList(); @@ -381,7 +381,7 @@ private string CalculateAllocationId(FeatureFlag flag) if (allocatedVariants.Any() && flag.Variants != null && flag.Variants.Any()) { - var sortedVariants = flag.Variants + IEnumerable sortedVariants = flag.Variants .Where(variant => allocatedVariants.Contains(variant.Name)) .OrderBy(variant => variant.Name) .ToList(); @@ -392,7 +392,7 @@ private string CalculateAllocationId(FeatureFlag flag) if (v.ConfigurationValue.ValueKind != JsonValueKind.Null && v.ConfigurationValue.ValueKind != JsonValueKind.Undefined) { - variantValue = JsonSerializer.Serialize(v.ConfigurationValue); + variantValue = v.ConfigurationValue.SerializeWithSortedKeys(); } return $"{v.Name.ToBase64String()},{(variantValue)}"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs new file mode 100644 index 000000000..a4be3b2a3 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs @@ -0,0 +1,87 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement +{ + internal static class JsonElementExtensions + { + public static string SerializeWithSortedKeys(this JsonElement rootElement) + { + using var stream = new MemoryStream(); + + using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = false })) + { + WriteElementWithSortedKeys(rootElement, writer); + } + + return Encoding.UTF8.GetString(stream.ToArray()); + } + + private static void WriteElementWithSortedKeys(JsonElement element, Utf8JsonWriter writer) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + + foreach (JsonProperty property in element.EnumerateObject().OrderBy(p => p.Name)) + { + writer.WritePropertyName(property.Name); + WriteElementWithSortedKeys(property.Value, writer); + } + + writer.WriteEndObject(); + break; + + case JsonValueKind.Array: + writer.WriteStartArray(); + + foreach (JsonElement item in element.EnumerateArray()) + { + WriteElementWithSortedKeys(item, writer); + } + + writer.WriteEndArray(); + break; + + case JsonValueKind.String: + writer.WriteStringValue(element.GetString()); + break; + + case JsonValueKind.Number: + if (element.TryGetInt32(out int intValue)) + { + writer.WriteNumberValue(intValue); + } + else if (element.TryGetInt64(out long longValue)) + { + writer.WriteNumberValue(longValue); + } + else + { + writer.WriteNumberValue(element.GetDouble()); + } + + break; + + case JsonValueKind.True: + writer.WriteBooleanValue(true); + break; + + case JsonValueKind.False: + writer.WriteBooleanValue(false); + break; + + case JsonValueKind.Null: + writer.WriteNullValue(); + break; + + default: + throw new InvalidOperationException($"Unsupported JsonValueKind: {element.ValueKind}"); + } + } + } +} diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 2bfc415e5..0a94c0f4a 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -658,10 +658,10 @@ public class FeatureManagementTests { ""name"": ""True_Override"", ""configuration_value"": { - ""someKey"": ""someValue"", ""someOtherKey"": { ""someSubKey"": ""someSubValue"" - } + }, + ""someKey"": ""someValue"" } } ], From cb4e61558b86125c4dda0b1c8fbb5975b80a404f Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Mon, 21 Oct 2024 15:08:09 -0700 Subject: [PATCH 03/16] use string empty --- .../FeatureManagement/FeatureManagementKeyValueAdapter.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index 3076cd996..dcfc70d55 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -342,7 +342,7 @@ private string CalculateAllocationId(FeatureFlag flag) StringBuilder inputBuilder = new StringBuilder(); // Seed - inputBuilder.Append($"seed={flag.Allocation.Seed ?? ""}"); + inputBuilder.Append($"seed={flag.Allocation.Seed ?? string.Empty}"); var allocatedVariants = new HashSet(); @@ -352,7 +352,7 @@ private string CalculateAllocationId(FeatureFlag flag) allocatedVariants.Add(flag.Allocation.DefaultWhenEnabled); } - inputBuilder.Append($"\ndefault_when_enabled={flag.Allocation.DefaultWhenEnabled ?? ""}"); + inputBuilder.Append($"\ndefault_when_enabled={flag.Allocation.DefaultWhenEnabled ?? string.Empty}"); // Percentiles inputBuilder.Append("\npercentiles="); @@ -388,7 +388,7 @@ private string CalculateAllocationId(FeatureFlag flag) inputBuilder.Append(string.Join(";", sortedVariants.Select(v => { - var variantValue = ""; + var variantValue = string.Empty; if (v.ConfigurationValue.ValueKind != JsonValueKind.Null && v.ConfigurationValue.ValueKind != JsonValueKind.Undefined) { From 49ef1fc8b8ae75911835d30a0f25fa3360cbffd1 Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Mon, 21 Oct 2024 15:11:51 -0700 Subject: [PATCH 04/16] nit --- .../FeatureManagementKeyValueAdapter.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index dcfc70d55..c29827b97 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Security.Cryptography; using System.Text; @@ -320,11 +321,14 @@ private List> ProcessMicrosoftSchemaFeatureFlag(Fea keyValues.Add(new KeyValuePair($"{telemetryPath}:{FeatureManagementConstants.Enabled}", telemetry.Enabled.ToString())); - string allocationId = CalculateAllocationId(featureFlag); - - if (allocationId != null) + if (featureFlag.Allocation != null) { - keyValues.Add(new KeyValuePair($"{telemetryPath}:{FeatureManagementConstants.Metadata}:{FeatureManagementConstants.AllocationId}", allocationId)); + string allocationId = CalculateAllocationId(featureFlag); + + if (allocationId != null) + { + keyValues.Add(new KeyValuePair($"{telemetryPath}:{FeatureManagementConstants.Metadata}:{FeatureManagementConstants.AllocationId}", allocationId)); + } } } } @@ -334,10 +338,7 @@ private List> ProcessMicrosoftSchemaFeatureFlag(Fea private string CalculateAllocationId(FeatureFlag flag) { - if (flag.Allocation == null) - { - return null; - } + Debug.Assert(flag.Allocation != null); StringBuilder inputBuilder = new StringBuilder(); From 86fde179dd388012c0a1523e3e8aa885a0500714 Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Mon, 21 Oct 2024 15:37:52 -0700 Subject: [PATCH 05/16] rename ff id to TelemetryVariantPercentile --- tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 0a94c0f4a..235aec842 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -652,7 +652,7 @@ public class FeatureManagementTests key: FeatureManagementConstants.FeatureFlagMarker + "TelemetryVariantPercentile", value: @" { - ""id"": ""TelemetryVariant"", + ""id"": ""TelemetryVariantPercentile"", ""enabled"": true, ""variants"": [ { @@ -2119,7 +2119,7 @@ public void WithAllocationId() // Validate TelemetryVariantPercentile Assert.Equal("True", config["feature_management:feature_flags:1:telemetry:enabled"]); - Assert.Equal("TelemetryVariant", config["feature_management:feature_flags:1:id"]); + Assert.Equal("TelemetryVariantPercentile", config["feature_management:feature_flags:1:id"]); Assert.Equal($"{TestHelpers.PrimaryConfigStoreEndpoint}kv/{FeatureManagementConstants.FeatureFlagMarker}TelemetryVariantPercentile?label=label", config["feature_management:feature_flags:1:telemetry:metadata:FeatureFlagReference"]); From c191d7f5666b11ba20156932ddff9c50f88c4a00 Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Mon, 21 Oct 2024 15:58:15 -0700 Subject: [PATCH 06/16] add more values --- .../FeatureManagement/JsonElementExtensions.cs | 7 +++++-- .../Tests.AzureAppConfiguration/FeatureManagementTests.cs | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs index a4be3b2a3..88fb4d5c3 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs @@ -60,11 +60,14 @@ private static void WriteElementWithSortedKeys(JsonElement element, Utf8JsonWrit { writer.WriteNumberValue(longValue); } - else + else if (element.TryGetDecimal(out decimal decimalValue)) + { + writer.WriteNumberValue(element.GetDecimal()); + } + else if (element.TryGetDouble(out double doubleValue)) { writer.WriteNumberValue(element.GetDouble()); } - break; case JsonValueKind.True: diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 235aec842..7e49e8abd 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -661,7 +661,10 @@ public class FeatureManagementTests ""someOtherKey"": { ""someSubKey"": ""someSubValue"" }, - ""someKey"": ""someValue"" + ""someKey4"": [3, 1, 4, true], + ""someKey"": ""someValue"", + ""someKey3"": 3.14, + ""someKey2"": 3 } } ], @@ -2123,7 +2126,7 @@ public void WithAllocationId() Assert.Equal($"{TestHelpers.PrimaryConfigStoreEndpoint}kv/{FeatureManagementConstants.FeatureFlagMarker}TelemetryVariantPercentile?label=label", config["feature_management:feature_flags:1:telemetry:metadata:FeatureFlagReference"]); - Assert.Equal("SCq5T7Bb1k6tWfWjI3qz", config["feature_management:feature_flags:1:telemetry:metadata:AllocationId"]); + Assert.Equal("YsdJ4pQpmhYa8KEhRLUn", config["feature_management:feature_flags:1:telemetry:metadata:AllocationId"]); // Validate Greeting Assert.Equal("True", config["feature_management:feature_flags:2:telemetry:enabled"]); From 603cd0f9901b7714d5ae89385d24433330111ee2 Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Mon, 21 Oct 2024 16:09:15 -0700 Subject: [PATCH 07/16] dotnet format --- .../FeatureManagement/JsonElementExtensions.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs index 88fb4d5c3..fc7f8b26d 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/JsonElementExtensions.cs @@ -68,6 +68,7 @@ private static void WriteElementWithSortedKeys(JsonElement element, Utf8JsonWrit { writer.WriteNumberValue(element.GetDouble()); } + break; case JsonValueKind.True: From 35386d0f18bc2cc000258f3da6d47d1ca2748708 Mon Sep 17 00:00:00 2001 From: Ross Grambo Date: Wed, 23 Oct 2024 11:48:59 -0700 Subject: [PATCH 08/16] Version bump --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index 1801f8d7d..a5d4dee4e 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -21,7 +21,7 @@ - 8.0.0-preview.3 + 8.1.0-preview diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index 3926c5fd8..4354b77ec 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 8.0.0-preview.3 + 8.1.0-preview diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 6387a82ff..aeedd6e67 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -35,7 +35,7 @@ - 8.0.0-preview.3 + 8.1.0-preview From 5e6a012e18959f4eed4a82057a7561b081f1d33e Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:51:56 -0800 Subject: [PATCH 09/16] Add `RegisterAll` API to enable monitoring collections of key-values for refresh (#574) * WIP * WIP testing out client extensions methods * WIP added selectors to multikeywatchers * remove unused property * WIP check for registerall changes to change refreshall * WIP * WIP fixing types and reslving errors * WIP fixing client extensions class * WIP * WIP update feature flag logic * WIP client extensions * WIP reload all flags on change * WIP * WIP fixing tests to return response for getconfigurationsettingsasync * WIP etag for tests * fix watchedcollections null * WIP tests, working for examples * remove unused variables * update to newest sdk version, remove unused * WIP fixing tests * WIP reworking testing to work with new etag approach * tests passing, fix mockasyncpageable * update sdk package version * fix loghelper, tests * WIP fixing aspages tests * revert watchesfeatureflags test * update test again * WIP * fixing watchconditions * separate selected key value collections from feature flag collections, separate selectors, add new methods to support new logic * comment and naming updates * fixing unit tests, namespace of defining/calling code needs to be same * fixing tests using AsPages * fix tests with pageablemanager * format * fix tests * fix tests * remove unused extension test class * fix comment, capitalization * check etag on 200, fix tests * add registerall test, fix refresh tests * fix condition for pages and old match conditions * WIP fixing PR comments, tests * check status after advancing existing etag enumerator * move around refresh logic * null check page etag, revert break to existing keys check in getrefreshedcollections * fix loadselected, replace selectedkvwatchers with registerall refresh time * fix comment in options * clean up tests * PR comments * PR comments * don't allow both registerall and register * fix check for calls to both register methods * PR comments for rename/small changes * fix compile error * simplify refreshasync path, fix naming from comments * remove redundant if check * simplify logic for minrefreshinterval * fix smaller comments * call loadselected when refreshing collection, separate data for individual refresh * in progress change to registerall include ff * fix load order * fix comments, rename logging constants to match new behavior * pr comments, refactor refreshasync * clean up etags dictionary creation * PR comments * add uncommitted changes to testhelper * update tests for registerall with feature flags, check ff keys to remove flags on refresh * PR comments * PR comments * use invalidoperationexception in configurerefresh, update loggingconstants to match behavior * remove unused changes --- .../AzureAppConfigurationOptions.cs | 113 ++- .../AzureAppConfigurationProvider.cs | 654 +++++++++++------- .../AzureAppConfigurationRefreshOptions.cs | 12 + .../ConfigurationSettingPageExtensions.cs | 33 + .../Constants/LoggingConstants.cs | 5 +- .../ConfigurationClientExtensions.cs | 129 +--- .../Extensions/StringExtensions.cs | 9 +- .../FeatureManagement/FeatureFlagOptions.cs | 3 +- .../GetKeyValueChangeCollectionOptions.cs | 13 - .../IConfigurationSettingPageIterator.cs | 13 + .../LogHelper.cs | 15 +- .../Models/KeyValueSelector.cs | 5 + .../Azure.Core.Testing/MockResponse.cs | 2 + .../FeatureManagementTests.cs | 100 ++- .../KeyVaultReferenceTests.cs | 4 +- .../RefreshTests.cs | 165 +++++ .../Tests.AzureAppConfiguration/TestHelper.cs | 60 +- 17 files changed, 865 insertions(+), 470 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs delete mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 7d9a9cad7..9391f21b1 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -23,12 +23,13 @@ public class AzureAppConfigurationOptions private const int MaxRetries = 2; private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); - private List _changeWatchers = new List(); - private List _multiKeyWatchers = new List(); + private List _individualKvWatchers = new List(); + private List _ffWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); - private List _kvSelectors = new List(); + private List _selectors; private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); + private bool _selectCalled = false; // The following set is sorted in descending order. // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. @@ -62,19 +63,29 @@ public class AzureAppConfigurationOptions internal TokenCredential Credential { get; private set; } /// - /// A collection of . + /// A collection of specified by user. /// - internal IEnumerable KeyValueSelectors => _kvSelectors; + internal IEnumerable Selectors => _selectors; + + /// + /// Indicates if was called. + /// + internal bool RegisterAllEnabled { get; private set; } + + /// + /// Refresh interval for selected key-value collections when is called. + /// + internal TimeSpan KvCollectionRefreshInterval { get; private set; } /// /// A collection of . /// - internal IEnumerable ChangeWatchers => _changeWatchers; + internal IEnumerable IndividualKvWatchers => _individualKvWatchers; /// /// A collection of . /// - internal IEnumerable MultiKeyWatchers => _multiKeyWatchers; + internal IEnumerable FeatureFlagWatchers => _ffWatchers; /// /// A collection of . @@ -96,11 +107,15 @@ internal IEnumerable Adapters internal IEnumerable KeyPrefixes => _keyPrefixes; /// - /// An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. + /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. /// - /// This property is used only for unit testing. internal IConfigurationClientManager ClientManager { get; set; } + /// + /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. + /// + internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } + /// /// An optional timespan value to set the minimum backoff duration to a value other than the default. /// @@ -142,6 +157,9 @@ public AzureAppConfigurationOptions() new JsonKeyValueAdapter(), new FeatureManagementKeyValueAdapter(FeatureFlagTracing) }; + + // Adds the default query to App Configuration if and are never called. + _selectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; } /// @@ -170,22 +188,30 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter throw new ArgumentNullException(nameof(keyFilter)); } + // Do not support * and , for label filter for now. + if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) + { + throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + } + if (string.IsNullOrWhiteSpace(labelFilter)) { labelFilter = LabelFilter.Null; } - // Do not support * and , for label filter for now. - if (labelFilter.Contains('*') || labelFilter.Contains(',')) + if (!_selectCalled) { - throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + _selectors.Clear(); + + _selectCalled = true; } - _kvSelectors.AppendUnique(new KeyValueSelector + _selectors.AppendUnique(new KeyValueSelector { KeyFilter = keyFilter, LabelFilter = labelFilter }); + return this; } @@ -201,7 +227,14 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) throw new ArgumentNullException(nameof(name)); } - _kvSelectors.AppendUnique(new KeyValueSelector + if (!_selectCalled) + { + _selectors.Clear(); + + _selectCalled = true; + } + + _selectors.AppendUnique(new KeyValueSelector { SnapshotName = name }); @@ -212,7 +245,7 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) /// /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. /// If no filtering is specified via the then all feature flags with no label are loaded. - /// All loaded feature flags will be automatically registered for refresh on an individual flag level. + /// All loaded feature flags will be automatically registered for refresh as a collection. /// /// A callback used to configure feature flag options. public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) @@ -237,25 +270,22 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c options.FeatureFlagSelectors.Add(new KeyValueSelector { KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", - LabelFilter = options.Label == null ? LabelFilter.Null : options.Label + LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, + IsFeatureFlagSelector = true }); } - foreach (var featureFlagSelector in options.FeatureFlagSelectors) + foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) { - var featureFlagFilter = featureFlagSelector.KeyFilter; - var labelFilter = featureFlagSelector.LabelFilter; + _selectors.AppendUnique(featureFlagSelector); - Select(featureFlagFilter, labelFilter); - - _multiKeyWatchers.AppendUnique(new KeyValueWatcher + _ffWatchers.AppendUnique(new KeyValueWatcher { - Key = featureFlagFilter, - Label = labelFilter, + Key = featureFlagSelector.KeyFilter, + Label = featureFlagSelector.LabelFilter, // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins RefreshInterval = options.RefreshInterval }); - } return this; @@ -376,18 +406,41 @@ public AzureAppConfigurationOptions ConfigureClientOptions(ActionA callback used to configure Azure App Configuration refresh options. public AzureAppConfigurationOptions ConfigureRefresh(Action configure) { + if (RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); + } + var refreshOptions = new AzureAppConfigurationRefreshOptions(); configure?.Invoke(refreshOptions); - if (!refreshOptions.RefreshRegistrations.Any()) + bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); + RegisterAllEnabled = refreshOptions.RegisterAllEnabled; + + if (!isRegisterCalled && !RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() must call either {nameof(AzureAppConfigurationRefreshOptions.Register)}()" + + $" or {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)}()"); + } + + // Check if both register methods are called at any point + if (RegisterAllEnabled && (_individualKvWatchers.Any() || isRegisterCalled)) { - throw new ArgumentException($"{nameof(ConfigureRefresh)}() must have at least one key-value registered for refresh."); + throw new InvalidOperationException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " + + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); } - foreach (var item in refreshOptions.RefreshRegistrations) + if (RegisterAllEnabled) + { + KvCollectionRefreshInterval = refreshOptions.RefreshInterval; + } + else { - item.RefreshInterval = refreshOptions.RefreshInterval; - _changeWatchers.Add(item); + foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) + { + item.RefreshInterval = refreshOptions.RefreshInterval; + _individualKvWatchers.Add(item); + } } return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index b5dd42f18..ebe8992c1 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -4,7 +4,6 @@ using Azure; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using Microsoft.Extensions.Logging; using System; @@ -32,9 +31,13 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; private Dictionary _mappedData; - private Dictionary _watchedSettings = new Dictionary(); + private Dictionary _watchedIndividualKvs = new Dictionary(); + private HashSet _ffKeys = new HashSet(); + private Dictionary> _kvEtags = new Dictionary>(); + private Dictionary> _ffEtags = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); + private DateTimeOffset _nextCollectionRefreshTime; private readonly TimeSpan MinRefreshInterval; @@ -108,11 +111,18 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; - IEnumerable watchers = options.ChangeWatchers.Union(options.MultiKeyWatchers); + IEnumerable watchers = options.IndividualKvWatchers.Union(options.FeatureFlagWatchers); - if (watchers.Any()) + bool hasWatchers = watchers.Any(); + TimeSpan minWatcherRefreshInterval = hasWatchers ? watchers.Min(w => w.RefreshInterval) : TimeSpan.MaxValue; + + if (options.RegisterAllEnabled) + { + MinRefreshInterval = TimeSpan.FromTicks(Math.Min(minWatcherRefreshInterval.Ticks, options.KvCollectionRefreshInterval.Ticks)); + } + else if (hasWatchers) { - MinRefreshInterval = watchers.Min(w => w.RefreshInterval); + MinRefreshInterval = minWatcherRefreshInterval; } else { @@ -194,13 +204,15 @@ public async Task RefreshAsync(CancellationToken cancellationToken) EnsureAssemblyInspected(); var utcNow = DateTimeOffset.UtcNow; - IEnumerable refreshableWatchers = _options.ChangeWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); - IEnumerable refreshableMultiKeyWatchers = _options.MultiKeyWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + IEnumerable refreshableIndividualKvWatchers = _options.IndividualKvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); + IEnumerable refreshableFfWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); + bool isRefreshDue = utcNow >= _nextCollectionRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && - !refreshableWatchers.Any() && - !refreshableMultiKeyWatchers.Any() && + !refreshableIndividualKvWatchers.Any() && + !refreshableFfWatchers.Any() && + !isRefreshDue && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) { return; @@ -249,179 +261,166 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary watchedSettings = null; + Dictionary> kvEtags = null; + Dictionary> ffEtags = null; + HashSet ffKeys = null; + Dictionary watchedIndividualKvs = null; List keyValueChanges = null; - List changedKeyValuesCollection = null; Dictionary data = null; + Dictionary ffCollectionData = null; + bool ffCollectionUpdated = false; bool refreshAll = false; StringBuilder logInfoBuilder = new StringBuilder(); StringBuilder logDebugBuilder = new StringBuilder(); await ExecuteWithFailOverPolicyAsync(clients, async (client) => - { - data = null; - watchedSettings = null; - keyValueChanges = new List(); - changedKeyValuesCollection = null; - refreshAll = false; - Uri endpoint = _configClientManager.GetEndpointForClient(client); - logDebugBuilder.Clear(); - logInfoBuilder.Clear(); - - foreach (KeyValueWatcher changeWatcher in refreshableWatchers) - { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; - - KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); - - KeyValueChange change = default; - - // - // Find if there is a change associated with watcher - if (_watchedSettings.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) - { - await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - else - { - // Load the key-value in case the previous load attempts had failed - - try - { - await CallWithRequestTracing( - async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) - { - watchedKv = null; - } - - if (watchedKv != null) - { - change = new KeyValueChange() - { - Key = watchedKv.Key, - Label = watchedKv.Label.NormalizeNull(), - Current = watchedKv, - ChangeType = KeyValueChangeType.Modified - }; - } - } - - // Check if a change has been detected in the key-value registered for refresh - if (change.ChangeType != KeyValueChangeType.None) - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); - keyValueChanges.Add(change); - - if (changeWatcher.RefreshAll) - { - refreshAll = true; - break; - } - } - else - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - } - } + { + kvEtags = null; + ffEtags = null; + ffKeys = null; + watchedIndividualKvs = null; + keyValueChanges = new List(); + data = null; + ffCollectionData = null; + ffCollectionUpdated = false; + refreshAll = false; + logDebugBuilder.Clear(); + logInfoBuilder.Clear(); + Uri endpoint = _configClientManager.GetEndpointForClient(client); - if (refreshAll) + if (_options.RegisterAllEnabled) + { + // Get key value collection changes if RegisterAll was called + if (isRefreshDue) { - // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - data = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); - logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); - return; + refreshAll = await HaveCollectionsChanged( + _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector), + _kvEtags, + client, + cancellationToken).ConfigureAwait(false); } + } + else + { + refreshAll = await RefreshIndividualKvWatchers( + client, + keyValueChanges, + refreshableIndividualKvWatchers, + endpoint, + logDebugBuilder, + logInfoBuilder, + cancellationToken).ConfigureAwait(false); + } - changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); + if (refreshAll) + { + // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true, + // or if any key-value collection change was detected. + kvEtags = new Dictionary>(); + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + data = await LoadSelected(client, kvEtags, ffEtags, _options.Selectors, ffKeys, cancellationToken).ConfigureAwait(false); + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); + return; + } - if (!changedKeyValuesCollection.Any()) + // Get feature flag changes + ffCollectionUpdated = await HaveCollectionsChanged( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } - }, - cancellationToken) - .ConfigureAwait(false); + KeyFilter = watcher.Key, + LabelFilter = watcher.Label, + IsFeatureFlagSelector = true + }), + _ffEtags, + client, + cancellationToken).ConfigureAwait(false); + + if (ffCollectionUpdated) + { + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + ffCollectionData = await LoadSelected( + client, + new Dictionary>(), + ffEtags, + _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), + ffKeys, + cancellationToken).ConfigureAwait(false); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + } + }, + cancellationToken) + .ConfigureAwait(false); - if (!refreshAll) + if (refreshAll) { - watchedSettings = new Dictionary(_watchedSettings); + _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableMultiKeyWatchers)) + // Invalidate all the cached KeyVault secrets + foreach (IKeyValueAdapter adapter in _options.Adapters) { - UpdateNextRefreshTime(changeWatcher); + adapter.OnChangeDetected(); } - foreach (KeyValueChange change in keyValueChanges.Concat(changedKeyValuesCollection)) + // Update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { - KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); - if (change.ChangeType == KeyValueChangeType.Modified) - { - ConfigurationSetting setting = change.Current; - ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - watchedSettings[changeIdentifier] = settingCopy; + UpdateNextRefreshTime(changeWatcher); + } + } + else + { + watchedIndividualKvs = new Dictionary(_watchedIndividualKvs); - foreach (Func> func in _options.Mappers) - { - setting = await func(setting).ConfigureAwait(false); - } + await ProcessKeyValueChangesAsync(keyValueChanges, _mappedData, watchedIndividualKvs).ConfigureAwait(false); - if (setting == null) - { - _mappedData.Remove(change.Key); - } - else - { - _mappedData[change.Key] = setting; - } - } - else if (change.ChangeType == KeyValueChangeType.Deleted) + if (ffCollectionUpdated) + { + // Remove all feature flag keys that are not present in the latest loading of feature flags, but were loaded previously + foreach (string key in _ffKeys.Except(ffKeys)) { - _mappedData.Remove(change.Key); - watchedSettings.Remove(changeIdentifier); + _mappedData.Remove(key); } - // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting - foreach (IKeyValueAdapter adapter in _options.Adapters) + Dictionary mappedFfData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + + foreach (KeyValuePair kvp in mappedFfData) { - // If the current setting is null, try to pass the previous setting instead - if (change.Current != null) - { - adapter.OnChangeDetected(change.Current); - } - else if (change.Previous != null) - { - adapter.OnChangeDetected(change.Previous); - } + _mappedData[kvp.Key] = kvp.Value; } } - } - else - { - _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - - // Invalidate all the cached KeyVault secrets - foreach (IKeyValueAdapter adapter in _options.Adapters) - { - adapter.OnChangeDetected(); - } - // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + // + // update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in refreshableIndividualKvWatchers.Concat(refreshableFfWatchers)) { UpdateNextRefreshTime(changeWatcher); } } - if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || changedKeyValuesCollection?.Any() == true || keyValueChanges.Any()) + if (isRefreshDue) { - _watchedSettings = watchedSettings; + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + + if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || refreshAll || ffCollectionUpdated) + { + _watchedIndividualKvs = watchedIndividualKvs ?? _watchedIndividualKvs; + + _ffEtags = ffEtags ?? _ffEtags; + + _kvEtags = kvEtags ?? _kvEtags; + + _ffKeys = ffKeys ?? _ffKeys; if (logDebugBuilder.Length > 0) { @@ -432,6 +431,7 @@ await CallWithRequestTracing( { _logger.LogInformation(logInfoBuilder.ToString().Trim()); } + // PrepareData makes calls to KeyVault and may throw exceptions. But, we still update watchers before // SetData because repeating appconfig calls (by not updating watchers) won't help anything for keyvault calls. // As long as adapter.NeedsRefresh is true, we will attempt to update keyvault again the next time RefreshAsync is called. @@ -555,14 +555,21 @@ private void SetDirty(TimeSpan? maxDelay) { DateTimeOffset nextRefreshTime = AddRandomDelay(DateTimeOffset.UtcNow, maxDelay ?? DefaultMaxSetDirtyDelay); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + if (_options.RegisterAllEnabled) { - changeWatcher.NextRefreshTime = nextRefreshTime; + _nextCollectionRefreshTime = nextRefreshTime; + } + else + { + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) + { + kvWatcher.NextRefreshTime = nextRefreshTime; + } } - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher featureFlagWatcher in _options.FeatureFlagWatchers) { - changeWatcher.NextRefreshTime = nextRefreshTime; + featureFlagWatcher.NextRefreshTime = nextRefreshTime; } } @@ -707,34 +714,44 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = null; - Dictionary watchedSettings = null; + Dictionary> kvEtags = new Dictionary>(); + Dictionary> ffEtags = new Dictionary>(); + Dictionary watchedIndividualKvs = null; + HashSet ffKeys = new HashSet(); await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - data = await LoadSelectedKeyValues( + data = await LoadSelected( client, + kvEtags, + ffEtags, + _options.Selectors, + ffKeys, cancellationToken) .ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh( + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh( client, data, cancellationToken) .ConfigureAwait(false); - - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); }, cancellationToken) .ConfigureAwait(false); // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } + if (_options.RegisterAllEnabled) + { + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + if (data != null) { // Invalidate all the cached KeyVault secrets @@ -744,51 +761,71 @@ await ExecuteWithFailOverPolicyAsync( } Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); + SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - _watchedSettings = watchedSettings; + _mappedData = mappedData; + _kvEtags = kvEtags; + _ffEtags = ffEtags; + _watchedIndividualKvs = watchedIndividualKvs; + _ffKeys = ffKeys; } } - private async Task> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) + private async Task> LoadSelected( + ConfigurationClient client, + Dictionary> kvEtags, + Dictionary> ffEtags, + IEnumerable selectors, + HashSet ffKeys, + CancellationToken cancellationToken) { - var serverData = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary data = new Dictionary(); - // Use default query if there are no key-values specified for use other than the feature flags - bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => selector.KeyFilter == null || - !selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)); - - if (useDefaultQuery) + foreach (KeyValueSelector loadOption in selectors) { - // Load all key-values with the null label. - var selector = new SettingSelector - { - KeyFilter = KeyFilter.Any, - LabelFilter = LabelFilter.Null - }; - - await CallWithRequestTracing(async () => + if (string.IsNullOrEmpty(loadOption.SnapshotName)) { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) + var selector = new SettingSelector() { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); - } + KeyFilter = loadOption.KeyFilter, + LabelFilter = loadOption.LabelFilter + }; - foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) - { - IAsyncEnumerable settingsEnumerable; + var matchConditions = new List(); - if (string.IsNullOrEmpty(loadOption.SnapshotName)) - { - settingsEnumerable = client.GetConfigurationSettingsAsync( - new SettingSelector + await CallWithRequestTracing(async () => + { + AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + + await foreach (Page page in pageableSettings.AsPages(_options.ConfigurationSettingPageIterator).ConfigureAwait(false)) { - KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter - }, - cancellationToken); + using Response response = page.GetRawResponse(); + + ETag serverEtag = (ETag)response.Headers.ETag; + + foreach (ConfigurationSetting setting in page.Values) + { + data[setting.Key] = setting; + + if (loadOption.IsFeatureFlagSelector) + { + ffKeys.Add(setting.Key); + } + } + + matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); + } + }).ConfigureAwait(false); + + if (loadOption.IsFeatureFlagSelector) + { + ffEtags[loadOption] = matchConditions; + } + else + { + kvEtags[loadOption] = matchConditions; + } } else { @@ -808,38 +845,42 @@ await CallWithRequestTracing(async () => throw new InvalidOperationException($"{nameof(snapshot.SnapshotComposition)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.SnapshotComposition}'."); } - settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( + IAsyncEnumerable settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( loadOption.SnapshotName, cancellationToken); - } - await CallWithRequestTracing(async () => - { - await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + await CallWithRequestTracing(async () => { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); + await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + { + data[setting.Key] = setting; + } + }).ConfigureAwait(false); + } } - return serverData; + return data; } - private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) + private async Task> LoadKeyValuesRegisteredForRefresh( + ConfigurationClient client, + IDictionary existingSettings, + CancellationToken cancellationToken) { - Dictionary watchedSettings = new Dictionary(); + var watchedIndividualKvs = new Dictionary(); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); // Skip the loading for the key-value in case it has already been loaded if (existingSettings.TryGetValue(watchedKey, out ConfigurationSetting loadedKv) && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -857,61 +898,84 @@ private async Task> LoadKey // If the key-value was found, store it for updating the settings if (watchedKv != null) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); existingSettings[watchedKey] = watchedKv; } } - return watchedSettings; + return watchedIndividualKvs; } - private Dictionary UpdateWatchedKeyValueCollections(Dictionary watchedSettings, IDictionary existingSettings) + private async Task RefreshIndividualKvWatchers( + ConfigurationClient client, + List keyValueChanges, + IEnumerable refreshableIndividualKvWatchers, + Uri endpoint, + StringBuilder logDebugBuilder, + StringBuilder logInfoBuilder, + CancellationToken cancellationToken) { - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher kvWatcher in refreshableIndividualKvWatchers) { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Values); + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; - foreach (ConfigurationSetting setting in currentKeyValues) + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); + + KeyValueChange change = default; + + // + // Find if there is a change associated with watcher + if (_watchedIndividualKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) { - watchedSettings[new KeyValueIdentifier(setting.Key, setting.Label)] = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } - } + else + { + // Load the key-value in case the previous load attempts had failed - return watchedSettings; - } + try + { + await CallWithRequestTracing( + async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) + { + watchedKv = null; + } - private async Task> GetRefreshedKeyValueCollections( - IEnumerable multiKeyWatchers, - ConfigurationClient client, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) - { - var keyValueChanges = new List(); + if (watchedKv != null) + { + change = new KeyValueChange() + { + Key = watchedKv.Key, + Label = watchedKv.Label.NormalizeNull(), + Current = watchedKv, + ChangeType = KeyValueChangeType.Modified + }; + } + } - foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) - { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, _watchedSettings.Values); + // Check if a change has been detected in the key-value registered for refresh + if (change.ChangeType != KeyValueChangeType.None) + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); + keyValueChanges.Add(change); - keyValueChanges.AddRange( - await client.GetKeyValueChangeCollection( - currentKeyValues, - new GetKeyValueChangeCollectionOptions - { - KeyFilter = changeWatcher.Key, - Label = changeWatcher.Label.NormalizeNull(), - RequestTracingEnabled = _requestTracingEnabled, - RequestTracingOptions = _requestTracingOptions - }, - logDebugBuilder, - logInfoBuilder, - endpoint, - cancellationToken) - .ConfigureAwait(false)); + if (kvWatcher.RefreshAll) + { + return true; + } + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + } } - return keyValueChanges; + return false; } private void SetData(IDictionary data) @@ -1179,30 +1243,6 @@ private async Task> MapConfigurationSet return mappedData; } - private IEnumerable GetCurrentKeyValueCollection(string key, string label, IEnumerable existingSettings) - { - IEnumerable currentKeyValues; - - if (key.EndsWith("*")) - { - // Get current application settings starting with changeWatcher.Key, excluding the last * character - string keyPrefix = key.Substring(0, key.Length - 1); - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.StartsWith(keyPrefix) && kv.Label == label.NormalizeNull(); - }); - } - else - { - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.Equals(key) && kv.Label == label.NormalizeNull(); - }); - } - - return currentKeyValues; - } - private void EnsureAssemblyInspected() { if (!_isAssemblyInspected) @@ -1248,6 +1288,88 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } + private async Task HaveCollectionsChanged( + IEnumerable selectors, + Dictionary> pageEtags, + ConfigurationClient client, + CancellationToken cancellationToken) + { + bool haveCollectionsChanged = false; + + foreach (KeyValueSelector selector in selectors) + { + if (pageEtags.TryGetValue(selector, out IEnumerable matchConditions)) + { + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => haveCollectionsChanged = await client.HaveCollectionsChanged( + selector, + matchConditions, + _options.ConfigurationSettingPageIterator, + cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + + if (haveCollectionsChanged) + { + return true; + } + } + + return haveCollectionsChanged; + } + + private async Task ProcessKeyValueChangesAsync( + IEnumerable keyValueChanges, + Dictionary mappedData, + Dictionary watchedIndividualKvs) + { + foreach (KeyValueChange change in keyValueChanges) + { + KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); + + if (change.ChangeType == KeyValueChangeType.Modified) + { + ConfigurationSetting setting = change.Current; + ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + watchedIndividualKvs[changeIdentifier] = settingCopy; + + foreach (Func> func in _options.Mappers) + { + setting = await func(setting).ConfigureAwait(false); + } + + if (setting == null) + { + mappedData.Remove(change.Key); + } + else + { + mappedData[change.Key] = setting; + } + } + else if (change.ChangeType == KeyValueChangeType.Deleted) + { + mappedData.Remove(change.Key); + + watchedIndividualKvs.Remove(changeIdentifier); + } + + // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting + foreach (IKeyValueAdapter adapter in _options.Adapters) + { + // If the current setting is null, try to pass the previous setting instead + if (change.Current != null) + { + adapter.OnChangeDetected(change.Current); + } + else if (change.Previous != null) + { + adapter.OnChangeDetected(change.Previous); + } + } + } + } + public void Dispose() { (_configClientManager as ConfigurationClientManager)?.Dispose(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs index f3fb6c4a1..cf2847a82 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs @@ -14,6 +14,7 @@ public class AzureAppConfigurationRefreshOptions { internal TimeSpan RefreshInterval { get; private set; } = RefreshConstants.DefaultRefreshInterval; internal ISet RefreshRegistrations = new HashSet(); + internal bool RegisterAllEnabled { get; private set; } /// /// Register the specified individual key-value to be refreshed when the configuration provider's triggers a refresh. @@ -50,6 +51,17 @@ public AzureAppConfigurationRefreshOptions Register(string key, string label = L return this; } + /// + /// Register all key-values loaded outside of to be refreshed when the configuration provider's triggers a refresh. + /// The instance can be obtained by calling . + /// + public AzureAppConfigurationRefreshOptions RegisterAll() + { + RegisterAllEnabled = true; + + return this; + } + /// /// Sets the cache expiration time for the key-values registered for refresh. Default value is 30 seconds. Must be greater than 1 second. /// Any refresh operation triggered using will not update the value for a key until the cached value for that key has expired. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs new file mode 100644 index 000000000..aba7684b3 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs @@ -0,0 +1,33 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + static class ConfigurationSettingPageExtensions + { + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable); + } + + return pageable.AsPages(); + } + + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator, IEnumerable matchConditions) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable, matchConditions); + } + + return pageable.AsPages(matchConditions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 86576a483..3bdcaecd5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -20,14 +20,15 @@ internal class LoggingConstants // Successful update, debug log level public const string RefreshKeyValueRead = "Key-value read from App Configuration."; public const string RefreshKeyVaultSecretRead = "Secret read from Key Vault for key-value."; - public const string RefreshFeatureFlagRead = "Feature flag read from App Configuration."; public const string RefreshFeatureFlagsUnchanged = "Feature flags read from App Configuration. Change:'None'"; + public const string RefreshSelectedKeyValueCollectionsUnchanged = "Selected key-value collections read from App Configuration. Change:'None'"; // Successful update, information log level public const string RefreshConfigurationUpdatedSuccess = "Configuration reloaded."; public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; - public const string RefreshFeatureFlagUpdated = "Feature flag updated."; + public const string RefreshFeatureFlagsUpdated = "Feature flags reloaded."; + public const string RefreshSelectedKeyValuesAndFeatureFlagsUpdated = "Selected key-value collections and feature flags reloaded."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible."; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index d479ad6bf..677122310 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -3,12 +3,10 @@ // using Azure; using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using System; using System.Collections.Generic; -using System.Linq; using System.Net; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -65,131 +63,50 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task> GetKeyValueChangeCollection( - this ConfigurationClient client, - IEnumerable keyValues, - GetKeyValueChangeCollectionOptions options, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) + public static async Task HaveCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, IConfigurationSettingPageIterator pageIterator, CancellationToken cancellationToken) { - if (options == null) + if (matchConditions == null) { - throw new ArgumentNullException(nameof(options)); + throw new ArgumentNullException(nameof(matchConditions)); } - if (keyValues == null) + if (keyValueSelector == null) { - keyValues = Enumerable.Empty(); + throw new ArgumentNullException(nameof(keyValueSelector)); } - if (options.KeyFilter == null) + if (keyValueSelector.SnapshotName != null) { - options.KeyFilter = string.Empty; + throw new ArgumentException("Cannot check snapshot for changes.", $"{nameof(keyValueSelector)}.{nameof(keyValueSelector.SnapshotName)}"); } - if (keyValues.Any(k => string.IsNullOrEmpty(k.Key))) + SettingSelector selector = new SettingSelector { - throw new ArgumentNullException($"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Key)}"); - } - - if (keyValues.Any(k => !string.Equals(k.Label.NormalizeNull(), options.Label.NormalizeNull()))) - { - throw new ArgumentException("All key-values registered for refresh must use the same label.", $"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Label)}"); - } - - if (keyValues.Any(k => k.Label != null && k.Label.Contains("*"))) - { - throw new ArgumentException("The label filter cannot contain '*'", $"{nameof(options)}.{nameof(options.Label)}"); - } - - var hasKeyValueCollectionChanged = false; - var selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label, - Fields = SettingFields.ETag | SettingFields.Key + KeyFilter = keyValueSelector.KeyFilter, + LabelFilter = keyValueSelector.LabelFilter }; - // Dictionary of eTags that we write to and use for comparison - var eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); + AsyncPageable pageable = client.GetConfigurationSettingsAsync(selector, cancellationToken); - // Fetch e-tags for prefixed key-values that can be used to detect changes - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - hasKeyValueCollectionChanged = true; - break; - } + using IEnumerator existingMatchConditionsEnumerator = matchConditions.GetEnumerator(); - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - // Check for any deletions - if (eTagMap.Any()) - { - hasKeyValueCollectionChanged = true; - } - - var changes = new List(); - - // If changes have been observed, refresh prefixed key-values - if (hasKeyValueCollectionChanged) + await foreach (Page page in pageable.AsPages(pageIterator, matchConditions).ConfigureAwait(false)) { - selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label - }; + using Response response = page.GetRawResponse(); - eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Modified, - Key = setting.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = setting - }); - string key = setting.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); - } + ETag serverEtag = (ETag)response.Headers.ETag; - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - foreach (var kvp in eTagMap) + // Return true if the lists of etags are different + if ((!existingMatchConditionsEnumerator.MoveNext() || + !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(serverEtag)) && + response.Status == (int)HttpStatusCode.OK) { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Deleted, - Key = kvp.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = null - }); - string key = kvp.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); + return true; } } - return changes; + // Need to check if pages were deleted and no change was found within the new shorter list of match conditions + return existingMatchConditionsEnumerator.MoveNext(); } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs index 8b2c488d0..615720305 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs @@ -5,18 +5,11 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { - internal static class LabelFilters - { - public static readonly string Null = "\0"; - - public static readonly string Any = "*"; - } - internal static class StringExtensions { public static string NormalizeNull(this string s) { - return s == LabelFilters.Null ? null : s; + return s == LabelFilter.Null ? null : s; } public static string ToBase64String(this string s) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs index 1e8beae69..263907624 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs @@ -102,7 +102,8 @@ public FeatureFlagOptions Select(string featureFlagFilter, string labelFilter = FeatureFlagSelectors.AppendUnique(new KeyValueSelector { KeyFilter = featureFlagPrefix, - LabelFilter = labelFilter + LabelFilter = labelFilter, + IsFeatureFlagSelector = true }); return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs deleted file mode 100644 index 5cb9b83d7..000000000 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -// -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - internal class GetKeyValueChangeCollectionOptions - { - public string KeyFilter { get; set; } - public string Label { get; set; } - public bool RequestTracingEnabled { get; set; } - public RequestTracingOptions RequestTracingOptions { get; set; } - } -} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs new file mode 100644 index 000000000..08c95751e --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs @@ -0,0 +1,13 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal interface IConfigurationSettingPageIterator + { + IAsyncEnumerable> IteratePages(AsyncPageable pageable); + + IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions); + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 11442dcb4..4f9994062 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -25,14 +25,19 @@ public static string BuildFeatureFlagsUnchangedMessage(string endpoint) return $"{LoggingConstants.RefreshFeatureFlagsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagReadMessage(string key, string label, string endpoint) - { - return $"{LoggingConstants.RefreshFeatureFlagRead} Key:'{key}' Label:'{label}' Endpoint:'{endpoint?.TrimEnd('/')}'"; + public static string BuildFeatureFlagsUpdatedMessage() + { + return LoggingConstants.RefreshFeatureFlagsUpdated; + } + + public static string BuildSelectedKeyValueCollectionsUnchangedMessage(string endpoint) + { + return $"{LoggingConstants.RefreshSelectedKeyValueCollectionsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagUpdatedMessage(string key) + public static string BuildSelectedKeyValuesAndFeatureFlagsUpdatedMessage() { - return $"{LoggingConstants.RefreshFeatureFlagUpdated} Key:'{key}'"; + return LoggingConstants.RefreshSelectedKeyValuesAndFeatureFlagsUpdated; } public static string BuildKeyVaultSecretReadMessage(string key, string label) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs index 5491d04de..54bda1a49 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs @@ -24,6 +24,11 @@ public class KeyValueSelector /// public string SnapshotName { get; set; } + /// + /// A boolean that signifies whether this selector is intended to select feature flags. + /// + public bool IsFeatureFlagSelector { get; set; } + /// /// Determines whether the specified object is equal to the current object. /// diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index 886d0a77b..aaee0b9c9 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -17,6 +17,8 @@ public MockResponse(int status, string reasonPhrase = null) { Status = status; ReasonPhrase = reasonPhrase; + + AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); } public override int Status { get; } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 7e49e8abd..f7ebf127e 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -768,19 +768,24 @@ public void UsesFeatureFlags() [Fact] public async Task WatchesFeatureFlags() { + var mockResponse = new MockResponse(200); + var featureFlags = new List { _kv }; - var mockResponse = new Mock(); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + var mockClient = new Mock(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -820,7 +825,7 @@ public async Task WatchesFeatureFlags() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); featureFlags.Add(_kv2); @@ -839,11 +844,12 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); var cacheExpirationInterval = TimeSpan.FromSeconds(1); @@ -852,6 +858,7 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = cacheExpirationInterval); refresher = options.GetRefresher(); @@ -910,17 +917,20 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(TimeSpan.FromSeconds(10))); refresher = options.GetRefresher(); @@ -977,17 +987,20 @@ public async Task SkipRefreshIfCacheNotExpired() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = TimeSpan.FromSeconds(10)); refresher = options.GetRefresher(); @@ -1096,17 +1109,22 @@ public void QueriesFeatureFlags() } [Fact] - public async Task UsesEtagForFeatureFlagRefresh() + public async Task DoesNotUseEtagForFeatureFlagRefresh() { + var mockAsyncPageable = new MockAsyncPageable(new List { _kv }); + var mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(new List { _kv })); + .Callback(() => mockAsyncPageable.UpdateCollection(new List { _kv })) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -1137,6 +1155,7 @@ public void SelectFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(testClient); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1486,18 +1505,19 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1) || - (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList()); - }); + (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(refreshInterval1); @@ -1656,18 +1676,18 @@ public async Task SelectAndRefreshSingleFeatureFlag() var label1 = "App1_Label"; IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => - s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList()); - }); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => + s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1720,8 +1740,17 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetIfChanged); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetTestKey); string informationalInvocation = ""; string verboseInvocation = ""; @@ -1745,6 +1774,7 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); }) @@ -1753,10 +1783,10 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( - key: FeatureManagementConstants.FeatureFlagMarker + "myFeature1", + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature2", value: @" { - ""id"": ""MyFeature"", + ""id"": ""MyFeature2"", ""description"": ""The new beta version of our web site."", ""display_name"": ""Beta Feature"", ""enabled"": true, @@ -1771,21 +1801,19 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); - Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); featureFlags.RemoveAt(0); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); Assert.Null(config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); } [Fact] @@ -1796,8 +1824,11 @@ public async Task ValidateFeatureFlagsUnchangedLogged() var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1821,6 +1852,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); options.ConfigureRefresh(refreshOptions => { @@ -1870,9 +1902,11 @@ public async Task MapTransformFeatureFlagWithRefresh() IConfigurationRefresher refresher = null; var featureFlags = new List { _kv }; var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1945,7 +1979,7 @@ public async Task MapTransformFeatureFlagWithRefresh() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs index 06c88040b..25edb546f 100644 --- a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs +++ b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs @@ -505,9 +505,9 @@ public void DoesNotThrowKeyVaultExceptionWhenProviderIsOptional() .Returns(new MockAsyncPageable(new List { _kv })); var mockKeyValueAdapter = new Mock(MockBehavior.Strict); - mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(_kv)) + mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(It.IsAny())) .Returns(true); - mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(_kv, It.IsAny(), It.IsAny(), It.IsAny())) + mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Throws(new KeyVaultReferenceException("Key vault error", null)); mockKeyValueAdapter.Setup(adapter => adapter.OnChangeDetected(null)); mockKeyValueAdapter.Setup(adapter => adapter.OnConfigUpdated()); diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index 6edc1a9ae..a7fdbd183 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -7,6 +7,7 @@ using Azure.Identity; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Logging.Abstractions; using Moq; using System; @@ -1056,6 +1057,170 @@ public void RefreshTests_RefreshIsCancelled() Assert.Equal("TestValue1", config["TestKey1"]); } + [Fact] + public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var mockAsyncPageable = new MockAsyncPageable(_kvCollection); + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Callback(() => mockAsyncPageable.UpdateCollection(_kvCollection)) + .Returns(mockAsyncPageable); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("TestValue3", config["TestKey3"]); + FirstKeyValue.Value = "newValue1"; + _kvCollection[2].Value = "newValue3"; + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("newValue3", config["TestKey3"]); + + _kvCollection.RemoveAt(2); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Null(config["TestKey3"]); + } + + [Fact] + public async Task RefreshTests_RegisterAllRefreshesFeatureFlags() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var featureFlags = new List { + ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""SuperUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")) + }; + + var mockAsyncPageableKv = new MockAsyncPageable(_kvCollection); + + var mockAsyncPageableFf = new MockAsyncPageable(featureFlags); + + MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) + { + if (selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)) + { + mockAsyncPageableFf.UpdateCollection(featureFlags); + + return mockAsyncPageableFf; + } + + mockAsyncPageableKv.UpdateCollection(_kvCollection); + + return mockAsyncPageableKv; + } + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns((Func)GetTestKeys); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + options.UseFeatureFlags(); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newValue1"; + featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""AllUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newerValue1"; + featureFlags.RemoveAt(0); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newerValue1", config["TestKey1"]); + Assert.Null(config["FeatureManagement:MyFeature"]); + } + #if NET8_0 [Fact] public void RefreshTests_ChainedConfigurationProviderUsedAsRootForRefresherProvider() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index bc7989b25..e3bf55006 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -3,6 +3,7 @@ // using Azure; using Azure.Core; +using Azure.Core.Testing; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; using Microsoft.Extensions.Logging; @@ -10,6 +11,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Text.Json; using System.Threading; @@ -155,19 +157,69 @@ public static bool ValidateLog(Mock logger, string expectedMessage, Log class MockAsyncPageable : AsyncPageable { - private readonly List _collection; + private readonly List _collection = new List(); + private int _status; public MockAsyncPageable(List collection) { - _collection = collection; + foreach (ConfigurationSetting setting in collection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + + _status = 200; + } + + public void UpdateCollection(List newCollection) + { + if (_collection.Count() == newCollection.Count() && + _collection.All(setting => newCollection.Any(newSetting => + setting.Key == newSetting.Key && + setting.Value == newSetting.Value && + setting.Label == newSetting.Label && + setting.ETag == newSetting.ETag))) + { + _status = 304; + } + else + { + _status = 200; + + _collection.Clear(); + + foreach (ConfigurationSetting setting in newCollection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + } } #pragma warning disable 1998 public async override IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) #pragma warning restore 1998 { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(_status)); + } + } + + internal class MockConfigurationSettingPageIterator : IConfigurationSettingPageIterator + { + public IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions) + { + return pageable.AsPages(); + } + public IAsyncEnumerable> IteratePages(AsyncPageable pageable) + { + return pageable.AsPages(); } } @@ -182,7 +234,7 @@ public MockPageable(List collection) public override IEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(200)); } } } From 6dc9ae2a42926952eb21339526264407bba5d5ee Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Thu, 6 Feb 2025 11:15:26 -0800 Subject: [PATCH 10/16] Give the users the ability to have control over ConfigurationClient instance(s) used by the provider (#598) (#617) * Introduced a new `AzureAppConfigurationClientFactory` class to handle the creation of `ConfigurationClient` instances * remove clients dictionary since we will not have hits and clients are already stored in ConfigurationClientManager * revert * add license + remove unused usings * ran dotnet format * add capability of fallback to different stores * add explicit type * address comments * remove scheme validation --------- Co-authored-by: Sami Sadfa <71456174+samsadsam@users.noreply.github.com> Co-authored-by: Sami Sadfa --- .../AzureAppConfigurationClientFactory.cs | 74 +++++++++++++++++++ .../AzureAppConfigurationOptions.cs | 17 +++++ .../AzureAppConfigurationSource.cs | 32 ++++---- .../ConfigurationClientManager.cs | 64 +++------------- ...Configuration.AzureAppConfiguration.csproj | 1 + .../FailoverTests.cs | 19 +++-- 6 files changed, 128 insertions(+), 79 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs new file mode 100644 index 000000000..6127822d8 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// +using Azure.Core; +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal class AzureAppConfigurationClientFactory : IAzureClientFactory + { + private readonly ConfigurationClientOptions _clientOptions; + + private readonly TokenCredential _credential; + private readonly IEnumerable _connectionStrings; + + public AzureAppConfigurationClientFactory( + IEnumerable connectionStrings, + ConfigurationClientOptions clientOptions) + { + if (connectionStrings == null || !connectionStrings.Any()) + { + throw new ArgumentNullException(nameof(connectionStrings)); + } + + _connectionStrings = connectionStrings; + + _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); + } + + public AzureAppConfigurationClientFactory( + TokenCredential credential, + ConfigurationClientOptions clientOptions) + { + _credential = credential ?? throw new ArgumentNullException(nameof(credential)); + _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); + } + + public ConfigurationClient CreateClient(string endpoint) + { + if (string.IsNullOrEmpty(endpoint)) + { + throw new ArgumentNullException(nameof(endpoint)); + } + + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out Uri uriResult)) + { + throw new ArgumentException("Invalid host URI."); + } + + if (_credential != null) + { + return new ConfigurationClient(uriResult, _credential, _clientOptions); + } + + string connectionString = _connectionStrings.FirstOrDefault(cs => ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection) == endpoint); + + // + // falback to the first connection string + if (connectionString == null) + { + string id = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.IdSection); + string secret = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.SecretSection); + + connectionString = ConnectionStringUtils.Build(uriResult, id, secret); + } + + return new ConfigurationClient(connectionString, _clientOptions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 9391f21b1..6e7f595a4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -3,6 +3,7 @@ // using Azure.Core; using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; @@ -146,6 +147,11 @@ internal IEnumerable Adapters /// internal StartupOptions Startup { get; set; } = new StartupOptions(); + /// + /// Client factory that is responsible for creating instances of ConfigurationClient. + /// + internal IAzureClientFactory ClientFactory { get; private set; } + /// /// Initializes a new instance of the class. /// @@ -162,6 +168,17 @@ public AzureAppConfigurationOptions() _selectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; } + /// + /// Sets the client factory used to create ConfigurationClient instances. + /// + /// The client factory. + /// The current instance. + public AzureAppConfigurationOptions SetClientFactory(IAzureClientFactory factory) + { + ClientFactory = factory ?? throw new ArgumentNullException(nameof(factory)); + return this; + } + /// /// Specify what key-values to include in the configuration provider. /// can be called multiple times to include multiple sets of key-values. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs index dee620060..83d20e2fb 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; using System; +using System.Collections.Generic; +using System.Linq; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { @@ -29,35 +33,33 @@ public IConfigurationProvider Build(IConfigurationBuilder builder) try { AzureAppConfigurationOptions options = _optionsProvider(); - IConfigurationClientManager clientManager; if (options.ClientManager != null) { - clientManager = options.ClientManager; + return new AzureAppConfigurationProvider(options.ClientManager, options, _optional); } - else if (options.ConnectionStrings != null) + + IEnumerable endpoints; + IAzureClientFactory clientFactory = options.ClientFactory; + + if (options.ConnectionStrings != null) { - clientManager = new ConfigurationClientManager( - options.ConnectionStrings, - options.ClientOptions, - options.ReplicaDiscoveryEnabled, - options.LoadBalancingEnabled); + endpoints = options.ConnectionStrings.Select(cs => new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection))); + + clientFactory ??= new AzureAppConfigurationClientFactory(options.ConnectionStrings, options.ClientOptions); } else if (options.Endpoints != null && options.Credential != null) { - clientManager = new ConfigurationClientManager( - options.Endpoints, - options.Credential, - options.ClientOptions, - options.ReplicaDiscoveryEnabled, - options.LoadBalancingEnabled); + endpoints = options.Endpoints; + + clientFactory ??= new AzureAppConfigurationClientFactory(options.Credential, options.ClientOptions); } else { throw new ArgumentException($"Please call {nameof(AzureAppConfigurationOptions)}.{nameof(AzureAppConfigurationOptions.Connect)} to specify how to connect to Azure App Configuration."); } - provider = new AzureAppConfigurationProvider(clientManager, options, _optional); + provider = new AzureAppConfigurationProvider(new ConfigurationClientManager(clientFactory, endpoints, options.ReplicaDiscoveryEnabled, options.LoadBalancingEnabled), options, _optional); } catch (InvalidOperationException ex) // InvalidOperationException is thrown when any problems are found while configuring AzureAppConfigurationOptions or when SDK fails to create a configurationClient. { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs index a0215ca37..61840d036 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs @@ -2,10 +2,10 @@ // Licensed under the MIT license. // -using Azure.Core; using Azure.Data.AppConfiguration; using DnsClient; using DnsClient.Protocol; +using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using System; using System.Collections.Generic; @@ -26,12 +26,11 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration /// internal class ConfigurationClientManager : IConfigurationClientManager, IDisposable { + private readonly IAzureClientFactory _clientFactory; private readonly IList _clients; + private readonly Uri _endpoint; - private readonly string _secret; - private readonly string _id; - private readonly TokenCredential _credential; - private readonly ConfigurationClientOptions _clientOptions; + private readonly bool _replicaDiscoveryEnabled; private readonly SrvLookupClient _srvLookupClient; private readonly string _validDomain; @@ -52,61 +51,20 @@ internal class ConfigurationClientManager : IConfigurationClientManager, IDispos internal int RefreshClientsCalled { get; set; } = 0; public ConfigurationClientManager( - IEnumerable connectionStrings, - ConfigurationClientOptions clientOptions, - bool replicaDiscoveryEnabled, - bool loadBalancingEnabled) - { - if (connectionStrings == null || !connectionStrings.Any()) - { - throw new ArgumentNullException(nameof(connectionStrings)); - } - - string connectionString = connectionStrings.First(); - _endpoint = new Uri(ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.EndpointSection)); - _secret = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.SecretSection); - _id = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.IdSection); - _clientOptions = clientOptions; - _replicaDiscoveryEnabled = replicaDiscoveryEnabled; - - // If load balancing is enabled, shuffle the passed in connection strings to randomize the endpoint used on startup - if (loadBalancingEnabled) - { - connectionStrings = connectionStrings.ToList().Shuffle(); - } - - _validDomain = GetValidDomain(_endpoint); - _srvLookupClient = new SrvLookupClient(); - - _clients = connectionStrings - .Select(cs => - { - var endpoint = new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection)); - return new ConfigurationClientWrapper(endpoint, new ConfigurationClient(cs, _clientOptions)); - }) - .ToList(); - } - - public ConfigurationClientManager( + IAzureClientFactory clientFactory, IEnumerable endpoints, - TokenCredential credential, - ConfigurationClientOptions clientOptions, bool replicaDiscoveryEnabled, bool loadBalancingEnabled) { + _clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory)); + if (endpoints == null || !endpoints.Any()) { throw new ArgumentNullException(nameof(endpoints)); } - if (credential == null) - { - throw new ArgumentNullException(nameof(credential)); - } - _endpoint = endpoints.First(); - _credential = credential; - _clientOptions = clientOptions; + _replicaDiscoveryEnabled = replicaDiscoveryEnabled; // If load balancing is enabled, shuffle the passed in endpoints to randomize the endpoint used on startup @@ -119,7 +77,7 @@ public ConfigurationClientManager( _srvLookupClient = new SrvLookupClient(); _clients = endpoints - .Select(endpoint => new ConfigurationClientWrapper(endpoint, new ConfigurationClient(endpoint, _credential, _clientOptions))) + .Select(endpoint => new ConfigurationClientWrapper(endpoint, clientFactory.CreateClient(endpoint.AbsoluteUri))) .ToList(); } @@ -289,9 +247,7 @@ private async Task RefreshFallbackClients(CancellationToken cancellationToken) { var targetEndpoint = new Uri($"https://{host}"); - var configClient = _credential == null - ? new ConfigurationClient(ConnectionStringUtils.Build(targetEndpoint, _id, _secret), _clientOptions) - : new ConfigurationClient(targetEndpoint, _credential, _clientOptions); + ConfigurationClient configClient = _clientFactory.CreateClient(targetEndpoint.AbsoluteUri); newDynamicClients.Add(new ConfigurationClientWrapper(targetEndpoint, configClient)); } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index aeedd6e67..369f5dbdf 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -19,6 +19,7 @@ + diff --git a/tests/Tests.AzureAppConfiguration/FailoverTests.cs b/tests/Tests.AzureAppConfiguration/FailoverTests.cs index 86ea96b97..929f9befc 100644 --- a/tests/Tests.AzureAppConfiguration/FailoverTests.cs +++ b/tests/Tests.AzureAppConfiguration/FailoverTests.cs @@ -268,10 +268,11 @@ public void FailOverTests_AutoFailover() [Fact] public void FailOverTests_ValidateEndpoints() { + var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); + var configClientManager = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.azconfig.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -285,9 +286,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager.IsValidEndpoint("azure.azconfig.bad.io")); var configClientManager2 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.appconfig.azure.com") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -301,9 +301,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager2.IsValidEndpoint("azure.appconfigbad.azure.com")); var configClientManager3 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.azconfig-test.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -311,9 +310,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager3.IsValidEndpoint("azure.azconfig.io")); var configClientManager4 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.z1.appconfig-test.azure.com") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -325,10 +323,11 @@ public void FailOverTests_ValidateEndpoints() [Fact] public void FailOverTests_GetNoDynamicClient() { + var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); + var configClientManager = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://azure.azconfig.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); From 4ab2dda65683af163f4309705e2309e092e22184 Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Wed, 12 Mar 2025 13:21:07 -0700 Subject: [PATCH 11/16] fix merge conflict errors --- .../AzureAppConfigurationProvider.cs | 7 ------- .../Extensions/StringExtensions.cs | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index bed5c70a0..d7d629f89 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -576,13 +576,6 @@ private void SetDirty(TimeSpan? maxDelay) kvWatcher.NextRefreshTime = nextRefreshTime; } } - else - { - foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) - { - kvWatcher.NextRefreshTime = nextRefreshTime; - } - } foreach (KeyValueWatcher featureFlagWatcher in _options.FeatureFlagWatchers) { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs index a451ae880..615720305 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs @@ -11,5 +11,12 @@ public static string NormalizeNull(this string s) { return s == LabelFilter.Null ? null : s; } + + public static string ToBase64String(this string s) + { + byte[] bytes = System.Text.Encoding.UTF8.GetBytes(s); + + return Convert.ToBase64String(bytes); + } } } From c2e3558e5dd1eb27fcfb44aec15e8486ad8dc50c Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Tue, 18 Mar 2025 12:45:53 -0700 Subject: [PATCH 12/16] revert versions --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index 4cd6bf4e7..a5d4dee4e 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -21,7 +21,7 @@ - 8.1.1 + 8.1.0-preview diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index e327421b7..4354b77ec 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 8.1.1 + 8.1.0-preview diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 5b941baac..369f5dbdf 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -36,7 +36,7 @@ - 8.1.1 + 8.1.0-preview From d7f5939c7fe1038c558b10561ea61af06ed768fb Mon Sep 17 00:00:00 2001 From: Sami Sadfa <71456174+samsadsam@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:18:35 -0400 Subject: [PATCH 13/16] upgrade to 8.2.0-preview (#638) --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index a5d4dee4e..fda406b99 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -21,7 +21,7 @@ - 8.1.0-preview + 8.2.0-preview diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index 4354b77ec..eb43831eb 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 8.1.0-preview + 8.2.0-preview diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 369f5dbdf..dc558788c 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -36,7 +36,7 @@ - 8.1.0-preview + 8.2.0-preview From d0d14a57d1e40fcba210a81720ef09a00af44496 Mon Sep 17 00:00:00 2001 From: Ross Grambo Date: Fri, 25 Apr 2025 15:43:17 -0700 Subject: [PATCH 14/16] Removed FeatureFlagId --- .../FeatureManagementConstants.cs | 1 - .../FeatureManagementKeyValueAdapter.cs | 18 ----------- .../FeatureManagementTests.cs | 31 ------------------- 3 files changed, 50 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs index af4647ee4..d344896ad 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs @@ -41,7 +41,6 @@ internal class FeatureManagementConstants // Telemetry metadata keys public const string ETag = "ETag"; - public const string FeatureFlagId = "FeatureFlagId"; public const string FeatureFlagReference = "FeatureFlagReference"; public const string AllocationId = "AllocationId"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index c6d284929..cbfa411aa 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -310,10 +310,6 @@ private List> ProcessMicrosoftSchemaFeatureFlag(Fea } } - string featureFlagId = CalculateFeatureFlagId(setting.Key, setting.Label); - - keyValues.Add(new KeyValuePair($"{telemetryPath}:{FeatureManagementConstants.Metadata}:{FeatureManagementConstants.FeatureFlagId}", featureFlagId)); - if (endpoint != null) { string featureFlagReference = $"{endpoint.AbsoluteUri}kv/{setting.Key}{(!string.IsNullOrWhiteSpace(setting.Label) ? $"?label={setting.Label}" : "")}"; @@ -1373,19 +1369,5 @@ private FeatureTelemetry ParseFeatureTelemetry(ref Utf8JsonReader reader, string return featureTelemetry; } - - private static string CalculateFeatureFlagId(string key, string label) - { - byte[] featureFlagIdHash; - - // Convert the value consisting of key, newline character, and label to a byte array using UTF8 encoding to hash it using SHA 256 - using (HashAlgorithm hashAlgorithm = SHA256.Create()) - { - featureFlagIdHash = hashAlgorithm.ComputeHash(Encoding.UTF8.GetBytes($"{key}\n{(string.IsNullOrWhiteSpace(label) ? null : label)}")); - } - - // Convert the hashed byte array to Base64Url - return featureFlagIdHash.ToBase64Url(); - } } } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 41e95959b..929bcbc5f 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -15,8 +15,6 @@ using System.Collections.Generic; using System.Diagnostics.Tracing; using System.Linq; -using System.Security.Cryptography; -using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -2151,19 +2149,6 @@ public void WithTelemetry() Assert.Equal("Tag2Value", config["feature_management:feature_flags:0:telemetry:metadata:Tags.Tag2"]); Assert.Equal("c3c231fd-39a0-4cb6-3237-4614474b92c1", config["feature_management:feature_flags:0:telemetry:metadata:ETag"]); - byte[] featureFlagIdHash; - - using (HashAlgorithm hashAlgorithm = SHA256.Create()) - { - featureFlagIdHash = hashAlgorithm.ComputeHash(Encoding.UTF8.GetBytes($"{FeatureManagementConstants.FeatureFlagMarker}TelemetryFeature1\nlabel")); - } - - string featureFlagId = Convert.ToBase64String(featureFlagIdHash) - .TrimEnd('=') - .Replace('+', '-') - .Replace('/', '_'); - - Assert.Equal(featureFlagId, config["feature_management:feature_flags:0:telemetry:metadata:FeatureFlagId"]); Assert.Equal($"{TestHelpers.PrimaryConfigStoreEndpoint}kv/{FeatureManagementConstants.FeatureFlagMarker}TelemetryFeature1?label=label", config["feature_management:feature_flags:0:telemetry:metadata:FeatureFlagReference"]); Assert.Equal("True", config["feature_management:feature_flags:1:telemetry:enabled"]); @@ -2189,24 +2174,10 @@ public void WithAllocationId() }) .Build(); - byte[] featureFlagIdHash; - - using (HashAlgorithm hashAlgorithm = SHA256.Create()) - { - featureFlagIdHash = hashAlgorithm.ComputeHash(Encoding.UTF8.GetBytes($"{FeatureManagementConstants.FeatureFlagMarker}TelemetryVariant\n")); - } - - string featureFlagId = Convert.ToBase64String(featureFlagIdHash) - .TrimEnd('=') - .Replace('+', '-') - .Replace('/', '_'); - // Validate TelemetryVariant Assert.Equal("True", config["feature_management:feature_flags:0:telemetry:enabled"]); Assert.Equal("TelemetryVariant", config["feature_management:feature_flags:0:id"]); - Assert.Equal(featureFlagId, config["feature_management:feature_flags:0:telemetry:metadata:FeatureFlagId"]); - Assert.Equal($"{TestHelpers.PrimaryConfigStoreEndpoint}kv/{FeatureManagementConstants.FeatureFlagMarker}TelemetryVariant", config["feature_management:feature_flags:0:telemetry:metadata:FeatureFlagReference"]); Assert.Equal("MExY1waco2tqen4EcJKK", config["feature_management:feature_flags:0:telemetry:metadata:AllocationId"]); @@ -2223,8 +2194,6 @@ public void WithAllocationId() Assert.Equal("True", config["feature_management:feature_flags:2:telemetry:enabled"]); Assert.Equal("Greeting", config["feature_management:feature_flags:2:id"]); - Assert.Equal("63pHsrNKDSi5Zfe_FvZPSegwbsEo5TS96hf4k7cc4Zw", config["feature_management:feature_flags:2:telemetry:metadata:FeatureFlagId"]); - Assert.Equal($"{TestHelpers.PrimaryConfigStoreEndpoint}kv/{FeatureManagementConstants.FeatureFlagMarker}Greeting", config["feature_management:feature_flags:2:telemetry:metadata:FeatureFlagReference"]); Assert.Equal("L0m7_ulkdsaQmz6dSw4r", config["feature_management:feature_flags:2:telemetry:metadata:AllocationId"]); From 17ed39c56051206ce17e9342d90c7f663d677fa3 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 7 May 2025 10:27:27 -0700 Subject: [PATCH 15/16] Clarify client factory comment (#660) * add comment to setclientfactory * update comment * update comment --- .../AzureAppConfigurationOptions.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 25180357b..3655bf7d8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -174,6 +174,9 @@ public AzureAppConfigurationOptions() /// /// Sets the client factory used to create ConfigurationClient instances. + /// If a client factory is provided using this method, a call to Connect is + /// still required to identify one or more Azure App Configuration stores but + /// will not be used to authenticate a . /// /// The client factory. /// The current instance. From a2c0bee5f5e7859b658545762abbd129d5d33c26 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 9 May 2025 14:36:48 -0700 Subject: [PATCH 16/16] renormalize options --- .../AzureAppConfigurationOptions.cs | 1064 ++++++++--------- 1 file changed, 532 insertions(+), 532 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index fc632c330..1e79daba2 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -1,208 +1,208 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -// -using Azure.Core; -using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Azure; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - /// - /// Options used to configure the behavior of an Azure App Configuration provider. - /// If neither nor is ever called, all key-values with no label are included in the configuration provider. - /// - public class AzureAppConfigurationOptions - { - private const int MaxRetries = 2; - private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// +using Azure.Core; +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + /// + /// Options used to configure the behavior of an Azure App Configuration provider. + /// If neither nor is ever called, all key-values with no label are included in the configuration provider. + /// + public class AzureAppConfigurationOptions + { + private const int MaxRetries = 2; + private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); private static readonly TimeSpan NetworkTimeout = TimeSpan.FromSeconds(10); - private static readonly KeyValueSelector DefaultQuery = new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null }; - - private List _individualKvWatchers = new List(); - private List _ffWatchers = new List(); - private List _adapters; - private List>> _mappers = new List>>(); - private List _selectors; - private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); - private bool _selectCalled = false; - - // The following set is sorted in descending order. - // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. - private SortedSet _keyPrefixes = new SortedSet(Comparer.Create((k1, k2) => -string.Compare(k1, k2, StringComparison.OrdinalIgnoreCase))); - - /// - /// Flag to indicate whether replica discovery is enabled. - /// - public bool ReplicaDiscoveryEnabled { get; set; } = true; - - /// - /// Flag to indicate whether load balancing is enabled. - /// - public bool LoadBalancingEnabled { get; set; } - - /// - /// The list of connection strings used to connect to an Azure App Configuration store and its replicas. - /// - internal IEnumerable ConnectionStrings { get; private set; } - - /// - /// The list of endpoints of an Azure App Configuration store. - /// If this property is set, the property also needs to be set. - /// - internal IEnumerable Endpoints { get; private set; } - - /// - /// The credential used to connect to the Azure App Configuration. - /// If this property is set, the property also needs to be set. - /// - internal TokenCredential Credential { get; private set; } - - /// - /// A collection of specified by user. - /// - internal IEnumerable Selectors => _selectors; - - /// - /// Indicates if was called. - /// - internal bool RegisterAllEnabled { get; private set; } - - /// - /// Refresh interval for selected key-value collections when is called. - /// - internal TimeSpan KvCollectionRefreshInterval { get; private set; } - - /// - /// A collection of . - /// - internal IEnumerable IndividualKvWatchers => _individualKvWatchers; - - /// - /// A collection of . - /// - internal IEnumerable FeatureFlagWatchers => _ffWatchers; - - /// - /// A collection of . - /// - internal IEnumerable Adapters - { - get => _adapters; - set => _adapters = value?.ToList(); - } - - /// - /// A collection of user defined functions that transform each . - /// - internal IEnumerable>> Mappers => _mappers; - - /// - /// A collection of key prefixes to be trimmed. - /// - internal IEnumerable KeyPrefixes => _keyPrefixes; - - /// - /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. - /// - internal IConfigurationClientManager ClientManager { get; set; } - - /// - /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. - /// - internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } - - /// - /// An optional timespan value to set the minimum backoff duration to a value other than the default. - /// - internal TimeSpan MinBackoffDuration { get; set; } = FailOverConstants.MinBackoffDuration; - - /// - /// Options used to configure the client used to communicate with Azure App Configuration. - /// - internal ConfigurationClientOptions ClientOptions { get; } = GetDefaultClientOptions(); - - /// - /// Flag to indicate whether Key Vault options have been configured. - /// - internal bool IsKeyVaultConfigured { get; private set; } = false; - - /// - /// Flag to indicate whether Key Vault secret values will be refreshed automatically. - /// - internal bool IsKeyVaultRefreshConfigured { get; private set; } = false; - - /// - /// Indicates all feature flag features used by the application. - /// - internal FeatureFlagTracing FeatureFlagTracing { get; set; } = new FeatureFlagTracing(); - - /// - /// Options used to configure provider startup. - /// - internal StartupOptions Startup { get; set; } = new StartupOptions(); - - /// - /// Client factory that is responsible for creating instances of ConfigurationClient. - /// - internal IAzureClientFactory ClientFactory { get; private set; } - - /// - /// Initializes a new instance of the class. - /// - public AzureAppConfigurationOptions() - { - _adapters = new List() - { - new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider()), - new JsonKeyValueAdapter(), - new FeatureManagementKeyValueAdapter(FeatureFlagTracing) - }; - - // Adds the default query to App Configuration if and are never called. - _selectors = new List { DefaultQuery }; - } - - /// - /// Sets the client factory used to create ConfigurationClient instances. + private static readonly KeyValueSelector DefaultQuery = new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null }; + + private List _individualKvWatchers = new List(); + private List _ffWatchers = new List(); + private List _adapters; + private List>> _mappers = new List>>(); + private List _selectors; + private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); + private bool _selectCalled = false; + + // The following set is sorted in descending order. + // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. + private SortedSet _keyPrefixes = new SortedSet(Comparer.Create((k1, k2) => -string.Compare(k1, k2, StringComparison.OrdinalIgnoreCase))); + + /// + /// Flag to indicate whether replica discovery is enabled. + /// + public bool ReplicaDiscoveryEnabled { get; set; } = true; + + /// + /// Flag to indicate whether load balancing is enabled. + /// + public bool LoadBalancingEnabled { get; set; } + + /// + /// The list of connection strings used to connect to an Azure App Configuration store and its replicas. + /// + internal IEnumerable ConnectionStrings { get; private set; } + + /// + /// The list of endpoints of an Azure App Configuration store. + /// If this property is set, the property also needs to be set. + /// + internal IEnumerable Endpoints { get; private set; } + + /// + /// The credential used to connect to the Azure App Configuration. + /// If this property is set, the property also needs to be set. + /// + internal TokenCredential Credential { get; private set; } + + /// + /// A collection of specified by user. + /// + internal IEnumerable Selectors => _selectors; + + /// + /// Indicates if was called. + /// + internal bool RegisterAllEnabled { get; private set; } + + /// + /// Refresh interval for selected key-value collections when is called. + /// + internal TimeSpan KvCollectionRefreshInterval { get; private set; } + + /// + /// A collection of . + /// + internal IEnumerable IndividualKvWatchers => _individualKvWatchers; + + /// + /// A collection of . + /// + internal IEnumerable FeatureFlagWatchers => _ffWatchers; + + /// + /// A collection of . + /// + internal IEnumerable Adapters + { + get => _adapters; + set => _adapters = value?.ToList(); + } + + /// + /// A collection of user defined functions that transform each . + /// + internal IEnumerable>> Mappers => _mappers; + + /// + /// A collection of key prefixes to be trimmed. + /// + internal IEnumerable KeyPrefixes => _keyPrefixes; + + /// + /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. + /// + internal IConfigurationClientManager ClientManager { get; set; } + + /// + /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. + /// + internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } + + /// + /// An optional timespan value to set the minimum backoff duration to a value other than the default. + /// + internal TimeSpan MinBackoffDuration { get; set; } = FailOverConstants.MinBackoffDuration; + + /// + /// Options used to configure the client used to communicate with Azure App Configuration. + /// + internal ConfigurationClientOptions ClientOptions { get; } = GetDefaultClientOptions(); + + /// + /// Flag to indicate whether Key Vault options have been configured. + /// + internal bool IsKeyVaultConfigured { get; private set; } = false; + + /// + /// Flag to indicate whether Key Vault secret values will be refreshed automatically. + /// + internal bool IsKeyVaultRefreshConfigured { get; private set; } = false; + + /// + /// Indicates all feature flag features used by the application. + /// + internal FeatureFlagTracing FeatureFlagTracing { get; set; } = new FeatureFlagTracing(); + + /// + /// Options used to configure provider startup. + /// + internal StartupOptions Startup { get; set; } = new StartupOptions(); + + /// + /// Client factory that is responsible for creating instances of ConfigurationClient. + /// + internal IAzureClientFactory ClientFactory { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + public AzureAppConfigurationOptions() + { + _adapters = new List() + { + new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider()), + new JsonKeyValueAdapter(), + new FeatureManagementKeyValueAdapter(FeatureFlagTracing) + }; + + // Adds the default query to App Configuration if and are never called. + _selectors = new List { DefaultQuery }; + } + + /// + /// Sets the client factory used to create ConfigurationClient instances. /// If a client factory is provided using this method, a call to Connect is /// still required to identify one or more Azure App Configuration stores but /// will not be used to authenticate a . - /// - /// The client factory. - /// The current instance. - public AzureAppConfigurationOptions SetClientFactory(IAzureClientFactory factory) - { - ClientFactory = factory ?? throw new ArgumentNullException(nameof(factory)); - return this; - } - - /// - /// Specify what key-values to include in the configuration provider. - /// can be called multiple times to include multiple sets of key-values. - /// - /// - /// The key filter to apply when querying Azure App Configuration for key-values. - /// An asterisk (*) can be added to the end to return all key-values whose key begins with the key filter. - /// e.g. key filter `abc*` returns all key-values whose key starts with `abc`. - /// A comma (,) can be used to select multiple key-values. Comma separated filters must exactly match a key to select it. - /// Using asterisk to select key-values that begin with a key filter while simultaneously using comma separated key filters is not supported. - /// E.g. the key filter `abc*,def` is not supported. The key filters `abc*` and `abc,def` are supported. - /// For all other cases the characters: asterisk (*), comma (,), and backslash (\) are reserved. Reserved characters must be escaped using a backslash (\). - /// e.g. the key filter `a\\b\,\*c*` returns all key-values whose key starts with `a\b,*c`. - /// Built-in key filter options: . - /// - /// - /// The label filter to apply when querying Azure App Configuration for key-values. By default the null label will be used. Built-in label filter options: - /// The characters asterisk (*) and comma (,) are not supported. Backslash (\) character is reserved and must be escaped using another backslash (\). - /// + /// + /// The client factory. + /// The current instance. + public AzureAppConfigurationOptions SetClientFactory(IAzureClientFactory factory) + { + ClientFactory = factory ?? throw new ArgumentNullException(nameof(factory)); + return this; + } + + /// + /// Specify what key-values to include in the configuration provider. + /// can be called multiple times to include multiple sets of key-values. + /// + /// + /// The key filter to apply when querying Azure App Configuration for key-values. + /// An asterisk (*) can be added to the end to return all key-values whose key begins with the key filter. + /// e.g. key filter `abc*` returns all key-values whose key starts with `abc`. + /// A comma (,) can be used to select multiple key-values. Comma separated filters must exactly match a key to select it. + /// Using asterisk to select key-values that begin with a key filter while simultaneously using comma separated key filters is not supported. + /// E.g. the key filter `abc*,def` is not supported. The key filters `abc*` and `abc,def` are supported. + /// For all other cases the characters: asterisk (*), comma (,), and backslash (\) are reserved. Reserved characters must be escaped using a backslash (\). + /// e.g. the key filter `a\\b\,\*c*` returns all key-values whose key starts with `a\b,*c`. + /// Built-in key filter options: . + /// + /// + /// The label filter to apply when querying Azure App Configuration for key-values. By default the null label will be used. Built-in label filter options: + /// The characters asterisk (*) and comma (,) are not supported. Backslash (\) character is reserved and must be escaped using another backslash (\). + /// /// /// In addition to key and label filters, key-values from Azure App Configuration can be filtered based on their tag names and values. /// Each tag filter must follow the format "tagName=tagValue". Only those key-values will be loaded whose tags match all the tags provided here. @@ -211,23 +211,23 @@ public AzureAppConfigurationOptions SetClientFactory(IAzureClientFactory public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter = LabelFilter.Null, IEnumerable tagFilters = null) - { - if (string.IsNullOrEmpty(keyFilter)) - { - throw new ArgumentNullException(nameof(keyFilter)); - } - - // Do not support * and , for label filter for now. - if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) - { - throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); - } - - if (string.IsNullOrWhiteSpace(labelFilter)) - { - labelFilter = LabelFilter.Null; - } - + { + if (string.IsNullOrEmpty(keyFilter)) + { + throw new ArgumentNullException(nameof(keyFilter)); + } + + // Do not support * and , for label filter for now. + if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) + { + throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + } + + if (string.IsNullOrWhiteSpace(labelFilter)) + { + labelFilter = LabelFilter.Null; + } + if (tagFilters != null) { foreach (string tag in tagFilters) @@ -239,321 +239,321 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter } } - if (!_selectCalled) - { - _selectors.Remove(DefaultQuery); - - _selectCalled = true; - } - - _selectors.AppendUnique(new KeyValueSelector - { - KeyFilter = keyFilter, + if (!_selectCalled) + { + _selectors.Remove(DefaultQuery); + + _selectCalled = true; + } + + _selectors.AppendUnique(new KeyValueSelector + { + KeyFilter = keyFilter, LabelFilter = labelFilter, TagFilters = tagFilters - }); - - return this; - } - - /// - /// Specify a snapshot and include its contained key-values in the configuration provider. - /// can be called multiple times to include key-values from multiple snapshots. - /// - /// The name of the snapshot in Azure App Configuration. - public AzureAppConfigurationOptions SelectSnapshot(string name) - { - if (string.IsNullOrEmpty(name)) - { - throw new ArgumentNullException(nameof(name)); - } - - if (!_selectCalled) - { - _selectors.Remove(DefaultQuery); - - _selectCalled = true; - } - - _selectors.AppendUnique(new KeyValueSelector - { - SnapshotName = name - }); - - return this; - } - - /// - /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. - /// If no filtering is specified via the then all feature flags with no label are loaded. - /// All loaded feature flags will be automatically registered for refresh as a collection. - /// - /// A callback used to configure feature flag options. - public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) - { - FeatureFlagOptions options = new FeatureFlagOptions(); - configure?.Invoke(options); - - if (options.RefreshInterval < RefreshConstants.MinimumFeatureFlagRefreshInterval) - { - throw new ArgumentOutOfRangeException(nameof(options.RefreshInterval), options.RefreshInterval.TotalMilliseconds, - string.Format(ErrorMessages.RefreshIntervalTooShort, RefreshConstants.MinimumFeatureFlagRefreshInterval.TotalMilliseconds)); - } - - if (options.FeatureFlagSelectors.Count() != 0 && options.Label != null) - { - throw new InvalidOperationException($"Please select feature flags by either the {nameof(options.Select)} method or by setting the {nameof(options.Label)} property, not both."); - } - - if (options.FeatureFlagSelectors.Count() == 0) - { - // Select clause is not present - options.FeatureFlagSelectors.Add(new KeyValueSelector - { - KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", - LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, - IsFeatureFlagSelector = true - }); - } - - foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) - { - _selectors.AppendUnique(featureFlagSelector); - - _ffWatchers.AppendUnique(new KeyValueWatcher - { - Key = featureFlagSelector.KeyFilter, - Label = featureFlagSelector.LabelFilter, + }); + + return this; + } + + /// + /// Specify a snapshot and include its contained key-values in the configuration provider. + /// can be called multiple times to include key-values from multiple snapshots. + /// + /// The name of the snapshot in Azure App Configuration. + public AzureAppConfigurationOptions SelectSnapshot(string name) + { + if (string.IsNullOrEmpty(name)) + { + throw new ArgumentNullException(nameof(name)); + } + + if (!_selectCalled) + { + _selectors.Remove(DefaultQuery); + + _selectCalled = true; + } + + _selectors.AppendUnique(new KeyValueSelector + { + SnapshotName = name + }); + + return this; + } + + /// + /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. + /// If no filtering is specified via the then all feature flags with no label are loaded. + /// All loaded feature flags will be automatically registered for refresh as a collection. + /// + /// A callback used to configure feature flag options. + public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) + { + FeatureFlagOptions options = new FeatureFlagOptions(); + configure?.Invoke(options); + + if (options.RefreshInterval < RefreshConstants.MinimumFeatureFlagRefreshInterval) + { + throw new ArgumentOutOfRangeException(nameof(options.RefreshInterval), options.RefreshInterval.TotalMilliseconds, + string.Format(ErrorMessages.RefreshIntervalTooShort, RefreshConstants.MinimumFeatureFlagRefreshInterval.TotalMilliseconds)); + } + + if (options.FeatureFlagSelectors.Count() != 0 && options.Label != null) + { + throw new InvalidOperationException($"Please select feature flags by either the {nameof(options.Select)} method or by setting the {nameof(options.Label)} property, not both."); + } + + if (options.FeatureFlagSelectors.Count() == 0) + { + // Select clause is not present + options.FeatureFlagSelectors.Add(new KeyValueSelector + { + KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", + LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, + IsFeatureFlagSelector = true + }); + } + + foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) + { + _selectors.AppendUnique(featureFlagSelector); + + _ffWatchers.AppendUnique(new KeyValueWatcher + { + Key = featureFlagSelector.KeyFilter, + Label = featureFlagSelector.LabelFilter, Tags = featureFlagSelector.TagFilters, - // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins - RefreshInterval = options.RefreshInterval - }); - } - - return this; - } - - /// - /// Connect the provider to the Azure App Configuration service via a connection string. - /// - /// - /// Used to authenticate with Azure App Configuration. - /// - public AzureAppConfigurationOptions Connect(string connectionString) - { - if (string.IsNullOrWhiteSpace(connectionString)) - { - throw new ArgumentNullException(nameof(connectionString)); - } - - return Connect(new List { connectionString }); - } - - /// - /// Connect the provider to an Azure App Configuration store and its replicas via a list of connection strings. - /// - /// - /// Used to authenticate with Azure App Configuration. - /// - public AzureAppConfigurationOptions Connect(IEnumerable connectionStrings) - { - if (connectionStrings == null || !connectionStrings.Any()) - { - throw new ArgumentNullException(nameof(connectionStrings)); - } - - if (connectionStrings.Distinct().Count() != connectionStrings.Count()) - { - throw new ArgumentException($"All values in '{nameof(connectionStrings)}' must be unique."); - } - - Endpoints = null; - Credential = null; - ConnectionStrings = connectionStrings; - return this; - } - - /// - /// Connect the provider to Azure App Configuration using endpoint and token credentials. - /// - /// The endpoint of the Azure App Configuration to connect to. - /// Token credentials to use to connect. - public AzureAppConfigurationOptions Connect(Uri endpoint, TokenCredential credential) - { - if (endpoint == null) - { - throw new ArgumentNullException(nameof(endpoint)); - } - - if (credential == null) - { - throw new ArgumentNullException(nameof(credential)); - } - - return Connect(new List() { endpoint }, credential); - } - - /// - /// Connect the provider to an Azure App Configuration store and its replicas using a list of endpoints and a token credential. - /// - /// The list of endpoints of an Azure App Configuration store and its replicas to connect to. - /// Token credential to use to connect. - public AzureAppConfigurationOptions Connect(IEnumerable endpoints, TokenCredential credential) - { - if (endpoints == null || !endpoints.Any()) - { - throw new ArgumentNullException(nameof(endpoints)); - } - - if (endpoints.Distinct(new EndpointComparer()).Count() != endpoints.Count()) - { - throw new ArgumentException($"All values in '{nameof(endpoints)}' must be unique."); - } - - Credential = credential ?? throw new ArgumentNullException(nameof(credential)); - - Endpoints = endpoints; - ConnectionStrings = null; - return this; - } - - /// - /// Trims the provided prefix from the keys of all key-values retrieved from Azure App Configuration. - /// - /// The prefix to be trimmed. - public AzureAppConfigurationOptions TrimKeyPrefix(string prefix) - { - if (string.IsNullOrEmpty(prefix)) - { - throw new ArgumentNullException(nameof(prefix)); - } - - _keyPrefixes.Add(prefix); - return this; - } - - /// - /// Configure the client(s) used to communicate with Azure App Configuration. - /// - /// A callback used to configure Azure App Configuration client options. - public AzureAppConfigurationOptions ConfigureClientOptions(Action configure) - { - configure?.Invoke(ClientOptions); - return this; - } - - /// - /// Configure refresh for key-values in the configuration provider. - /// - /// A callback used to configure Azure App Configuration refresh options. - public AzureAppConfigurationOptions ConfigureRefresh(Action configure) - { - if (RegisterAllEnabled) - { - throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); - } - - var refreshOptions = new AzureAppConfigurationRefreshOptions(); - configure?.Invoke(refreshOptions); - - bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); - RegisterAllEnabled = refreshOptions.RegisterAllEnabled; - - if (!isRegisterCalled && !RegisterAllEnabled) - { - throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() must call either {nameof(AzureAppConfigurationRefreshOptions.Register)}()" + - $" or {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)}()"); - } - - // Check if both register methods are called at any point - if (RegisterAllEnabled && (_individualKvWatchers.Any() || isRegisterCalled)) - { - throw new InvalidOperationException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " - + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); - } - - if (RegisterAllEnabled) - { - KvCollectionRefreshInterval = refreshOptions.RefreshInterval; - } - else - { - foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) - { - item.RefreshInterval = refreshOptions.RefreshInterval; - _individualKvWatchers.Add(item); - } - } - - return this; - } - - /// - /// Get an instance of that can be used to trigger a refresh for the registered key-values. - /// - /// An instance of . - public IConfigurationRefresher GetRefresher() - { - return _refresher; - } - - /// - /// Configures the Azure App Configuration provider to use the provided Key Vault configuration to resolve key vault references. - /// - /// A callback used to configure Azure App Configuration key vault options. - public AzureAppConfigurationOptions ConfigureKeyVault(Action configure) - { - var keyVaultOptions = new AzureAppConfigurationKeyVaultOptions(); - configure?.Invoke(keyVaultOptions); - - if (keyVaultOptions.Credential != null && keyVaultOptions.SecretResolver != null) - { - throw new InvalidOperationException($"Cannot configure both default credentials and secret resolver for Key Vault references. Please call either {nameof(keyVaultOptions.SetCredential)} or {nameof(keyVaultOptions.SetSecretResolver)} method, not both."); - } - - _adapters.RemoveAll(a => a is AzureKeyVaultKeyValueAdapter); - _adapters.Add(new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider(keyVaultOptions))); - - IsKeyVaultRefreshConfigured = keyVaultOptions.IsKeyVaultRefreshConfigured; - IsKeyVaultConfigured = true; - return this; - } - - /// - /// Provides a way to transform settings retrieved from App Configuration before they are processed by the configuration provider. - /// - /// A callback registered by the user to transform each configuration setting. - public AzureAppConfigurationOptions Map(Func> mapper) - { - if (mapper == null) - { - throw new ArgumentNullException(nameof(mapper)); - } - - _mappers.Add(mapper); - return this; - } - - /// - /// Configure the provider behavior when loading data from Azure App Configuration on startup. - /// - /// A callback used to configure Azure App Configuration startup options. - public AzureAppConfigurationOptions ConfigureStartupOptions(Action configure) - { - configure?.Invoke(Startup); - return this; - } - - private static ConfigurationClientOptions GetDefaultClientOptions() - { - var clientOptions = new ConfigurationClientOptions(ConfigurationClientOptions.ServiceVersion.V2023_10_01); - clientOptions.Retry.MaxRetries = MaxRetries; - clientOptions.Retry.MaxDelay = MaxRetryDelay; - clientOptions.Retry.Mode = RetryMode.Exponential; + // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins + RefreshInterval = options.RefreshInterval + }); + } + + return this; + } + + /// + /// Connect the provider to the Azure App Configuration service via a connection string. + /// + /// + /// Used to authenticate with Azure App Configuration. + /// + public AzureAppConfigurationOptions Connect(string connectionString) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentNullException(nameof(connectionString)); + } + + return Connect(new List { connectionString }); + } + + /// + /// Connect the provider to an Azure App Configuration store and its replicas via a list of connection strings. + /// + /// + /// Used to authenticate with Azure App Configuration. + /// + public AzureAppConfigurationOptions Connect(IEnumerable connectionStrings) + { + if (connectionStrings == null || !connectionStrings.Any()) + { + throw new ArgumentNullException(nameof(connectionStrings)); + } + + if (connectionStrings.Distinct().Count() != connectionStrings.Count()) + { + throw new ArgumentException($"All values in '{nameof(connectionStrings)}' must be unique."); + } + + Endpoints = null; + Credential = null; + ConnectionStrings = connectionStrings; + return this; + } + + /// + /// Connect the provider to Azure App Configuration using endpoint and token credentials. + /// + /// The endpoint of the Azure App Configuration to connect to. + /// Token credentials to use to connect. + public AzureAppConfigurationOptions Connect(Uri endpoint, TokenCredential credential) + { + if (endpoint == null) + { + throw new ArgumentNullException(nameof(endpoint)); + } + + if (credential == null) + { + throw new ArgumentNullException(nameof(credential)); + } + + return Connect(new List() { endpoint }, credential); + } + + /// + /// Connect the provider to an Azure App Configuration store and its replicas using a list of endpoints and a token credential. + /// + /// The list of endpoints of an Azure App Configuration store and its replicas to connect to. + /// Token credential to use to connect. + public AzureAppConfigurationOptions Connect(IEnumerable endpoints, TokenCredential credential) + { + if (endpoints == null || !endpoints.Any()) + { + throw new ArgumentNullException(nameof(endpoints)); + } + + if (endpoints.Distinct(new EndpointComparer()).Count() != endpoints.Count()) + { + throw new ArgumentException($"All values in '{nameof(endpoints)}' must be unique."); + } + + Credential = credential ?? throw new ArgumentNullException(nameof(credential)); + + Endpoints = endpoints; + ConnectionStrings = null; + return this; + } + + /// + /// Trims the provided prefix from the keys of all key-values retrieved from Azure App Configuration. + /// + /// The prefix to be trimmed. + public AzureAppConfigurationOptions TrimKeyPrefix(string prefix) + { + if (string.IsNullOrEmpty(prefix)) + { + throw new ArgumentNullException(nameof(prefix)); + } + + _keyPrefixes.Add(prefix); + return this; + } + + /// + /// Configure the client(s) used to communicate with Azure App Configuration. + /// + /// A callback used to configure Azure App Configuration client options. + public AzureAppConfigurationOptions ConfigureClientOptions(Action configure) + { + configure?.Invoke(ClientOptions); + return this; + } + + /// + /// Configure refresh for key-values in the configuration provider. + /// + /// A callback used to configure Azure App Configuration refresh options. + public AzureAppConfigurationOptions ConfigureRefresh(Action configure) + { + if (RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); + } + + var refreshOptions = new AzureAppConfigurationRefreshOptions(); + configure?.Invoke(refreshOptions); + + bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); + RegisterAllEnabled = refreshOptions.RegisterAllEnabled; + + if (!isRegisterCalled && !RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() must call either {nameof(AzureAppConfigurationRefreshOptions.Register)}()" + + $" or {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)}()"); + } + + // Check if both register methods are called at any point + if (RegisterAllEnabled && (_individualKvWatchers.Any() || isRegisterCalled)) + { + throw new InvalidOperationException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " + + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); + } + + if (RegisterAllEnabled) + { + KvCollectionRefreshInterval = refreshOptions.RefreshInterval; + } + else + { + foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) + { + item.RefreshInterval = refreshOptions.RefreshInterval; + _individualKvWatchers.Add(item); + } + } + + return this; + } + + /// + /// Get an instance of that can be used to trigger a refresh for the registered key-values. + /// + /// An instance of . + public IConfigurationRefresher GetRefresher() + { + return _refresher; + } + + /// + /// Configures the Azure App Configuration provider to use the provided Key Vault configuration to resolve key vault references. + /// + /// A callback used to configure Azure App Configuration key vault options. + public AzureAppConfigurationOptions ConfigureKeyVault(Action configure) + { + var keyVaultOptions = new AzureAppConfigurationKeyVaultOptions(); + configure?.Invoke(keyVaultOptions); + + if (keyVaultOptions.Credential != null && keyVaultOptions.SecretResolver != null) + { + throw new InvalidOperationException($"Cannot configure both default credentials and secret resolver for Key Vault references. Please call either {nameof(keyVaultOptions.SetCredential)} or {nameof(keyVaultOptions.SetSecretResolver)} method, not both."); + } + + _adapters.RemoveAll(a => a is AzureKeyVaultKeyValueAdapter); + _adapters.Add(new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider(keyVaultOptions))); + + IsKeyVaultRefreshConfigured = keyVaultOptions.IsKeyVaultRefreshConfigured; + IsKeyVaultConfigured = true; + return this; + } + + /// + /// Provides a way to transform settings retrieved from App Configuration before they are processed by the configuration provider. + /// + /// A callback registered by the user to transform each configuration setting. + public AzureAppConfigurationOptions Map(Func> mapper) + { + if (mapper == null) + { + throw new ArgumentNullException(nameof(mapper)); + } + + _mappers.Add(mapper); + return this; + } + + /// + /// Configure the provider behavior when loading data from Azure App Configuration on startup. + /// + /// A callback used to configure Azure App Configuration startup options. + public AzureAppConfigurationOptions ConfigureStartupOptions(Action configure) + { + configure?.Invoke(Startup); + return this; + } + + private static ConfigurationClientOptions GetDefaultClientOptions() + { + var clientOptions = new ConfigurationClientOptions(ConfigurationClientOptions.ServiceVersion.V2023_10_01); + clientOptions.Retry.MaxRetries = MaxRetries; + clientOptions.Retry.MaxDelay = MaxRetryDelay; + clientOptions.Retry.Mode = RetryMode.Exponential; clientOptions.Retry.NetworkTimeout = NetworkTimeout; - clientOptions.AddPolicy(new UserAgentHeaderPolicy(), HttpPipelinePosition.PerCall); - - return clientOptions; - } - } -} + clientOptions.AddPolicy(new UserAgentHeaderPolicy(), HttpPipelinePosition.PerCall); + + return clientOptions; + } + } +}