From 2836f66d9c1b462559a9ac5b1641daf2da5ce98a Mon Sep 17 00:00:00 2001 From: Aditya Narayanaswamy Date: Wed, 24 Jun 2026 23:25:14 -0400 Subject: [PATCH] azure: Fix the dualstack errors Fixing a few errors like DNS conflict with ipv4 and ipv6 entry, trying to force CCM to use ipv6 primary etc. --- pkg/infrastructure/azure/azure.go | 60 +++++++++------- pkg/infrastructure/azure/network.go | 47 +++---------- pkg/types/validation/installconfig.go | 24 ++++++- pkg/types/validation/installconfig_test.go | 79 ++++++++++++++++++++++ 4 files changed, 149 insertions(+), 61 deletions(-) diff --git a/pkg/infrastructure/azure/azure.go b/pkg/infrastructure/azure/azure.go index 4d3995fda4d..09749757a73 100644 --- a/pkg/infrastructure/azure/azure.go +++ b/pkg/infrastructure/azure/azure.go @@ -448,9 +448,14 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput var extLBFQDN string if in.InstallConfig.Config.PublicAPI() { var publicIPv6 *armnetwork.PublicIPAddress + v4InfraID := in.InfraID + v6InfraID := "" + if in.InstallConfig.Config.Azure.IPFamily.DualStackEnabled() { + v6InfraID = fmt.Sprintf("%s-v6", in.InfraID) + } publicIP, err := createPublicIP(ctx, &pipInput{ name: fmt.Sprintf("%s-pip-v4", in.InfraID), - infraID: in.InfraID, + infraID: v4InfraID, region: in.InstallConfig.Config.Azure.Region, resourceGroup: resourceGroupName, pipClient: networkClientFactory.NewPublicIPAddressesClient(), @@ -464,7 +469,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput if in.InstallConfig.Config.Azure.IPFamily.DualStackEnabled() { publicIPv6, err = createPublicIP(ctx, &pipInput{ name: fmt.Sprintf("%s-pip-v6", in.InfraID), - infraID: in.InfraID, + infraID: v6InfraID, region: in.InstallConfig.Config.Azure.Region, resourceGroup: resourceGroupName, pipClient: networkClientFactory.NewPublicIPAddressesClient(), @@ -585,6 +590,36 @@ func (p *Provider) PostProvision(ctx context.Context, in clusterapi.PostProvisio return fmt.Errorf("failed to associate control plane VMs with external load balancer: %w", err) } + if in.InstallConfig.Config.Azure.IPFamily.DualStackEnabled() { + bootstrapNicName := fmt.Sprintf("%s-bootstrap-nic", in.InfraID) + nicClient := p.NetworkClientFactory.NewInterfacesClient() + bootstrapNic, err := nicClient.Get(ctx, p.ResourceGroupName, bootstrapNicName, nil) + if err != nil { + return fmt.Errorf("failed to get bootstrap nic: %w", err) + } + for _, ipconfig := range bootstrapNic.Properties.IPConfigurations { + if ipconfig.Properties.PrivateIPAddressVersion != nil && *ipconfig.Properties.PrivateIPAddressVersion == armnetwork.IPVersionIPv6 { + for _, pool := range p.lbBackendAddressPools { + if pool.Name != nil && strings.HasSuffix(*pool.Name, "-v6") { + ipconfig.Properties.LoadBalancerBackendAddressPools = append( + ipconfig.Properties.LoadBalancerBackendAddressPools, + pool, + ) + } + } + } + } + pollerResp, err := nicClient.BeginCreateOrUpdate(ctx, p.ResourceGroupName, bootstrapNicName, bootstrapNic.Interface, nil) + if err != nil { + return fmt.Errorf("failed to update bootstrap nic with IPv6 backend pools: %w", err) + } + _, err = pollerResp.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to update bootstrap nic with IPv6 backend pools: %w", err) + } + logrus.Debugf("associated bootstrap NIC with IPv6 backend pools") + } + sshRuleName := fmt.Sprintf("%s_ssh_in", in.InfraID) loadBalancerName := in.InfraID @@ -630,27 +665,6 @@ func (p *Provider) PostProvision(ctx context.Context, in clusterapi.PostProvisio // For dual-stack, create IPv6 inbound rule for SSH access to bootstrap. if in.InstallConfig.Config.Azure.IPFamily.DualStackEnabled() { - publicIPv6outbound, err := createPublicIP(ctx, &pipInput{ - name: fmt.Sprintf("%s-pip-v6-outbound-lb", in.InfraID), - infraID: in.InfraID, - region: in.InstallConfig.Config.Azure.Region, - resourceGroup: p.ResourceGroupName, - pipClient: p.NetworkClientFactory.NewPublicIPAddressesClient(), - tags: p.Tags, - ipversion: armnetwork.IPVersionIPv6, - }) - if err != nil { - return fmt.Errorf("failed to create public ipv6 for outbound ipv6 lb: %w", err) - } - logrus.Debugf("created public ipv6 for outbound ipv6 lb: %s", *publicIPv6outbound.ID) - - // Update the outbound node IPv6 load balancer. - outboundLBName := fmt.Sprintf("%s-ipv6-outbound-node-lb", in.InfraID) - err = updateOutboundIPv6LoadBalancer(ctx, publicIPv6outbound, p.NetworkClientFactory.NewLoadBalancersClient(), p.ResourceGroupName, outboundLBName, in.InfraID) - if err != nil { - return fmt.Errorf("failed to set public ipv6 to outbound ipv6 lb: %w", err) - } - logrus.Debugf("updated outbound ipv6 lb %s with public ipv6: %s", outboundLBName, *publicIPv6outbound.ID) frontendIPv6ConfigName := "public-lb-ip-v6" sshRuleNameV6 := fmt.Sprintf("%s_ssh_in_v6", in.InfraID) frontendIPv6ConfigID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", diff --git a/pkg/infrastructure/azure/network.go b/pkg/infrastructure/azure/network.go index 46890e81624..4fa107d264c 100644 --- a/pkg/infrastructure/azure/network.go +++ b/pkg/infrastructure/azure/network.go @@ -73,6 +73,15 @@ type inboundNatRuleInput struct { } func createPublicIP(ctx context.Context, in *pipInput) (*armnetwork.PublicIPAddress, error) { + properties := &armnetwork.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: to.Ptr(in.ipversion), + PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodStatic), + } + if in.infraID != "" { + properties.DNSSettings = &armnetwork.PublicIPAddressDNSSettings{ + DomainNameLabel: to.Ptr(in.infraID), + } + } pollerResp, err := in.pipClient.BeginCreateOrUpdate( ctx, in.resourceGroup, @@ -84,14 +93,8 @@ func createPublicIP(ctx context.Context, in *pipInput) (*armnetwork.PublicIPAddr Name: to.Ptr(armnetwork.PublicIPAddressSKUNameStandard), Tier: to.Ptr(armnetwork.PublicIPAddressSKUTierRegional), }, - Properties: &armnetwork.PublicIPAddressPropertiesFormat{ - PublicIPAddressVersion: to.Ptr(in.ipversion), - PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodStatic), - DNSSettings: &armnetwork.PublicIPAddressDNSSettings{ - DomainNameLabel: to.Ptr(in.infraID), - }, - }, - Tags: in.tags, + Properties: properties, + Tags: in.tags, }, nil, ) @@ -764,31 +767,3 @@ func associateNatGatewayToSubnet(ctx context.Context, in natGatewayInput) error } return nil } - -func updateOutboundIPv6LoadBalancer(ctx context.Context, pipv6 *armnetwork.PublicIPAddress, lbClient *armnetwork.LoadBalancersClient, resourceGroup, loadBalancerName, infraID string) error { - outboundIPv6LB, err := lbClient.Get(ctx, resourceGroup, loadBalancerName, nil) - if err != nil { - return fmt.Errorf("failed to get external load balancer: %w", err) - } - - loadBalancer := outboundIPv6LB.LoadBalancer - loadBalancer.Properties.FrontendIPConfigurations = append(loadBalancer.Properties.FrontendIPConfigurations, &armnetwork.FrontendIPConfiguration{ - Name: to.Ptr(fmt.Sprintf("%s-frontend-ipv6", infraID)), - Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodDynamic), - PublicIPAddress: pipv6, - }, - }) - - pollerResp, err := lbClient.BeginCreateOrUpdate(ctx, - resourceGroup, - loadBalancerName, - loadBalancer, nil) - - if err != nil { - return fmt.Errorf("cannot update outbound node ipv6 load balancer: %w", err) - } - - _, err = pollerResp.PollUntilDone(ctx, nil) - return err -} diff --git a/pkg/types/validation/installconfig.go b/pkg/types/validation/installconfig.go index a43e29711b1..f89195d7cd1 100644 --- a/pkg/types/validation/installconfig.go +++ b/pkg/types/validation/installconfig.go @@ -381,6 +381,9 @@ func validateNetworkingIPVersion(n *types.Networking, p *types.Platform) field.E switch { case p.Azure != nil && experimentalDualStackEnabled: logrus.Warnf("Using experimental Azure dual-stack support") + if p.Azure.IPFamily == network.DualStackIPv6Primary { + allowV6Primary = true + } case p.BareMetal != nil: // We now support ipv6-primary dual stack on baremetal allowV6Primary = true @@ -424,7 +427,7 @@ func validateNetworkingIPVersion(n *types.Networking, p *types.Platform) field.E allErrs = append(allErrs, field.Invalid(field.NewPath("networking", k), strings.Join(ipnetworksToStrings(addresses[k]), ", "), "dual-stack IPv4/IPv6 requires an IPv4 network in this list")) } - allErrs = append(allErrs, validateNetworkEntryOrder(p, v, addresses[k], allowV6Primary, field.NewPath("networking", k))...) + allErrs = append(allErrs, validateNetworkEntryOrder(p, v, addresses[k], allowV6Primary, k, field.NewPath("networking", k))...) } case hasIPv6: @@ -475,7 +478,7 @@ func validateNetworkingIPVersion(n *types.Networking, p *types.Platform) field.E // - IPv4 primary dual-stack: IPv4 CIDR first in list // - IPv6 primary dual-stack: IPv6 CIDR first in list // Some platforms have an explicit field to define the dual-stack variant, for example, platform.aws.ipFamily on AWS. -func validateNetworkEntryOrder(p *types.Platform, ipAddressType ipAddressType, networks []ipnet.IPNet, allowV6Primary bool, fldPath *field.Path) field.ErrorList { +func validateNetworkEntryOrder(p *types.Platform, ipAddressType ipAddressType, networks []ipnet.IPNet, allowV6Primary bool, networkType string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} // If missing either IPv4 or IPv6 CIDR, order validation is not applicable @@ -495,6 +498,23 @@ func validateNetworkEntryOrder(p *types.Platform, ipAddressType ipAddressType, n if ipFamily == network.DualStackIPv6Primary && ipAddressType.Primary == corev1.IPv4Protocol { allErrs = append(allErrs, field.Invalid(fldPath, strings.Join(ipnetworksToStrings(networks), ", "), "DualStackIPv6Primary requires an IPv6 network first in this list")) } + case p.Azure != nil: + ipFamily := p.Azure.IPFamily + + // Azure nodes always have IPv4 as the primary NIC address, so serviceNetwork + // must have IPv4 first regardless of ipFamily. The kube-apiserver requires the + // primary service IP family to match the node's address family. + if networkType == networkTypeService && ipAddressType.Primary != corev1.IPv4Protocol { + allErrs = append(allErrs, field.Invalid(fldPath, strings.Join(ipnetworksToStrings(networks), ", "), "Azure requires an IPv4 service network first in this list because node primary addresses are always IPv4")) + } + + if ipFamily == network.DualStackIPv4Primary && ipAddressType.Primary == corev1.IPv6Protocol && networkType != networkTypeService { + allErrs = append(allErrs, field.Invalid(fldPath, strings.Join(ipnetworksToStrings(networks), ", "), "DualStackIPv4Primary requires an IPv4 network first in this list")) + } + + if ipFamily == network.DualStackIPv6Primary && ipAddressType.Primary == corev1.IPv4Protocol && networkType != networkTypeService { + allErrs = append(allErrs, field.Invalid(fldPath, strings.Join(ipnetworksToStrings(networks), ", "), "DualStackIPv6Primary requires an IPv6 network first in this list")) + } default: // For platforms that don't support IPv6-primary dual-stack, reject configurations with IPv6 CIDRs listed first. if !allowV6Primary && ipAddressType.Primary != corev1.IPv4Protocol { diff --git a/pkg/types/validation/installconfig_test.go b/pkg/types/validation/installconfig_test.go index 61dbf481ab1..88343e92f7d 100644 --- a/pkg/types/validation/installconfig_test.go +++ b/pkg/types/validation/installconfig_test.go @@ -3,6 +3,7 @@ package validation import ( "fmt" "net" + "os" "testing" "github.com/aws/smithy-go/ptr" @@ -1976,6 +1977,84 @@ func TestValidateInstallConfig(t *testing.T) { }(), expectedError: `networking.serviceNetwork: Invalid value: "ffd1::/112": when installing dual-stack IPv4/IPv6 you must provide two service networks, one for each IP address type`, }, + { + name: "azure: valid dual-stack with DualStackIPv6Primary and IPv4-first serviceNetwork", + installConfig: func() *types.InstallConfig { + c := validInstallConfig() + c.Platform = types.Platform{Azure: validAzurePlatform()} + c.Platform.Azure.IPFamily = network.DualStackIPv6Primary + c.FeatureSet = configv1.CustomNoUpgrade + c.FeatureGates = []string{"AzureDualStackInstall=true"} + c.Networking = validPrimaryV6DualStackNetworkingConfig() + c.Networking.ServiceNetwork = []ipnet.IPNet{ + *ipnet.MustParseCIDR("172.30.0.0/16"), + *ipnet.MustParseCIDR("ffd1::/112"), + } + return c + }(), + restoreFnFactory: func(_ *types.InstallConfig) func() { + os.Setenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK", "true") + return func() { + os.Unsetenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK") + } + }, + }, + { + name: "azure: valid dual-stack with DualStackIPv4Primary", + installConfig: func() *types.InstallConfig { + c := validInstallConfig() + c.Platform = types.Platform{Azure: validAzurePlatform()} + c.Platform.Azure.IPFamily = network.DualStackIPv4Primary + c.FeatureSet = configv1.CustomNoUpgrade + c.FeatureGates = []string{"AzureDualStackInstall=true"} + c.Networking = validDualStackNetworkingConfig() + return c + }(), + restoreFnFactory: func(_ *types.InstallConfig) func() { + os.Setenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK", "true") + return func() { + os.Unsetenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK") + } + }, + }, + { + name: "azure: invalid dual-stack with IPv6-first serviceNetwork", + installConfig: func() *types.InstallConfig { + c := validInstallConfig() + c.Platform = types.Platform{Azure: validAzurePlatform()} + c.Platform.Azure.IPFamily = network.DualStackIPv6Primary + c.FeatureSet = configv1.CustomNoUpgrade + c.FeatureGates = []string{"AzureDualStackInstall=true"} + c.Networking = validPrimaryV6DualStackNetworkingConfig() + return c + }(), + expectedError: `networking.serviceNetwork: Invalid value: "ffd1::/112, 172.30.0.0/16": Azure requires an IPv4 service network first in this list because node primary addresses are always IPv4`, + restoreFnFactory: func(_ *types.InstallConfig) func() { + os.Setenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK", "true") + return func() { + os.Unsetenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK") + } + }, + }, + { + name: "azure: invalid dual-stack with DualStackIPv4Primary but IPv6-primary networks", + installConfig: func() *types.InstallConfig { + c := validInstallConfig() + c.Platform = types.Platform{Azure: validAzurePlatform()} + c.Platform.Azure.IPFamily = network.DualStackIPv4Primary + c.FeatureSet = configv1.CustomNoUpgrade + c.FeatureGates = []string{"AzureDualStackInstall=true"} + c.Networking = validPrimaryV6DualStackNetworkingConfig() + return c + }(), + expectedError: `^\Q[networking.clusterNetwork: Invalid value: "ffd2::/48, 192.168.1.0/24": DualStackIPv4Primary requires an IPv4 network first in this list, networking.machineNetwork: Invalid value: "ffd0::/48, 10.0.0.0/16": DualStackIPv4Primary requires an IPv4 network first in this list, networking.serviceNetwork: Invalid value: "ffd1::/112, 172.30.0.0/16": Azure requires an IPv4 service network first in this list because node primary addresses are always IPv4]\E$`, + restoreFnFactory: func(_ *types.InstallConfig) func() { + os.Setenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK", "true") + return func() { + os.Unsetenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK") + } + }, + }, { name: "invalid IPv6 hostprefix", installConfig: func() *types.InstallConfig {