WIP Add retry diagnostic logging and enable pprof on all metrics servers

go-controller/cmd/ovnkube/ovnkube.go

@@ -639,12 +639,12 @@ func runOvnKube(ctx context.Context, runMode *ovnkubeRunMode, ovnClientset *util
 				EnableOVNControllerMetrics: true,
 				EnableOVNNorthdMetrics:     true,
 				EnableOVNDBMetrics:         true,
+				EnablePprof:                config.Metrics.EnablePprof,
 			}
 
 			if combineMetricsEndpoints(runMode) {
 				// Reuse the default registry (and its gatherer) so ovnkube-node metrics and OVN metrics share one endpoint.
 				opts.Registerer = prometheus.DefaultRegisterer
-				opts.EnablePprof = config.Metrics.EnablePprof
 			}
 
 			if !config.OVNKubernetesFeature.EnableInterconnect {

go-controller/pkg/cni/cni.go

@@ -451,7 +451,7 @@ func HandlePodRequest(
 	var response *Response
 	var err, err1 error
 
-	klog.Infof("%s %s starting CNI request %+v", request, request.Command, request)
+	klog.V(5).Infof("%s %s starting CNI request %+v", request, request.Command, request)
 	switch request.Command {
 	case CNIAdd:
 		response, err = request.cmdAdd(kubeAuth, clientset, networkManager, ovsClient)

go-controller/pkg/ovn/base_network_controller_policy.go

@@ -563,7 +563,7 @@ func (bnc *BaseNetworkController) handleNetPolNamespaceUpdate(namespace string,
 		if err != nil {
 			return fmt.Errorf("unable to update ACL for network policy %s: %v", npKey, err)
 		}
-		klog.Infof("ACL for network policy: %s, updated to new log level: %s", npKey, nsInfo.aclLogging.Allow)
+		klog.V(5).Infof("ACL for network policy: %s, updated to new log level: %s", npKey, nsInfo.aclLogging.Allow)
 	}
 	return nil
 }
@@ -653,7 +653,7 @@ func (bnc *BaseNetworkController) getNewLocalPolicyPorts(np *networkPolicy,
 // getExistingLocalPolicyPorts will find and return port info for every given pod obj, that is present in np.localPods.
 func (bnc *BaseNetworkController) getExistingLocalPolicyPorts(np *networkPolicy,
 	objs ...interface{}) (policyPortsToUUIDs map[string]string, policyPortUUIDs []string, err error) {
-	klog.Infof("Processing NetworkPolicy %s/%s to delete %d local pods...", np.namespace, np.name, len(objs))
+	klog.V(5).Infof("Processing NetworkPolicy %s/%s to delete %d local pods...", np.namespace, np.name, len(objs))
 
 	policyPortUUIDs = []string{}
 	policyPortsToUUIDs = map[string]string{}
@@ -762,7 +762,7 @@ func (bnc *BaseNetworkController) denyPGDeletePorts(np *networkPolicy, portNames
 		sharedPGs, ok := bnc.sharedNetpolPortGroups.Load(pgKey)
 		if !ok {
 			// Port group doesn't exist, nothing to clean up
-			klog.Infof("Skip delete ports from default deny port group: port group doesn't exist")
+			klog.V(5).Infof("Skip delete ports from default deny port group: port group doesn't exist")
 		} else {
 			ingressDenyPorts, egressDenyPorts := sharedPGs.deletePortsForPolicy(np, portNamesToUUIDs)
 			// counters were updated, update back to initial values on error
@@ -1034,7 +1034,7 @@ func (bnc *BaseNetworkController) createNetworkPolicy(policy *knet.NetworkPolicy
 		var err error
 
 		if aclLogging.Deny != "" || aclLogging.Allow != "" {
-			klog.Infof("ACL logging for network policy %s in namespace %s set to deny=%s, allow=%s",
+			klog.V(5).Infof("ACL logging for network policy %s in namespace %s set to deny=%s, allow=%s",
 				policy.Name, policy.Namespace, aclLogging.Deny, aclLogging.Allow)
 		}
 
@@ -1099,7 +1099,7 @@ func (bnc *BaseNetworkController) createNetworkPolicy(policy *knet.NetworkPolicy
 				}
 			}
 		}
-		klog.Infof("Policy %s added to peer address sets %v", npKey, np.peerAddressSets)
+		klog.V(5).Infof("Policy %s added to peer address sets %v", npKey, np.peerAddressSets)
 
 		// 3. Add policy to default deny port group
 		// Pods are not added to default deny port groups yet, this is just a preparation step
@@ -1224,7 +1224,7 @@ func (bnc *BaseNetworkController) setupGressPolicy(np *networkPolicy, gp *gressP
 // ports from Kubernetes NetworkPolicy objects using OVN Port Groups
 // if addNetworkPolicy fails, create or delete operation can be retried
 func (bnc *BaseNetworkController) addNetworkPolicy(policy *knet.NetworkPolicy) error {
-	klog.Infof("Adding network policy %s for network %s", getPolicyKey(policy), bnc.GetNetworkName())
+	klog.V(5).Infof("Adding network policy %s for network %s", getPolicyKey(policy), bnc.GetNetworkName())
 	if !bnc.IsUserDefinedNetwork() && config.Metrics.EnableScaleMetrics {
 		start := time.Now()
 		defer func() {
@@ -1257,27 +1257,27 @@ func (bnc *BaseNetworkController) addNetworkPolicy(policy *knet.NetworkPolicy) e
 	np, err = bnc.createNetworkPolicy(policy, &aclLogging)
 	defer func() {
 		if err != nil {
-			klog.Infof("Create network policy %s failed, try to cleanup", npKey)
+			klog.V(5).Infof("Create network policy %s failed, try to cleanup", npKey)
 			// try to cleanup network policy straight away
 			// it will be retried later with add/delete network policy handlers if it fails
 			cleanupErr := bnc.networkPolicies.DoWithLock(npKey, func(npKey string) error {
 				np, ok := bnc.networkPolicies.Load(npKey)
 				if !ok {
-					klog.Infof("Deleting policy %s that is already deleted", npKey)
+					klog.V(5).Infof("Deleting policy %s that is already deleted", npKey)
 					return nil
 				}
 				return bnc.cleanupNetworkPolicy(np)
 			})
 			if cleanupErr != nil {
-				klog.Infof("Cleanup for failed create network policy %s returned an error: %v",
+				klog.V(5).Infof("Cleanup for failed create network policy %s returned an error: %v",
 					npKey, cleanupErr)
 			}
 		}
 	}()
 	if err != nil {
 		return fmt.Errorf("failed to create Network Policy %s: %v", npKey, err)
 	}
-	klog.Infof("Create network policy %s resources completed, update namespace loglevel", npKey)
+	klog.V(5).Infof("Create network policy %s resources completed, update namespace loglevel", npKey)
 
 	// 3. lock namespace
 	nsInfo, nsUnlock = bnc.getNamespaceLocked(policy.Namespace, false)
@@ -1298,15 +1298,15 @@ func (bnc *BaseNetworkController) addNetworkPolicy(policy *knet.NetworkPolicy) e
 		if err = bnc.updateACLLoggingForDefaultACLs(policy.Namespace, nsInfo); err != nil {
 			return fmt.Errorf("network policy %s failed to be created: update default deny ACLs failed: %v", npKey, err)
 		} else {
-			klog.Infof("Policy %s: ACL logging setting updated to deny=%s allow=%s",
+			klog.V(5).Infof("Policy %s: ACL logging setting updated to deny=%s allow=%s",
 				npKey, nsInfo.aclLogging.Deny, nsInfo.aclLogging.Allow)
 		}
 	}
 	if nsInfo.aclLogging.Allow != aclLogging.Allow {
 		if err = bnc.updateACLLoggingForPolicy(np, &nsInfo.aclLogging); err != nil {
 			return fmt.Errorf("network policy %s failed to be created: update policy ACLs failed: %v", npKey, err)
 		} else {
-			klog.Infof("Policy %s: ACL logging setting updated to deny=%s allow=%s",
+			klog.V(5).Infof("Policy %s: ACL logging setting updated to deny=%s allow=%s",
 				npKey, nsInfo.aclLogging.Deny, nsInfo.aclLogging.Allow)
 		}
 	}
@@ -1336,7 +1336,7 @@ func (bnc *BaseNetworkController) buildNetworkPolicyACLs(np *networkPolicy, aclL
 // It only uses Namespace and Name from given network policy
 func (bnc *BaseNetworkController) deleteNetworkPolicy(policy *knet.NetworkPolicy) error {
 	npKey := getPolicyKey(policy)
-	klog.Infof("Deleting network policy %s", npKey)
+	klog.V(5).Infof("Deleting network policy %s", npKey)
 	if config.Metrics.EnableScaleMetrics {
 		start := time.Now()
 		defer func() {
@@ -1372,7 +1372,7 @@ func (bnc *BaseNetworkController) deleteNetworkPolicy(policy *knet.NetworkPolicy
 // that information.
 func (bnc *BaseNetworkController) cleanupNetworkPolicy(np *networkPolicy) error {
 	npKey := np.getKey()
-	klog.Infof("Cleaning up network policy %s", npKey)
+	klog.V(5).Infof("Cleaning up network policy %s", npKey)
 	np.Lock()
 	defer np.Unlock()
 

go-controller/pkg/ovn/base_network_controller_user_defined.go

@@ -721,7 +721,7 @@ func (bsnc *BaseUserDefinedNetworkController) addPodToNamespaceForUserDefinedNet
 
 // AddNamespaceForUserDefinedNetwork creates corresponding addressset in ovn db for User Defined Network
 func (bsnc *BaseUserDefinedNetworkController) AddNamespaceForUserDefinedNetwork(ns *corev1.Namespace) error {
-	klog.Infof("[%s] adding namespace for network %s", ns.Name, bsnc.GetNetworkName())
+	klog.V(5).Infof("[%s] adding namespace for network %s", ns.Name, bsnc.GetNetworkName())
 	// Keep track of how long syncs take.
 	start := time.Now()
 	defer func() {

go-controller/pkg/ovn/namespace.go

@@ -94,11 +94,11 @@ func isNamespaceMulticastEnabled(annotations map[string]string) bool {
 
 // AddNamespace creates corresponding addressset in ovn db
 func (oc *DefaultNetworkController) AddNamespace(ns *corev1.Namespace) error {
-	klog.Infof("[%s] adding namespace", ns.Name)
+	klog.V(5).Infof("[%s] adding namespace", ns.Name)
 	// Keep track of how long syncs take.
 	start := time.Now()
 	defer func() {
-		klog.Infof("[%s] adding namespace took %v", ns.Name, time.Since(start))
+		klog.V(5).Infof("[%s] adding namespace took %v", ns.Name, time.Since(start))
 	}()
 
 	_, nsUnlock, err := oc.ensureNamespaceLocked(ns.Name, false, ns)
@@ -138,7 +138,7 @@ func (oc *DefaultNetworkController) configureNamespace(nsInfo *namespaceInfo, ns
 
 func (oc *DefaultNetworkController) updateNamespace(old, newer *corev1.Namespace) error {
 	var errors []error
-	klog.Infof("[%s] updating namespace", old.Name)
+	klog.V(5).Infof("[%s] updating namespace", old.Name)
 
 	nsInfo, nsUnlock := oc.getNamespaceLocked(old.Name, false)
 	if nsInfo == nil {
@@ -282,7 +282,7 @@ func (oc *DefaultNetworkController) updateNamespace(old, newer *corev1.Namespace
 					klog.Errorf("Failed to get key for EgressFirewall %s/%s, will not update ACL logging: %v", old.Name, fwKey, err)
 					continue
 				}
-				klog.Infof("Namespace %s: EgressFirewall ACL logging setting updating to deny=%s allow=%s",
+				klog.V(5).Infof("Namespace %s: EgressFirewall ACL logging setting updating to deny=%s allow=%s",
 					old.Name, nsInfo.aclLogging.Deny, nsInfo.aclLogging.Allow)
 				oc.efController.Reconcile(fwKey)
 			}
@@ -296,7 +296,7 @@ func (oc *DefaultNetworkController) updateNamespace(old, newer *corev1.Namespace
 }
 
 func (oc *DefaultNetworkController) deleteNamespace(ns *corev1.Namespace) error {
-	klog.Infof("[%s] deleting namespace", ns.Name)
+	klog.V(5).Infof("[%s] deleting namespace", ns.Name)
 
 	nsInfo, err := oc.deleteNamespaceLocked(ns.Name)
 	if err != nil {

go-controller/pkg/ovn/pods.go

@@ -232,7 +232,7 @@ func (oc *DefaultNetworkController) deleteLogicalPort(pod *corev1.Pod, portInfo
 	// and we dont know if the IP was released or not, and subsequently could accidentally release the IP
 	// while it is now on another pod. Releasing IPs may fail at this point if cache knows nothing about it,
 	// which is okay since node may have been deleted.
-	klog.Infof("Attempting to release IPs for pod: %s/%s, ips: %s", pod.Namespace, pod.Name,
+	klog.V(5).Infof("Attempting to release IPs for pod: %s/%s, ips: %s", pod.Namespace, pod.Name,
 		util.JoinIPNetIPs(pInfo.ips, " "))
 	return oc.releasePodIPs(pInfo)
 }