diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index 63216abc915..afd0aec526c 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -57,6 +57,7 @@ import ( ) type Backend struct { + clock utilsclock.PassiveClock options *BackendOptions } @@ -138,6 +139,7 @@ func (o *BackendOptions) NewBackend() (*Backend, error) { return nil, err } return &Backend{ + clock: utilsclock.RealClock{}, options: o, }, nil } @@ -397,18 +399,19 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, managementClusterDumpController := datadumpcontrollers.NewManagementClusterDataDumpController(b.options.FleetDBClient, managementClusterLister, fleetInformers) doNothingController := controllers.NewDoNothingExampleController(b.options.ResourcesDBClient, subscriptionLister) dispatchRequestCredentialController := operationcontrollers.NewDispatchRequestCredentialController( - utilsclock.RealClock{}, + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, activeOperationInformer, ) dispatchRevokeCredentialsController := operationcontrollers.NewDispatchRevokeCredentialsController( - utilsclock.RealClock{}, + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, activeOperationInformer, ) operationClusterCreateController := operationcontrollers.NewOperationClusterCreateController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, @@ -416,12 +419,14 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, backendInformers, ) operationClusterUpdateController := operationcontrollers.NewOperationClusterUpdateController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, activeOperationInformer, ) operationClusterDeleteController := operationcontrollers.NewOperationClusterDeleteController( + b.clock, b.options.ResourcesDBClient, b.options.BillingDBClient, b.options.ClustersServiceClient, @@ -429,48 +434,56 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, activeOperationInformer, ) operationNodePoolCreateController := operationcontrollers.NewOperationNodePoolCreateController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, activeOperationInformer, ) operationNodePoolUpdateController := operationcontrollers.NewOperationNodePoolUpdateController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, activeOperationInformer, ) operationNodePoolDeleteController := operationcontrollers.NewOperationNodePoolDeleteController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, activeOperationInformer, ) operationExternalAuthCreateController := operationcontrollers.NewOperationExternalAuthCreateController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, activeOperationInformer, ) operationExternalAuthUpdateController := operationcontrollers.NewOperationExternalAuthUpdateController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, activeOperationInformer, ) operationExternalAuthDeleteController := operationcontrollers.NewOperationExternalAuthDeleteController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, activeOperationInformer, ) operationRequestCredentialController := operationcontrollers.NewOperationRequestCredentialController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, activeOperationInformer, ) operationRevokeCredentialsController := operationcontrollers.NewOperationRevokeCredentialsController( + b.clock, b.options.ResourcesDBClient, b.options.ClustersServiceClient, http.DefaultClient, @@ -479,7 +492,7 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, clusterServiceMatchingClusterController := mismatchcontrollers.NewClusterServiceClusterMatchingController(b.options.ResourcesDBClient, subscriptionLister, b.options.ClustersServiceClient) cosmosMatchingNodePoolController := mismatchcontrollers.NewCosmosNodePoolMatchingController(b.options.ResourcesDBClient, b.options.ClustersServiceClient, backendInformers) cosmosMatchingExternalAuthController := mismatchcontrollers.NewCosmosExternalAuthMatchingController(b.options.ResourcesDBClient, b.options.ClustersServiceClient, backendInformers) - cosmosMatchingClusterController := mismatchcontrollers.NewCosmosClusterMatchingController(utilsclock.RealClock{}, b.options.ResourcesDBClient, b.options.BillingDBClient, b.options.ClustersServiceClient, backendInformers) + cosmosMatchingClusterController := mismatchcontrollers.NewCosmosClusterMatchingController(b.clock, b.options.ResourcesDBClient, b.options.BillingDBClient, b.options.ClustersServiceClient, backendInformers) alwaysSuccessClusterValidationController := validationcontrollers.NewClusterValidationController( validations.NewAlwaysSuccessValidation(), activeOperationLister, @@ -489,11 +502,11 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, deleteOrphanedCosmosResourcesController := mismatchcontrollers.NewDeleteOrphanedCosmosResourcesController(b.options.ResourcesDBClient, b.options.KubeApplierDBClients, subscriptionLister, managementClusterLister) backfillClusterUIDController := controllerutils.NewClusterWatchingController( "BackfillClusterUID", b.options.ResourcesDBClient, backendInformers, 60*time.Minute, - mismatchcontrollers.NewBackfillClusterUIDController(utilsclock.RealClock{}, b.options.ResourcesDBClient, b.options.BillingDBClient, clusterLister)) - orphanedBillingCleanupController := billingcontrollers.NewOrphanedBillingCleanupController(utilsclock.RealClock{}, b.options.BillingDBClient, clusterLister, billingLister) + mismatchcontrollers.NewBackfillClusterUIDController(b.clock, b.options.ResourcesDBClient, b.options.BillingDBClient, clusterLister)) + orphanedBillingCleanupController := billingcontrollers.NewOrphanedBillingCleanupController(b.clock, b.options.BillingDBClient, clusterLister, billingLister) createBillingDocController := controllerutils.NewClusterWatchingController( "CreateBillingDoc", b.options.ResourcesDBClient, backendInformers, 60*time.Second, - billingcontrollers.NewCreateBillingDocController(utilsclock.RealClock{}, b.options.AzureLocation, b.options.ResourcesDBClient, b.options.BillingDBClient, clusterLister, billingLister)) + billingcontrollers.NewCreateBillingDocController(b.clock, b.options.AzureLocation, b.options.ResourcesDBClient, b.options.BillingDBClient, clusterLister, billingLister)) controlPlaneActiveVersionController := upgradecontrollers.NewControlPlaneActiveVersionController( b.options.ResourcesDBClient, activeOperationLister, diff --git a/backend/pkg/controllers/operationcontrollers/operation_cluster_create.go b/backend/pkg/controllers/operationcontrollers/operation_cluster_create.go index 17f00e8c3fb..00e7d2b799c 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_cluster_create.go +++ b/backend/pkg/controllers/operationcontrollers/operation_cluster_create.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/tools/cache" + utilsclock "k8s.io/utils/clock" configv1 "github.com/openshift/api/config/v1" "github.com/openshift/hypershift/api/hypershift/v1beta1" @@ -43,6 +44,7 @@ import ( ) type operationClusterCreate struct { + clock utilsclock.PassiveClock clusterLister listers.ClusterLister clusterManagementClusterContentLister listers.ManagementClusterContentLister resourcesDBClient database.ResourcesDBClient @@ -65,6 +67,7 @@ type operationClusterCreate struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationClusterCreateController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, @@ -74,6 +77,7 @@ func NewOperationClusterCreateController( _, clusterLister := informers.Clusters() _, clusterManagementClusterContentLister := informers.ManagementClusterContents() syncer := &operationClusterCreate{ + clock: clock, clusterLister: clusterLister, clusterManagementClusterContentLister: clusterManagementClusterContentLister, resourcesDBClient: resourcesDBClient, @@ -151,7 +155,7 @@ func (c *operationClusterCreate) SynchronizeOperation(ctx context.Context, key c } logger.Info("updating status") - err = UpdateOperationStatus(ctx, c.resourcesDBClient, operation, newOperationStatus, opError, postAsyncNotificationFn(c.notificationClient)) + err = UpdateOperationStatus(ctx, c.clock, c.resourcesDBClient, operation, newOperationStatus, opError, postAsyncNotificationFn(c.notificationClient)) if err != nil { return utils.TrackError(err) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_cluster_create_test.go b/backend/pkg/controllers/operationcontrollers/operation_cluster_create_test.go index 58c1e575abd..d12c0038387 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_cluster_create_test.go +++ b/backend/pkg/controllers/operationcontrollers/operation_cluster_create_test.go @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kruntime "k8s.io/apimachinery/pkg/runtime" + utilsclock "k8s.io/utils/clock" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" @@ -123,6 +124,7 @@ func TestOperationClusterCreate_SynchronizeOperation(t *testing.T) { }) controller := &operationClusterCreate{ + clock: utilsclock.RealClock{}, resourcesDBClient: mockResourcesDBClient, clusterServiceClient: mockCSClient, notificationClient: nil, diff --git a/backend/pkg/controllers/operationcontrollers/operation_cluster_delete.go b/backend/pkg/controllers/operationcontrollers/operation_cluster_delete.go index 05ad7dff504..9286e885c04 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_cluster_delete.go +++ b/backend/pkg/controllers/operationcontrollers/operation_cluster_delete.go @@ -57,6 +57,7 @@ type operationClusterDelete struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationClusterDeleteController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, billingDBClient database.BillingDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, @@ -64,7 +65,7 @@ func NewOperationClusterDeleteController( activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationClusterDelete{ - clock: utilsclock.RealClock{}, + clock: clock, resourcesDBClient: resourcesDBClient, billingDBClient: billingDBClient, clusterServiceClient: clusterServiceClient, @@ -132,7 +133,7 @@ func (c *operationClusterDelete) SynchronizeOperation(ctx context.Context, key c return utils.TrackError(err) } - err = SetDeleteOperationAsCompleted(ctx, c.resourcesDBClient, operation, postAsyncNotificationFn(c.notificationClient)) + err = SetDeleteOperationAsCompleted(ctx, c.clock, c.resourcesDBClient, operation, postAsyncNotificationFn(c.notificationClient)) if err != nil { return utils.TrackError(err) } @@ -148,7 +149,7 @@ func (c *operationClusterDelete) SynchronizeOperation(ctx context.Context, key c return utils.TrackError(err) } - err = UpdateOperationStatus(ctx, c.resourcesDBClient, operation, newOperationStatus, newOperationError, postAsyncNotificationFn(c.notificationClient)) + err = UpdateOperationStatus(ctx, c.clock, c.resourcesDBClient, operation, newOperationStatus, newOperationError, postAsyncNotificationFn(c.notificationClient)) if err != nil { return utils.TrackError(err) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_cluster_update.go b/backend/pkg/controllers/operationcontrollers/operation_cluster_update.go index d5d4eb720e3..72fe3fb14c8 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_cluster_update.go +++ b/backend/pkg/controllers/operationcontrollers/operation_cluster_update.go @@ -28,7 +28,7 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" - "k8s.io/utils/clock" + utilsclock "k8s.io/utils/clock" "k8s.io/utils/lru" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" @@ -40,10 +40,10 @@ import ( ) type operationClusterUpdate struct { + clock utilsclock.PassiveClock resourcesDBClient database.ResourcesDBClient clusterServiceClient ocm.ClusterServiceClientSpec notificationClient *http.Client - clock clock.PassiveClock desiredVersionMismatchFirstSeen *lru.Cache } @@ -62,16 +62,17 @@ type operationClusterUpdate struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationClusterUpdateController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationClusterUpdate{ + clock: clock, resourcesDBClient: resourcesDBClient, clusterServiceClient: clusterServiceClient, notificationClient: notificationClient, - clock: clock.RealClock{}, desiredVersionMismatchFirstSeen: lru.New(100000), } @@ -133,7 +134,7 @@ func (c *operationClusterUpdate) SynchronizeOperation(ctx context.Context, key c } logger.Info("updating status") - if err := UpdateOperationStatus(ctx, c.resourcesDBClient, operation, operationalState.provisioningState, persistErr, postAsyncNotificationFn(c.notificationClient)); err != nil { + if err := UpdateOperationStatus(ctx, c.clock, c.resourcesDBClient, operation, operationalState.provisioningState, persistErr, postAsyncNotificationFn(c.notificationClient)); err != nil { return utils.TrackError(err) } return nil diff --git a/backend/pkg/controllers/operationcontrollers/operation_external_auth_create.go b/backend/pkg/controllers/operationcontrollers/operation_external_auth_create.go index 7aefb3d6529..36d2a06cff1 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_external_auth_create.go +++ b/backend/pkg/controllers/operationcontrollers/operation_external_auth_create.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/client-go/tools/cache" + utilsclock "k8s.io/utils/clock" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/internal/api" @@ -31,6 +32,7 @@ import ( ) type operationExternalAuthCreate struct { + clock utilsclock.PassiveClock resourcesDBClient database.ResourcesDBClient clusterServiceClient ocm.ClusterServiceClientSpec notificationClient *http.Client @@ -51,12 +53,14 @@ type operationExternalAuthCreate struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationExternalAuthCreateController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationExternalAuthCreate{ + clock: clock, resourcesDBClient: resourcesDBClient, clusterServiceClient: clusterServiceClient, notificationClient: notificationClient, @@ -101,5 +105,5 @@ func (c *operationExternalAuthCreate) SynchronizeOperation(ctx context.Context, return nil // no work to do } - return pollExternalAuthStatus(ctx, c.resourcesDBClient, c.clusterServiceClient, operation, c.notificationClient) + return pollExternalAuthStatus(ctx, c.clock, c.resourcesDBClient, c.clusterServiceClient, operation, c.notificationClient) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_external_auth_create_test.go b/backend/pkg/controllers/operationcontrollers/operation_external_auth_create_test.go index 5d33f528ef0..1b1022e9195 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_external_auth_create_test.go +++ b/backend/pkg/controllers/operationcontrollers/operation_external_auth_create_test.go @@ -24,6 +24,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + utilsclock "k8s.io/utils/clock" + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" "github.com/Azure/ARO-HCP/internal/api/arm" @@ -97,6 +99,7 @@ func TestOperationExternalAuthCreate_SynchronizeOperation(t *testing.T) { mockCSClient := tt.setupMock(ctrl, fixture) controller := &operationExternalAuthCreate{ + clock: utilsclock.RealClock{}, resourcesDBClient: mockResourcesDBClient, clusterServiceClient: mockCSClient, notificationClient: nil, diff --git a/backend/pkg/controllers/operationcontrollers/operation_external_auth_delete.go b/backend/pkg/controllers/operationcontrollers/operation_external_auth_delete.go index d7641e1bbf0..0723ef90e48 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_external_auth_delete.go +++ b/backend/pkg/controllers/operationcontrollers/operation_external_auth_delete.go @@ -23,6 +23,7 @@ import ( "time" "k8s.io/client-go/tools/cache" + utilsclock "k8s.io/utils/clock" ocmerrors "github.com/openshift-online/ocm-sdk-go/errors" @@ -34,6 +35,7 @@ import ( ) type operationExternalAuthDelete struct { + clock utilsclock.PassiveClock resourcesDBClient database.ResourcesDBClient clusterServiceClient ocm.ClusterServiceClientSpec notificationClient *http.Client @@ -54,12 +56,14 @@ type operationExternalAuthDelete struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationExternalAuthDeleteController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationExternalAuthDelete{ + clock: clock, resourcesDBClient: resourcesDBClient, clusterServiceClient: clusterServiceClient, notificationClient: notificationClient, @@ -109,7 +113,7 @@ func (c *operationExternalAuthDelete) SynchronizeOperation(ctx context.Context, if err != nil && errors.As(err, &ocmGetExternalAuthError) && ocmGetExternalAuthError.Status() == http.StatusNotFound { logger.Info("external auth was deleted") - err = SetDeleteOperationAsCompleted(ctx, c.resourcesDBClient, operation, postAsyncNotificationFn(c.notificationClient)) + err = SetDeleteOperationAsCompleted(ctx, c.clock, c.resourcesDBClient, operation, postAsyncNotificationFn(c.notificationClient)) if err != nil { return utils.TrackError(err) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_external_auth_delete_test.go b/backend/pkg/controllers/operationcontrollers/operation_external_auth_delete_test.go index 7ecc38cc554..e8a918f4b6b 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_external_auth_delete_test.go +++ b/backend/pkg/controllers/operationcontrollers/operation_external_auth_delete_test.go @@ -24,6 +24,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + utilsclock "k8s.io/utils/clock" + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" ocmerrors "github.com/openshift-online/ocm-sdk-go/errors" @@ -107,6 +109,7 @@ func TestOperationExternalAuthDelete_SynchronizeOperation(t *testing.T) { mockCSClient := tt.setupMock(ctrl, fixture) controller := &operationExternalAuthDelete{ + clock: utilsclock.RealClock{}, resourcesDBClient: mockResourcesDBClient, clusterServiceClient: mockCSClient, notificationClient: nil, diff --git a/backend/pkg/controllers/operationcontrollers/operation_external_auth_update.go b/backend/pkg/controllers/operationcontrollers/operation_external_auth_update.go index dc69f2f3755..77a55560104 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_external_auth_update.go +++ b/backend/pkg/controllers/operationcontrollers/operation_external_auth_update.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/client-go/tools/cache" + utilsclock "k8s.io/utils/clock" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/internal/api" @@ -31,6 +32,7 @@ import ( ) type operationExternalAuthUpdate struct { + clock utilsclock.PassiveClock resourcesDBClient database.ResourcesDBClient clusterServiceClient ocm.ClusterServiceClientSpec notificationClient *http.Client @@ -51,12 +53,14 @@ type operationExternalAuthUpdate struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationExternalAuthUpdateController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationExternalAuthUpdate{ + clock: clock, resourcesDBClient: resourcesDBClient, clusterServiceClient: clusterServiceClient, notificationClient: notificationClient, @@ -101,5 +105,5 @@ func (c *operationExternalAuthUpdate) SynchronizeOperation(ctx context.Context, return nil // no work to do } - return pollExternalAuthStatus(ctx, c.resourcesDBClient, c.clusterServiceClient, operation, c.notificationClient) + return pollExternalAuthStatus(ctx, c.clock, c.resourcesDBClient, c.clusterServiceClient, operation, c.notificationClient) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_external_auth_update_test.go b/backend/pkg/controllers/operationcontrollers/operation_external_auth_update_test.go index 670b3f4b320..8b117d23ac5 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_external_auth_update_test.go +++ b/backend/pkg/controllers/operationcontrollers/operation_external_auth_update_test.go @@ -24,6 +24,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + utilsclock "k8s.io/utils/clock" + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" "github.com/Azure/ARO-HCP/internal/api/arm" @@ -97,6 +99,7 @@ func TestOperationExternalAuthUpdate_SynchronizeOperation(t *testing.T) { mockCSClient := tt.setupMock(ctrl, fixture) controller := &operationExternalAuthUpdate{ + clock: utilsclock.RealClock{}, resourcesDBClient: mockResourcesDBClient, clusterServiceClient: mockCSClient, notificationClient: nil, diff --git a/backend/pkg/controllers/operationcontrollers/operation_node_pool_create.go b/backend/pkg/controllers/operationcontrollers/operation_node_pool_create.go index b19817e626d..ebdd9ea7e5e 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_node_pool_create.go +++ b/backend/pkg/controllers/operationcontrollers/operation_node_pool_create.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/client-go/tools/cache" + utilsclock "k8s.io/utils/clock" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/internal/api" @@ -31,6 +32,7 @@ import ( ) type operationNodePoolCreate struct { + clock utilsclock.PassiveClock resourcesDBClient database.ResourcesDBClient clusterServiceClient ocm.ClusterServiceClientSpec notificationClient *http.Client @@ -51,12 +53,14 @@ type operationNodePoolCreate struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationNodePoolCreateController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationNodePoolCreate{ + clock: clock, resourcesDBClient: resourcesDBClient, clusterServiceClient: clusterServiceClient, notificationClient: notificationClient, @@ -101,5 +105,5 @@ func (c *operationNodePoolCreate) SynchronizeOperation(ctx context.Context, key return nil // no work to do } - return pollNodePoolStatus(ctx, c.resourcesDBClient, c.clusterServiceClient, operation, c.notificationClient) + return pollNodePoolStatus(ctx, c.clock, c.resourcesDBClient, c.clusterServiceClient, operation, c.notificationClient) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_node_pool_create_test.go b/backend/pkg/controllers/operationcontrollers/operation_node_pool_create_test.go index ec8551fb43c..e9a677aae72 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_node_pool_create_test.go +++ b/backend/pkg/controllers/operationcontrollers/operation_node_pool_create_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + utilsclock "k8s.io/utils/clock" + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" "github.com/Azure/ARO-HCP/internal/api/arm" @@ -142,6 +144,7 @@ func TestOperationNodePoolCreate_SynchronizeOperation(t *testing.T) { Return(nodePoolStatus, nil) controller := &operationNodePoolCreate{ + clock: utilsclock.RealClock{}, resourcesDBClient: mockResourcesDBClient, clusterServiceClient: mockCSClient, notificationClient: nil, diff --git a/backend/pkg/controllers/operationcontrollers/operation_node_pool_delete.go b/backend/pkg/controllers/operationcontrollers/operation_node_pool_delete.go index e21924cbfb2..0272e1acfa8 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_node_pool_delete.go +++ b/backend/pkg/controllers/operationcontrollers/operation_node_pool_delete.go @@ -23,6 +23,7 @@ import ( "time" "k8s.io/client-go/tools/cache" + utilsclock "k8s.io/utils/clock" ocmerrors "github.com/openshift-online/ocm-sdk-go/errors" @@ -34,6 +35,7 @@ import ( ) type operationNodePoolDelete struct { + clock utilsclock.PassiveClock resourcesDBClient database.ResourcesDBClient clusterServiceClient ocm.ClusterServiceClientSpec notificationClient *http.Client @@ -54,12 +56,14 @@ type operationNodePoolDelete struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationNodePoolDeleteController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationNodePoolDelete{ + clock: clock, resourcesDBClient: resourcesDBClient, clusterServiceClient: clusterServiceClient, notificationClient: notificationClient, @@ -109,7 +113,7 @@ func (c *operationNodePoolDelete) SynchronizeOperation(ctx context.Context, key if err != nil && errors.As(err, &ocmGetNodePoolError) && ocmGetNodePoolError.Status() == http.StatusNotFound { logger.Info("node pool was deleted") - err = SetDeleteOperationAsCompleted(ctx, c.resourcesDBClient, operation, postAsyncNotificationFn(c.notificationClient)) + err = SetDeleteOperationAsCompleted(ctx, c.clock, c.resourcesDBClient, operation, postAsyncNotificationFn(c.notificationClient)) if err != nil { return utils.TrackError(err) } @@ -124,7 +128,7 @@ func (c *operationNodePoolDelete) SynchronizeOperation(ctx context.Context, key return utils.TrackError(err) } - err = UpdateOperationStatus(ctx, c.resourcesDBClient, operation, newOperationStatus, newOperationError, postAsyncNotificationFn(c.notificationClient)) + err = UpdateOperationStatus(ctx, c.clock, c.resourcesDBClient, operation, newOperationStatus, newOperationError, postAsyncNotificationFn(c.notificationClient)) if err != nil { return utils.TrackError(err) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_node_pool_delete_test.go b/backend/pkg/controllers/operationcontrollers/operation_node_pool_delete_test.go index fd82fd8d2db..a56014635da 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_node_pool_delete_test.go +++ b/backend/pkg/controllers/operationcontrollers/operation_node_pool_delete_test.go @@ -24,6 +24,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + utilsclock "k8s.io/utils/clock" + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" ocmerrors "github.com/openshift-online/ocm-sdk-go/errors" @@ -157,6 +159,7 @@ func TestOperationNodePoolDelete_SynchronizeOperation(t *testing.T) { mockCSClient := tt.setupMock(ctrl, fixture) controller := &operationNodePoolDelete{ + clock: utilsclock.RealClock{}, resourcesDBClient: mockResourcesDBClient, clusterServiceClient: mockCSClient, notificationClient: nil, diff --git a/backend/pkg/controllers/operationcontrollers/operation_node_pool_update.go b/backend/pkg/controllers/operationcontrollers/operation_node_pool_update.go index 042996f5eeb..7b2dc80ed8d 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_node_pool_update.go +++ b/backend/pkg/controllers/operationcontrollers/operation_node_pool_update.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/client-go/tools/cache" + utilsclock "k8s.io/utils/clock" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/internal/api" @@ -31,6 +32,7 @@ import ( ) type operationNodePoolUpdate struct { + clock utilsclock.PassiveClock resourcesDBClient database.ResourcesDBClient clusterServiceClient ocm.ClusterServiceClientSpec notificationClient *http.Client @@ -51,12 +53,14 @@ type operationNodePoolUpdate struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationNodePoolUpdateController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationNodePoolUpdate{ + clock: clock, resourcesDBClient: resourcesDBClient, clusterServiceClient: clusterServiceClient, notificationClient: notificationClient, @@ -101,5 +105,5 @@ func (c *operationNodePoolUpdate) SynchronizeOperation(ctx context.Context, key return nil // no work to do } - return pollNodePoolStatus(ctx, c.resourcesDBClient, c.clusterServiceClient, operation, c.notificationClient) + return pollNodePoolStatus(ctx, c.clock, c.resourcesDBClient, c.clusterServiceClient, operation, c.notificationClient) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_node_pool_update_test.go b/backend/pkg/controllers/operationcontrollers/operation_node_pool_update_test.go index fb7da1768a2..a7ca6671bf9 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_node_pool_update_test.go +++ b/backend/pkg/controllers/operationcontrollers/operation_node_pool_update_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + utilsclock "k8s.io/utils/clock" + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" "github.com/Azure/ARO-HCP/internal/api/arm" @@ -136,6 +138,7 @@ func TestOperationNodePoolUpdate_SynchronizeOperation(t *testing.T) { Return(nodePoolStatus, nil) controller := &operationNodePoolUpdate{ + clock: utilsclock.RealClock{}, resourcesDBClient: mockResourcesDBClient, clusterServiceClient: mockCSClient, notificationClient: nil, diff --git a/backend/pkg/controllers/operationcontrollers/operation_request_credential.go b/backend/pkg/controllers/operationcontrollers/operation_request_credential.go index ff7b63334f9..1e36badbcda 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_request_credential.go +++ b/backend/pkg/controllers/operationcontrollers/operation_request_credential.go @@ -21,6 +21,7 @@ import ( "time" "k8s.io/client-go/tools/cache" + utilsclock "k8s.io/utils/clock" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" @@ -33,6 +34,7 @@ import ( ) type operationRequestCredential struct { + clock utilsclock.PassiveClock resourcesDBClient database.ResourcesDBClient clustersServiceClient ocm.ClusterServiceClientSpec notificationClient *http.Client @@ -54,12 +56,14 @@ type operationRequestCredential struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationRequestCredentialController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clustersServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationRequestCredential{ + clock: clock, resourcesDBClient: resourcesDBClient, clustersServiceClient: clustersServiceClient, notificationClient: notificationClient, @@ -133,7 +137,7 @@ func (opsync *operationRequestCredential) SynchronizeOperation(ctx context.Conte return nil } - err = patchOperation(ctx, opsync.resourcesDBClient, oldOperation, newOperationStatus, newOperationError, postAsyncNotificationFn(opsync.notificationClient)) + err = patchOperation(ctx, opsync.clock, opsync.resourcesDBClient, oldOperation, newOperationStatus, newOperationError, postAsyncNotificationFn(opsync.notificationClient)) if err != nil { return utils.TrackError(err) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_request_credential_test.go b/backend/pkg/controllers/operationcontrollers/operation_request_credential_test.go index e75edcdefbf..9b416024924 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_request_credential_test.go +++ b/backend/pkg/controllers/operationcontrollers/operation_request_credential_test.go @@ -24,6 +24,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + utilsclock "k8s.io/utils/clock" + cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" "github.com/Azure/ARO-HCP/internal/api" @@ -187,6 +189,7 @@ func TestOperationRequestCredential_SynchronizeOperation(t *testing.T) { } controller := &operationRequestCredential{ + clock: utilsclock.RealClock{}, resourcesDBClient: mockResourcesDBClient, clustersServiceClient: mockCSClient, } diff --git a/backend/pkg/controllers/operationcontrollers/operation_revoke_credentials.go b/backend/pkg/controllers/operationcontrollers/operation_revoke_credentials.go index 3a5c041ac31..e4228f47851 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_revoke_credentials.go +++ b/backend/pkg/controllers/operationcontrollers/operation_revoke_credentials.go @@ -21,6 +21,7 @@ import ( "time" "k8s.io/client-go/tools/cache" + utilsclock "k8s.io/utils/clock" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" @@ -33,6 +34,7 @@ import ( ) type operationRevokeCredentials struct { + clock utilsclock.PassiveClock resourcesDBClient database.ResourcesDBClient clustersServiceClient ocm.ClusterServiceClientSpec notificationClient *http.Client @@ -53,12 +55,14 @@ type operationRevokeCredentials struct { // any of "Succeeded", "Failed", or "Canceled". Once the operation status reaches // a terminal value, there will be no further updates to the operation document. func NewOperationRevokeCredentialsController( + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clustersServiceClient ocm.ClusterServiceClientSpec, notificationClient *http.Client, activeOperationInformer cache.SharedIndexInformer, ) controllerutils.Controller { syncer := &operationRevokeCredentials{ + clock: clock, resourcesDBClient: resourcesDBClient, clustersServiceClient: clustersServiceClient, notificationClient: notificationClient, @@ -197,7 +201,7 @@ func (opsync *operationRevokeCredentials) SynchronizeOperation(ctx context.Conte } logger.Info("updating status") - err = patchOperation(ctx, opsync.resourcesDBClient, oldOperation, newOperationStatus, newOperationError, postAsyncNotificationFn(opsync.notificationClient)) + err = patchOperation(ctx, opsync.clock, opsync.resourcesDBClient, oldOperation, newOperationStatus, newOperationError, postAsyncNotificationFn(opsync.notificationClient)) if err != nil { return utils.TrackError(err) } diff --git a/backend/pkg/controllers/operationcontrollers/operation_revoke_credentials_test.go b/backend/pkg/controllers/operationcontrollers/operation_revoke_credentials_test.go index 17e6cde0e1e..9041bf47673 100644 --- a/backend/pkg/controllers/operationcontrollers/operation_revoke_credentials_test.go +++ b/backend/pkg/controllers/operationcontrollers/operation_revoke_credentials_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + utilsclock "k8s.io/utils/clock" + cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" "github.com/Azure/ARO-HCP/internal/api" @@ -252,6 +254,7 @@ func TestOperationRevokeCredentials_SynchronizeOperation(t *testing.T) { } controller := &operationRevokeCredentials{ + clock: utilsclock.RealClock{}, resourcesDBClient: mockResourcesDBClient, clustersServiceClient: mockCSClient, } diff --git a/backend/pkg/controllers/operationcontrollers/utils.go b/backend/pkg/controllers/operationcontrollers/utils.go index e5449fede92..2354f821f91 100644 --- a/backend/pkg/controllers/operationcontrollers/utils.go +++ b/backend/pkg/controllers/operationcontrollers/utils.go @@ -24,7 +24,7 @@ import ( "github.com/go-logr/logr" - "k8s.io/utils/clock" + utilsclock "k8s.io/utils/clock" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" @@ -42,8 +42,6 @@ const ( InflightChecksFailedProvisionErrorCode = "OCM4001" ) -var localClock clock.Clock = clock.RealClock{} - type PostAsyncNotificationFunc func(ctx context.Context, operation *api.Operation) error // Copied from uhc-clusters-service, because the @@ -77,7 +75,7 @@ const ( // // In all of these cases the operation document is still persisted and ARM is // notified, so the operation reaches its terminal state and does not get stuck. -func UpdateOperationStatus(ctx context.Context, resourcesDBClient database.ResourcesDBClient, existingOperation *api.Operation, newOperationStatus arm.ProvisioningState, newOperationError *arm.CloudErrorBody, postAsyncNotificationFn PostAsyncNotificationFunc) error { +func UpdateOperationStatus(ctx context.Context, clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, existingOperation *api.Operation, newOperationStatus arm.ProvisioningState, newOperationError *arm.CloudErrorBody, postAsyncNotificationFn PostAsyncNotificationFunc) error { logger := utils.LoggerFromContext(ctx) if existingOperation == nil { return nil @@ -88,7 +86,7 @@ func UpdateOperationStatus(ctx context.Context, resourcesDBClient database.Resou } updatedOperation := existingOperation.DeepCopy() - updatedOperation.LastTransitionTime = localClock.Now() + updatedOperation.LastTransitionTime = clock.Now() updatedOperation.Status = newOperationStatus if newOperationError != nil { updatedOperation.Error = newOperationError @@ -283,7 +281,7 @@ func needToPatchOperation(oldOperation *api.Operation, newOperationStatus arm.Pr } // patchOperation patches the status and error fields of an OperationDocument. -func patchOperation(ctx context.Context, resourcesDBClient database.ResourcesDBClient, oldOperation *api.Operation, newOperationStatus arm.ProvisioningState, newOperationError *arm.CloudErrorBody, postAsyncNotificationFn PostAsyncNotificationFunc) error { +func patchOperation(ctx context.Context, clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, oldOperation *api.Operation, newOperationStatus arm.ProvisioningState, newOperationError *arm.CloudErrorBody, postAsyncNotificationFn PostAsyncNotificationFunc) error { logger := utils.LoggerFromContext(ctx) if !needToPatchOperation(oldOperation, newOperationStatus, newOperationError) { @@ -293,7 +291,7 @@ func patchOperation(ctx context.Context, resourcesDBClient database.ResourcesDBC } operationToWrite := oldOperation.DeepCopy() - operationToWrite.LastTransitionTime = localClock.Now() + operationToWrite.LastTransitionTime = clock.Now() operationToWrite.Status = newOperationStatus if newOperationError != nil { operationToWrite.Error = newOperationError @@ -483,6 +481,7 @@ func convertClusterStatus(ctx context.Context, clusterServiceClient ocm.ClusterS // Service to info for an Azure async operation status endpoint. func pollNodePoolStatus( ctx context.Context, + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, operation *api.Operation, @@ -507,7 +506,7 @@ func pollNodePoolStatus( logger.Info("new status", "newStatus", newOperationStatus) logger.Info("updating status") - err = UpdateOperationStatus(ctx, resourcesDBClient, operation, newOperationStatus, newOperationError, postAsyncNotificationFn(notificationClient)) + err = UpdateOperationStatus(ctx, clock, resourcesDBClient, operation, newOperationStatus, newOperationError, postAsyncNotificationFn(notificationClient)) if err != nil { return utils.TrackError(err) } @@ -565,6 +564,7 @@ func convertNodePoolStatus(operation *api.Operation, nodePoolStatus *arohcpv1alp // Service to info for an Azure async operation status endpoint. func pollExternalAuthStatus( ctx context.Context, + clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, clusterServiceClient ocm.ClusterServiceClientSpec, operation *api.Operation, @@ -586,7 +586,7 @@ func pollExternalAuthStatus( logger.Info("new status", "newStatus", newOperationStatus) logger.Info("updating status") - err = UpdateOperationStatus(ctx, resourcesDBClient, operation, newOperationStatus, nil, postAsyncNotificationFn(notificationClient)) + err = UpdateOperationStatus(ctx, clock, resourcesDBClient, operation, newOperationStatus, nil, postAsyncNotificationFn(notificationClient)) if err != nil { return utils.TrackError(err) } @@ -658,8 +658,8 @@ func convertInflightCheckDetails(inflightCheck *arohcpv1alpha1.InflightCheck) (s return "", false } -// setDeleteOperationAsCompleted updates Cosmos DB to reflect a completed resource deletion. -func SetDeleteOperationAsCompleted(ctx context.Context, resourcesDBClient database.ResourcesDBClient, operation *api.Operation, postAsyncNotificationFn PostAsyncNotificationFunc) error { +// SetDeleteOperationAsCompleted updates Cosmos DB to reflect a completed resource deletion. +func SetDeleteOperationAsCompleted(ctx context.Context, clock utilsclock.PassiveClock, resourcesDBClient database.ResourcesDBClient, operation *api.Operation, postAsyncNotificationFn PostAsyncNotificationFunc) error { // Delete the resource document first. If it fails the backend will retry // by virtue of the operation document still having a non-terminal status. untypedCRUD, err := resourcesDBClient.UntypedCRUD(*operation.ExternalID) @@ -705,7 +705,7 @@ func SetDeleteOperationAsCompleted(ctx context.Context, resourcesDBClient databa } // Save a final "succeeded" operation status until TTL expires. - err = patchOperation(ctx, resourcesDBClient, operation, arm.ProvisioningStateSucceeded, nil, postAsyncNotificationFn) + err = patchOperation(ctx, clock, resourcesDBClient, operation, arm.ProvisioningStateSucceeded, nil, postAsyncNotificationFn) if err != nil { return utils.TrackError(err) } diff --git a/test-integration/utils/integrationutils/utils.go b/test-integration/utils/integrationutils/utils.go index 5543c2058c1..142fca54101 100644 --- a/test-integration/utils/integrationutils/utils.go +++ b/test-integration/utils/integrationutils/utils.go @@ -31,6 +31,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/goleak" + utilsclock "k8s.io/utils/clock" "k8s.io/utils/set" adminApiServer "github.com/Azure/ARO-HCP/admin/server/server" @@ -183,7 +184,7 @@ func MarkOperationsCompleteForName(ctx context.Context, resourcesDBClient databa if operation.ExternalID.Name != resourceName { continue } - err := operationcontrollers.UpdateOperationStatus(ctx, resourcesDBClient, operation, arm.ProvisioningStateSucceeded, nil, nil) + err := operationcontrollers.UpdateOperationStatus(ctx, utilsclock.RealClock{}, resourcesDBClient, operation, arm.ProvisioningStateSucceeded, nil, nil) if err != nil { return err }