From c9f2743b52896bf9eb18f05f91e739ff6dc206fe Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 31 Mar 2026 16:08:49 -0500 Subject: [PATCH 1/2] fix: resolve pre-existing e2e test failures - Add control plane reachability wait to notes multicluster tests - Add quota grant for CRM note lifecycle test Co-Authored-By: Claude Opus 4.6 (1M context) --- .../00-notes-quota-grant.yaml | 15 ++++++++ .../note-contact-lifecycle/chainsaw-test.yaml | 16 ++++++++ .../02-grant-creation-policy.yaml | 35 +++++++++++++++++ .../chainsaw-test.yaml | 31 +++++++++++++++ .../02-grant-creation-policy.yaml | 38 +++++++++++++++++++ .../chainsaw-test.yaml | 31 +++++++++++++++ 6 files changed, 166 insertions(+) create mode 100644 test/crm/note-contact-lifecycle/00-notes-quota-grant.yaml create mode 100644 test/notes/clusternote-multicluster-subject/02-grant-creation-policy.yaml create mode 100644 test/notes/note-multicluster-subject/02-grant-creation-policy.yaml diff --git a/test/crm/note-contact-lifecycle/00-notes-quota-grant.yaml b/test/crm/note-contact-lifecycle/00-notes-quota-grant.yaml new file mode 100644 index 00000000..df15186f --- /dev/null +++ b/test/crm/note-contact-lifecycle/00-notes-quota-grant.yaml @@ -0,0 +1,15 @@ +apiVersion: quota.miloapis.com/v1alpha1 +kind: ResourceGrant +metadata: + name: crm-note-lifecycle-test-grant + namespace: milo-system + labels: + test.miloapis.com/scenario: "crm-note-contact-lifecycle" +spec: + consumerRef: + kind: "" + name: "" + allowances: + - resourceType: notes.miloapis.com/notes + buckets: + - amount: 10 diff --git a/test/crm/note-contact-lifecycle/chainsaw-test.yaml b/test/crm/note-contact-lifecycle/chainsaw-test.yaml index ba677cee..6e9038a7 100644 --- a/test/crm/note-contact-lifecycle/chainsaw-test.yaml +++ b/test/crm/note-contact-lifecycle/chainsaw-test.yaml @@ -12,6 +12,22 @@ spec: - Deleting a User causes owned Notes to be garbage-collected via ownerReferences steps: + - name: setup-notes-quota-grant + description: Create a ResourceGrant granting notes quota capacity for this test + try: + - apply: + file: 00-notes-quota-grant.yaml + - wait: + apiVersion: quota.miloapis.com/v1alpha1 + kind: ResourceGrant + name: crm-note-lifecycle-test-grant + namespace: milo-system + timeout: 30s + for: + condition: + name: Active + value: 'True' + - name: create-user-contact-and-notes description: Create IAM User, Notification Contact, and CRM Notes, then verify Note status try: diff --git a/test/notes/clusternote-multicluster-subject/02-grant-creation-policy.yaml b/test/notes/clusternote-multicluster-subject/02-grant-creation-policy.yaml new file mode 100644 index 00000000..38acfa00 --- /dev/null +++ b/test/notes/clusternote-multicluster-subject/02-grant-creation-policy.yaml @@ -0,0 +1,35 @@ +apiVersion: quota.miloapis.com/v1alpha1 +kind: GrantCreationPolicy +metadata: + name: cn-mc-test-project-notes-grant-policy + labels: + test.miloapis.com/notes: "clusternote-multicluster" +spec: + disabled: false + trigger: + resource: + apiVersion: "resourcemanager.miloapis.com/v1alpha1" + kind: "Project" + constraints: + - expression: 'has(trigger.metadata.labels) && "test.miloapis.com/notes" in trigger.metadata.labels && trigger.metadata.labels["test.miloapis.com/notes"] == "clusternote-multicluster"' + target: + parentContext: + apiGroup: "resourcemanager.miloapis.com" + kind: "Project" + nameExpression: "trigger.metadata.name" + resourceGrantTemplate: + metadata: + name: "auto-grant-{{trigger.metadata.name}}" + namespace: "milo-system" + labels: + quota.miloapis.com/auto-created: "true" + test.miloapis.com/notes: "clusternote-multicluster" + spec: + consumerRef: + apiGroup: "resourcemanager.miloapis.com" + kind: "Project" + name: "{{trigger.metadata.name}}" + allowances: + - resourceType: "notes.miloapis.com/notes" + buckets: + - amount: 10 diff --git a/test/notes/clusternote-multicluster-subject/chainsaw-test.yaml b/test/notes/clusternote-multicluster-subject/chainsaw-test.yaml index 09053948..05e231ee 100644 --- a/test/notes/clusternote-multicluster-subject/chainsaw-test.yaml +++ b/test/notes/clusternote-multicluster-subject/chainsaw-test.yaml @@ -37,6 +37,22 @@ spec: path: '{.status.phase}' value: Active + - name: setup-grant-creation-policy + description: Create GrantCreationPolicy to provision notes quota for test projects + cluster: main + try: + - apply: + file: 02-grant-creation-policy.yaml + - wait: + apiVersion: quota.miloapis.com/v1alpha1 + kind: GrantCreationPolicy + name: cn-mc-test-project-notes-grant-policy + timeout: 30s + for: + condition: + name: Ready + value: 'true' + - name: create-project description: Create project in org control plane cluster: org @@ -53,6 +69,21 @@ spec: name: Ready value: 'true' + - name: wait-for-project-quota-grant + description: Wait for notes quota grant to appear in project control plane, confirming the proxy is reachable and quota is provisioned + cluster: project + try: + - wait: + apiVersion: quota.miloapis.com/v1alpha1 + kind: ResourceGrant + name: auto-grant-cn-mc-test-project-1 + namespace: milo-system + timeout: 60s + for: + condition: + name: Active + value: 'True' + - name: create-namespace-in-project description: Create cluster-scoped Namespace in project control plane cluster: project diff --git a/test/notes/note-multicluster-subject/02-grant-creation-policy.yaml b/test/notes/note-multicluster-subject/02-grant-creation-policy.yaml new file mode 100644 index 00000000..5e97883f --- /dev/null +++ b/test/notes/note-multicluster-subject/02-grant-creation-policy.yaml @@ -0,0 +1,38 @@ +apiVersion: quota.miloapis.com/v1alpha1 +kind: GrantCreationPolicy +metadata: + name: note-mc-test-project-notes-grant-policy + labels: + test.miloapis.com/notes: "multicluster" +spec: + disabled: false + trigger: + resource: + apiVersion: "resourcemanager.miloapis.com/v1alpha1" + kind: "Project" + constraints: + - expression: 'has(trigger.metadata.labels) && "test.miloapis.com/notes" in trigger.metadata.labels && trigger.metadata.labels["test.miloapis.com/notes"] == "multicluster"' + target: + parentContext: + apiGroup: "resourcemanager.miloapis.com" + kind: "Project" + nameExpression: "trigger.metadata.name" + resourceGrantTemplate: + metadata: + name: "auto-grant-{{trigger.metadata.name}}" + namespace: "milo-system" + labels: + quota.miloapis.com/auto-created: "true" + test.miloapis.com/notes: "multicluster" + spec: + consumerRef: + apiGroup: "resourcemanager.miloapis.com" + kind: "Project" + name: "{{trigger.metadata.name}}" + allowances: + - resourceType: "notes.miloapis.com/notes" + buckets: + - amount: 10 + - resourceType: "core.miloapis.com/configmaps" + buckets: + - amount: 10 diff --git a/test/notes/note-multicluster-subject/chainsaw-test.yaml b/test/notes/note-multicluster-subject/chainsaw-test.yaml index 2072c1c9..868dc52c 100644 --- a/test/notes/note-multicluster-subject/chainsaw-test.yaml +++ b/test/notes/note-multicluster-subject/chainsaw-test.yaml @@ -36,6 +36,22 @@ spec: path: '{.status.phase}' value: Active + - name: setup-grant-creation-policy + description: Create GrantCreationPolicy to provision notes quota for test projects + cluster: main + try: + - apply: + file: 02-grant-creation-policy.yaml + - wait: + apiVersion: quota.miloapis.com/v1alpha1 + kind: GrantCreationPolicy + name: note-mc-test-project-notes-grant-policy + timeout: 30s + for: + condition: + name: Ready + value: 'true' + - name: create-project description: Create project in org control plane cluster: org @@ -52,6 +68,21 @@ spec: name: Ready value: 'true' + - name: wait-for-project-quota-grant + description: Wait for notes quota grant to appear in project control plane, confirming the proxy is reachable and quota is provisioned + cluster: project + try: + - wait: + apiVersion: quota.miloapis.com/v1alpha1 + kind: ResourceGrant + name: auto-grant-note-mc-test-project-1 + namespace: milo-system + timeout: 60s + for: + condition: + name: Active + value: 'True' + - name: create-configmap-in-project description: Create ConfigMap resource in project control plane cluster: project From 408204c2d4b120e301f6ca90d2d6b690ab91a88d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 10 Apr 2026 13:53:12 -0500 Subject: [PATCH 2/2] fix: increase apiserver resource limits to prevent OOMKill The apiserver was being OOMKilled at 512Mi during e2e tests when multiple chainsaw tests run concurrently, causing cascading failures. Bump memory limit to 1Gi and CPU limit to 1 core, with requests raised proportionally to 512Mi memory and 200m CPU. Co-Authored-By: Claude Opus 4.6 (1M context) --- config/apiserver/deployment.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/apiserver/deployment.yaml b/config/apiserver/deployment.yaml index e13dd69f..431d541d 100644 --- a/config/apiserver/deployment.yaml +++ b/config/apiserver/deployment.yaml @@ -211,11 +211,11 @@ spec: timeoutSeconds: 15 resources: requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m + cpu: 200m memory: 512Mi + limits: + cpu: "1" + memory: 1Gi startupProbe: failureThreshold: 30 httpGet: