From a9d1602657f5712d8d7b4b1f5141303239da2ecf Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Tue, 12 May 2026 14:19:22 +0000 Subject: [PATCH 01/10] add: acl-resource-consume scenario --- .../CRI Benchmark/acl-resource-consume.yml | 48 +++++++ pipelines/system/new-pipeline-test.yml | 55 ++++--- .../perf-eval/acl-resource-consume/README.md | 54 +++++++ .../terraform-inputs/azure.tfvars | 134 ++++++++++++++++++ .../terraform-test-inputs/azure.json | 4 + 5 files changed, 279 insertions(+), 16 deletions(-) create mode 100644 pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml create mode 100644 scenarios/perf-eval/acl-resource-consume/README.md create mode 100644 scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars create mode 100644 scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json diff --git a/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml new file mode 100644 index 0000000000..8377ab15c4 --- /dev/null +++ b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml @@ -0,0 +1,48 @@ +trigger: none +pr: none + +variables: + SCENARIO_TYPE: perf-eval + SCENARIO_NAME: acl-resource-consume + ACL_IMAGE_SUBSCRIPTION_ID: b3e01d89-bd55-414f-bbb4-cdfeb2628caa + ACL_IMAGE_RESOURCE_GROUP: ACL-IMAGES + ACL_IMAGE_GALLERY: acl + ACL_IMAGE_NAME: acl-aks + ACL_IMAGE_VERSION: 1.1775601122.6691 + AKS_CLI_CUSTOM_HEADERS: >- + AKSHTTPCustomFeatures=Microsoft.ContainerService/UseCustomizedOSImage, + OSImageSubscriptionID=$(ACL_IMAGE_SUBSCRIPTION_ID), + OSImageResourceGroup=$(ACL_IMAGE_RESOURCE_GROUP), + OSImageGallery=$(ACL_IMAGE_GALLERY), + OSImageName=$(ACL_IMAGE_NAME), + OSImageVersion=$(ACL_IMAGE_VERSION), + OSSKU=AzureContainerLinux, + OSDistro=CustomizedImageLinuxGuard + +stages: + - stage: acl_resource_consume + displayName: ACL CRI resource consume (westus2) + dependsOn: [] + jobs: + - template: /jobs/competitive-test.yml + parameters: + cloud: azure + regions: + - westus2 + engine: clusterloader2 + engine_input: + image: "ghcr.io/azure/clusterloader2:v20250513" + topology: cri-resource-consume + matrix: + n10-p300-memory: + node_count: 10 + max_pods: 30 + repeats: 1 + operation_timeout: 3m + load_type: memory + scrape_kubelets: True + os_type: linux + max_parallel: 1 + timeout_in_minutes: 120 + credential_type: service_connection + ssh_key_enabled: false diff --git a/pipelines/system/new-pipeline-test.yml b/pipelines/system/new-pipeline-test.yml index 63d55f02d9..8377ab15c4 100644 --- a/pipelines/system/new-pipeline-test.yml +++ b/pipelines/system/new-pipeline-test.yml @@ -1,25 +1,48 @@ trigger: none +pr: none variables: - SCENARIO_TYPE: - SCENARIO_NAME: + SCENARIO_TYPE: perf-eval + SCENARIO_NAME: acl-resource-consume + ACL_IMAGE_SUBSCRIPTION_ID: b3e01d89-bd55-414f-bbb4-cdfeb2628caa + ACL_IMAGE_RESOURCE_GROUP: ACL-IMAGES + ACL_IMAGE_GALLERY: acl + ACL_IMAGE_NAME: acl-aks + ACL_IMAGE_VERSION: 1.1775601122.6691 + AKS_CLI_CUSTOM_HEADERS: >- + AKSHTTPCustomFeatures=Microsoft.ContainerService/UseCustomizedOSImage, + OSImageSubscriptionID=$(ACL_IMAGE_SUBSCRIPTION_ID), + OSImageResourceGroup=$(ACL_IMAGE_RESOURCE_GROUP), + OSImageGallery=$(ACL_IMAGE_GALLERY), + OSImageName=$(ACL_IMAGE_NAME), + OSImageVersion=$(ACL_IMAGE_VERSION), + OSSKU=AzureContainerLinux, + OSDistro=CustomizedImageLinuxGuard stages: - - stage: # format: [_]+ (e.g. azure_eastus2, aws_eastus_westus) + - stage: acl_resource_consume + displayName: ACL CRI resource consume (westus2) dependsOn: [] jobs: - - template: /jobs/competitive-test.yml # must keep as is + - template: /jobs/competitive-test.yml parameters: - cloud: # e.g. azure, aws - regions: # list of regions - - region1 # e.g. eastus2 - topology: # e.g. cluster-autoscaler - engine: # e.g. clusterloader2 - matrix: # list of test parameters to customize the provisioned resources - : - : - : - max_parallel: # required - credential_type: service_connection # required + cloud: azure + regions: + - westus2 + engine: clusterloader2 + engine_input: + image: "ghcr.io/azure/clusterloader2:v20250513" + topology: cri-resource-consume + matrix: + n10-p300-memory: + node_count: 10 + max_pods: 30 + repeats: 1 + operation_timeout: 3m + load_type: memory + scrape_kubelets: True + os_type: linux + max_parallel: 1 + timeout_in_minutes: 120 + credential_type: service_connection ssh_key_enabled: false - timeout_in_minutes: 60 # if not specified, default is 60 diff --git a/scenarios/perf-eval/acl-resource-consume/README.md b/scenarios/perf-eval/acl-resource-consume/README.md new file mode 100644 index 0000000000..1ba74afa7c --- /dev/null +++ b/scenarios/perf-eval/acl-resource-consume/README.md @@ -0,0 +1,54 @@ +# ACL CRI Resource Consume + +This scenario is the first Telescope v1 harness for ACL AKS node performance validation. It reuses the existing `cri-resource-consume` topology and ClusterLoader2 runner, then publishes the raw ClusterLoader2 JUnit file into Azure DevOps test results. + +## Pipeline + +Register this YAML in Azure DevOps: + +```text +pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml +``` + +Queue it manually for the first validation run. + +## Azure DevOps Inputs + +The pipeline follows the existing Telescope v1 style and keeps defaults as YAML variables rather than queue-time parameters: + +| Variable | Default | Purpose | +|---|---|---| +| `ACL_IMAGE_SUBSCRIPTION_ID` | `b3e01d89-bd55-414f-bbb4-cdfeb2628caa` | Subscription containing the ACL Beta BYOI image. | +| `ACL_IMAGE_RESOURCE_GROUP` | `ACL-IMAGES` | Resource group containing the ACL Beta BYOI image. | +| `ACL_IMAGE_GALLERY` | `acl` | Gallery containing the ACL Beta BYOI image. | +| `ACL_IMAGE_NAME` | `acl-aks` | ACL image definition. Use `acl-aks-arm64` for ARM64. | +| `ACL_IMAGE_VERSION` | `1.1775601122.6691` | ACL image version to test. | + +The New Pipeline Test environment must provide `AZURE_SERVICE_CONNECTION`, `AZURE_SUBSCRIPTION_ID`, and `AZURE_TELESCOPE_STORAGE_ACCOUNT_NAME`. The service connection identity needs Contributor on the subscription for temporary resource groups and Storage Blob Data Contributor on the result storage account. Ensure the storage account has a `perf-eval` container, because Telescope uploads results to a container named after `SCENARIO_TYPE`. + +For temporary testing, override these as Azure DevOps variables when queuing the New Pipeline Test instead of changing the scenario file. + +## ACL OS Selector And Image + +The current Terraform input uses the AKS CLI path and sets `--os-sku AzureContainerLinux`. For ACL Beta BYOI, the pipeline composes `AKS_CLI_CUSTOM_HEADERS` from the ACL image variables and Telescope passes those headers into both `az aks create` and `az aks nodepool add`. + +The fields prepared for that are in `terraform-inputs/azure.tfvars`: + +```hcl +use_aks_preview_cli_extension = true +aks_custom_headers = [] +optional_parameters = [...] +``` + +For ACL Alpha, queue the run with `AKS_CLI_CUSTOM_HEADERS` set to an empty value and use an AKS/RP region where `--os-sku AzureContainerLinux` is already available. With empty custom headers, Telescope passes `aks_custom_headers = []` to Terraform and the AKS CLI module does not emit `--aks-custom-headers`. + +For ACL Beta ARM64, queue with an ARM64-capable region/SKU setup and override: + +```text +ACL_IMAGE_NAME=acl-aks-arm64 +ACL_IMAGE_VERSION=1.1776195556.10516 +``` + +## First Run + +The initial matrix intentionally runs only `n10-p300-memory`. After it creates an AKS cluster, runs ClusterLoader2, publishes JUnit, uploads results, and cleans up reliably, add the higher pressure cases from `azurelinux-resource-consume`. \ No newline at end of file diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars new file mode 100644 index 0000000000..41d5eb1705 --- /dev/null +++ b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars @@ -0,0 +1,134 @@ +scenario_type = "perf-eval" +scenario_name = "acl-resource-consume" +deletion_delay = "2h" +owner = "aks" + +# This scenario intentionally uses the AKS CLI Terraform path instead of the +# azurerm AKS resource path. ACL Beta uses AKS BYOI custom headers to select the +# Azure Compute Gallery image, while ACL Alpha can use the same shape with the +# custom headers omitted and --os-sku AzureContainerLinux. +aks_cli_config_list = [ + { + role = "client" + aks_name = "acl-resource-consume" + sku_tier = "standard" + kubernetes_version = "1.35" + use_aks_preview_cli_extension = true + aks_custom_headers = [] + default_node_pool = { + name = "default" + node_count = 3 + vm_size = "Standard_D16_v5" + } + extra_node_pool = [ + { + name = "prompool" + node_count = 1 + vm_size = "Standard_D16_v5" + optional_parameters = [ + { + name = "labels" + value = "prometheus=true" + }, + { + name = "os-sku" + value = "AzureContainerLinux" + }, + { + name = "enable-secure-boot" + value = "" + }, + { + name = "enable-vtpm" + value = "" + }, + { + name = "nodepool-tags" + value = "AzSecPackAutoConfigReady=true" + } + ] + }, + { + name = "userpool0" + node_count = 10 + vm_size = "Standard_D16ds_v6" + optional_parameters = [ + { + name = "node-osdisk-type" + value = "Ephemeral" + }, + { + name = "os-sku" + value = "AzureContainerLinux" + }, + { + name = "enable-secure-boot" + value = "" + }, + { + name = "enable-vtpm" + value = "" + }, + { + name = "nodepool-tags" + value = "AzSecPackAutoConfigReady=true" + }, + { + name = "node-taints" + value = "cri-resource-consume=true:NoSchedule,cri-resource-consume=true:NoExecute" + }, + { + name = "labels" + value = "cri-resource-consume=true" + } + ] + } + ] + optional_parameters = [ + { + name = "network-plugin" + value = "azure" + }, + { + name = "network-plugin-mode" + value = "overlay" + }, + { + name = "node-init-taints" + value = "CriticalAddonsOnly=true:NoSchedule" + }, + { + name = "pod-cidr" + value = "10.0.0.0/9" + }, + { + name = "service-cidr" + value = "192.168.0.0/16" + }, + { + name = "dns-service-ip" + value = "192.168.0.10" + }, + { + name = "os-sku" + value = "AzureContainerLinux" + }, + { + name = "enable-secure-boot" + value = "" + }, + { + name = "enable-vtpm" + value = "" + }, + { + name = "nodepool-tags" + value = "AzSecPackAutoConfigReady=true" + }, + { + name = "node-os-upgrade-channel" + value = "None" + } + ] + } +] \ No newline at end of file diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json b/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json new file mode 100644 index 0000000000..f76620d05e --- /dev/null +++ b/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json @@ -0,0 +1,4 @@ +{ + "run_id" : "123456789", + "region" : "westus2" +} \ No newline at end of file From fa7367bf3743c25bdf1be02d425f9970dfdc6e1f Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Fri, 15 May 2026 12:22:30 +0000 Subject: [PATCH 02/10] update: aks-preview extension --- modules/terraform/azure/aks-cli/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/terraform/azure/aks-cli/main.tf b/modules/terraform/azure/aks-cli/main.tf index 47395fcab6..2b0eee0985 100644 --- a/modules/terraform/azure/aks-cli/main.tf +++ b/modules/terraform/azure/aks-cli/main.tf @@ -281,7 +281,7 @@ resource "terraform_data" "enable_aks_cli_preview_extension" { EOT ) : ( < Date: Fri, 15 May 2026 12:55:53 +0000 Subject: [PATCH 03/10] fix: use tags for acl extra node pools --- .../acl-resource-consume/terraform-inputs/azure.tfvars | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars index 41d5eb1705..1aacfc4416 100644 --- a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars +++ b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars @@ -43,7 +43,7 @@ aks_cli_config_list = [ value = "" }, { - name = "nodepool-tags" + name = "tags" value = "AzSecPackAutoConfigReady=true" } ] @@ -70,7 +70,7 @@ aks_cli_config_list = [ value = "" }, { - name = "nodepool-tags" + name = "tags" value = "AzSecPackAutoConfigReady=true" }, { From 77ffea5b28207550dc5aa844c0d4fe2ae5a2a3e3 Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Fri, 15 May 2026 14:36:50 +0000 Subject: [PATCH 04/10] update: try westus3 as region --- pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml | 4 ++-- pipelines/system/new-pipeline-test.yml | 4 ++-- .../acl-resource-consume/terraform-test-inputs/azure.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml index 8377ab15c4..cb46028b95 100644 --- a/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml +++ b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml @@ -21,14 +21,14 @@ variables: stages: - stage: acl_resource_consume - displayName: ACL CRI resource consume (westus2) + displayName: ACL CRI resource consume (westus3) dependsOn: [] jobs: - template: /jobs/competitive-test.yml parameters: cloud: azure regions: - - westus2 + - westus3 engine: clusterloader2 engine_input: image: "ghcr.io/azure/clusterloader2:v20250513" diff --git a/pipelines/system/new-pipeline-test.yml b/pipelines/system/new-pipeline-test.yml index 8377ab15c4..cb46028b95 100644 --- a/pipelines/system/new-pipeline-test.yml +++ b/pipelines/system/new-pipeline-test.yml @@ -21,14 +21,14 @@ variables: stages: - stage: acl_resource_consume - displayName: ACL CRI resource consume (westus2) + displayName: ACL CRI resource consume (westus3) dependsOn: [] jobs: - template: /jobs/competitive-test.yml parameters: cloud: azure regions: - - westus2 + - westus3 engine: clusterloader2 engine_input: image: "ghcr.io/azure/clusterloader2:v20250513" diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json b/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json index f76620d05e..b7d9808166 100644 --- a/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json +++ b/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json @@ -1,4 +1,4 @@ { "run_id" : "123456789", - "region" : "westus2" + "region" : "westus3" } \ No newline at end of file From 4ad69cc079199a950c3e664ff9c8567b8f64db84 Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Mon, 18 May 2026 11:12:16 +0000 Subject: [PATCH 05/10] update: add other resource consume tests --- .../CRI Benchmark/acl-resource-consume.yml | 16 ++++++++++++++++ pipelines/system/new-pipeline-test.yml | 16 ++++++++++++++++ .../perf-eval/acl-resource-consume/README.md | 4 ++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml index cb46028b95..d0fc73bb03 100644 --- a/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml +++ b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml @@ -42,6 +42,22 @@ stages: load_type: memory scrape_kubelets: True os_type: linux + n10-p700-memory: + node_count: 10 + max_pods: 70 + repeats: 1 + operation_timeout: 7m + load_type: memory + scrape_kubelets: True + os_type: linux + n10-p1100-memory: + node_count: 10 + max_pods: 110 + repeats: 1 + operation_timeout: 11m + load_type: memory + scrape_kubelets: True + os_type: linux max_parallel: 1 timeout_in_minutes: 120 credential_type: service_connection diff --git a/pipelines/system/new-pipeline-test.yml b/pipelines/system/new-pipeline-test.yml index cb46028b95..d0fc73bb03 100644 --- a/pipelines/system/new-pipeline-test.yml +++ b/pipelines/system/new-pipeline-test.yml @@ -42,6 +42,22 @@ stages: load_type: memory scrape_kubelets: True os_type: linux + n10-p700-memory: + node_count: 10 + max_pods: 70 + repeats: 1 + operation_timeout: 7m + load_type: memory + scrape_kubelets: True + os_type: linux + n10-p1100-memory: + node_count: 10 + max_pods: 110 + repeats: 1 + operation_timeout: 11m + load_type: memory + scrape_kubelets: True + os_type: linux max_parallel: 1 timeout_in_minutes: 120 credential_type: service_connection diff --git a/scenarios/perf-eval/acl-resource-consume/README.md b/scenarios/perf-eval/acl-resource-consume/README.md index 1ba74afa7c..eba2d8403d 100644 --- a/scenarios/perf-eval/acl-resource-consume/README.md +++ b/scenarios/perf-eval/acl-resource-consume/README.md @@ -49,6 +49,6 @@ ACL_IMAGE_NAME=acl-aks-arm64 ACL_IMAGE_VERSION=1.1776195556.10516 ``` -## First Run +## Benchmark Matrix -The initial matrix intentionally runs only `n10-p300-memory`. After it creates an AKS cluster, runs ClusterLoader2, publishes JUnit, uploads results, and cleans up reliably, add the higher pressure cases from `azurelinux-resource-consume`. \ No newline at end of file +The ACL pipeline runs the same initial Linux memory pressure cases as `azurelinux-resource-consume`: `n10-p300-memory`, `n10-p700-memory`, and `n10-p1100-memory`. \ No newline at end of file From fd6477049b5079d03a9eb7303a091ac76906670f Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Mon, 18 May 2026 13:10:09 +0000 Subject: [PATCH 06/10] update: remove acl custom BYOI scenario --- .../CRI Benchmark/acl-resource-consume.yml | 14 ------ pipelines/system/new-pipeline-test.yml | 14 ------ .../perf-eval/acl-resource-consume/README.md | 43 +++---------------- .../terraform-inputs/azure.tfvars | 5 +-- 4 files changed, 7 insertions(+), 69 deletions(-) diff --git a/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml index d0fc73bb03..b11d588e93 100644 --- a/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml +++ b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml @@ -4,20 +4,6 @@ pr: none variables: SCENARIO_TYPE: perf-eval SCENARIO_NAME: acl-resource-consume - ACL_IMAGE_SUBSCRIPTION_ID: b3e01d89-bd55-414f-bbb4-cdfeb2628caa - ACL_IMAGE_RESOURCE_GROUP: ACL-IMAGES - ACL_IMAGE_GALLERY: acl - ACL_IMAGE_NAME: acl-aks - ACL_IMAGE_VERSION: 1.1775601122.6691 - AKS_CLI_CUSTOM_HEADERS: >- - AKSHTTPCustomFeatures=Microsoft.ContainerService/UseCustomizedOSImage, - OSImageSubscriptionID=$(ACL_IMAGE_SUBSCRIPTION_ID), - OSImageResourceGroup=$(ACL_IMAGE_RESOURCE_GROUP), - OSImageGallery=$(ACL_IMAGE_GALLERY), - OSImageName=$(ACL_IMAGE_NAME), - OSImageVersion=$(ACL_IMAGE_VERSION), - OSSKU=AzureContainerLinux, - OSDistro=CustomizedImageLinuxGuard stages: - stage: acl_resource_consume diff --git a/pipelines/system/new-pipeline-test.yml b/pipelines/system/new-pipeline-test.yml index d0fc73bb03..b11d588e93 100644 --- a/pipelines/system/new-pipeline-test.yml +++ b/pipelines/system/new-pipeline-test.yml @@ -4,20 +4,6 @@ pr: none variables: SCENARIO_TYPE: perf-eval SCENARIO_NAME: acl-resource-consume - ACL_IMAGE_SUBSCRIPTION_ID: b3e01d89-bd55-414f-bbb4-cdfeb2628caa - ACL_IMAGE_RESOURCE_GROUP: ACL-IMAGES - ACL_IMAGE_GALLERY: acl - ACL_IMAGE_NAME: acl-aks - ACL_IMAGE_VERSION: 1.1775601122.6691 - AKS_CLI_CUSTOM_HEADERS: >- - AKSHTTPCustomFeatures=Microsoft.ContainerService/UseCustomizedOSImage, - OSImageSubscriptionID=$(ACL_IMAGE_SUBSCRIPTION_ID), - OSImageResourceGroup=$(ACL_IMAGE_RESOURCE_GROUP), - OSImageGallery=$(ACL_IMAGE_GALLERY), - OSImageName=$(ACL_IMAGE_NAME), - OSImageVersion=$(ACL_IMAGE_VERSION), - OSSKU=AzureContainerLinux, - OSDistro=CustomizedImageLinuxGuard stages: - stage: acl_resource_consume diff --git a/scenarios/perf-eval/acl-resource-consume/README.md b/scenarios/perf-eval/acl-resource-consume/README.md index eba2d8403d..e5f6004c3a 100644 --- a/scenarios/perf-eval/acl-resource-consume/README.md +++ b/scenarios/perf-eval/acl-resource-consume/README.md @@ -1,6 +1,6 @@ # ACL CRI Resource Consume -This scenario is the first Telescope v1 harness for ACL AKS node performance validation. It reuses the existing `cri-resource-consume` topology and ClusterLoader2 runner, then publishes the raw ClusterLoader2 JUnit file into Azure DevOps test results. +This scenario runs ACL AKS node performance validation through Telescope v1. ## Pipeline @@ -10,45 +10,12 @@ Register this YAML in Azure DevOps: pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml ``` -Queue it manually for the first validation run. +Queue it manually for validation runs. ## Azure DevOps Inputs -The pipeline follows the existing Telescope v1 style and keeps defaults as YAML variables rather than queue-time parameters: +The pipeline environment must provide `AZURE_SERVICE_CONNECTION`, `AZURE_SUBSCRIPTION_ID`, and `AZURE_TELESCOPE_STORAGE_ACCOUNT_NAME`. The service connection identity needs Contributor on the subscription for temporary resource groups and Storage Blob Data Contributor on the result storage account. Ensure the storage account has a `perf-eval` container, because Telescope uploads results to a container named after `SCENARIO_TYPE`. -| Variable | Default | Purpose | -|---|---|---| -| `ACL_IMAGE_SUBSCRIPTION_ID` | `b3e01d89-bd55-414f-bbb4-cdfeb2628caa` | Subscription containing the ACL Beta BYOI image. | -| `ACL_IMAGE_RESOURCE_GROUP` | `ACL-IMAGES` | Resource group containing the ACL Beta BYOI image. | -| `ACL_IMAGE_GALLERY` | `acl` | Gallery containing the ACL Beta BYOI image. | -| `ACL_IMAGE_NAME` | `acl-aks` | ACL image definition. Use `acl-aks-arm64` for ARM64. | -| `ACL_IMAGE_VERSION` | `1.1775601122.6691` | ACL image version to test. | +## ACL OS Selector -The New Pipeline Test environment must provide `AZURE_SERVICE_CONNECTION`, `AZURE_SUBSCRIPTION_ID`, and `AZURE_TELESCOPE_STORAGE_ACCOUNT_NAME`. The service connection identity needs Contributor on the subscription for temporary resource groups and Storage Blob Data Contributor on the result storage account. Ensure the storage account has a `perf-eval` container, because Telescope uploads results to a container named after `SCENARIO_TYPE`. - -For temporary testing, override these as Azure DevOps variables when queuing the New Pipeline Test instead of changing the scenario file. - -## ACL OS Selector And Image - -The current Terraform input uses the AKS CLI path and sets `--os-sku AzureContainerLinux`. For ACL Beta BYOI, the pipeline composes `AKS_CLI_CUSTOM_HEADERS` from the ACL image variables and Telescope passes those headers into both `az aks create` and `az aks nodepool add`. - -The fields prepared for that are in `terraform-inputs/azure.tfvars`: - -```hcl -use_aks_preview_cli_extension = true -aks_custom_headers = [] -optional_parameters = [...] -``` - -For ACL Alpha, queue the run with `AKS_CLI_CUSTOM_HEADERS` set to an empty value and use an AKS/RP region where `--os-sku AzureContainerLinux` is already available. With empty custom headers, Telescope passes `aks_custom_headers = []` to Terraform and the AKS CLI module does not emit `--aks-custom-headers`. - -For ACL Beta ARM64, queue with an ARM64-capable region/SKU setup and override: - -```text -ACL_IMAGE_NAME=acl-aks-arm64 -ACL_IMAGE_VERSION=1.1776195556.10516 -``` - -## Benchmark Matrix - -The ACL pipeline runs the same initial Linux memory pressure cases as `azurelinux-resource-consume`: `n10-p300-memory`, `n10-p700-memory`, and `n10-p1100-memory`. \ No newline at end of file +The current Terraform input uses the AKS CLI path and sets `--os-sku AzureContainerLinux` for the cluster and ACL node pools. \ No newline at end of file diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars index 1aacfc4416..9f35c24e79 100644 --- a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars +++ b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars @@ -4,9 +4,8 @@ deletion_delay = "2h" owner = "aks" # This scenario intentionally uses the AKS CLI Terraform path instead of the -# azurerm AKS resource path. ACL Beta uses AKS BYOI custom headers to select the -# Azure Compute Gallery image, while ACL Alpha can use the same shape with the -# custom headers omitted and --os-sku AzureContainerLinux. +# azurerm AKS resource path so it can pass ACL-specific AKS CLI options such as +# --os-sku AzureContainerLinux. aks_cli_config_list = [ { role = "client" From 7ee16c15b2abffa7b593d0599d51e1c6f357d5ac Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Mon, 18 May 2026 13:13:50 +0000 Subject: [PATCH 07/10] fix: add missing newline at end of files in terraform inputs --- .../acl-resource-consume/terraform-inputs/azure.tfvars | 2 +- .../acl-resource-consume/terraform-test-inputs/azure.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars index 9f35c24e79..aa4b9872a1 100644 --- a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars +++ b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars @@ -130,4 +130,4 @@ aks_cli_config_list = [ } ] } -] \ No newline at end of file +] diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json b/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json index b7d9808166..40b8d650c4 100644 --- a/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json +++ b/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json @@ -1,4 +1,4 @@ { "run_id" : "123456789", "region" : "westus3" -} \ No newline at end of file +} From dc21f4d4c4222f8487c7ff75f84549f099a2ec89 Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Fri, 22 May 2026 06:44:26 +0000 Subject: [PATCH 08/10] update: change region to swedencentral --- pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml | 4 ++-- pipelines/system/new-pipeline-test.yml | 4 ++-- .../acl-resource-consume/terraform-inputs/azure.tfvars | 3 --- .../acl-resource-consume/terraform-test-inputs/azure.json | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml index b11d588e93..16ced277cf 100644 --- a/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml +++ b/pipelines/perf-eval/CRI Benchmark/acl-resource-consume.yml @@ -7,14 +7,14 @@ variables: stages: - stage: acl_resource_consume - displayName: ACL CRI resource consume (westus3) + displayName: ACL CRI resource consume (swedencentral) dependsOn: [] jobs: - template: /jobs/competitive-test.yml parameters: cloud: azure regions: - - westus3 + - swedencentral engine: clusterloader2 engine_input: image: "ghcr.io/azure/clusterloader2:v20250513" diff --git a/pipelines/system/new-pipeline-test.yml b/pipelines/system/new-pipeline-test.yml index b11d588e93..16ced277cf 100644 --- a/pipelines/system/new-pipeline-test.yml +++ b/pipelines/system/new-pipeline-test.yml @@ -7,14 +7,14 @@ variables: stages: - stage: acl_resource_consume - displayName: ACL CRI resource consume (westus3) + displayName: ACL CRI resource consume (swedencentral) dependsOn: [] jobs: - template: /jobs/competitive-test.yml parameters: cloud: azure regions: - - westus3 + - swedencentral engine: clusterloader2 engine_input: image: "ghcr.io/azure/clusterloader2:v20250513" diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars index aa4b9872a1..8669d28b25 100644 --- a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars +++ b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars @@ -3,9 +3,6 @@ scenario_name = "acl-resource-consume" deletion_delay = "2h" owner = "aks" -# This scenario intentionally uses the AKS CLI Terraform path instead of the -# azurerm AKS resource path so it can pass ACL-specific AKS CLI options such as -# --os-sku AzureContainerLinux. aks_cli_config_list = [ { role = "client" diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json b/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json index 40b8d650c4..0a6ba308b6 100644 --- a/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json +++ b/scenarios/perf-eval/acl-resource-consume/terraform-test-inputs/azure.json @@ -1,4 +1,4 @@ { "run_id" : "123456789", - "region" : "westus3" + "region" : "swedencentral" } From d445b94b1130a0244cacfc9df0e3b7b29359ca2b Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Fri, 22 May 2026 10:25:55 +0000 Subject: [PATCH 09/10] update: remove azsecpack config --- .../terraform-inputs/azure.tfvars | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars index 8669d28b25..a38f20d146 100644 --- a/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars +++ b/scenarios/perf-eval/acl-resource-consume/terraform-inputs/azure.tfvars @@ -38,10 +38,6 @@ aks_cli_config_list = [ name = "enable-vtpm" value = "" }, - { - name = "tags" - value = "AzSecPackAutoConfigReady=true" - } ] }, { @@ -65,10 +61,6 @@ aks_cli_config_list = [ name = "enable-vtpm" value = "" }, - { - name = "tags" - value = "AzSecPackAutoConfigReady=true" - }, { name = "node-taints" value = "cri-resource-consume=true:NoSchedule,cri-resource-consume=true:NoExecute" @@ -117,10 +109,6 @@ aks_cli_config_list = [ name = "enable-vtpm" value = "" }, - { - name = "nodepool-tags" - value = "AzSecPackAutoConfigReady=true" - }, { name = "node-os-upgrade-channel" value = "None" From 68a87ad480db83c7f20c16ea2350fbd75ade05cd Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Fri, 22 May 2026 12:53:12 +0000 Subject: [PATCH 10/10] chore: restore new pipeline test template --- pipelines/system/new-pipeline-test.yml | 57 ++++++++------------------ 1 file changed, 16 insertions(+), 41 deletions(-) diff --git a/pipelines/system/new-pipeline-test.yml b/pipelines/system/new-pipeline-test.yml index 16ced277cf..63d55f02d9 100644 --- a/pipelines/system/new-pipeline-test.yml +++ b/pipelines/system/new-pipeline-test.yml @@ -1,50 +1,25 @@ trigger: none -pr: none variables: - SCENARIO_TYPE: perf-eval - SCENARIO_NAME: acl-resource-consume + SCENARIO_TYPE: + SCENARIO_NAME: stages: - - stage: acl_resource_consume - displayName: ACL CRI resource consume (swedencentral) + - stage: # format: [_]+ (e.g. azure_eastus2, aws_eastus_westus) dependsOn: [] jobs: - - template: /jobs/competitive-test.yml + - template: /jobs/competitive-test.yml # must keep as is parameters: - cloud: azure - regions: - - swedencentral - engine: clusterloader2 - engine_input: - image: "ghcr.io/azure/clusterloader2:v20250513" - topology: cri-resource-consume - matrix: - n10-p300-memory: - node_count: 10 - max_pods: 30 - repeats: 1 - operation_timeout: 3m - load_type: memory - scrape_kubelets: True - os_type: linux - n10-p700-memory: - node_count: 10 - max_pods: 70 - repeats: 1 - operation_timeout: 7m - load_type: memory - scrape_kubelets: True - os_type: linux - n10-p1100-memory: - node_count: 10 - max_pods: 110 - repeats: 1 - operation_timeout: 11m - load_type: memory - scrape_kubelets: True - os_type: linux - max_parallel: 1 - timeout_in_minutes: 120 - credential_type: service_connection + cloud: # e.g. azure, aws + regions: # list of regions + - region1 # e.g. eastus2 + topology: # e.g. cluster-autoscaler + engine: # e.g. clusterloader2 + matrix: # list of test parameters to customize the provisioned resources + : + : + : + max_parallel: # required + credential_type: service_connection # required ssh_key_enabled: false + timeout_in_minutes: 60 # if not specified, default is 60