feat(crud-machine-ci): wire create-machine/scale-machine into ADO pipeline

pipelines/perf-eval/AKS Machine API Benchmark/k8s-cluster-crud-machine.yml

@@ -0,0 +1,59 @@
+trigger: none
+schedules:
+  - cron: "0 12 * * *"
+    displayName: "Daily at 12:00 UTC"
+    branches:
+      include:
+        - main
+    always: true
+
+variables:
+  SCENARIO_TYPE: perf-eval
+  SCENARIO_NAME: k8s-cluster-crud-machine
+
+stages:
+  - stage: azure_machine_api_crud_test
+    dependsOn: []
+    jobs:
+      - template: /jobs/competitive-test.yml
+        parameters:
+          cloud: azure
+          regions:
+            - canadacentral
+          topology: k8s-crud-machine
+          engine: crud
+          matrix:
+            crud_small_machine_pool:
+              pool_name: smallpool
+              vm_size: Standard_D2_v3
+              scale_machine_count: 1
+              machine_workers: 1
+              use_batch_api: false
+              step_time_out: 600
+              readiness_wait_timeout: 1200
+              system_node_pool: >-
+                {"name":"default","vm_size":"Standard_D2_v3","node_count":8,"vm_set_type":"VirtualMachineScaleSets"}
+            crud_medium_machine_pool:
+              pool_name: mediumpool
+              vm_size: Standard_D2_v3
+              scale_machine_count: 50
+              machine_workers: 50
+              use_batch_api: false
+              step_time_out: 600
+              readiness_wait_timeout: 1200
+              system_node_pool: >-
+                {"name":"default","vm_size":"Standard_D2_v3","node_count":8,"vm_set_type":"VirtualMachineScaleSets"}
+            crud_large_machine_pool:
+              pool_name: largepool
+              vm_size: Standard_D2_v3
+              scale_machine_count: 100
+              machine_workers: 100
+              use_batch_api: true
+              step_time_out: 900
+              readiness_wait_timeout: 1200
+              system_node_pool: >-
+                {"name":"default","vm_size":"Standard_D2_v3","node_count":8,"vm_set_type":"VirtualMachineScaleSets"}
+          max_parallel: 1
+          credential_type: service_connection
+          ssh_key_enabled: false
+          timeout_in_minutes: 180

scenarios/perf-eval/k8s-cluster-crud-machine/terraform-inputs/azure.tfvars

@@ -0,0 +1,29 @@
+scenario_type  = "perf-eval"
+scenario_name  = "k8s-cluster-crud-machine"
+deletion_delay = "2h"
+owner          = "aks"
+
+# AKS Machine API CRUD scenario.
+#
+# The cluster is provisioned with `default_node_pool = null` because the
+# Machine API requires the user-mode agent pool to be created with
+# `mode = Machines` via the ARM PUT call performed at test time by
+# `modules/python/crud/main.py create-machine`. Terraform only owns the
+# cluster shell; the system pool is injected at job runtime via the
+# `SYSTEM_NODE_POOL` matrix variable (see
+# `steps/terraform/set-input-variables-azure.yml`), and the
+# machine-mode user pool is born during the `create-machine` step.
+aks_cli_config_list = [
+  {
+    role                          = "client"
+    aks_name                      = "mchapi"
+    sku_tier                      = "standard"
+    use_aks_preview_cli_extension = true
+    default_node_pool             = null
+
+    optional_parameters = [
+      { name = "network-plugin", value = "azure" },
+      { name = "network-plugin-mode", value = "overlay" }
+    ]
+  }
+]

scenarios/perf-eval/k8s-cluster-crud-machine/terraform-test-inputs/azure.json

@@ -0,0 +1,10 @@
+{
+    "run_id": "123456789",
+    "region": "canadacentral",
+    "aks_cli_system_node_pool": {
+        "name": "default",
+        "node_count": 8,
+        "vm_size": "Standard_D2_v3",
+        "vm_set_type": "VirtualMachineScaleSets"
+    }
+}

steps/engine/crud/k8s/execute-machine.yml

@@ -0,0 +1,66 @@
+parameters:
+  cloud: ""
+  result_dir: $(System.DefaultWorkingDirectory)/$(RUN_ID)
+  pool_name: ""
+  vm_size: ""
+  scale_machine_count: 0
+  machine_workers: 1
+  use_batch_api: false
+  step_time_out: 600
+  readiness_wait_timeout: 1200
+
+steps:
+- script: |
+    set -eo pipefail
+
+    # Create machine-mode agent pool (ARM PUT mode=Machines)
+    PYTHONPATH=$PYTHONPATH:$(pwd) python3 "$PYTHON_SCRIPT_FILE" create-machine \
+      --cloud "$CLOUD" \
+      --run-id "$RUN_ID" \
+      --result-dir "$RESULT_DIR" \
+      --node-pool-name "$POOL_NAME" \
+      --vm-size "$VM_SIZE" \
+      --step-timeout "$STEP_TIME_OUT"
+  displayName: 'Create AKS Machine Agent Pool for ${{ parameters.cloud }}'
+  workingDirectory: modules/python
+  env:
+    PYTHON_SCRIPT_FILE: $(Pipeline.Workspace)/s/modules/python/crud/main.py
+    CLOUD: ${{ parameters.cloud }}
+    POOL_NAME: ${{ parameters.pool_name }}
+    VM_SIZE: ${{ parameters.vm_size }}
+    STEP_TIME_OUT: ${{ parameters.step_time_out }}
+    RESULT_DIR: ${{ parameters.result_dir }}
+
+- script: |
+    set -eo pipefail
+
+    USE_BATCH_API_FLAG=""
+    if [ "${USE_BATCH_API,,}" = "true" ]; then
+      USE_BATCH_API_FLAG="--use-batch-api"
+    fi
+
+    # Add N machines to the machine-mode agent pool
+    PYTHONPATH=$PYTHONPATH:$(pwd) python3 "$PYTHON_SCRIPT_FILE" scale-machine \
+      --cloud "$CLOUD" \
+      --run-id "$RUN_ID" \
+      --result-dir "$RESULT_DIR" \
+      --node-pool-name "$POOL_NAME" \
+      --vm-size "$VM_SIZE" \
+      --scale-machine-count "$SCALE_MACHINE_COUNT" \
+      --machine-workers "$MACHINE_WORKERS" \
+      --readiness-wait-timeout "$READINESS_WAIT_TIMEOUT" \
+      --step-timeout "$STEP_TIME_OUT" \
+      $USE_BATCH_API_FLAG
+  displayName: 'Scale AKS Machine Agent Pool for ${{ parameters.cloud }}'
+  workingDirectory: modules/python
+  env:
+    PYTHON_SCRIPT_FILE: $(Pipeline.Workspace)/s/modules/python/crud/main.py
+    CLOUD: ${{ parameters.cloud }}
+    POOL_NAME: ${{ parameters.pool_name }}
+    VM_SIZE: ${{ parameters.vm_size }}
+    SCALE_MACHINE_COUNT: ${{ parameters.scale_machine_count }}
+    MACHINE_WORKERS: ${{ parameters.machine_workers }}
+    READINESS_WAIT_TIMEOUT: ${{ parameters.readiness_wait_timeout }}
+    STEP_TIME_OUT: ${{ parameters.step_time_out }}
+    RESULT_DIR: ${{ parameters.result_dir }}
+    USE_BATCH_API: ${{ parameters.use_batch_api }}

steps/topology/k8s-crud-machine/collect-crud.yml

@@ -0,0 +1,15 @@
+parameters:
+- name: cloud
+  type: string
+- name: regions
+  type: object
+  default: {}
+- name: engine_input
+  type: object
+  default: {}
+
+steps:
+- template: /steps/engine/crud/k8s/collect.yml
+  parameters:
+    cloud: ${{ parameters.cloud }}
+    region: ${{ parameters.regions[0] }}

steps/topology/k8s-crud-machine/execute-crud.yml

@@ -0,0 +1,23 @@
+parameters:
+- name: cloud
+  type: string
+  default: ''
+- name: engine_input
+  type: object
+  default: {}
+- name: regions
+  type: object
+  default: {}
+
+steps:
+- template: /steps/engine/crud/k8s/execute-machine.yml
+  parameters:
+    cloud: ${{ parameters.cloud }}
+    pool_name: $(POOL_NAME)
+    vm_size: $(VM_SIZE)
+    scale_machine_count: $(SCALE_MACHINE_COUNT)
+    machine_workers: $(MACHINE_WORKERS)
+    use_batch_api: $(USE_BATCH_API)
+    step_time_out: $(STEP_TIME_OUT)
+    readiness_wait_timeout: $(READINESS_WAIT_TIMEOUT)
+    result_dir: $(System.DefaultWorkingDirectory)/$(RUN_ID)

steps/topology/k8s-crud-machine/validate-resources.yml

@@ -0,0 +1,13 @@
+parameters:
+- name: cloud
+  type: string
+- name: regions
+  type: object
+- name: engine
+  type: string
+
+steps:
+- template: /steps/cloud/${{ parameters.cloud }}/update-kubeconfig.yml
+  parameters:
+    role: client
+    region: ${{ parameters.regions[0] }}