-
Notifications
You must be signed in to change notification settings - Fork 115
ama-logs deployment and test in dev clusters #1625
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: ci_prod
Are you sure you want to change the base?
Changes from all commits
f404bad
8c58fc6
bf2c196
c8f57c1
94f7b2e
63ed3d6
cadf2d6
02fb781
1b91fe8
fd8b687
8cd773f
5cabe38
2eb48ba
ecd4a3b
e615f10
d293b23
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,6 +43,7 @@ | |
| - ES365AIMigrationTooling | ||
| stages: | ||
| - stage: stage | ||
| displayName: 'Build and Publish Container Images' | ||
| jobs: | ||
| - job: common | ||
| pool: | ||
|
|
@@ -276,7 +277,8 @@ | |
| export TRIVY_JAVA_DB_REPOSITORY=$PRIMARY_TRIVY_JAVA_DB_REPOSITORY | ||
| # Function to run Trivy scan and handle output | ||
| run_trivy_scan() { | ||
| trivy image --exit-code 1 --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM "${{ variables.repoImageName }}:$(linuxImagetag)" > trivy_output.log 2>&1 | ||
| #trivy image --exit-code 1 --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM "${{ variables.repoImageName }}:$(linuxImagetag)" > trivy_output.log 2>&1 | ||
| trivy image --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM "${{ variables.repoImageName }}:$(linuxImagetag)" > trivy_output.log 2>&1 | ||
| return $? | ||
| } | ||
| # Attempt scan up to 5 times with repository fallback | ||
|
|
@@ -881,3 +883,74 @@ | |
| FileDirPath: '$(Build.ArtifactStagingDirectory)' | ||
| DisableRemediation: false | ||
| AcceptableOutdatedSignatureInHours: 72 | ||
|
|
||
| - stage: Deploy_and_Test_Images_In_Dev_Clusters | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. two steps for each cluster:
|
||
| displayName: Deploy and Test Images in Dev Clusters | ||
| lockBehavior: sequential | ||
| dependsOn: | ||
| - stage | ||
| condition: | | ||
| eq(dependencies.stage.result, 'Succeeded') | ||
| variables: | ||
| # Override the helm chart's default image repository (/azuremonitor/containerinsights/ciprod) to use cidev | ||
| ImageRepositoryOverride: '/azuremonitor/containerinsights/cidev' | ||
| # Use image tags built from the previous build stage | ||
| linuxImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.linuxImagetag']] | ||
| windowsImageTagUnderTest: $[stageDependencies.stage.common.outputs['setup.windowsImageTag']] | ||
| jobs: | ||
| # TODO: remomve the two temp cluster and add more clusters from test automation framework when the tests are stable | ||
Check noticeCode scanning / devskim A "TODO" or similar was left in source code, possibly indicating incomplete functionality Note
Suspicious comment
|
||
| # ============================================================ | ||
| # Cluster 1: zane-test — Deploy via Helm | ||
| # ============================================================ | ||
| - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self | ||
| parameters: | ||
| clusterName: 'zane-test' | ||
| resourceGroup: 'zane-test' | ||
| region: 'westus2' | ||
| subscriptionId: $(CI_BUILD_SUB_ID) | ||
| workspaceId: $(ZANE_TEST_LA2_LAW_ID) # zane-test-la2 | ||
| amalogsLinuxImage: $(linuxImageTagUnderTest) | ||
| amalogsWindowsImage: $(windowsImageTagUnderTest) | ||
| imageRepository: $(ImageRepositoryOverride) | ||
| environment: 'CI-Agent-Dev' | ||
| azureSubscription: 'ContainerInsights_Build_Subscription_CI' | ||
|
|
||
| # Cluster 1: zane-test — Run E2E Tests | ||
| - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self | ||
| parameters: | ||
| clusterName: 'zane-test' | ||
| resourceGroup: 'zane-test' | ||
| azureSubscription: 'ContainerInsights_Build_Subscription_CI' | ||
| environmentName: 'CI-Agent-Dev' | ||
| dependsOnDeployJob: 'Deploy_AmaLogs_zane_test' | ||
| azureClientId: $(ZANE_TEST_CLIENT_ID) | ||
| azureTenantId: $(CI_BUILD_AZURE_TENANT_ID) | ||
| teamsWebhookUri: $(TeamsWebhookUri) | ||
|
|
||
| # ============================================================ | ||
| # Cluster 2: zane-test2 — Deploy via Helm | ||
| # ============================================================ | ||
| - template: /.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self | ||
| parameters: | ||
| clusterName: 'zane-test2' | ||
| resourceGroup: 'zane-test' | ||
| region: 'centralus' | ||
| subscriptionId: $(CI_BUILD_SUB_ID) | ||
| workspaceId: $(ZANE_TEST_LA2_LAW_ID) # zane-test2-la2 | ||
| amalogsLinuxImage: $(linuxImageTagUnderTest) | ||
| amalogsWindowsImage: $(windowsImageTagUnderTest) | ||
| imageRepository: $(ImageRepositoryOverride) | ||
| environment: 'CI-Agent-Dev2' | ||
| azureSubscription: 'ContainerInsights_Build_Subscription_CI' | ||
|
|
||
| # Cluster 2: zane-test2 — Run E2E Tests | ||
| - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self | ||
| parameters: | ||
| clusterName: 'zane-test2' | ||
| resourceGroup: 'zane-test' | ||
| azureSubscription: 'ContainerInsights_Build_Subscription_CI' | ||
| environmentName: 'CI-Agent-Dev2' | ||
| dependsOnDeployJob: 'Deploy_AmaLogs_zane_test2' | ||
| azureClientId: $(ZANE_TEST2_CLIENT_ID) | ||
| azureTenantId: $(CI_BUILD_AZURE_TENANT_ID) | ||
| teamsWebhookUri: $(TeamsWebhookUri) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| parameters: | ||
| - name: clusterName | ||
| type: string | ||
| - name: resourceGroup | ||
| type: string | ||
| - name: azureSubscription | ||
| type: string | ||
| default: 'ContainerInsights_Build_Subscription_CI' | ||
| - name: environmentName | ||
| type: string | ||
| - name: dependsOnDeployJob | ||
| type: string | ||
| displayName: 'Name of the deploy job this test depends on (e.g., Deploy_AmaLogs_zane_test)' | ||
| - name: azureClientId | ||
| type: string | ||
| - name: azureTenantId | ||
| type: string | ||
| - name: teamsWebhookUri | ||
| type: string | ||
| default: '$(TeamsWebhookUri)' | ||
| - name: additionalTestParams | ||
| type: string | ||
| default: '' | ||
|
|
||
| jobs: | ||
| - deployment: Test_${{ replace(parameters.clusterName, '-', '_') }} | ||
| displayName: 'Test: ${{ parameters.clusterName }}' | ||
| environment: ${{ parameters.environmentName }} | ||
| dependsOn: ${{ parameters.dependsOnDeployJob }} | ||
| pool: | ||
| name: Azure-Pipelines-CI-Test-EO | ||
| image: ci-1es-managed-ubuntu-2204 | ||
| os: linux | ||
| variables: | ||
| skipComponentGovernanceDetection: true | ||
| strategy: | ||
| runOnce: | ||
| deploy: | ||
| steps: | ||
| - checkout: self | ||
| persistCredentials: true | ||
|
|
||
| - script: | | ||
| set -euo pipefail | ||
| echo "Ensuring kubectl is installed" | ||
| if ! command -v kubectl >/dev/null 2>&1; then | ||
| echo "Installing kubectl" | ||
| sudo az aks install-cli | ||
| else | ||
| echo "kubectl already installed: $(kubectl version --client --short || true)" | ||
| fi | ||
| displayName: 'Install kubectl' | ||
|
|
||
| - task: AzureCLI@2 | ||
| displayName: 'Get credentials for ${{ parameters.clusterName }}' | ||
| inputs: | ||
| azureSubscription: ${{ parameters.azureSubscription }} | ||
| scriptLocation: 'inlineScript' | ||
| scriptType: 'bash' | ||
| inlineScript: 'az aks get-credentials -g ${{ parameters.resourceGroup }} -n ${{ parameters.clusterName }}' | ||
|
|
||
| - task: Bash@3 | ||
| displayName: 'Wait for logs to be ingested into Log Analytics (20 min)' | ||
| inputs: | ||
| targetType: 'inline' | ||
| script: | | ||
| echo "========================================" | ||
| echo "Waiting for Log Analytics Ingestion" | ||
| echo "========================================" | ||
| echo "Cluster: ${{ parameters.clusterName }}" | ||
| echo "" | ||
| echo "Waiting 10 minutes to allow logs to be ingested..." | ||
| echo "This ensures queries will find logs from the newly deployed containers." | ||
| echo "" | ||
|
|
||
| wait_time=600 | ||
| interval=60 | ||
| elapsed=0 | ||
|
|
||
| while [ $elapsed -lt $wait_time ]; do | ||
| remaining=$((wait_time - elapsed)) | ||
| minutes_elapsed=$((elapsed / 60)) | ||
| minutes_remaining=$((remaining / 60)) | ||
| echo "⏳ Waiting... ($minutes_elapsed/$((wait_time / 60)) minutes elapsed, $minutes_remaining minutes remaining)" | ||
| sleep $interval | ||
| elapsed=$((elapsed + interval)) | ||
| done | ||
|
|
||
| echo "" | ||
| echo "✓ Wait complete! Logs should now be available in Log Analytics." | ||
| echo "========================================" | ||
|
|
||
| - bash: | | ||
| echo "Running tests for cluster: ${{ parameters.clusterName }}" | ||
|
|
||
| chmod +x ./install-and-execute-testkube-tests.sh | ||
| ./install-and-execute-testkube-tests.sh \ | ||
| AzureClientId=${{ parameters.azureClientId }} \ | ||
| AzureTenantId=${{ parameters.azureTenantId }} \ | ||
| TeamsWebhookUri=${{ parameters.teamsWebhookUri }} \ | ||
| ${{ parameters.additionalTestParams }} | ||
| workingDirectory: $(Build.SourcesDirectory)/test/testkube/ | ||
| displayName: 'Install Testkube and run E2E tests' | ||
|
|
||
| # Log completion | ||
| - bash: | | ||
| echo "=========================================" | ||
| echo "TEST COMPLETE" | ||
| echo "=========================================" | ||
| echo "Cluster: ${{ parameters.clusterName }}" | ||
| echo "Build ID: $(Build.BuildId)" | ||
| echo "✓ Testing finished for: ${{ parameters.clusterName }}" | ||
| echo "=========================================" | ||
| displayName: 'Test Completion' | ||
| condition: always() |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,10 +4,11 @@ | |
| # parameters: | ||
| # clusterName: 'my-cluster' | ||
| # resourceGroup: 'my-rg' | ||
| # region: 'eastus' | ||
| # subscriptionId: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb' | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. do not want to expose sub id, so remove it. |
||
| # region: e.g. 'eastus' | ||
| # subscriptionId: 'your-subscription-id' | ||
| # workspaceId: 'your-workspace-id' | ||
| # imageTag: '$(AgentImageTagSuffix)' # e.g., 3.1.32 | ||
| # amalogsLinuxImage: "ama-logs linux image tag" | ||
| # amalogsWindowsImage: "ama-logs windows image tag" | ||
| # environment: 'deployment environment' | ||
|
|
||
| parameters: | ||
|
|
@@ -23,9 +24,16 @@ parameters: | |
| - name: workspaceId | ||
| type: string | ||
| displayName: 'Log Analytics Workspace ID' | ||
| - name: imageTag | ||
| - name: amalogsLinuxImage | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. separate win and linux image as build pipeline outputs these two vars separately. note: due to this change, the ci-aks-prod-release.yaml is also changed to accommodate this new. |
||
| type: string | ||
| displayName: 'Image tag suffix (e.g., 3.1.32)' | ||
| - name: amalogsWindowsImage | ||
| type: string | ||
| displayName: 'Image tag suffix (e.g., win-3.1.32)' | ||
| - name: imageRepository | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. add imageRepository, because in build pipeline, we need use cidev, and in prod release pipeline, we need use ciprod |
||
| type: string | ||
| default: '/azuremonitor/containerinsights/ciprod' | ||
| displayName: 'Image repository path (e.g., /azuremonitor/containerinsights/ciprod or /azuremonitor/containerinsights/cidev)' | ||
| - name: environment | ||
| type: string | ||
| displayName: 'Azure DevOps Environment name' | ||
|
|
@@ -34,9 +42,6 @@ parameters: | |
| - name: cloudEnvironment | ||
| type: string | ||
| default: 'azurepubliccloud' | ||
| - name: kubernetesVersion | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. unused and useless, removed. |
||
| type: string | ||
| default: '1.32.7' | ||
| - name: azureSubscription | ||
| type: string | ||
| default: 'ContainerInsights_Build_Subscription_CI' | ||
|
|
@@ -46,9 +51,6 @@ parameters: | |
| - name: releaseName | ||
| type: string | ||
| default: 'azuremonitor-containers' | ||
| - name: helmVersion | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. remove version pin, always use latest verison. |
||
| type: string | ||
| default: '3.12.3' | ||
| - name: dependsOn | ||
| type: object | ||
| default: [] | ||
|
|
@@ -83,7 +85,7 @@ jobs: | |
| - task: HelmInstaller@1 | ||
| displayName: Install Helm | ||
| inputs: | ||
| helmVersionToInstall: '${{ parameters.helmVersion }}' | ||
| helmVersionToInstall: 'latest' | ||
| - task: HelmDeploy@0 | ||
| displayName: 'Helm Deploy: ama-logs to ${{ parameters.clusterName }}' | ||
| inputs: | ||
|
|
@@ -99,7 +101,7 @@ jobs: | |
| # TODO: When it is merged to ci_prod, the following chartPath will be updated accordingly. | ||
| chartPath: '$(Build.SourcesDirectory)/charts/azuremonitor-containerinsights-for-prod-clusters' | ||
| releaseName: '${{ parameters.releaseName }}' | ||
| overrideValues: 'global.commonGlobals.CloudEnvironment=${{ parameters.cloudEnvironment }},global.commonGlobals.Region=${{ parameters.region }},OmsAgent.aksResourceID=$(AKS_RESOURCE_ID),OmsAgent.workspaceID=${{ parameters.workspaceId }},OmsAgent.imageTagLinux=${{ parameters.imageTag }},OmsAgent.imageTagWindows=win-${{ parameters.imageTag }}' | ||
| overrideValues: 'global.commonGlobals.CloudEnvironment=${{ parameters.cloudEnvironment }},global.commonGlobals.Region=${{ parameters.region }},OmsAgent.aksResourceID=$(AKS_RESOURCE_ID),OmsAgent.workspaceID=${{ parameters.workspaceId }},OmsAgent.imageRepository=${{ parameters.imageRepository }},OmsAgent.imageTagLinux=${{ parameters.amalogsLinuxImage }},OmsAgent.imageTagWindows=${{ parameters.amalogsWindowsImage }}' | ||
| waitForExecution: false | ||
| arguments: '--timeout 10m --install' | ||
| - task: AzureCLI@2 | ||
|
|
@@ -115,8 +117,8 @@ jobs: | |
| echo "Cluster: ${{ parameters.clusterName }}" | ||
| echo "Resource Group: ${{ parameters.resourceGroup }}" | ||
| echo "Region: ${{ parameters.region }}" | ||
| echo "Linux Image Tag: ${{ parameters.imageTag }}" | ||
| echo "Windows Image Tag: win-${{ parameters.imageTag }}" | ||
| echo "Linux Image Tag: ${{ parameters.amalogsLinuxImage }}" | ||
| echo "Windows Image Tag: ${{ parameters.amalogsWindowsImage }}" | ||
| echo "" | ||
|
|
||
| echo "Getting AKS credentials..." | ||
|
|
@@ -151,8 +153,8 @@ jobs: | |
| echo "==========================================" | ||
| echo "Image Verification" | ||
| echo "==========================================" | ||
| EXPECTED_LINUX_TAG="${{ parameters.imageTag }}" | ||
| EXPECTED_WINDOWS_TAG="win-${{ parameters.imageTag }}" | ||
| EXPECTED_LINUX_TAG="${{ parameters.amalogsLinuxImage }}" | ||
| EXPECTED_WINDOWS_TAG="${{ parameters.amalogsWindowsImage }}" | ||
| VERIFICATION_PASSED=true | ||
|
|
||
| echo "Expected Linux image tag: $EXPECTED_LINUX_TAG" | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -34,6 +34,7 @@ OmsAgent: | |
| workspaceKey: "<your_workspace_key>" | ||
|
|
||
| # Image configuration | ||
| imageRepository: "/azuremonitor/containerinsights/ciprod" | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. add this allow the helm chart can be used for both build pipeline and prod release pipelines. |
||
| imageTagLinux: <image_to_be_deployed_for_linux> | ||
| imageTagWindows: <image_to_be_deployed_for_windows> | ||
| isImagePullPolicyAlways: false | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
temporarily remove trivy failure and let test can run. helm deployment and tests depend on successful image build.