ci: integrate VSIX packaging with pre-release into CI/CD pipeline

.github/workflows/_build.yml

@@ -1,16 +1,19 @@
 # AgentOps Toolkit — Reusable Build Workflow
 #
 # Workflows:
-#   1. ci.yml          — Lint + test on every push/PR; publish dev builds to TestPyPI on develop
-#   2. _build.yml      — Reusable build (test + package), called by staging and release
-#   3. staging.yml     — Staging: release/* branch → TestPyPI → verify
-#   4. release.yml     — Production: v* tag → TestPyPI → verify → PyPI → GitHub Release
+#   1. ci.yml          — Lint + test on every push/PR; build VSIX validation
+#   2. _build.yml      — Reusable Python build (test + package), called by staging and release
+#   3. staging.yml     — Staging: release/* → TestPyPI + VSIX pre-release
+#   4. release.yml     — Production: v* tag → PyPI + VSIX stable + GitHub Release
 #   5. cut-release.yml — Manual dispatch: create release branch + PR from develop
 #
 # Called by staging.yml and release.yml via workflow_call.
-# Runs tests, builds the package (version via setuptools-scm), and uploads
+# Runs tests, builds the Python package (version via setuptools-scm), and uploads
 # the dist/ artifacts for downstream jobs.
 #
+# Note: VSIX packaging is handled directly in ci/staging/release workflows
+# (requires Node.js + @vscode/vsce), not in this Python-focused reusable build.
+#
 # Usage in caller workflows:
 #   jobs:
 #     build:

.github/workflows/ci.yml

@@ -3,8 +3,8 @@
 # Workflows:
 #   1. ci.yml          — Lint + test on every push/PR; publish dev builds to TestPyPI on develop
 #   2. _build.yml      — Reusable build (test + package), called by staging and release
-#   3. staging.yml     — Staging: release/* branch → TestPyPI → verify
-#   4. release.yml     — Production: v* tag → TestPyPI → verify → PyPI → GitHub Release
+#   3. staging.yml     — Staging: release/* branch → TestPyPI → verify; VSIX pre-release → Marketplace
+#   4. release.yml     — Production: v* tag → TestPyPI → verify → PyPI → GitHub Release; VSIX stable → Marketplace
 #   5. cut-release.yml — Manual dispatch: create release branch + PR from develop
 
 name: CI
@@ -186,3 +186,26 @@ jobs:
           echo "- TestPyPI: https://test.pypi.org/project/agentops-toolkit/${{ steps.version.outputs.version }}/" >> "$GITHUB_STEP_SUMMARY"
           echo "" >> "$GITHUB_STEP_SUMMARY"
           echo "Install: \`pip install agentops-toolkit==${{ steps.version.outputs.version }} --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/\`" >> "$GITHUB_STEP_SUMMARY"
+
+  # Validate that the VSIX extension packages correctly
+  build-vsix:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install vsce
+        run: npm install -g @vscode/vsce
+
+      - name: Package VSIX (dry run)
+        working-directory: plugins/agentops
+        run: vsce package -o agentops-skills.vsix
+
+      - name: Show VSIX info
+        run: |
+          ls -la plugins/agentops/*.vsix
+          echo "✅ VSIX packaging validated"

.github/workflows/cut-release.yml

@@ -1,14 +1,15 @@
 # AgentOps Toolkit — Cut Release
 #
 # Workflows:
-#   1. ci.yml          — Lint + test on every push/PR
+#   1. ci.yml          — Lint + test on every push/PR; VSIX build validation
 #   2. _build.yml      — Reusable build (test + package), called by staging and release
-#   3. staging.yml     — Staging: release/* branch → TestPyPI → verify
-#   4. release.yml     — Production: v* tag → TestPyPI → verify → PyPI → GitHub Release
+#   3. staging.yml     — Staging: release/* → TestPyPI → verify; VSIX pre-release → Marketplace
+#   4. release.yml     — Production: v* tag → TestPyPI → verify → PyPI → GH Release; VSIX stable → Marketplace
 #   5. cut-release.yml — Manual dispatch: create release branch + PR from develop
 #
 # One-click release branch creation. Triggered manually from the Actions tab.
-# Creates a release branch from develop, updates CHANGELOG.md, and opens a PR to main.
+# Creates a release branch from develop, updates CHANGELOG.md, syncs the
+# VS Code extension version in package.json, and opens a PR to main.
 # The branch push then triggers staging.yml automatically.
 #
 # Usage:
@@ -72,14 +73,21 @@ jobs:
           # Replace [Unreleased] with versioned section, add fresh Unreleased above
           sed -i "s/## \[Unreleased\]/## [Unreleased]\n\n## [${{ env.version }}] - $DATE/" CHANGELOG.md
 
+      - name: Sync VS Code extension version
+        run: |
+          jq --arg v "${{ env.version }}" '.version = $v' \
+            plugins/agentops/package.json > plugins/agentops/package.json.tmp
+          mv plugins/agentops/package.json.tmp plugins/agentops/package.json
+          echo "VSIX version set to ${{ env.version }}"
+
       - name: Configure git
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
       - name: Commit and push
         run: |
-          git add CHANGELOG.md
+          git add CHANGELOG.md plugins/agentops/package.json
           git commit -m "chore: prepare release ${{ env.version }}"
           git push origin "release/v${{ env.version }}"
 
@@ -98,22 +106,24 @@ jobs:
           ### What happened
           - Branch \`release/v${{ env.version }}\` created from \`develop\`
           - \`CHANGELOG.md\` updated: \`[Unreleased]\` → \`[${{ env.version }}]\`
-          - Staging pipeline triggered automatically (build → TestPyPI → verify)
+          - \`plugins/agentops/package.json\` version synced to \`${{ env.version }}\`
+          - Staging pipeline triggered automatically (build → TestPyPI + VSIX pre-release → verify)
 
           ### Next steps
           1. Wait for the **Staging** pipeline to pass
           2. Review and approve this PR
           3. Merge to \`main\`
           4. Tag and push: \`git tag v${{ env.version }} && git push origin v${{ env.version }}\`
-          5. Approve the PyPI publish in the **Release** workflow
+          5. Approve the PyPI publish and VSIX stable publish in the **Release** workflow
           6. Sync develop: \`git checkout develop && git merge main && git push origin develop\`
 
           ### Checklist
-          - [ ] Staging pipeline passes (build + TestPyPI + verify)
+          - [ ] Staging pipeline passes (build + TestPyPI + VSIX pre-release + verify)
           - [ ] CHANGELOG entries reviewed
           - [ ] PR approved and merged to main
           - [ ] Tag \`v${{ env.version }}\` pushed
           - [ ] PyPI publish approved
+          - [ ] VSIX stable publish approved
           - [ ] develop synced from main"
 
       - name: Summary
@@ -122,6 +132,7 @@ jobs:
           echo "" >> "$GITHUB_STEP_SUMMARY"
           echo "- Branch: \`release/v${{ env.version }}\`" >> "$GITHUB_STEP_SUMMARY"
           echo "- CHANGELOG updated with version **${{ env.version }}**" >> "$GITHUB_STEP_SUMMARY"
+          echo "- VS Code extension version synced to **${{ env.version }}**" >> "$GITHUB_STEP_SUMMARY"
           echo "- PR opened: \`release/v${{ env.version }}\` → \`main\`" >> "$GITHUB_STEP_SUMMARY"
           echo "- Staging pipeline triggered automatically" >> "$GITHUB_STEP_SUMMARY"
           echo "" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/release.yml

@@ -1,33 +1,38 @@
-# AgentOps Toolkit — Production Release
+# AgentOps Toolkit — Production Release (PyPI + VSIX Stable)
 #
 # Workflows:
-#   1. ci.yml          — Lint + test on every push/PR
+#   1. ci.yml          — Lint + test on every push/PR; VSIX build validation
 #   2. _build.yml      — Reusable build (test + package), called by staging and release
-#   3. staging.yml     — Staging: release/* branch → TestPyPI → verify
-#   4. release.yml     — Production: v* tag → TestPyPI → verify → PyPI → GitHub Release
+#   3. staging.yml     — Staging: release/* → TestPyPI → verify; VSIX pre-release → Marketplace
+#   4. release.yml     — Production: v* tag → TestPyPI → verify → PyPI → GH Release; VSIX stable → Marketplace
+#   5. cut-release.yml — Manual dispatch: create release branch + PR from develop
 #
 # Triggered by v* tag pushes (e.g. v0.2.0).
 # Calls the reusable _build.yml, then publishes to TestPyPI for final
 # verification, then to PyPI (requires 'release' environment approval),
-# and finally creates a GitHub Release.
+# publishes the VS Code extension as stable to the Marketplace,
+# and finally creates a GitHub Release (with both dist and VSIX attached).
 #
 # Versioning:
 #   Uses setuptools-scm — version is derived from the git tag automatically.
 #   Tagged commit v0.2.0 → version 0.2.0. No manual version in pyproject.toml.
+#   VSIX version is managed in plugins/agentops/package.json (synced by cut-release).
 #
 # Required GitHub secrets (in respective environments):
 #   TEST_PYPI_TOKEN  — TestPyPI API token (environment: staging)
 #   PYPI_TOKEN       — PyPI API token     (environment: release)
+#   VSCE_PAT         — VS Code Marketplace PAT (environment: release)
 #
 # Required GitHub environments:
 #   staging  — for TestPyPI publish (optional approval)
-#   release  — for PyPI publish (requires approval from designated reviewers)
+#   release  — for PyPI + VSIX publish (requires approval from designated reviewers)
 #
 # Setup:
 #   1. https://test.pypi.org/manage/account/token/  → Create TEST_PYPI_TOKEN
 #   2. https://pypi.org/manage/account/token/        → Create PYPI_TOKEN
-#   3. GitHub repo → Settings → Secrets → Actions    → Add secrets to environments
-#   4. GitHub repo → Settings → Environments         → Create "release" with required reviewers
+#   3. https://dev.azure.com/ → PAT with Marketplace scope → Create VSCE_PAT
+#   4. GitHub repo → Settings → Secrets → Actions    → Add secrets to environments
+#   5. GitHub repo → Settings → Environments         → Create "release" with required reviewers
 
 name: Release
 
@@ -133,26 +138,64 @@ jobs:
           password: ${{ secrets.PYPI_TOKEN }}
           verbose: true
 
-  # Create GitHub Release with built artifacts
+  # ── VSIX Stable Publish ──────────────────────────────────────────────
+  # Publish the VS Code extension as a stable release to the Marketplace.
+  # Runs in parallel with the TestPyPI→PyPI flow (only needs source checkout).
+  publish-vsix:
+    needs: build          # gate on successful lint + test
+    runs-on: ubuntu-latest
+    environment: release  # same approval gate as PyPI
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install vsce
+        run: npm install -g @vscode/vsce
+
+      - name: Package VSIX
+        working-directory: plugins/agentops
+        run: vsce package -o agentops-skills.vsix
+
+      - name: Publish stable to VS Code Marketplace
+        working-directory: plugins/agentops
+        run: vsce publish --packagePath agentops-skills.vsix -p "${{ secrets.VSCE_PAT }}"
+
+      - name: Upload VSIX artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: vsix
+          path: plugins/agentops/agentops-skills.vsix
+
+  # Create GitHub Release with built artifacts (Python dist + VSIX)
   github-release:
-    needs: publish-pypi
+    needs: [publish-pypi, publish-vsix]
     runs-on: ubuntu-latest
     permissions:
       contents: write
     steps:
       - uses: actions/checkout@v4
 
-      - name: Download build artifacts
+      - name: Download Python dist artifacts
         uses: actions/download-artifact@v4
         with:
           name: dist
           path: dist/
 
+      - name: Download VSIX artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: vsix
+          path: vsix/
+
       - name: Create GitHub Release
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          gh release create "${{ github.ref_name }}" dist/* \
+          gh release create "${{ github.ref_name }}" dist/* vsix/* \
             --repo "${{ github.repository }}" \
             --title "${{ github.ref_name }}" \
             --generate-notes

.github/workflows/staging.yml

@@ -1,32 +1,36 @@
-# AgentOps Toolkit — Staging (TestPyPI)
+# AgentOps Toolkit — Staging (TestPyPI + VSIX Pre-release)
 #
 # Workflows:
-#   1. ci.yml          — Lint + test on every push/PR
+#   1. ci.yml          — Lint + test on every push/PR; VSIX build validation
 #   2. _build.yml      — Reusable build (test + package), called by staging and release
-#   3. staging.yml     — Staging: release/* branch → TestPyPI → verify
-#   4. release.yml     — Production: v* tag → TestPyPI → verify → PyPI → GitHub Release
+#   3. staging.yml     — Staging: release/* → TestPyPI → verify; VSIX pre-release → Marketplace
+#   4. release.yml     — Production: v* tag → TestPyPI → verify → PyPI → GH Release; VSIX stable → Marketplace
+#   5. cut-release.yml — Manual dispatch: create release branch + PR from develop
 #
 # Triggered by pushes to release/* branches.
-# Calls the reusable _build.yml, publishes to TestPyPI, and verifies the
-# package installs correctly with a CLI smoke test.
-#
-# This workflow lets you iterate on a release branch and validate the
-# built package before tagging for production.
+# Calls the reusable _build.yml, publishes to TestPyPI, verifies the
+# package installs correctly with a CLI smoke test, and publishes the
+# VS Code extension as a pre-release to the Marketplace.
 #
 # Branch flow:
 #   develop → release/v0.2.0 → push → this workflow
 #     → build → TestPyPI → verify install → ✅ ready to merge and tag
+#     → VSIX pre-release → Marketplace (early access channel)
 #
 # Versioning:
 #   Uses setuptools-scm — on a release branch 5 commits after the last tag,
 #   the version will be something like 0.2.0.dev5 (PEP 440 pre-release).
+#   VSIX version is managed in plugins/agentops/package.json.
 #
 # Required GitHub secrets (environment: staging):
 #   TEST_PYPI_TOKEN  — TestPyPI API token
+#   VSCE_PAT         — VS Code Marketplace Personal Access Token
 #
 # Setup:
 #   1. https://test.pypi.org/manage/account/token/  → Create TEST_PYPI_TOKEN
 #   2. GitHub repo → Settings → Secrets → Actions    → Add to staging environment
+#   3. https://dev.azure.com/ → PAT with Marketplace scope → Create VSCE_PAT
+#   4. Add VSCE_PAT to staging environment
 
 name: Staging
 
@@ -110,3 +114,35 @@ jobs:
           test -f .agentops/config.yaml
           test -f .agentops/run.yaml
           echo "✅ agentops init succeeded"
+
+  # ── VSIX Pre-release ─────────────────────────────────────────────────
+  # Publish the VS Code extension as a pre-release to the Marketplace.
+  # Runs in parallel with the TestPyPI flow (only needs source checkout).
+  publish-vsix-prerelease:
+    needs: build        # gate on successful lint + test
+    runs-on: ubuntu-latest
+    environment: staging
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install vsce
+        run: npm install -g @vscode/vsce
+
+      - name: Package VSIX (pre-release)
+        working-directory: plugins/agentops
+        run: vsce package --pre-release -o agentops-skills.vsix
+
+      - name: Publish pre-release to VS Code Marketplace
+        working-directory: plugins/agentops
+        run: vsce publish --pre-release --packagePath agentops-skills.vsix -p "${{ secrets.VSCE_PAT }}"
+
+      - name: Show VSIX info
+        working-directory: plugins/agentops
+        run: |
+          ls -lh agentops-skills.vsix
+          echo "✅ VSIX pre-release published to Marketplace"

plugins/agentops/.vscodeignore

@@ -0,0 +1,11 @@
+.git
+node_modules
+*.vsix
+
+# Keep these (explicit include after exclude)
+!README.md
+!CHANGELOG.md
+!skills/**/SKILL.md
+!package.json
+!LICENSE
+!icon.png

plugins/agentops/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+All notable changes to the **AgentOps Skills for GitHub Copilot** extension
+will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/).
+
+## [Unreleased]
+
+## [0.1.0] - 2025-07-08
+
+### Added
+
+- Initial pre-release with six Copilot agent skills:
+  - **Workspace Setup** — initialize `.agentops/`, create configs, manage bundles and datasets
+  - **Run Evals** — execute evaluations, multi-model benchmarks, N-run comparisons
+  - **Investigate Regression** — compare runs, analyze row-level scores, root-cause regressions
+  - **Observability & Triage** — OTLP tracing setup, interpret evaluation outputs
+  - **Browse & Inspect** — list/inspect runs, view per-row scores, browse history
+  - **Dataset Management** — validate, describe, and import datasets

plugins/agentops/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Microsoft Corporation.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.