promptdiff/.github/workflows/test-action.yml at main · latryee/promptdiff · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
name: Test GitHub Action

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  test-action:
    name: Integration Test Composite Action
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Run promptdiff action (Mock regression test)
        uses: ./
        with:
          v1: "examples/prompts/support_bot_v1.txt"
          v2: "examples/prompts/support_bot_v2.txt"
          inputs: "examples/testcases.jsonl"
          eval: "json_validity,latency,cost,similarity"
          mock: "true"
          export-html: "action-test-report.html"
          export-markdown: "action-test-report.md"
          export-bundle: "action-test-bundle.html"
          export-json: "action-test-report.json"
          fail-on-regression: "false"

      - name: Verify generated report artifacts
        shell: bash
        run: |
          test -f action-test-report.html
          test -f action-test-report.md
          test -f action-test-bundle.html
          test -f action-test-report.json
          echo "PromptDiff Action successfully executed and produced all expected artifacts."

      - name: Run promptdiff action with command injection attempt
        uses: ./
        with:
          v1: "examples/prompts/support_bot_v1.txt"
          v2: "examples/prompts/support_bot_v2.txt"
          inputs: "examples/testcases.jsonl"
          mock: "true"
          export-html: "safe-report.html; touch /tmp/pwned.txt"
          fail-on-regression: "false"

      - name: Verify command injection was neutralized
        shell: bash
        run: |
          if [ -f /tmp/pwned.txt ]; then
            echo "SECURITY VULNERABILITY DETECTED: Command injection succeeded!"
            exit 1
          fi
          echo "Command injection attempt was successfully neutralized."