diff --git a/.github/workflows/reusable.yml b/.github/workflows/reusable.yml index acd1496..14e990f 100644 --- a/.github/workflows/reusable.yml +++ b/.github/workflows/reusable.yml @@ -121,11 +121,27 @@ jobs: ARGS="$ARGS --json-output yamlspec-results.json" ARGS="$ARGS --emd-output yamlspec-results.emd.md" ARGS="$ARGS --junit-output yamlspec-results.xml" + # GITHUB_ACTIONS=true auto-enables --github-annotations via env binding, + # but we pass it explicitly for clarity in the run log. + ARGS="$ARGS --github-annotations" ARGS="$ARGS ${{ inputs.extra-args }}" echo "Running: yamlspec $ARGS" yamlspec $ARGS && echo "success=true" >> "$GITHUB_OUTPUT" || echo "success=false" >> "$GITHUB_OUTPUT" + # --- Step summary (visible on the workflow run page) --- + + - name: Write results to step summary + if: always() + run: | + if [ -f yamlspec-results.emd.md ]; then + cat yamlspec-results.emd.md >> "$GITHUB_STEP_SUMMARY" + else + echo "## yamlspec" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo ":warning: Results file not produced — check the run logs." >> "$GITHUB_STEP_SUMMARY" + fi + # --- Upload artifacts --- - name: Upload test results diff --git a/.structlint.yaml b/.structlint.yaml index fb533f5..2197223 100644 --- a/.structlint.yaml +++ b/.structlint.yaml @@ -33,6 +33,8 @@ file_naming_pattern: - "*.md" - "*.xml" - "*.txt" + - "*.gif" + - "*.cast" - "Makefile" - ".gitignore" - ".goreleaser.yml" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8fc5c00 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,81 @@ +# Changelog + +All notable changes to yamlspec are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- `docs/index.md` documentation landing page with cross-links between docs. +- `docs/recipes.md` cookbook of common assertion patterns + (multi-environment, security baseline, image pinning, label conventions, + HPA/PDB invariants, NetworkPolicy, etc.). +- `docs/troubleshooting.md` with the common authoring/runtime errors and how + to read them. +- `examples/README.md` describing each worked example and the exact command + to run it. +- `CONTRIBUTING.md` covering dev setup, layer conventions, how to add an + operator/formatter/command, commit conventions, and the release flow. +- README field-path quick reference (dotted, `[index]`, `[*]` wildcard, + bracket notation, leading-dot JQ form) and cross-links to the docs. +- GitHub Actions reusable workflow now writes the EMD output to + `$GITHUB_STEP_SUMMARY` so results show on the workflow run page, not only + as a PR comment. +- `--github-annotations` output mode (auto-enabled when `GITHUB_ACTIONS=true`) + emits `::error file=...,line=...::` lines so failing assertions appear + inline in the GitHub "Files changed" diff view. + +### Changed +- README install instructions now reflect that no GitHub release exists yet — + use `go install` from source until v0.1.0 is cut. + +## [0.1.0] — Initial release + +The first usable yamlspec build, including a correctness-hardening pass. + +### Added +- RSpec-like `describe`/`it`/`should` test syntax in `spec.yaml`. +- 22 assertion operators: equality (`toEqual`, `toNotEqual`), numeric + comparison (`toBeGreaterThan`/`Less`/`OrEqual`), string + (`toContain`/`Start`/`End`/`Match` and negations), existence (`toExist`, + `toBeNull`), object (`toHaveKey`), set membership + (`toBeOneOf`/`toNotBeOneOf`), array (`toContainItem`, `toHaveLength`, + `toHaveMinLength`, `toHaveMaxLength`). +- Wildcard array iteration in field paths + (`spec.containers[*].image`) — assertion runs against every element. +- Distinction between "field missing" and "field is null" for + `toExist`/`toBeNull` semantics. +- Six output formats: console (colored RSpec-style tree), JSON, YAML, + Markdown, enriched Markdown (collapsible `
` for PR comments), + JUnit XML. +- Parallel execution via `--workers N`. +- `--fail-fast` for fast feedback on sequential runs. +- Configurable `--pre-run-timeout` (default 60s) for `pre_run` shell + commands. +- Tag filtering via repeatable `--tag` flag. +- Strict YAML decoding for `spec.yaml` — typos in field names are rejected + rather than silently ignored. +- Comprehensive spec validation: empty `describe`, missing `should`/`expect`, + assertions with no operator are caught at parse time. +- `pre_run` execution kills the entire process group on timeout — orphaned + `sleep`-style children no longer keep the command alive past its deadline. +- Cross-platform process management (Unix `setpgid` + `kill -PGID`; Windows + fallback). +- Reusable GitHub Actions workflow with one-line opt-in via + `uses: AxeForging/yamlspec/.github/workflows/reusable.yml@main`. Posts + enriched-markdown PR comments and updates them in place on subsequent + pushes. +- `init` command to scaffold new specs. +- `list` / `list --tags` for spec/tag discovery (deterministic ordering). +- `ai-help` command emitting a comprehensive reference for AI assistants. +- Built on Go 1.25.8 (covers the recent stdlib CVE). + +### Validation +- 118 tests passing across 6 packages. +- Examples for plain manifests, Helm charts, Kustomize overlays, + multi-resource specs, and a security baseline. + +[Unreleased]: https://github.com/AxeForging/yamlspec/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/AxeForging/yamlspec/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4458767 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,171 @@ +# Contributing + +Thanks for considering a contribution. yamlspec aims to stay small, fast, and +opinionated — pull requests that keep it that way are very welcome. + +## Dev setup + +Requirements: + +- Go 1.25+ (the minimum tracks the upstream stdlib security floor) +- `golangci-lint` for `make lint` +- [`lefthook`](https://github.com/evilmartians/lefthook) for pre-commit/pre-push hooks (optional but recommended) +- `helm` and/or `kustomize` if you want to run those examples locally + +```bash +git clone git@github.com:AxeForging/yamlspec.git +cd yamlspec +go mod download +make build-local # produces ./yamlspec +make test # runs unit + integration tests +``` + +Install hooks (optional): + +```bash +lefthook install +``` + +This wires up `gofmt`, `go vet`, `golangci-lint`, `structlint`, and +conventional-commits message validation on `pre-commit`, plus `govulncheck` on +`pre-push`. + +## Project layout + +``` +yamlspec/ +├── main.go # CLI entry point (urfave/cli v1) +├── flags.go # CLI flag definitions +├── actions/ # Command handlers (validate, list, init, version, ai-help) +├── domain/ # Models (Spec, Results, Config) — no business logic +├── services/ # Business logic +│ ├── assertion.go # Assertion engine (operator evaluation, field paths) +│ ├── discovery.go # Spec file discovery + strict YAML decode +│ ├── runner.go # Test execution (sequential + parallel) +│ ├── runner_unix.go # Unix process-group handling for pre_run timeouts +│ ├── runner_windows.go +│ ├── formatter.go # Formatter interface +│ └── fmt_*.go # Output formatters +├── helpers/ # Utilities (errors, logger, terminal detection) +├── integration/ # E2E tests with testdata fixtures +├── examples/ # Worked examples +└── docs/ # Public-facing documentation +``` + +Keep the layers honest: `actions/` orchestrates, `services/` does the work, +`domain/` is data only. New cross-cutting code generally belongs in +`services/`. + +## Common contribution patterns + +### Adding a new assertion operator + +1. Add the field to `domain.Assertion` in `domain/models.go`. Pointer types + (`*float64`, `*int`, `*bool`) for "not set" / "set to zero" disambiguation. +2. If it's value-based, add it to `Assertion.HasValueOperators()`. If it can + stand alone (like `toExist` / `toBeNull`), wire it into `HasAnyOperator()`. +3. Add the evaluation logic to `services/assertion.go` `evaluateOperators()` — + each operator gets its own short helper. +4. Add unit tests in `services/assertion_test.go`. Cover: passing case, + failing case, type mismatch, missing field. +5. Update [docs/spec-format.md](docs/spec-format.md) and the README operator + table. + +### Adding a new output format + +1. Create `services/fmt_.go` implementing the `Formatter` interface + (single method: `Format(*SuiteResult) ([]byte, error)`). +2. Add a constructor `NewXFormatter()`. +3. Wire it into the `formatters` map in `actions/validate.go`. +4. Add the CLI flag in `flags.go` and reference it from `main.go`. +5. Add an integration test in `integration/`. + +### Adding a new CLI command + +1. Create `actions/.go` with a struct + `Execute(c *cli.Context)` + method. +2. Add a constructor `NewXAction()`. +3. Wire it into the `app.Commands` slice in `main.go`. + +## Testing + +```bash +make test # everything (unit + integration) +make test-unit # services/ only — fast +make test-e2e # integration/ — builds the binary first +make test-coverage # writes coverage.html +``` + +Integration tests live in `integration/` and use `testdata/` fixtures. They +build the binary from source and run it as a subprocess against real spec +files, so they catch regressions in the actual CLI surface, not just the +library code. + +## Linting and formatting + +```bash +make lint # golangci-lint +gofmt -l -w . # auto-format (or let lefthook do it) +``` + +Pre-commit hooks (via `lefthook install`) catch these automatically. + +## Commit conventions + +Conventional Commits are enforced on the commit message. Format: + +``` +[optional scope][!]: +``` + +Allowed types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore`, +`build`, `ci`, `perf`, `revert`. + +Examples: + +``` +feat: add toContainItem operator +fix(assertion): handle nil values in toEqual +docs: add troubleshooting guide +test: add integration tests for tag filtering +``` + +Use `!` after the type/scope to mark a breaking change: + +``` +feat!: rename --test-dir to --suite-dir +``` + +## Pull requests + +- Keep changes focused. One feature or fix per PR. +- Update tests. New code without tests will not be merged. +- Update docs. If you change user-facing behavior, update the README, the + relevant `docs/*.md` page, and `CHANGELOG.md` under `## [Unreleased]`. +- The CI workflow (`.github/workflows/pr.yml`) runs lint, tests, and a + govulncheck. All must pass. + +## Releasing + +Releases are cut via the workflow_dispatch `release` workflow: + +1. Land all changes on `main`. Make sure `CHANGELOG.md` has a populated + `## [Unreleased]` section. +2. Run the **Release** workflow from the GitHub Actions tab. Optional `tag` + input — if omitted, the patch version auto-bumps. +3. The workflow runs tests, creates the tag, and runs GoReleaser, which + publishes platform binaries (`linux/darwin/windows × amd64/arm64`) and a + checksums file. +4. After the release lands, move the `## [Unreleased]` entries under a new + `## [vX.Y.Z]` heading in `CHANGELOG.md` and commit. + +GoReleaser config lives in `.goreleaser.yml`. + +## Code of conduct + +Be respectful, give constructive feedback, assume good intent. Standard stuff. + +## Questions? + +Open an issue, or start a draft PR with a question — early feedback beats a +finished PR going the wrong direction. diff --git a/README.md b/README.md index dc7c0db..c7cdd66 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,16 @@ # yamlspec +[![CI](https://github.com/AxeForging/yamlspec/actions/workflows/pr.yml/badge.svg)](https://github.com/AxeForging/yamlspec/actions/workflows/pr.yml) +[![Go Reference](https://pkg.go.dev/badge/github.com/AxeForging/yamlspec.svg)](https://pkg.go.dev/github.com/AxeForging/yamlspec) +[![Go Report Card](https://goreportcard.com/badge/github.com/AxeForging/yamlspec)](https://goreportcard.com/report/github.com/AxeForging/yamlspec) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) + YAML test framework with RSpec-like assertions. Validate any YAML manifests — Kubernetes, Helm, Kustomize, or plain files — with a clean, readable syntax. +![yamlspec demo](docs/demo.gif) + +**Documentation:** [docs index](docs/index.md) · [spec.yaml format](docs/spec-format.md) · [recipes](docs/recipes.md) · [troubleshooting](docs/troubleshooting.md) · [CI workflow](docs/reusable-workflow.md) + ## Why yamlspec? - **RSpec-like syntax** — `describe`/`it`/`should` vocabulary developers already know @@ -14,15 +23,28 @@ YAML test framework with RSpec-like assertions. Validate any YAML manifests — ## Install +No tagged release yet — install from source: + ```bash -# From source -go install github.com/AxeForging/yamlspec@latest +# Latest commit on main +go install github.com/AxeForging/yamlspec@main + +# Or clone and build +git clone https://github.com/AxeForging/yamlspec.git +cd yamlspec +make install # builds and installs to /usr/local/bin +``` + +Once v0.1.0 is published you'll also be able to use: -# Or download a release +```bash +# Direct download (post-release) curl -sSL https://github.com/AxeForging/yamlspec/releases/latest/download/yamlspec-linux-amd64.tar.gz | tar xz sudo mv yamlspec /usr/local/bin/ ``` +See [CHANGELOG.md](CHANGELOG.md) for release status. + ## Quick Start ```bash @@ -42,6 +64,36 @@ yamlspec validate --json-output results.json --junit-output results.xml yamlspec validate --workers 4 ``` +## Sample output + +``` + ✓ Production deployment + Deployment configuration + [select: select(.kind == "Deployment")] + ✓ have 3 replicas + ✓ use a pinned image tag + ✓ have resource limits + ✓ be in production namespace + + ✗ Staging overlay + Deployment + [select: select(.kind == "Deployment")] + ✓ be in staging namespace + ✗ have 2 replicas + +Failures: + + 1) Staging overlay > Deployment > have 2 replicas + expected 2, got 1 + +26 assertions, 25 passed, 1 failed +Finished in 0.18s +``` + +In CI, the same run also produces a JUnit XML, an enriched-Markdown summary +(rendered as a GitHub PR comment and on the workflow run page), and inline +`::error file=...,line=...::` annotations against `spec.yaml` lines. + ## Usage with Helm Charts Use `pre_run` to render templates before validation: @@ -108,6 +160,8 @@ tests/my-feature/ ## spec.yaml +Full schema reference: [docs/spec-format.md](docs/spec-format.md). + Tests are defined in `spec.yaml` files with an RSpec-like vocabulary: ```yaml @@ -150,6 +204,20 @@ tests/ values.yaml # Optional: Helm values for pre_run ``` +## Field paths + +The `expect:` field accepts these syntaxes — mix freely: + +```yaml +expect: spec.replicas # dotted path +expect: spec.template.spec.containers[0].image # array index +expect: spec.template.spec.containers[*].image # wildcard — assertion runs against every element +expect: metadata.labels["app.kubernetes.io/name"] # bracket notation for keys with dots/special chars +expect: .spec.replicas # leading dot (JQ-style) also works +``` + +Wildcards (`[*]`) iterate every element — useful for "every container must have resource limits" style checks. See [docs/spec-format.md](docs/spec-format.md#field-path-syntax) for the full reference. + ## Assertion Operators | Operator | Example | Description | @@ -212,12 +280,15 @@ VALIDATE FLAGS: --tag, -t Filter by tag (repeatable) --workers, -w Parallel workers (default: 1) --fail-fast Stop on first failure + --pre-run-timeout Max duration for each pre_run command (default: 60s) --quiet, -q Summary only --json-output JSON output file --yaml-output YAML output file --markdown-output Markdown output file --emd-output Enriched markdown output file --junit-output JUnit XML output file + --github-annotations Emit ::error file=...,line=... for failing assertions + (auto-enabled when GITHUB_ACTIONS=true) ``` ## CI/CD — Reusable Workflow @@ -241,8 +312,14 @@ jobs: This installs yamlspec, runs your specs, generates JSON/EMD/JUnit artifacts, and posts a collapsible results comment on PRs. See [docs/reusable-workflow.md](docs/reusable-workflow.md) for all options. +## Examples + +Five runnable examples live under [`examples/`](examples) — see the [examples README](examples/README.md) for what each demonstrates and the exact command to run it. + ## Development +See [CONTRIBUTING.md](CONTRIBUTING.md) for the full guide. Quick reference: + ```bash make build-local # Build binary make test # Run all tests @@ -252,6 +329,8 @@ make lint # Linter make install # Install to /usr/local/bin ``` +Release history is in [CHANGELOG.md](CHANGELOG.md). + ## License MIT diff --git a/actions/validate.go b/actions/validate.go index 7c3da8b..e4d9ab5 100644 --- a/actions/validate.go +++ b/actions/validate.go @@ -27,18 +27,19 @@ func NewValidateAction() *ValidateAction { // Execute runs test specs func (a *ValidateAction) Execute(c *cli.Context) error { config := &domain.Config{ - TestDir: c.String("test-dir"), - Tags: c.StringSlice("tag"), - Workers: c.Int("workers"), - FailFast: c.Bool("fail-fast"), - PreRunTimeout: c.Duration("pre-run-timeout"), - Verbose: c.GlobalBool("verbose"), - Quiet: c.Bool("quiet"), - JSONOutput: c.String("json-output"), - YAMLOutput: c.String("yaml-output"), - MarkdownOutput: c.String("markdown-output"), - EMDOutput: c.String("emd-output"), - JUnitOutput: c.String("junit-output"), + TestDir: c.String("test-dir"), + Tags: c.StringSlice("tag"), + Workers: c.Int("workers"), + FailFast: c.Bool("fail-fast"), + PreRunTimeout: c.Duration("pre-run-timeout"), + Verbose: c.GlobalBool("verbose"), + Quiet: c.Bool("quiet"), + JSONOutput: c.String("json-output"), + YAMLOutput: c.String("yaml-output"), + MarkdownOutput: c.String("markdown-output"), + EMDOutput: c.String("emd-output"), + JUnitOutput: c.String("junit-output"), + GitHubAnnotations: c.Bool("github-annotations"), } if config.FailFast && config.Workers > 1 { @@ -86,6 +87,16 @@ func (a *ValidateAction) Execute(c *cli.Context) error { } } + // GitHub Actions annotations on stdout — picked up by the runner and + // rendered inline in the PR diff view. + if config.GitHubAnnotations { + gha := services.NewGitHubAnnotationsFormatter() + out, err := gha.Format(result) + if err == nil && len(out) > 0 { + fmt.Print(string(out)) + } + } + if !result.Summary.Success { return cli.NewExitError("", 1) } diff --git a/docs/demo.cast b/docs/demo.cast new file mode 100644 index 0000000..e850404 --- /dev/null +++ b/docs/demo.cast @@ -0,0 +1,244 @@ +{"version": 2, "width": 100, "height": 28, "timestamp": 1778092906, "idle_time_limit": 3.0, "env": {"SHELL": "/usr/bin/zsh", "TERM": "tmux-256color"}, "title": "yamlspec demo"} +[0.017382, "o", "\u001b[H\u001b[J\u001b[3J"] +[0.519245, "o", "\u001b[2;36m# yamlspec — RSpec-like assertions for any YAML\u001b[0m\r\n"] +[2.021317, "o", "\u001b[2;36m# Define what 'good' looks like in spec.yaml:\u001b[0m\r\n"] +[2.923289, "o", "\u001b[1;32m$\u001b[0m c"] +[2.965548, "o", "a"] +[3.008012, "o", "t"] +[3.050562, "o", " "] +[3.102221, "o", "t"] +[3.154026, "o", "e"] +[3.20557, "o", "s"] +[3.257781, "o", "t"] +[3.30949, "o", "s"] +[3.361744, "o", "/"] +[3.413876, "o", "p"] +[3.465519, "o", "r"] +[3.51744, "o", "o"] +[3.569625, "o", "d"] +[3.621317, "o", "u"] +[3.673569, "o", "c"] +[3.725469, "o", "t"] +[3.776986, "o", "i"] +[3.829255, "o", "o"] +[3.881898, "o", "n"] +[3.934725, "o", "/"] +[3.987241, "o", "s"] +[4.040092, "o", "p"] +[4.078561, "o", "e"] +[4.116327, "o", "c"] +[4.154401, "o", "."] +[4.192575, "o", "y"] +[4.230672, "o", "a"] +[4.269022, "o", "m"] +[4.307314, "o", "l"] +[4.597326, "o", "\r\n"] +[4.59877, "o", "name: \"Production deployment\"\r\ntags: [\"production\", \"deployment\"]\r\n\r\ndescribe:\r\n - name: \"Deployment\"\r\n select: 'select(.kind == \"Deployment\")'\r\n it:\r\n - should: \"have 3 replicas\"\r\n expect: spec.replicas\r\n toEqual: 3\r\n\r\n - should: \"be in production namespace\"\r\n expect: metadata.namespace\r\n toEqual: \"production\"\r\n\r\n - should: \"use a pinned image tag\"\r\n expect: spec.template.spec.containers[0].image\r\n toNotEndWith: \":latest\"\r\n\r\n - should: \"have CPU limits\"\r\n expect: spec.template.spec.containers[0].resources.limits.cpu\r\n toExist: true\r\n"] +[7.6026, "o", "\u001b[H\u001b[J\u001b[3J"] +[7.602851, "o", "\u001b[2;36m# And the manifest we want to validate:\u001b[0m\r\n"] +[8.50459, "o", "\u001b[1;32m$\u001b[0m c"] +[8.563744, "o", "a"] +[8.623326, "o", "t"] +[8.682932, "o", " "] +[8.742304, "o", "t"] +[8.801497, "o", "e"] +[8.861339, "o", "s"] +[8.920428, "o", "t"] +[8.979703, "o", "s"] +[9.038877, "o", "/"] +[9.081938, "o", "p"] +[9.125309, "o", "r"] +[9.169028, "o", "o"] +[9.212399, "o", "d"] +[9.255465, "o", "u"] +[9.299072, "o", "c"] +[9.342282, "o", "t"] +[9.385472, "o", "i"] +[9.428355, "o", "o"] +[9.471569, "o", "n"] +[9.515123, "o", "/"] +[9.55879, "o", "m"] +[9.601586, "o", "a"] +[9.644671, "o", "n"] +[9.687421, "o", "i"] +[9.730464, "o", "f"] +[9.773898, "o", "e"] +[9.817123, "o", "s"] +[9.860338, "o", "t"] +[9.903677, "o", "s"] +[9.947112, "o", "/"] +[9.990527, "o", "d"] +[10.033767, "o", "e"] +[10.077224, "o", "p"] +[10.11736, "o", "l"] +[10.157408, "o", "o"] +[10.197256, "o", "y"] +[10.237481, "o", "m"] +[10.277463, "o", "e"] +[10.317522, "o", "n"] +[10.357653, "o", "t"] +[10.397907, "o", "."] +[10.438188, "o", "y"] +[10.478387, "o", "a"] +[10.518927, "o", "m"] +[10.559493, "o", "l"] +[10.851579, "o", "\r\n"] +[10.853001, "o", "apiVersion: apps/v1\r\nkind: Deployment\r\nmetadata:\r\n name: web\r\n namespace: production\r\nspec:\r\n replicas: 3\r\n selector:\r\n matchLabels:\r\n app: web\r\n template:\r\n metadata:\r\n labels:\r\n app: web\r\n spec:\r\n containers:\r\n - name: web\r\n image: nginx:1.25.3\r\n resources:\r\n limits:\r\n cpu: \"500m\"\r\n memory: \"256Mi\"\r\n"] +[13.856621, "o", "\u001b[H\u001b[J\u001b[3J"] +[13.856868, "o", "\u001b[2;36m# Run the tests:\u001b[0m\r\n"] +[14.758491, "o", "\u001b[1;32m$\u001b[0m ."] +[14.800197, "o", "/"] +[14.842251, "o", "y"] +[14.884455, "o", "a"] +[14.926849, "o", "m"] +[14.969755, "o", "l"] +[15.012345, "o", "s"] +[15.054748, "o", "p"] +[15.099137, "o", "e"] +[15.143491, "o", "c"] +[15.188164, "o", " "] +[15.232539, "o", "v"] +[15.276755, "o", "a"] +[15.321101, "o", "l"] +[15.365347, "o", "i"] +[15.409151, "o", "d"] +[15.453398, "o", "a"] +[15.497029, "o", "t"] +[15.540565, "o", "e"] +[15.5852, "o", " "] +[15.629629, "o", "-"] +[15.67457, "o", "-"] +[15.718674, "o", "t"] +[15.762538, "o", "e"] +[15.806429, "o", "s"] +[15.850778, "o", "t"] +[15.894986, "o", "-"] +[15.938513, "o", "d"] +[15.98282, "o", "i"] +[16.027188, "o", "r"] +[16.071227, "o", " "] +[16.131612, "o", "t"] +[16.191799, "o", "e"] +[16.251748, "o", "s"] +[16.311971, "o", "t"] +[16.372346, "o", "s"] +[16.684761, "o", "\r\n"] +[16.690924, "o", "\r\n \u001b[32m✓\u001b[0m \u001b[1mProduction deployment\u001b[0m\r\n Deployment\r\n \u001b[36m[select: select(.kind == \"Deployment\")]\u001b[0m\r\n \u001b[32m✓\u001b[0m have 3 replicas\r\n \u001b[32m✓\u001b[0m be in production namespace\r\n \u001b[32m✓\u001b[0m use a pinned image tag\r\n \u001b[32m✓\u001b[0m have CPU limits\r\n\r\n\r\n4 assertions, \u001b[32m4 passed\u001b[0m\r\nFinished in 0.00s\r\n\r\n"] +[19.193184, "o", "\u001b[2;36m# Break it — drop replicas to 1:\u001b[0m\r\n"] +[20.095501, "o", "\u001b[1;32m$\u001b[0m s"] +[20.154429, "o", "e"] +[20.214446, "o", "d"] +[20.273673, "o", " "] +[20.332705, "o", "-"] +[20.391701, "o", "i"] +[20.451065, "o", " "] +[20.510616, "o", "'"] +[20.569722, "o", "s"] +[20.62924, "o", "/"] +[20.689011, "o", "r"] +[20.748685, "o", "e"] +[20.807841, "o", "p"] +[20.867684, "o", "l"] +[20.927103, "o", "i"] +[20.986392, "o", "c"] +[21.044952, "o", "a"] +[21.107656, "o", "s"] +[21.169726, "o", ":"] +[21.232243, "o", " "] +[21.294896, "o", "3"] +[21.357125, "o", "/"] +[21.419141, "o", "r"] +[21.481428, "o", "e"] +[21.544104, "o", "p"] +[21.606423, "o", "l"] +[21.668626, "o", "i"] +[21.730607, "o", "c"] +[21.792844, "o", "a"] +[21.854484, "o", "s"] +[21.917117, "o", ":"] +[21.979397, "o", " "] +[22.041778, "o", "1"] +[22.097601, "o", "/"] +[22.154065, "o", "'"] +[22.210159, "o", " "] +[22.266485, "o", "t"] +[22.322307, "o", "e"] +[22.37897, "o", "s"] +[22.434537, "o", "t"] +[22.491311, "o", "s"] +[22.546857, "o", "/"] +[22.603137, "o", "p"] +[22.65952, "o", "r"] +[22.715623, "o", "o"] +[22.771467, "o", "d"] +[22.826911, "o", "u"] +[22.882697, "o", "c"] +[22.937937, "o", "t"] +[22.994003, "o", "i"] +[23.049504, "o", "o"] +[23.107298, "o", "n"] +[23.165433, "o", "/"] +[23.223791, "o", "m"] +[23.281587, "o", "a"] +[23.339826, "o", "n"] +[23.398051, "o", "i"] +[23.456333, "o", "f"] +[23.51511, "o", "e"] +[23.57315, "o", "s"] +[23.630826, "o", "t"] +[23.689381, "o", "s"] +[23.747784, "o", "/"] +[23.806031, "o", "d"] +[23.864387, "o", "e"] +[23.923197, "o", "p"] +[23.981028, "o", "l"] +[24.039514, "o", "o"] +[24.081617, "o", "y"] +[24.124449, "o", "m"] +[24.166406, "o", "e"] +[24.208994, "o", "n"] +[24.251553, "o", "t"] +[24.29365, "o", "."] +[24.334789, "o", "y"] +[24.377155, "o", "a"] +[24.419641, "o", "m"] +[24.461846, "o", "l"] +[24.756435, "o", "\r\n"] +[25.760824, "o", "\u001b[1;32m$\u001b[0m ."] +[25.820645, "o", "/"] +[25.882122, "o", "y"] +[25.942938, "o", "a"] +[26.003745, "o", "m"] +[26.064272, "o", "l"] +[26.112765, "o", "s"] +[26.160711, "o", "p"] +[26.208545, "o", "e"] +[26.2559, "o", "c"] +[26.303597, "o", " "] +[26.351976, "o", "v"] +[26.400069, "o", "a"] +[26.448033, "o", "l"] +[26.495905, "o", "i"] +[26.543907, "o", "d"] +[26.592661, "o", "a"] +[26.640751, "o", "t"] +[26.68903, "o", "e"] +[26.737372, "o", " "] +[26.785714, "o", "-"] +[26.83373, "o", "-"] +[26.882395, "o", "t"] +[26.930264, "o", "e"] +[26.978462, "o", "s"] +[27.026581, "o", "t"] +[27.074896, "o", "-"] +[27.135129, "o", "d"] +[27.195433, "o", "i"] +[27.255473, "o", "r"] +[27.315584, "o", " "] +[27.375216, "o", "t"] +[27.434953, "o", "e"] +[27.494735, "o", "s"] +[27.554574, "o", "t"] +[27.614694, "o", "s"] +[27.926854, "o", "\r\n"] +[27.932694, "o", "\r\n \u001b[31m✗\u001b[0m \u001b[1mProduction deployment\u001b[0m\r\n Deployment\r\n \u001b[36m[select: select(.kind == \"Deployment\")]\u001b[0m\r\n \u001b[31m✗\u001b[0m have 3 replicas\r\n \u001b[2m\u001b[31mexpected 'spec.replicas' to equal 3, got 1\u001b[0m\r\n \u001b[32m✓\u001b[0m be in production namespace\r\n \u001b[32m✓\u001b[0m use a pinned image tag\r\n \u001b[32m✓\u001b[0m have CPU limits\r\n\r\n\r\n\u001b[31m\u001b[1mFailures:\u001b[0m\r\n\r\n \u001b[31m1) Production deployment > Deployment > have 3 replicas\u001b[0m\r\n \u001b[2mexpected 'spec.replicas' to equal 3, got 1\u001b[0m\r\n\r\n\r\n4 assertions, \u001b[32m3 passed\u001b[0m, \u001b[31m1 failed\u001b[0m\r\nFinished in 0.00s\r\n\r\n"] diff --git a/docs/demo.gif b/docs/demo.gif new file mode 100644 index 0000000..b3e706e Binary files /dev/null and b/docs/demo.gif differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..a636731 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,29 @@ +# yamlspec documentation + +A YAML test framework with RSpec-like assertions. Validate plain YAML, Helm +charts, Kustomize overlays, or anything else that produces YAML. + +If you're new, start with the project [README](../README.md) for install and a +60-second tour. + +## Reference + +| Page | What's in it | +|------|--------------| +| [spec-format.md](spec-format.md) | Full `spec.yaml` schema — fields, field-path syntax, all 22 assertion operators, selector examples | +| [recipes.md](recipes.md) | Cookbook of common patterns: multi-environment Helm values, security policy checks, label conventions, ConfigMap data, image pinning | +| [troubleshooting.md](troubleshooting.md) | Common failure modes and how to read the error messages | +| [reusable-workflow.md](reusable-workflow.md) | GitHub Actions reusable workflow — one-line CI integration with PR comments | + +## Project meta + +| Page | What's in it | +|------|--------------| +| [../CONTRIBUTING.md](../CONTRIBUTING.md) | Dev setup, how to add an operator or output format, release process | +| [../CHANGELOG.md](../CHANGELOG.md) | Release history (Keep a Changelog format) | + +## Examples + +Five worked examples live under [`../examples/`](../examples) — see the +[examples README](../examples/README.md) for what each one demonstrates and the +exact command to run it. diff --git a/docs/recipes.md b/docs/recipes.md new file mode 100644 index 0000000..28b4184 --- /dev/null +++ b/docs/recipes.md @@ -0,0 +1,306 @@ +# Recipes + +Patterns that come up over and over. Copy, adapt, run. + +For the full schema and operator list see [spec-format.md](spec-format.md). + +## Multi-environment Helm values + +One spec per environment, each rendering against its own values file. +Filesystem layout: + +``` +tests/ + default/ + spec.yaml + staging/ + spec.yaml + values.yaml + production/ + spec.yaml + values.yaml +``` + +```yaml +# tests/production/spec.yaml +name: "Production overrides" +tags: ["production"] + +pre_run: + - helm template prod ../../chart -f values.yaml > manifests/rendered.yaml + +describe: + - name: "Deployment" + select: 'select(.kind == "Deployment")' + it: + - should: "scale up for prod traffic" + expect: spec.replicas + toBeGreaterOrEqual: 3 + + - should: "land in the prod namespace" + expect: metadata.namespace + toEqual: "production" +``` + +Run all environments at once with `yamlspec validate --test-dir tests`, or +filter with `--tag production`. + +## Every container has resource limits (wildcard iteration) + +`[*]` runs the assertion against every element. Failure on any one fails the +assertion. + +```yaml +- should: "every container has CPU limits" + expect: spec.template.spec.containers[*].resources.limits.cpu + toExist: true + +- should: "every container has memory limits" + expect: spec.template.spec.containers[*].resources.limits.memory + toExist: true + +- should: "no container uses :latest" + expect: spec.template.spec.containers[*].image + toNotEndWith: ":latest" +``` + +This is one of the most useful patterns — multi-container pods (sidecars, +init containers) silently regressing is a common production incident. + +## Image pinning with semver + +```yaml +- should: "image is pinned to a semver tag" + expect: spec.template.spec.containers[0].image + toMatch: ':v?\d+\.\d+\.\d+(-[a-z0-9.-]+)?$' + toNotEndWith: ":latest" +``` + +For SHA-pinned images: + +```yaml +- should: "image is pinned to a digest" + expect: spec.template.spec.containers[0].image + toMatch: '@sha256:[a-f0-9]{64}$' +``` + +## Required labels and annotations (the standard k8s set) + +```yaml +- name: "Standard recommended labels" + select: 'select(.kind == "Deployment")' + it: + - should: "have app.kubernetes.io/name" + expect: metadata.labels["app.kubernetes.io/name"] + toExist: true + + - should: "have app.kubernetes.io/instance" + expect: metadata.labels["app.kubernetes.io/instance"] + toExist: true + + - should: "have app.kubernetes.io/version" + expect: metadata.labels["app.kubernetes.io/version"] + toMatch: '^v?\d+\.\d+\.\d+' + + - should: "have app.kubernetes.io/managed-by" + expect: metadata.labels["app.kubernetes.io/managed-by"] + toBeOneOf: ["Helm", "kustomize", "argocd"] +``` + +Bracket notation is required for keys containing dots or slashes. + +## ConfigMap data validation + +```yaml +- name: "App configuration" + select: 'select(.kind == "ConfigMap" and .metadata.name == "app-config")' + it: + - should: "set the right environment" + expect: data.APP_ENV + toEqual: "production" + + - should: "use a valid log level" + expect: data.LOG_LEVEL + toBeOneOf: ["debug", "info", "warn", "error"] + + - should: "not enable debug in prod" + expect: data.DEBUG + toNotEqual: "true" + + - should: "have a sane request timeout" + expect: data.REQUEST_TIMEOUT_MS + toMatch: '^\d+$' +``` + +## Pod security baseline + +```yaml +- name: "Pod security context" + select: 'select(.kind == "Deployment")' + it: + - should: "run as non-root" + expect: spec.template.spec.securityContext.runAsNonRoot + toEqual: true + + - should: "set a non-root UID" + expect: spec.template.spec.securityContext.runAsUser + toBeGreaterThan: 0 + + - should: "disable service account token automount" + expect: spec.template.spec.automountServiceAccountToken + toEqual: false + +- name: "Container security context" + select: 'select(.kind == "Deployment")' + it: + - should: "every container has a read-only root filesystem" + expect: spec.template.spec.containers[*].securityContext.readOnlyRootFilesystem + toEqual: true + + - should: "every container drops all capabilities" + expect: spec.template.spec.containers[*].securityContext.capabilities.drop + toContainItem: "ALL" + + - should: "no container allows privilege escalation" + expect: spec.template.spec.containers[*].securityContext.allowPrivilegeEscalation + toEqual: false +``` + +## HPA / PDB invariants + +```yaml +- name: "HorizontalPodAutoscaler" + select: 'select(.kind == "HorizontalPodAutoscaler")' + it: + - should: "have min replicas in [3, 10]" + expect: spec.minReplicas + toBeGreaterOrEqual: 3 + toBeLessOrEqual: 10 + + - should: "have max replicas <= 50" + expect: spec.maxReplicas + toBeLessOrEqual: 50 + +- name: "PodDisruptionBudget" + select: 'select(.kind == "PodDisruptionBudget")' + it: + - should: "guarantee at least 2 pods stay available" + expect: spec.minAvailable + toBeGreaterOrEqual: 2 +``` + +## NetworkPolicy egress rules + +```yaml +- name: "Default-deny + allow-list" + select: 'select(.kind == "NetworkPolicy")' + it: + - should: "include Egress in policy types" + expect: spec.policyTypes + toContainItem: "Egress" + + - should: "include Ingress in policy types" + expect: spec.policyTypes + toContainItem: "Ingress" + + - should: "have at least one egress rule (not pure deny-all)" + expect: spec.egress + toHaveMinLength: 1 +``` + +## Service / Ingress sanity + +```yaml +- name: "Service" + select: 'select(.kind == "Service" and .metadata.name == "web")' + it: + - should: "be ClusterIP (not LoadBalancer in this env)" + expect: spec.type + toEqual: "ClusterIP" + + - should: "expose exactly one port" + expect: spec.ports + toHaveLength: 1 + + - should: "select pods by app label" + expect: spec.selector.app + toEqual: "web" + +- name: "Ingress" + select: 'select(.kind == "Ingress")' + it: + - should: "use TLS" + expect: spec.tls + toHaveMinLength: 1 + + - should: "use the prod ingress class" + expect: spec.ingressClassName + toEqual: "nginx-prod" +``` + +## Cross-overlay assertion (Kustomize) + +```yaml +# tests/staging/spec.yaml +pre_run: + - kustomize build ../../overlays/staging > manifests/rendered.yaml + +describe: + - name: "Staging-specific overrides" + select: 'select(.kind == "Deployment" and .metadata.name == "api")' + it: + - should: "use the staging image registry" + expect: spec.template.spec.containers[0].image + toStartWith: "registry.staging.example.com/" + + - should: "have the env label patched in" + expect: metadata.labels.env + toEqual: "staging" +``` + +## Multi-document manifests + +yamlspec parses every `---`-separated document in every YAML file under +`manifests/`. No special config needed: + +```yaml +# manifests/rendered.yaml (output of helm template or kustomize build) +apiVersion: v1 +kind: Service +... +--- +apiVersion: apps/v1 +kind: Deployment +... +--- +apiVersion: v1 +kind: ConfigMap +... +``` + +Each `describe` block uses its `select:` to pick which documents the +assertions apply to. + +## Combining multiple operators + +Operators on a single assertion are AND'd — all must pass: + +```yaml +- should: "image is on our registry, semver-tagged, and not :latest" + expect: spec.template.spec.containers[0].image + toStartWith: "registry.example.com/" + toMatch: ':v\d+\.\d+\.\d+$' + toNotEndWith: ":latest" +``` + +For OR semantics, write multiple assertions or use `toBeOneOf`. + +## Reusable security baseline + +The [`examples/security-checks`](../examples/security-checks) spec is a +ready-to-run baseline you can drop into any chart or kustomize repo as +`tests/security/spec.yaml`. + +```bash +yamlspec validate --test-dir tests --tag security +``` diff --git a/docs/reusable-workflow.md b/docs/reusable-workflow.md index c78aab5..4dd68d8 100644 --- a/docs/reusable-workflow.md +++ b/docs/reusable-workflow.md @@ -1,5 +1,8 @@ # Reusable Workflow +[← back to docs index](index.md) + + yamlspec provides a reusable GitHub Actions workflow that any repo can call to run tests and post results as PR comments. ## Quick Start diff --git a/docs/spec-format.md b/docs/spec-format.md index 285f0b5..06404fc 100644 --- a/docs/spec-format.md +++ b/docs/spec-format.md @@ -1,5 +1,8 @@ # spec.yaml Format Reference +[← back to docs index](index.md) + + ## Structure ```yaml diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..4530396 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,184 @@ +# Troubleshooting + +Common failure modes and how to read what yamlspec is telling you. + +## "spec must contain at least one describe block" + +``` +Error: discovery failed: parse '.../tests/foo/spec.yaml': spec must contain at least one describe block +``` + +Your `spec.yaml` parsed cleanly but has no `describe:` entries (or the field +is missing entirely). Add at least one `describe` block with at least one +`it` assertion. + +## "describe[0] (\"X\").it[0] (\"Y\") has no assertion operator" + +``` +describe[0] ("Deployment").it[2] ("be in production namespace") has no assertion operator +(add one of toEqual, toExist, toContain, etc.) +``` + +The assertion has a `should:` and `expect:` but no operator like `toEqual:`. +This is the most common authoring mistake — easy to forget the operator line. +Add one (or more): + +```yaml +- should: "be in production namespace" + expect: metadata.namespace + toEqual: "production" # ← was missing +``` + +## "field X not found in type domain.Spec" + +``` +Error: yaml: unmarshal errors: + line 7: field describes not found in type domain.Spec +``` + +yamlspec uses **strict YAML decoding** — typos in field names are rejected, +not silently ignored. Fix the typo (`describes` → `describe`, `pre-run` → +`pre_run`, etc.). The valid top-level fields are `name`, `tags`, `pre_run`, +`describe`. + +## "selector 'X' matched no resources" + +``` +Status: FAILED +Error: selector 'select(.kind == "Deployment")' matched no resources +``` + +The JQ selector ran fine but matched zero documents. Possible causes: + +1. **No manifests were discovered** — yamlspec looks for YAML in + `/manifests/` first, falling back to YAML files alongside + `spec.yaml` (excluding `spec.yaml` and `values.yaml` themselves). Confirm + the files exist and have a `.yaml` or `.yml` extension. + +2. **`pre_run` ran but didn't write where you expected** — make sure your + redirect targets `manifests/`: + + ```yaml + pre_run: + - mkdir -p manifests + - helm template app ../../chart > manifests/rendered.yaml + ``` + +3. **The selector itself is wrong** — typo in the kind, namespace mismatch, + etc. Try a broader selector first (`select: ""` matches everything) and + narrow from there. + +## "pre_run failed: command 'X' timed out after Ys" + +``` +Status: ERROR +Error: pre_run failed: command 'helm template ...' timed out after 1m0s +``` + +The default per-command timeout is 60 seconds. Bump it for slow renders: + +```bash +yamlspec validate --pre-run-timeout 5m +``` + +If a command was hanging on a child process (e.g. `sleep` left over from a +test), the timeout now kills the entire process group — no orphaned children. + +## "no manifests found" + +``` +Status: ERROR +Error: no manifests found +``` + +The spec ran its `pre_run` (if any) but yamlspec found zero YAML files to +test. Check: + +- Files have `.yaml` / `.yml` extension (not `.txt`, not extension-less) +- If using `manifests/` subdirectory, the directory exists and contains files +- If your `pre_run` writes to a different path, point yamlspec at the right + directory by writing into `manifests/` + +## "field path not found" vs "field is null" + +These are different. yamlspec distinguishes: + +| Situation | `toExist: true` | `toExist: false` | `toBeNull: true` | `toBeNull: false` | +|-----------|-----------------|------------------|------------------|-------------------| +| Field absent | FAIL | PASS | FAIL | FAIL | +| Field present, value `null` | FAIL | PASS | PASS | FAIL | +| Field present, non-null value | PASS | FAIL | FAIL | PASS | + +If you're checking that a field both exists *and* is non-null, `toExist: true` +is what you want. `toBeNull: true` only passes when the field is explicitly +present but null. + +## "--fail-fast cannot be used with --workers > 1" + +``` +Error: --fail-fast cannot be used with --workers > 1 +(parallel workers can't honor ordered short-circuit; pick one) +``` + +These are mutually exclusive — parallel execution can't honor a +deterministic short-circuit. Pick one: + +- `--fail-fast` (sequential, stops on first failure — fast feedback) +- `--workers N` (parallel, runs everything — fast wall time) + +## Empty or malformed spec.yaml + +``` +Error: discovery failed: parse '.../tests/foo/spec.yaml': spec file is empty +``` + +The file was readable but had no content (or only whitespace). Re-scaffold +with `yamlspec init ` if you want a starter template. + +## YAML parsing errors in manifests + +``` +parse '.../manifests/rendered.yaml': yaml: line 42: did not find expected node content +``` + +Your manifests YAML is malformed. If this came from a `pre_run` rendering +step, run that command manually and inspect the output — `helm template` +sometimes emits empty documents or comment-only blocks that confuse strict +parsers. yamlspec uses `gopkg.in/yaml.v3`, which is stricter than `yaml.v2`. + +## Wildcard matched nothing + +A wildcard path like `spec.containers[*].image` over an empty array yields +zero values. yamlspec treats this as "no values to assert against" — your +assertion will pass vacuously for value-based operators (`toEqual`, +`toMatch`, etc.) but `toExist: true` will still fail because the path +produced no values. + +If you want to assert "there is at least one container," add a separate +length check: + +```yaml +- should: "has at least one container" + expect: spec.containers + toHaveMinLength: 1 +``` + +## Verbose mode + +When in doubt, run with `--verbose` to see debug logs (pre_run commands, +discovered specs, etc.): + +```bash +yamlspec --verbose validate --test-dir tests +``` + +## Still stuck? + +Open an issue with: + +- Your `spec.yaml` +- A minimal manifest that reproduces the problem +- The exact command you ran +- The full error output + +The smaller the repro, the faster the fix. diff --git a/domain/config.go b/domain/config.go index 48fc4c4..68d06ab 100644 --- a/domain/config.go +++ b/domain/config.go @@ -18,6 +18,11 @@ type Config struct { MarkdownOutput string EMDOutput string JUnitOutput string + + // GitHubAnnotations emits ::error file=...,line=...:: lines on stdout for + // failing assertions, so they appear inline in GitHub PR diffs. Auto-enabled + // when GITHUB_ACTIONS=true. + GitHubAnnotations bool } // DefaultConfig returns a Config with sensible defaults diff --git a/domain/models.go b/domain/models.go index d7bb81c..66cd11f 100644 --- a/domain/models.go +++ b/domain/models.go @@ -6,6 +6,9 @@ type Spec struct { Tags []string `yaml:"tags,omitempty"` PreRun []string `yaml:"pre_run,omitempty"` Describe []DescBlock `yaml:"describe"` + + // SourceFile is the spec.yaml path. Populated by discovery; not parsed from YAML. + SourceFile string `yaml:"-"` } // HasTag returns true if the spec has at least one of the given tags @@ -32,6 +35,10 @@ type Assertion struct { Should string `yaml:"should"` Expect string `yaml:"expect"` + // SourceLine is the 1-indexed line of this assertion's `should:` in spec.yaml. + // Populated by discovery; not parsed from YAML. + SourceLine int `yaml:"-"` + // Equality ToEqual interface{} `yaml:"toEqual,omitempty"` ToNotEqual interface{} `yaml:"toNotEqual,omitempty"` diff --git a/domain/results.go b/domain/results.go index 0c14e73..a8d6c4a 100644 --- a/domain/results.go +++ b/domain/results.go @@ -20,12 +20,13 @@ type SuiteResult struct { // SpecResult is the result of one spec.yaml file type SpecResult struct { - Name string `json:"name" yaml:"name"` - Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` - Status Status `json:"status" yaml:"status"` - Duration time.Duration `json:"duration" yaml:"duration"` - Describes []DescribeResult `json:"describes,omitempty" yaml:"describes,omitempty"` - Error string `json:"error,omitempty" yaml:"error,omitempty"` + Name string `json:"name" yaml:"name"` + Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` + Status Status `json:"status" yaml:"status"` + Duration time.Duration `json:"duration" yaml:"duration"` + Describes []DescribeResult `json:"describes,omitempty" yaml:"describes,omitempty"` + Error string `json:"error,omitempty" yaml:"error,omitempty"` + SourceFile string `json:"source_file,omitempty" yaml:"source_file,omitempty"` } // DescribeResult is the result of one describe block @@ -38,11 +39,12 @@ type DescribeResult struct { // AssertionResult is the result of a single assertion type AssertionResult struct { - Should string `json:"should" yaml:"should"` - Status Status `json:"status" yaml:"status"` - Expected interface{} `json:"expected,omitempty" yaml:"expected,omitempty"` - Actual interface{} `json:"actual,omitempty" yaml:"actual,omitempty"` - Error string `json:"error,omitempty" yaml:"error,omitempty"` + Should string `json:"should" yaml:"should"` + Status Status `json:"status" yaml:"status"` + Expected interface{} `json:"expected,omitempty" yaml:"expected,omitempty"` + Actual interface{} `json:"actual,omitempty" yaml:"actual,omitempty"` + Error string `json:"error,omitempty" yaml:"error,omitempty"` + SourceLine int `json:"source_line,omitempty" yaml:"source_line,omitempty"` } // Summary aggregates counts across the run diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..ede7b44 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,106 @@ +# Examples + +Five runnable examples covering the main use cases. Each lives in its own +directory and is a complete, self-contained spec — clone the repo and run them +locally to see what passing/failing output looks like. + +## Running an example + +From the repo root, after `make build-local` (binary lands at `./yamlspec`): + +```bash +./yamlspec validate --test-dir examples/ +``` + +Or run all examples at once: + +```bash +./yamlspec validate --test-dir examples +``` + +Filter by tag: + +```bash +./yamlspec validate --test-dir examples --tag security +``` + +## What each example shows + +### [`plain-manifests/`](plain-manifests) + +The simplest case — no `pre_run`, no rendering. Just YAML files in +`manifests/` and a `spec.yaml` next to them. + +Demonstrates: deployment replica count, image-tag policy (no `:latest`, +semver-only), resource limits/requests, health-probe presence, ConfigMap +key/value checks, set-membership (`toBeOneOf` for log levels). + +```bash +./yamlspec validate --test-dir examples/plain-manifests +``` + +### [`helm-chart/`](helm-chart) + +Full Helm pipeline — `pre_run` runs `helm template` against a real chart with +a values override, writes the output to `manifests/rendered.yaml`, then +yamlspec asserts on the rendered result. + +Demonstrates: Helm rendering integration, asserting that `values.yaml` +overrides actually take effect, multi-resource validation in one spec. + +Requires `helm` on `$PATH`. + +```bash +./yamlspec validate --test-dir examples/helm-chart +``` + +### [`kustomize-overlay/`](kustomize-overlay) + +Kustomize equivalent — `pre_run` runs `kustomize build` against a base, asserts +on the built output. + +Demonstrates: kustomize integration, namespace/label patches, replica patches. + +Requires `kustomize` on `$PATH`. + +```bash +./yamlspec validate --test-dir examples/kustomize-overlay +``` + +### [`multi-resource/`](multi-resource) + +Several resources of different kinds (Deployment + Service + HPA + PDB) tested +in a single spec, each isolated via JQ selector. + +Demonstrates: `select: 'select(.kind == "X")'` filtering, asserting +relationships between resources (HPA targets the Deployment, PDB selector +matches the pods), array-length checks (`toHaveLength`, `toHaveMinLength`). + +```bash +./yamlspec validate --test-dir examples/multi-resource +``` + +### [`security-checks/`](security-checks) + +A reusable security baseline — runAsNonRoot, readOnlyRootFilesystem, dropped +capabilities, no `:latest` tags, mandatory probes, mandatory resource limits. + +Demonstrates: how to write a security-policy spec you can drop into any chart +or kustomize repo. No `pre_run` — point it at any rendered manifest with +`./yamlspec validate --test-dir examples/security-checks`. + +This example is also a good template for organization-wide policy checks. + +## Output formats + +Try the different formatters on any example: + +```bash +./yamlspec validate --test-dir examples/multi-resource \ + --json-output results.json \ + --emd-output results.md \ + --junit-output results.xml +``` + +See [docs/spec-format.md](../docs/spec-format.md) for the full assertion +reference and [docs/recipes.md](../docs/recipes.md) for more patterns. diff --git a/flags.go b/flags.go index f6a458f..b1b6390 100644 --- a/flags.go +++ b/flags.go @@ -58,4 +58,9 @@ var ( Name: "junit-output", Usage: "Write JUnit XML results to file", } + githubAnnotationsFlag = cli.BoolFlag{ + Name: "github-annotations", + Usage: "Emit ::error file=...,line=...:: lines on stdout for failing assertions (auto-enabled when GITHUB_ACTIONS=true)", + EnvVar: "GITHUB_ACTIONS", + } ) diff --git a/integration/validate_test.go b/integration/validate_test.go index c53eb94..8ec5d9f 100644 --- a/integration/validate_test.go +++ b/integration/validate_test.go @@ -1022,3 +1022,85 @@ describe: t.Errorf("pre-run timeout didn't cancel: took %s", elapsed) } } + +// --- GitHub Actions annotations --- + +func TestE2E_GitHubAnnotations_FailingAssertions(t *testing.T) { + bin := buildBinary(t) + tmpDir := t.TempDir() + specDir := filepath.Join(tmpDir, "annotated") + manifestsDir := filepath.Join(specDir, "manifests") + if err := os.MkdirAll(manifestsDir, 0755); err != nil { + t.Fatal(err) + } + + // Note line numbers below; the assertion `should:` lines are referenced. + spec := `name: "Annotated" +tags: ["annotated"] +describe: + - name: "Deployment" + select: 'select(.kind == "Deployment")' + it: + - should: "have 99 replicas" + expect: spec.replicas + toEqual: 99 + - should: "be in fictional namespace" + expect: metadata.namespace + toEqual: "nonexistent" +` + specPath := filepath.Join(specDir, "spec.yaml") + if err := os.WriteFile(specPath, []byte(spec), 0644); err != nil { + t.Fatal(err) + } + + manifest := `apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + namespace: production +spec: + replicas: 1 +` + if err := os.WriteFile(filepath.Join(manifestsDir, "deployment.yaml"), []byte(manifest), 0644); err != nil { + t.Fatal(err) + } + + output, exitCode := run(t, bin, "validate", "--test-dir", tmpDir, "--github-annotations") + + if exitCode != 1 { + t.Errorf("expected exit 1 (failures), got %d\n%s", exitCode, output) + } + + // Both failing assertions should produce ::error lines pointing at spec.yaml + if !strings.Contains(output, "::error file=") { + t.Errorf("expected ::error annotations in output, got:\n%s", output) + } + if !strings.Contains(output, "spec.yaml") { + t.Errorf("expected spec.yaml in annotation paths, got:\n%s", output) + } + // Line numbers for the two `should:` lines (7 and 10 in the spec above). + for _, frag := range []string{"line=7", "line=10"} { + if !strings.Contains(output, frag) { + t.Errorf("expected %q in annotations, got:\n%s", frag, output) + } + } + // Both assertion descriptions should appear in the annotation messages. + for _, frag := range []string{"have 99 replicas", "be in fictional namespace"} { + if !strings.Contains(output, frag) { + t.Errorf("expected %q in annotations, got:\n%s", frag, output) + } + } +} + +func TestE2E_GitHubAnnotations_PassingSuiteEmitsNothing(t *testing.T) { + bin := buildBinary(t) + + output, exitCode := run(t, bin, "validate", "--test-dir", "integration/testdata", "--tag", "service", "--github-annotations") + + if exitCode != 0 { + t.Errorf("expected exit 0 for passing suite, got %d\n%s", exitCode, output) + } + if strings.Contains(output, "::error") { + t.Errorf("expected no ::error lines for a passing suite, got:\n%s", output) + } +} diff --git a/main.go b/main.go index 0762a38..1512bec 100644 --- a/main.go +++ b/main.go @@ -57,6 +57,7 @@ func main() { markdownOutputFlag, emdOutputFlag, junitOutputFlag, + githubAnnotationsFlag, }, Action: validateAction.Execute, }, diff --git a/services/assertion.go b/services/assertion.go index a274949..d6fb20b 100644 --- a/services/assertion.go +++ b/services/assertion.go @@ -74,8 +74,9 @@ func (ae *AssertionEngine) evaluateDescribe(desc *domain.DescBlock, manifests [] func (ae *AssertionEngine) evaluateAssertion(assertion *domain.Assertion, resources []interface{}) domain.AssertionResult { result := domain.AssertionResult{ - Should: assertion.Should, - Status: domain.StatusPassed, + Should: assertion.Should, + Status: domain.StatusPassed, + SourceLine: assertion.SourceLine, } fieldPath := assertion.Expect diff --git a/services/discovery.go b/services/discovery.go index 2ff1176..4627e9c 100644 --- a/services/discovery.go +++ b/services/discovery.go @@ -155,6 +155,12 @@ func (ds *DiscoveryService) parseSpec(path string) (*domain.Spec, error) { spec.Name = filepath.Base(filepath.Dir(path)) } + spec.SourceFile = path + if err := annotateSourceLines(&spec, data); err != nil { + // Line annotations are best-effort — failure here shouldn't block validation. + helpers.Log.Debug().Err(err).Str("path", path).Msg("could not annotate source lines") + } + if err := validateSpec(&spec); err != nil { return nil, err } @@ -162,6 +168,76 @@ func (ds *DiscoveryService) parseSpec(path string) (*domain.Spec, error) { return &spec, nil } +// annotateSourceLines re-decodes the YAML as a Node tree and stamps each +// assertion's SourceLine. Used for emitting GitHub Actions annotations that +// link back to the right line in spec.yaml. +func annotateSourceLines(spec *domain.Spec, data []byte) error { + var root yaml.Node + if err := yaml.Unmarshal(data, &root); err != nil { + return err + } + if len(root.Content) == 0 { + return nil + } + doc := root.Content[0] + if doc.Kind != yaml.MappingNode { + return nil + } + + describeNode := findMapValue(doc, "describe") + if describeNode == nil || describeNode.Kind != yaml.SequenceNode { + return nil + } + + for di, descNode := range describeNode.Content { + if di >= len(spec.Describe) || descNode.Kind != yaml.MappingNode { + continue + } + itNode := findMapValue(descNode, "it") + if itNode == nil || itNode.Kind != yaml.SequenceNode { + continue + } + for ai, assertNode := range itNode.Content { + if ai >= len(spec.Describe[di].It) || assertNode.Kind != yaml.MappingNode { + continue + } + // Prefer the line of `should:` itself; fall back to the mapping's start. + line := assertNode.Line + if shouldNode := findMapKey(assertNode, "should"); shouldNode != nil { + line = shouldNode.Line + } + spec.Describe[di].It[ai].SourceLine = line + } + } + return nil +} + +// findMapValue returns the value node for the given key in a MappingNode, or nil. +func findMapValue(m *yaml.Node, key string) *yaml.Node { + if m.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i+1] + } + } + return nil +} + +// findMapKey returns the key node for the given key in a MappingNode, or nil. +func findMapKey(m *yaml.Node, key string) *yaml.Node { + if m.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i] + } + } + return nil +} + // validateSpec ensures the spec is structurally sound. // It catches empty describes, missing fields, and assertions with no operators // set — all of which would otherwise silently pass. diff --git a/services/fmt_ghactions.go b/services/fmt_ghactions.go new file mode 100644 index 0000000..f10a9eb --- /dev/null +++ b/services/fmt_ghactions.go @@ -0,0 +1,106 @@ +package services + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/AxeForging/yamlspec/domain" +) + +// GitHubAnnotationsFormatter emits GitHub Actions workflow commands +// (::error file=...,line=...,title=...::message) for failing assertions and +// errored specs. When written to stdout in a workflow, GitHub renders these +// inline in the PR "Files changed" view. +// +// Spec format: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message +type GitHubAnnotationsFormatter struct{} + +// NewGitHubAnnotationsFormatter creates a new GitHubAnnotationsFormatter. +func NewGitHubAnnotationsFormatter() *GitHubAnnotationsFormatter { + return &GitHubAnnotationsFormatter{} +} + +func (f *GitHubAnnotationsFormatter) Format(result *domain.SuiteResult) ([]byte, error) { + cwd, _ := os.Getwd() + + var b strings.Builder + for _, spec := range result.Specs { + path := relPath(cwd, spec.SourceFile) + + if spec.Error != "" { + fmt.Fprintf(&b, "::error file=%s,title=yamlspec: %s::%s\n", + escapePath(path), escapeProp(spec.Name), escapeMsg(spec.Error)) + continue + } + + for _, desc := range spec.Describes { + for _, ar := range desc.Assertions { + if ar.Status != domain.StatusFailed && ar.Status != domain.StatusError { + continue + } + title := fmt.Sprintf("yamlspec: %s — %s", spec.Name, desc.Name) + msg := fmt.Sprintf("%s: %s", ar.Should, ar.Error) + if ar.Error == "" { + msg = ar.Should + } + + if ar.SourceLine > 0 && path != "" { + fmt.Fprintf(&b, "::error file=%s,line=%d,title=%s::%s\n", + escapePath(path), ar.SourceLine, escapeProp(title), escapeMsg(msg)) + } else if path != "" { + fmt.Fprintf(&b, "::error file=%s,title=%s::%s\n", + escapePath(path), escapeProp(title), escapeMsg(msg)) + } else { + fmt.Fprintf(&b, "::error title=%s::%s\n", escapeProp(title), escapeMsg(msg)) + } + } + } + } + return []byte(b.String()), nil +} + +// relPath returns specPath relative to cwd. Falls back to specPath unchanged +// if Rel fails or specPath is empty. +func relPath(cwd, specPath string) string { + if specPath == "" { + return "" + } + if cwd == "" { + return specPath + } + rel, err := filepath.Rel(cwd, specPath) + if err != nil { + return specPath + } + return rel +} + +// escapeProp escapes workflow-command property values. +// See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-using-properties +func escapeProp(s string) string { + r := strings.NewReplacer( + "%", "%25", + "\r", "%0D", + "\n", "%0A", + ":", "%3A", + ",", "%2C", + ) + return r.Replace(s) +} + +// escapeMsg escapes workflow-command message bodies. +func escapeMsg(s string) string { + r := strings.NewReplacer( + "%", "%25", + "\r", "%0D", + "\n", "%0A", + ) + return r.Replace(s) +} + +// escapePath is a path-safe variant of escapeProp — keeps slashes intact. +func escapePath(s string) string { + return escapeProp(s) +} diff --git a/services/runner.go b/services/runner.go index 1369e74..c7f8bf0 100644 --- a/services/runner.go +++ b/services/runner.go @@ -51,9 +51,10 @@ func (rs *RunnerService) RunAll(ctx context.Context, specs []DiscoveredSpec, con func (rs *RunnerService) RunOne(ctx context.Context, spec DiscoveredSpec, config *domain.Config) *domain.SpecResult { start := time.Now() result := &domain.SpecResult{ - Name: spec.Spec.Name, - Tags: spec.Spec.Tags, - Status: domain.StatusPassed, + Name: spec.Spec.Name, + Tags: spec.Spec.Tags, + Status: domain.StatusPassed, + SourceFile: spec.Spec.SourceFile, } // Execute pre_run commands