From 019bb26d27da012e74572d029c7b933fce72bb91 Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Fri, 7 Oct 2022 22:50:16 +0200 Subject: [PATCH 01/12] feat: import CLI app from benchttp/engine Some package renamings made sense: - configparse -> configfile - configflags -> configflag --- .github/ISSUE_TEMPLATE.md | 18 + .github/PULL_REQUEST_TEMPLATE.md | 16 + .github/workflows/ci.yml | 42 ++ .gitignore | 11 + .golangci.yml | 161 ++++++++ LICENSE | 23 ++ Makefile | 55 +++ README.md | 1 - TODO.md | 14 + cmd/benchttp/main.go | 46 +++ cmd/benchttp/run.go | 137 +++++++ cmd/benchttp/version.go | 21 + examples/config/default.yml | 13 + examples/config/full.yml | 22 ++ go.mod | 10 + go.sum | 10 + internal/ansi/style.go | 70 ++++ internal/cli/state.go | 88 +++++ internal/configfile/error.go | 26 ++ internal/configfile/find.go | 14 + internal/configfile/find_test.go | 31 ++ internal/configfile/parse.go | 372 ++++++++++++++++++ internal/configfile/parse_test.go | 264 +++++++++++++ internal/configfile/parser.go | 182 +++++++++ internal/configfile/parser_internal_test.go | 146 +++++++ .../testdata/extends/extends-circular-0.yml | 1 + .../testdata/extends/extends-circular-1.yml | 1 + .../testdata/extends/extends-circular-2.yml | 1 + .../extends/extends-circular-self.yml | 1 + .../testdata/extends/extends-valid-child.yml | 4 + .../testdata/extends/extends-valid-parent.yml | 3 + .../nest-0/nest-1/extends-valid-nested.yml | 4 + .../configfile/testdata/invalid/badext.yams | 2 + .../testdata/invalid/badfields.json | 7 + .../configfile/testdata/invalid/badfields.yml | 4 + .../testdata/valid/benchttp-zeros.yml | 3 + .../configfile/testdata/valid/benchttp.json | 47 +++ .../configfile/testdata/valid/benchttp.yaml | 38 ++ .../configfile/testdata/valid/benchttp.yml | 35 ++ internal/configflag/bind.go | 83 ++++ internal/configflag/bind_test.go | 74 ++++ internal/configflag/body.go | 50 +++ internal/configflag/header.go | 30 ++ internal/configflag/url.go | 29 ++ internal/configflag/which.go | 19 + internal/configflag/which_test.go | 52 +++ internal/errorutil/errorutil.go | 24 ++ internal/signals/signals.go | 17 + script/build | 35 ++ script/build-healthcheck | 17 + 50 files changed, 2373 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 TODO.md create mode 100644 cmd/benchttp/main.go create mode 100644 cmd/benchttp/run.go create mode 100644 cmd/benchttp/version.go create mode 100644 examples/config/default.yml create mode 100644 examples/config/full.yml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/ansi/style.go create mode 100644 internal/cli/state.go create mode 100644 internal/configfile/error.go create mode 100644 internal/configfile/find.go create mode 100644 internal/configfile/find_test.go create mode 100644 internal/configfile/parse.go create mode 100644 internal/configfile/parse_test.go create mode 100644 internal/configfile/parser.go create mode 100644 internal/configfile/parser_internal_test.go create mode 100644 internal/configfile/testdata/extends/extends-circular-0.yml create mode 100644 internal/configfile/testdata/extends/extends-circular-1.yml create mode 100644 internal/configfile/testdata/extends/extends-circular-2.yml create mode 100644 internal/configfile/testdata/extends/extends-circular-self.yml create mode 100644 internal/configfile/testdata/extends/extends-valid-child.yml create mode 100644 internal/configfile/testdata/extends/extends-valid-parent.yml create mode 100644 internal/configfile/testdata/extends/nest-0/nest-1/extends-valid-nested.yml create mode 100644 internal/configfile/testdata/invalid/badext.yams create mode 100644 internal/configfile/testdata/invalid/badfields.json create mode 100644 internal/configfile/testdata/invalid/badfields.yml create mode 100644 internal/configfile/testdata/valid/benchttp-zeros.yml create mode 100644 internal/configfile/testdata/valid/benchttp.json create mode 100644 internal/configfile/testdata/valid/benchttp.yaml create mode 100644 internal/configfile/testdata/valid/benchttp.yml create mode 100644 internal/configflag/bind.go create mode 100644 internal/configflag/bind_test.go create mode 100644 internal/configflag/body.go create mode 100644 internal/configflag/header.go create mode 100644 internal/configflag/url.go create mode 100644 internal/configflag/which.go create mode 100644 internal/configflag/which_test.go create mode 100644 internal/errorutil/errorutil.go create mode 100644 internal/signals/signals.go create mode 100755 script/build create mode 100755 script/build-healthcheck diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..5116628 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,18 @@ +## Description + + + +## Tasks + + + +## Suggestions + + + +## Notes + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..d359971 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ + + +## Description + + + +## Changes + + + +## Notes + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5f2bf95 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: Lint & Test & Build + +on: + pull_request: + push: + branches: + - main + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.17 + + - name: Install coverage tool + run: go get github.com/ory/go-acc + + # Check #1: Lint + - name: Lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.47.2 + + # Check #2: Test & generate coverage report + - name: Test & coverage + run: make test-cov + + # Check #3: Build binaries + - name: Build + run: make build + + - name: Upload coverage report + uses: codecov/codecov-action@v1.0.2 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.txt + flags: unittests diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a925db --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Binary files +/bin + +# IDE files +/.vscode +/.idea + +# Benchttp configs +/.benchttp.yml +/.benchttp.yaml +/.benchttp.json diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..64c7726 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,161 @@ +run: + timeout: 1m + uniq-by-line: false + +linters-settings: + dupl: + threshold: 100 + + errcheck: + exclude-functions: + - (net/http.ResponseWriter).Write + - (io.StringWriter).WriteString + - (io.Writer).Write + + gocognit: + min-complexity: 15 + + gocritic: + enabled-checks: + - appendAssign + - appendCombine + - argOrder + - assignOp + - badCall + - badCond + - badLock + - badRegexp + - boolExprSimplify + - builtinShadow + - builtinShadowDecl + - captLocal + - caseOrder + - codegenComment + - commentedOutCode + - commentedOutImport + - commentFormatting + - defaultCaseOrder + - deferUnlambda + - deprecatedComment + - docStub + - dupArg + - dupBranchBody + - dupCase + - dupImport + - dupSubExpr + - elseif + - emptyFallthrough + - emptyStringTest + - equalFold + - evalOrder + - exitAfterDefer + - filepathJoin + - flagDeref + - flagName + - hexLiteral + - hugeParam + - ifElseChain + - importShadow + - indexAlloc + - initClause + - mapKey + - methodExprCall + - nestingReduce + - newDeref + - nilValReturn + - octalLiteral + - offBy1 + - paramTypeCombine + - ptrToRefParam + - rangeExprCopy + - rangeValCopy + - regexpMust + - regexpPattern + - regexpSimplify + - ruleguard + - singleCaseSwitch + - sloppyLen + - sloppyReassign + - sloppyTypeAssert + - sortSlice + - sqlQuery + - stringXbytes + - switchTrue + - tooManyResultsChecker + - truncateCmp + - typeAssertChain + - typeDefFirst + - typeSwitchVar + - typeUnparen + - underef + # - unamedResult + - unlabelStmt + - unlambda + - unnecessaryBlock + - unnecessaryDefer + - unslice + - valSwap + - weakCond + # - whynolint + - wrapperFunc + - yodaStyleExpr + + settings: + hugeParam: + sizeThreshold: 256 + rangeValCopy: + sizeThreshold: 256 + + gofumpt: + lang-version: "1.17" + extra-rules: true + + goimports: + local-prefixes: github.com/benchttp/cli + + misspell: + locale: US + + revive: + enableAllRules: true + + staticcheck: + go: "1.17" + checks: [all] + + stylecheck: + go: "1.17" + checks: [all] + +linters: + disable-all: true + enable: + - bodyclose # enforce resp.Body.Close() + - deadcode + - dupl # duplicate code + - errcheck + - exportloopref + - gocognit # smart complexity analyzer + - gocritic # opinionated linter + - gofumpt # stricter gofmt + - goimports # imports order + - gosec # security checks + - govet + - misspell # typos in strings, comments + - prealloc # enforce capacity allocation when possible + - revive # golint enhancement + - staticcheck # go vet enhancement + - structcheck # unused struct fields + - testpackage # checks on tests (*_test) + - thelper # enforce t.Helper() + - varcheck # unused global var and const + - wastedassign + fast: false + +issues: + exclude-rules: + - path: _test\.go + linters: + - dupl + - gocognit + - gocyclo diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..73ec2c6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +MIT License + +Copyright (c) 2021 +Gregory Albouy, Clara Gonnon, Damien Mathieu +Alex Mongeot, Thomas Moreira, Kerian Pelat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a5bb555 --- /dev/null +++ b/Makefile @@ -0,0 +1,55 @@ +# Default command + +.PHONY: default +default: + @make check + +# Check code + +.PHONY: check +check: + @make lint + @make tests + +.PHONY: lint +lint: + @golangci-lint run + +.PHONY: tests +tests: + @go test -race -timeout 10s ./... + +TEST_FUNC=^.*$$ +ifdef t +TEST_FUNC=$(t) +endif +TEST_PKG=./... +ifdef p +TEST_PKG=./$(p) +endif + +.PHONY: test +test: + @go test -race -timeout 10s -run $(TEST_FUNC) $(TEST_PKG) + + +.PHONY: test-cov +test-cov: + @go-acc ./... + +# Build +.PHONY: build +build: + @./script/build + @./script/build-healthcheck + +.PHONY: clear +clear: + @rm -rf ./bin/* + +# Docs + +.PHONY: docs +docs: + @echo "\033[4mhttp://localhost:9995/pkg/github.com/benchttp/cli/\033[0m" + @godoc -http=localhost:9995 diff --git a/README.md b/README.md index 2c38811..e69de29 100644 --- a/README.md +++ b/README.md @@ -1 +0,0 @@ -# benchttp/cli diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..44a72fb --- /dev/null +++ b/TODO.md @@ -0,0 +1,14 @@ +# TODO + +## `refactor/config` + +- `runner.ConfigField` -> `runner.Option` +- `Config.WithFelds` -> `Config.WithOptionsSet` +- commit `test(config): fix breaking` -> fixup `feat: implement config.SetFields` +- rename `interface configfiler` -> `interface configParser` (renaming error) +- remove dependency to `testx` +- embed token.txt? + +## `poc/test-suite` + +- move `configfile.requireConfigField` into `configfile.requireConfigFields` diff --git a/cmd/benchttp/main.go b/cmd/benchttp/main.go new file mode 100644 index 0000000..e3c11b3 --- /dev/null +++ b/cmd/benchttp/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "os" +) + +// errUsage reports an incorrect usage of the benchttp command. +var errUsage = errors.New("usage") + +func main() { + if err := run(); err != nil { + fmt.Println(err) + if errors.Is(err, errUsage) { + flag.Usage() + } + os.Exit(1) + } +} + +func run() error { + if len(os.Args) < 2 { + return fmt.Errorf("%w: no command specified", errUsage) + } + + var cmd command + args := os.Args[1:] + + switch sub := args[0]; sub { + case "run": + cmd = &cmdRun{flagset: flag.NewFlagSet("run", flag.ExitOnError)} + case "version": + cmd = &cmdVersion{} + default: + return fmt.Errorf("%w: unknown command: %s", errUsage, sub) + } + + return cmd.execute(args) +} + +// command is the interface that all benchttp subcommands must implement. +type command interface { + execute(args []string) error +} diff --git a/cmd/benchttp/run.go b/cmd/benchttp/run.go new file mode 100644 index 0000000..d9fd1b0 --- /dev/null +++ b/cmd/benchttp/run.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/cli" + "github.com/benchttp/cli/internal/configfile" + "github.com/benchttp/cli/internal/configflag" + "github.com/benchttp/cli/internal/signals" +) + +// cmdRun handles subcommand "benchttp run [options]". +type cmdRun struct { + flagset *flag.FlagSet + + // configFile is the parsed value for flag -configFile + configFile string + + // config is the runner config resulting from parsing CLI flags. + config runner.Config +} + +// init initializes cmdRun with default values. +func (cmd *cmdRun) init() { + cmd.config = runner.DefaultConfig() + cmd.configFile = configfile.Find([]string{ + "./.benchttp.yml", + "./.benchttp.yaml", + "./.benchttp.json", + }) +} + +// execute runs the benchttp runner: it parses CLI flags, loads config +// from config file and parsed flags, then runs the benchmark and outputs +// it according to the config. +func (cmd *cmdRun) execute(args []string) error { + cmd.init() + + // Set CLI config from flags and retrieve fields that were set + fieldsSet := cmd.parseArgs(args) + + // Generate merged config (defaults < config file < CLI flags) + cfg, err := cmd.makeConfig(fieldsSet) + if err != nil { + return err + } + + // Prepare graceful shutdown in case of os.Interrupt (Ctrl+C) + ctx, cancel := context.WithCancel(context.Background()) + go signals.ListenOSInterrupt(cancel) + + // Run the benchmark + out, err := runner. + New(onRecordingProgress(cfg.Output.Silent)). + Run(ctx, cfg) + if err != nil { + return err + } + + // Write results to stdout + if _, err := out.Write(os.Stdout); err != nil { + return err + } + + if !out.Tests.Pass { + return errors.New("test suite failed") + } + + return nil +} + +// parseArgs parses input args as config fields and returns +// a slice of fields that were set by the user. +func (cmd *cmdRun) parseArgs(args []string) []string { + // first arg is subcommand "run" + // skip parsing if no flags are provided + if len(args) <= 1 { + return []string{} + } + + // config file path + cmd.flagset.StringVar(&cmd.configFile, + "configFile", + cmd.configFile, + "Config file path", + ) + + // attach config options flags to the flagset + // and bind their value to the config struct + configflag.Bind(cmd.flagset, &cmd.config) + + cmd.flagset.Parse(args[1:]) //nolint:errcheck // never occurs due to flag.ExitOnError + + return configflag.Which(cmd.flagset) +} + +// makeConfig returns a runner.ConfigGlobal initialized with config file +// options if found, overridden with CLI options listed in fields +// slice param. +func (cmd *cmdRun) makeConfig(fields []string) (cfg runner.Config, err error) { + // configFile not set and default ones not found: + // skip the merge and return the cli config + if cmd.configFile == "" { + return cmd.config, cmd.config.Validate() + } + + fileConfig, err := configfile.Parse(cmd.configFile) + if err != nil && !errors.Is(err, configfile.ErrFileNotFound) { + // config file is not mandatory: discard ErrFileNotFound. + // other errors are critical + return + } + + mergedConfig := fileConfig.Override(cmd.config, fields...) + + return mergedConfig, mergedConfig.Validate() +} + +func onRecordingProgress(silent bool) func(runner.RecordingProgress) { + if silent { + return func(runner.RecordingProgress) {} + } + + // hack: write a blank line as cli.WriteRecordingProgress always + // erases the previous line + fmt.Println() + + return func(progress runner.RecordingProgress) { + cli.WriteRecordingProgress(os.Stdout, progress) //nolint: errcheck + } +} diff --git a/cmd/benchttp/version.go b/cmd/benchttp/version.go new file mode 100644 index 0000000..94d3527 --- /dev/null +++ b/cmd/benchttp/version.go @@ -0,0 +1,21 @@ +package main + +import "fmt" + +// benchttpVersion is the current version of benchttp +// as output by `benchttp version`. It is assumed to be set +// by `go build -ldflags "-X main.benchttpVersion="`, +// allowing us to set the value dynamically at build time +// using latest git tag. +// +// Its default value "development" is only used when the app +// is ran locally without a build (e.g. `go run ./cmd/benchttp`). +var benchttpVersion = "development" + +// cmdVersion handles subcommand "benchttp version". +type cmdVersion struct{} + +func (cmdVersion) execute([]string) error { + fmt.Println("benchttp", benchttpVersion) + return nil +} diff --git a/examples/config/default.yml b/examples/config/default.yml new file mode 100644 index 0000000..07e6bea --- /dev/null +++ b/examples/config/default.yml @@ -0,0 +1,13 @@ +request: + method: GET + url: "" # empty + +runner: + requests: 100 + concurrency: 10 + interval: 0ms + requestTimeout: 5s + globalTimeout: 30s + +output: + silent: false diff --git a/examples/config/full.yml b/examples/config/full.yml new file mode 100644 index 0000000..c20fb5e --- /dev/null +++ b/examples/config/full.yml @@ -0,0 +1,22 @@ +request: + method: POST + url: http://localhost:8080/users + queryParams: + page: 3 + sort: asc + header: + key0: [val0, val1] + key1: [val0] + body: + type: raw # only "raw" accepted at the moment + content: '{"key0":"val0","key1":"val1"}' + +runner: + requests: 100 + concurrency: 1 + interval: 50ms + requestTimeout: 2s + globalTimeout: 60s + +output: + silent: true diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..417262f --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/benchttp/cli + +go 1.17 + +require ( + github.com/benchttp/engine v0.0.0-20221006130541-30d09b451066 + gopkg.in/yaml.v3 v3.0.1 +) + +require golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..da7311a --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/benchttp/engine v0.0.0-20221006130541-30d09b451066 h1:epXygHg38XvHZ41VXf/mvDWKg33UcuJE7qKSVhwZwtg= +github.com/benchttp/engine v0.0.0-20221006130541-30d09b451066/go.mod h1:FRfUnUjoL1s0aHVGlrxB3pdPAEDLNCnWh6cVOur24hM= +github.com/drykit-go/cond v0.1.0 h1:y7MNxREQLT83vGfcfSKjyFPLC/ZDjYBNp6KuaVVjOg4= +github.com/drykit-go/testx v1.2.0 h1:UsH+tFd24z3Xu+mwvwPY+9eBEg9CUyMsUeMYyUprG0o= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/ansi/style.go b/internal/ansi/style.go new file mode 100644 index 0000000..a7c9227 --- /dev/null +++ b/internal/ansi/style.go @@ -0,0 +1,70 @@ +package ansi + +import ( + "fmt" + "strings" +) + +type StyleFunc func(in string) string + +type style string + +const ( + reset style = "\033[0m" + + bold style = "\033[1m" + + grey style = "\033[1;30m" + red style = "\033[1;31m" + green style = "\033[1;32m" + yellow style = "\033[1;33m" + cyan style = "\033[1;36m" + + erase style = "\033[1A" +) + +func withStyle(in string, s style) string { + return fmt.Sprintf("%s%s%s", s, in, reset) +} + +// Bold returns the bold version of the input string. +func Bold(in string) string { + return withStyle(in, bold) +} + +// Green returns the green version of the input string. +func Green(in string) string { + return withStyle(in, green) +} + +// Yellow returns the yellow version of the input string. +func Yellow(in string) string { + return withStyle(in, yellow) +} + +// Cyan returns the cyan version of the input string. +func Cyan(in string) string { + return withStyle(in, cyan) +} + +// Red returns the red version of the input string. +func Red(in string) string { + return withStyle(in, red) +} + +// Grey returns the grey version of the input string. +func Grey(in string) string { + return withStyle(in, grey) +} + +// Erase returns a string that erases the previous line n times. +func Erase(n int) string { + if n < 1 { + return "" + } + var b strings.Builder + for i := 0; i < n; i++ { + b.Write([]byte(erase)) + } + return b.String() +} diff --git a/internal/cli/state.go b/internal/cli/state.go new file mode 100644 index 0000000..255a3b5 --- /dev/null +++ b/internal/cli/state.go @@ -0,0 +1,88 @@ +package cli + +import ( + "fmt" + "io" + "strconv" + "strings" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/ansi" +) + +// WriteRecordingProgress renders a fancy representation of p as a string +// and writes the result to w. +func WriteRecordingProgress(w io.Writer, p runner.RecordingProgress) (int, error) { + return fmt.Fprint(w, renderProgress(p)) +} + +// renderProgress returns a string representation of runner.RecordingProgress +// for a fancy display in a CLI: +// +// RUNNING ◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎ 50% | 50/100 requests | 27s timeout +func renderProgress(s runner.RecordingProgress) string { + var ( + countdown = s.Timeout - s.Elapsed + reqmax = strconv.Itoa(s.MaxCount) + pctdone = s.Percent() + timeline = renderTimeline(pctdone) + ) + + if reqmax == "-1" { + reqmax = "∞" + } + if countdown < 0 { + countdown = 0 + } + + return fmt.Sprintf( + "%s%s %s %d%% | %d/%s requests | %.0fs timeout \n", + ansi.Erase(1), // replace previous line + renderStatus(s.Status()), timeline, pctdone, // progress + s.DoneCount, reqmax, // requests + countdown.Seconds(), // timeout + ) +} + +var ( + tlBlock = "◼︎" + tlBlockGrey = ansi.Grey(tlBlock) + tlBlockGreen = ansi.Green(tlBlock) + tlLen = 10 +) + +// renderTimeline returns a colored representation of the progress as a string: +// +// ◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎ +func renderTimeline(pctdone int) string { + tl := strings.Repeat(tlBlockGrey, tlLen) + for i := 0; i < tlLen; i++ { + if pctdone >= (tlLen * i) { + tl = strings.Replace(tl, tlBlockGrey, tlBlockGreen, 1) + } + } + return tl +} + +// renderStatus returns a string representing the status, +// depending on whether the run is done or not and the value +// of its context error. +func renderStatus(status runner.RecordingStatus) string { + color := statusStyle(status) + return color(string(status)) +} + +func statusStyle(status runner.RecordingStatus) ansi.StyleFunc { + switch status { + case runner.StatusRunning: + return ansi.Yellow + case runner.StatusDone: + return ansi.Green + case runner.StatusCanceled: + return ansi.Red + case runner.StatusTimeout: + return ansi.Cyan + } + return ansi.Grey // should not occur +} diff --git a/internal/configfile/error.go b/internal/configfile/error.go new file mode 100644 index 0000000..61fdcf6 --- /dev/null +++ b/internal/configfile/error.go @@ -0,0 +1,26 @@ +package configfile + +import ( + "errors" +) + +var ( + // ErrFileNotFound signals a config file not found. + ErrFileNotFound = errors.New("file not found") + + // ErrFileRead signals an error trying to read a config file. + // It can be due to a corrupted file or an invalid permission + // for instance. + ErrFileRead = errors.New("invalid file") + + // ErrFileExt signals an unsupported extension for the config file. + ErrFileExt = errors.New("invalid extension") + + // ErrParse signals an error parsing a retrieved config file. + // It is returned if it contains an unexpected field or an unexpected + // value for a field. + ErrParse = errors.New("parsing error: invalid config file") + + // ErrCircularExtends signals a circular reference in the config file. + ErrCircularExtends = errors.New("circular reference detected") +) diff --git a/internal/configfile/find.go b/internal/configfile/find.go new file mode 100644 index 0000000..755e39a --- /dev/null +++ b/internal/configfile/find.go @@ -0,0 +1,14 @@ +package configfile + +import "os" + +// Find returns the first name tham matches a file path. +// If no match is found, it returns an empty string. +func Find(names []string) string { + for _, path := range names { + if _, err := os.Stat(path); err == nil { // err IS nil: file exists + return path + } + } + return "" +} diff --git a/internal/configfile/find_test.go b/internal/configfile/find_test.go new file mode 100644 index 0000000..df6490d --- /dev/null +++ b/internal/configfile/find_test.go @@ -0,0 +1,31 @@ +package configfile_test + +import ( + "testing" + + "github.com/benchttp/cli/internal/configfile" +) + +var ( + goodFileYML = configPath("valid/benchttp.yml") + goodFileJSON = configPath("valid/benchttp.json") + badFile = configPath("does-not-exist.json") +) + +func TestFind(t *testing.T) { + t.Run("return first existing file", func(t *testing.T) { + files := []string{badFile, goodFileYML, goodFileJSON} + + if got := configfile.Find(files); got != goodFileYML { + t.Errorf("did not retrieve good file: exp %s, got %s", goodFileYML, got) + } + }) + + t.Run("return empty string when no match", func(t *testing.T) { + files := []string{badFile} + + if got := configfile.Find(files); got != "" { + t.Errorf("retrieved unexpected file: %s", got) + } + }) +} diff --git a/internal/configfile/parse.go b/internal/configfile/parse.go new file mode 100644 index 0000000..0cfeb10 --- /dev/null +++ b/internal/configfile/parse.go @@ -0,0 +1,372 @@ +package configfile + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/errorutil" +) + +// UnmarshaledConfig is a raw data model for runner config files. +// It serves as a receiver for unmarshaling processes and for that reason +// its types are kept simple (certain types are incompatible with certain +// unmarshalers). +type UnmarshaledConfig struct { + Extends *string `yaml:"extends" json:"extends"` + + Request struct { + Method *string `yaml:"method" json:"method"` + URL *string `yaml:"url" json:"url"` + QueryParams map[string]string `yaml:"queryParams" json:"queryParams"` + Header map[string][]string `yaml:"header" json:"header"` + Body *struct { + Type string `yaml:"type" json:"type"` + Content string `yaml:"content" json:"content"` + } `yaml:"body" json:"body"` + } `yaml:"request" json:"request"` + + Runner struct { + Requests *int `yaml:"requests" json:"requests"` + Concurrency *int `yaml:"concurrency" json:"concurrency"` + Interval *string `yaml:"interval" json:"interval"` + RequestTimeout *string `yaml:"requestTimeout" json:"requestTimeout"` + GlobalTimeout *string `yaml:"globalTimeout" json:"globalTimeout"` + } `yaml:"runner" json:"runner"` + + Output struct { + Silent *bool `yaml:"silent" json:"silent"` + } `yaml:"output" json:"output"` + + Tests []struct { + Name *string `yaml:"name" json:"name"` + Field *string `yaml:"field" json:"field"` + Predicate *string `yaml:"predicate" json:"predicate"` + Target interface{} `yaml:"target" json:"target"` + } `yaml:"tests" json:"tests"` +} + +// Parse parses a benchttp runner config file into a runner.ConfigGlobal +// and returns it or the first non-nil error occurring in the process, +// which can be any of the values declared in the package. +func Parse(filename string) (cfg runner.Config, err error) { + uconfs, err := parseFileRecursive(filename, []UnmarshaledConfig{}, set{}) + if err != nil { + return + } + return parseAndMergeConfigs(uconfs) +} + +// set is a collection of unique string values. +type set map[string]bool + +// add adds v to the receiver. If v is already set, it returns a non-nil +// error instead. +func (s set) add(v string) error { + if _, exists := s[v]; exists { + return errors.New("value already set") + } + s[v] = true + return nil +} + +// parseFileRecursive parses a config file and its parent found from key +// "extends" recursively until the root config file is reached. +// It returns the list of all parsed configs or the first non-nil error +// occurring in the process. +func parseFileRecursive( + filename string, + uconfs []UnmarshaledConfig, + seen set, +) ([]UnmarshaledConfig, error) { + // avoid infinite recursion caused by circular reference + if err := seen.add(filename); err != nil { + return uconfs, ErrCircularExtends + } + + // parse current file, append parsed config + uconf, err := parseFile(filename) + if err != nil { + return uconfs, err + } + uconfs = append(uconfs, uconf) + + // root config reached: stop now and return the parsed configs + if uconf.Extends == nil { + return uconfs, nil + } + + // config has parent: resolve its path and parse it recursively + parentPath := filepath.Join(filepath.Dir(filename), *uconf.Extends) + return parseFileRecursive(parentPath, uconfs, seen) +} + +// parseFile parses a single config file and returns the result as an +// unmarshaledConfig and an appropriate error predeclared in the package. +func parseFile(filename string) (uconf UnmarshaledConfig, err error) { + b, err := os.ReadFile(filename) + switch { + case err == nil: + case errors.Is(err, os.ErrNotExist): + return uconf, errorutil.WithDetails(ErrFileNotFound, filename) + default: + return uconf, errorutil.WithDetails(ErrFileRead, filename, err) + } + + ext := extension(filepath.Ext(filename)) + parser, err := newParser(ext) + if err != nil { + return uconf, errorutil.WithDetails(ErrFileExt, ext, err) + } + + if err = parser.parse(b, &uconf); err != nil { + return uconf, errorutil.WithDetails(ErrParse, filename, err) + } + + return uconf, nil +} + +// parseAndMergeConfigs iterates backwards over uconfs, parsing them +// as runner.ConfigGlobal and merging them into a single one. +// It returns the merged result or the first non-nil error occurring in the +// process. +func parseAndMergeConfigs(uconfs []UnmarshaledConfig) (cfg runner.Config, err error) { + if len(uconfs) == 0 { // supposedly catched upstream, should not occur + return cfg, errors.New( + "an unacceptable error occurred parsing the config file, " + + "please visit https://github.com/benchttp/runner/issues/new " + + "and insult us properly", + ) + } + + cfg = runner.DefaultConfig() + + for i := len(uconfs) - 1; i >= 0; i-- { + uconf := uconfs[i] + pconf, err := newParsedConfig(uconf) + if err != nil { + return cfg, errorutil.WithDetails(ErrParse, err) + } + cfg = cfg.Override(pconf.config, pconf.fields...) + } + + return cfg, nil +} + +// parsedConfig embeds a parsed runner.ConfigGlobal and the list of its set fields. +type parsedConfig struct { + // TODO: do not embed, use field config + config runner.Config + fields []string +} + +// addField adds a field to the list of set fields. +func (pconf *parsedConfig) add(field string) { + pconf.fields = append(pconf.fields, field) +} + +// newParsedConfig parses an input raw config as a runner.ConfigGlobal and returns +// a parsedConfig or the first non-nil error occurring in the process. +func newParsedConfig(uconf UnmarshaledConfig) (parsedConfig, error) { //nolint:gocognit // acceptable complexity for a parsing func + maxFields := len(runner.ConfigFieldsUsage) + pconf := parsedConfig{fields: make([]string, 0, maxFields)} + cfg := &pconf.config + + abort := func(err error) (parsedConfig, error) { + return parsedConfig{}, err + } + + if method := uconf.Request.Method; method != nil { + cfg.Request.Method = *method + pconf.add(runner.ConfigFieldMethod) + } + + if rawURL := uconf.Request.URL; rawURL != nil { + parsedURL, err := parseAndBuildURL(*uconf.Request.URL, uconf.Request.QueryParams) + if err != nil { + return abort(err) + } + cfg.Request.URL = parsedURL + pconf.add(runner.ConfigFieldURL) + } + + if header := uconf.Request.Header; header != nil { + httpHeader := http.Header{} + for key, val := range header { + httpHeader[key] = val + } + cfg.Request.Header = httpHeader + pconf.add(runner.ConfigFieldHeader) + } + + if body := uconf.Request.Body; body != nil { + cfg.Request.Body = runner.RequestBody{ + Type: body.Type, + Content: []byte(body.Content), + } + pconf.add(runner.ConfigFieldBody) + } + + if requests := uconf.Runner.Requests; requests != nil { + cfg.Runner.Requests = *requests + pconf.add(runner.ConfigFieldRequests) + } + + if concurrency := uconf.Runner.Concurrency; concurrency != nil { + cfg.Runner.Concurrency = *concurrency + pconf.add(runner.ConfigFieldConcurrency) + } + + if interval := uconf.Runner.Interval; interval != nil { + parsedInterval, err := parseOptionalDuration(*interval) + if err != nil { + return abort(err) + } + cfg.Runner.Interval = parsedInterval + pconf.add(runner.ConfigFieldInterval) + } + + if requestTimeout := uconf.Runner.RequestTimeout; requestTimeout != nil { + parsedTimeout, err := parseOptionalDuration(*requestTimeout) + if err != nil { + return abort(err) + } + cfg.Runner.RequestTimeout = parsedTimeout + pconf.add(runner.ConfigFieldRequestTimeout) + } + + if globalTimeout := uconf.Runner.GlobalTimeout; globalTimeout != nil { + parsedGlobalTimeout, err := parseOptionalDuration(*globalTimeout) + if err != nil { + return abort(err) + } + cfg.Runner.GlobalTimeout = parsedGlobalTimeout + pconf.add(runner.ConfigFieldGlobalTimeout) + } + + if silent := uconf.Output.Silent; silent != nil { + cfg.Output.Silent = *silent + pconf.add(runner.ConfigFieldSilent) + } + + testSuite := uconf.Tests + if len(testSuite) == 0 { + return pconf, nil + } + + cases := make([]runner.TestCase, len(testSuite)) + for i, t := range testSuite { + fieldPath := func(caseField string) string { + return fmt.Sprintf("tests[%d].%s", i, caseField) + } + + if err := requireConfigFields(map[string]interface{}{ + fieldPath("name"): t.Name, + fieldPath("field"): t.Field, + fieldPath("predicate"): t.Predicate, + fieldPath("target"): t.Target, + }); err != nil { + return abort(err) + } + + field := runner.MetricsField(*t.Field) + if err := field.Validate(); err != nil { + return abort(fmt.Errorf("%s: %s", fieldPath("field"), err)) + } + + predicate := runner.TestPredicate(*t.Predicate) + if err := predicate.Validate(); err != nil { + return abort(fmt.Errorf("%s: %s", fieldPath("predicate"), err)) + } + + target, err := parseMetricValue(field, fmt.Sprint(t.Target)) + if err != nil { + return abort(fmt.Errorf("%s: %s", fieldPath("target"), err)) + } + + cases[i] = runner.TestCase{ + Name: *t.Name, + Field: field, + Predicate: predicate, + Target: target, + } + } + cfg.Tests = cases + pconf.add(runner.ConfigFieldTests) + + return pconf, nil +} + +// helpers + +// parseAndBuildURL parses a raw string as a *url.URL and adds any extra +// query parameters. It returns the first non-nil error occurring in the +// process. +func parseAndBuildURL(raw string, qp map[string]string) (*url.URL, error) { + u, err := url.ParseRequestURI(raw) + if err != nil { + return nil, err + } + + // retrieve url query, add extra params, re-attach to url + if qp != nil { + q := u.Query() + for k, v := range qp { + q.Add(k, v) + } + u.RawQuery = q.Encode() + } + + return u, nil +} + +// parseOptionalDuration parses the raw string as a time.Duration +// and returns the parsed value or a non-nil error. +// Contrary to time.ParseDuration, it does not return an error +// if raw == "". +func parseOptionalDuration(raw string) (time.Duration, error) { + if raw == "" { + return 0, nil + } + return time.ParseDuration(raw) +} + +func parseMetricValue( + field runner.MetricsField, + inputValue string, +) (runner.MetricsValue, error) { + fieldType := field.Type() + handleError := func(v interface{}, err error) (runner.MetricsValue, error) { + if err != nil { + return nil, fmt.Errorf( + "value %q is incompatible with field %s (want %s)", + inputValue, field, fieldType, + ) + } + return v, nil + } + switch fieldType { + case "int": + return handleError(strconv.Atoi(inputValue)) + case "time.Duration": + return handleError(time.ParseDuration(inputValue)) + default: + return nil, fmt.Errorf("unknown field: %s", field) + } +} + +func requireConfigFields(fields map[string]interface{}) error { + for name, value := range fields { + if value == nil { + return fmt.Errorf("%s: missing field", name) + } + } + return nil +} diff --git a/internal/configfile/parse_test.go b/internal/configfile/parse_test.go new file mode 100644 index 0000000..7b910db --- /dev/null +++ b/internal/configfile/parse_test.go @@ -0,0 +1,264 @@ +package configfile_test + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/configfile" +) + +const ( + testdataConfigPath = "./testdata" + testURL = "http://localhost:9999?fib=30&delay=200ms" // value from testdata files +) + +var supportedExt = []string{ + ".yml", + ".yaml", + ".json", +} + +// TestParse ensures the config file is open, read, and correctly parsed. +func TestParse(t *testing.T) { + t.Run("return file errors early", func(t *testing.T) { + testcases := []struct { + label string + path string + expErr error + }{ + { + label: "not found", + path: configPath("invalid/bad path"), + expErr: configfile.ErrFileNotFound, + }, + { + label: "unsupported extension", + path: configPath("invalid/badext.yams"), + expErr: configfile.ErrFileExt, + }, + { + label: "yaml invalid fields", + path: configPath("invalid/badfields.yml"), + expErr: configfile.ErrParse, + }, + { + label: "json invalid fields", + path: configPath("invalid/badfields.json"), + expErr: configfile.ErrParse, + }, + { + label: "self reference", + path: configPath("extends/extends-circular-self.yml"), + expErr: configfile.ErrCircularExtends, + }, + { + label: "circular reference", + path: configPath("extends/extends-circular-0.yml"), + expErr: configfile.ErrCircularExtends, + }, + } + + for _, tc := range testcases { + t.Run(tc.label, func(t *testing.T) { + gotCfg, gotErr := configfile.Parse(tc.path) + + if gotErr == nil { + t.Fatal("exp non-nil error, got nil") + } + + if !errors.Is(gotErr, tc.expErr) { + t.Errorf("\nexp %v\ngot %v", tc.expErr, gotErr) + } + + if !reflect.DeepEqual(gotCfg, runner.Config{}) { + t.Errorf("\nexp empty config\ngot %v", gotCfg) + } + }) + } + }) + + t.Run("happy path for all extensions", func(t *testing.T) { + for _, ext := range supportedExt { + expCfg := newExpConfig() + fname := configPath("valid/benchttp" + ext) + + gotCfg, err := configfile.Parse(fname) + if err != nil { + // critical error, stop the test + t.Fatal(err) + } + + expURL, gotURL := expCfg.Request.URL, gotCfg.Request.URL + + // compare *url.URLs separately, as they contain unpredictable values + // they need special checks + if !sameURL(gotURL, expURL) { + t.Errorf("unexpected parsed URL:\nexp %v, got %v", expURL, gotURL) + } + + // replace unpredictable values (undetermined query params order) + restoreGotCfg := setTempValue(&gotURL.RawQuery, "replaced by test") + restoreExpCfg := setTempValue(&expURL.RawQuery, "replaced by test") + + if !reflect.DeepEqual(gotCfg, expCfg) { + t.Errorf("unexpected parsed config for %s file:\nexp %v\ngot %v", ext, expCfg, gotCfg) + } + + restoreExpCfg() + restoreGotCfg() + } + }) + + t.Run("override default values", func(t *testing.T) { + const ( + expRequests = 0 // default is -1 + expGlobalTimeout = 42 * time.Millisecond + ) + + fname := configPath("valid/benchttp-zeros.yml") + + cfg, err := configfile.Parse(fname) + if err != nil { + t.Fatal(err) + } + + if gotRequests := cfg.Runner.Requests; gotRequests != expRequests { + t.Errorf("did not override Requests: exp %d, got %d", expRequests, gotRequests) + } + + if gotGlobalTimeout := cfg.Runner.GlobalTimeout; gotGlobalTimeout != expGlobalTimeout { + t.Errorf("did not override GlobalTimeout: exp %d, got %d", expGlobalTimeout, gotGlobalTimeout) + } + + t.Log(cfg) + }) + + t.Run("extend config files", func(t *testing.T) { + testcases := []struct { + label string + cfname string + cfpath string + }{ + { + label: "same directory", + cfname: "child", + cfpath: configPath("extends/extends-valid-child.yml"), + }, + { + label: "nested directory", + cfname: "nested", + cfpath: configPath("extends/nest-0/nest-1/extends-valid-nested.yml"), + }, + } + + for _, tc := range testcases { + t.Run(tc.label, func(t *testing.T) { + cfg, err := configfile.Parse(tc.cfpath) + if err != nil { + t.Fatal(err) + } + + var ( + expMethod = "POST" + expURL = fmt.Sprintf("http://%s.config", tc.cfname) + ) + + if gotMethod := cfg.Request.Method; gotMethod != expMethod { + t.Errorf("method: exp %s, got %s", expMethod, gotMethod) + } + + if gotURL := cfg.Request.URL.String(); gotURL != expURL { + t.Errorf("method: exp %s, got %s", expURL, gotURL) + } + }) + } + }) +} + +// helpers + +// newExpConfig returns the expected runner.ConfigConfig result after parsing +// one of the config files in testdataConfigPath. +func newExpConfig() runner.Config { + u, _ := url.ParseRequestURI(testURL) + return runner.Config{ + Request: runner.RequestConfig{ + Method: "POST", + URL: u, + Header: http.Header{ + "key0": []string{"val0", "val1"}, + "key1": []string{"val0"}, + }, + Body: runner.NewRequestBody("raw", `{"key0":"val0","key1":"val1"}`), + }, + Runner: runner.RecorderConfig{ + Requests: 100, + Concurrency: 1, + Interval: 50 * time.Millisecond, + RequestTimeout: 2 * time.Second, + GlobalTimeout: 60 * time.Second, + }, + Output: runner.OutputConfig{ + Silent: true, + }, + Tests: []runner.TestCase{ + { + Name: "minimum response time", + Field: "ResponseTimes.Min", + Predicate: "GT", + Target: 80 * time.Millisecond, + }, + { + Name: "maximum response time", + Field: "ResponseTimes.Max", + Predicate: "LTE", + Target: 120 * time.Millisecond, + }, + { + Name: "100% availability", + Field: "RequestFailureCount", + Predicate: "EQ", + Target: 0, + }, + }, + } +} + +// sameURL returns true if a and b are the same *url.URL, taking into account +// the undeterministic nature of their RawQuery. +func sameURL(a, b *url.URL) bool { + // check query params equality via Query() rather than RawQuery + if !reflect.DeepEqual(a.Query(), b.Query()) { + return false + } + + // temporarily set RawQuery to a determined value + for _, u := range []*url.URL{a, b} { + defer setTempValue(&u.RawQuery, "replaced by test")() + } + + // we can now rely on deep equality check + return reflect.DeepEqual(a, b) +} + +// setTempValue sets *ptr to val and returns a restore func that sets *ptr +// back to its previous value. +func setTempValue(ptr *string, val string) (restore func()) { + previousValue := *ptr + *ptr = val + return func() { + *ptr = previousValue + } +} + +func configPath(name string) string { + return filepath.Join(testdataConfigPath, name) +} diff --git a/internal/configfile/parser.go b/internal/configfile/parser.go new file mode 100644 index 0000000..3103f13 --- /dev/null +++ b/internal/configfile/parser.go @@ -0,0 +1,182 @@ +package configfile + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "regexp" + + "gopkg.in/yaml.v3" +) + +type extension string + +const ( + extYML extension = ".yml" + extYAML extension = ".yaml" + extJSON extension = ".json" +) + +// configParser exposes a method parse to read bytes as a raw config. +type configParser interface { + // parse parses a raw bytes input as a raw config and stores + // the resulting value into dst. + parse(in []byte, dst *UnmarshaledConfig) error +} + +// newParser returns an appropriate parser according to ext, or a non-nil +// error if ext is not an expected extension. +func newParser(ext extension) (configParser, error) { + switch ext { + case extYML, extYAML: + return yamlParser{}, nil + case extJSON: + return jsonParser{}, nil + default: + return nil, errors.New("unsupported config format") + } +} + +// yamlParser implements configParser. +type yamlParser struct{} + +// parse decodes a raw yaml input in strict mode (unknown fields disallowed) +// and stores the resulting value into dst. +func (p yamlParser) parse(in []byte, dst *UnmarshaledConfig) error { + decoder := yaml.NewDecoder(bytes.NewReader(in)) + decoder.KnownFields(true) + return p.handleError(decoder.Decode(dst)) +} + +// handleError handles a raw yaml decoder.Decode error, filters it, +// and return the resulting error. +func (p yamlParser) handleError(err error) error { + // yaml.TypeError errors require special handling, other errors + // (nil included) can be returned as is. + var typeError *yaml.TypeError + if !errors.As(err, &typeError) { + return err + } + + // filter out unwanted errors + filtered := &yaml.TypeError{} + for _, msg := range typeError.Errors { + // With decoder.KnownFields set to true, Decode reports any field + // that do not match the destination structure as a non-nil error. + // It is a wanted behavior but prevents the usage of custom aliases. + // To work around this we allow an exception for that rule with fields + // starting with x- (inspired by docker compose api). + if p.isCustomFieldError(msg) { + continue + } + filtered.Errors = append(filtered.Errors, p.prettyErrorMessage(msg)) + } + + if len(filtered.Errors) != 0 { + return filtered + } + + return nil +} + +// isCustomFieldError returns true if the raw error message is due +// to an allowed custom field. +func (p yamlParser) isCustomFieldError(raw string) bool { + customFieldRgx := regexp.MustCompile( + // raw output example: + // line 9: field x-my-alias not found in type struct { ... } + `^line \d+: field (x-\S+) not found in type`, + ) + return customFieldRgx.MatchString(raw) +} + +// prettyErrorMessage transforms a raw Decode error message into a more +// user-friendly one by removing noisy information and returns the resulting +// value. +func (p yamlParser) prettyErrorMessage(raw string) string { + // field not found error + fieldNotFoundRgx := regexp.MustCompile( + // raw output example (type unmarshaledConfig is entirely exposed): + // line 11: field interval not found in type struct { ... } + `^line (\d+): field (\S+) not found in type`, + ) + if matches := fieldNotFoundRgx.FindStringSubmatch(raw); len(matches) >= 3 { + line, field := matches[1], matches[2] + return fmt.Sprintf(`line %s: invalid field ("%s"): does not exist`, line, field) + } + + // wrong field type error + fieldBadValueRgx := regexp.MustCompile( + // raw output examples: + // line 9: cannot unmarshal !!seq into int // unknown input value + // line 10: cannot unmarshal !!str `hello` into int // known input value + `^line (\d+): cannot unmarshal !!\w+(?: ` + "`" + `(\S+)` + "`" + `)? into (\S+)$`, + ) + if matches := fieldBadValueRgx.FindStringSubmatch(raw); len(matches) >= 3 { + line, value, exptype := matches[1], matches[2], matches[3] + if value == "" { + return fmt.Sprintf("line %s: wrong type: want %s", line, exptype) + } + return fmt.Sprintf(`line %s: wrong type ("%s"): want %s`, line, value, exptype) + } + + // we may not have covered all cases, return raw output in this case + return raw +} + +// jsonParser implements configParser. +type jsonParser struct{} + +// parse decodes a raw JSON input in strict mode (unknown fields disallowed) +// and stores the resulting value into dst. +func (p jsonParser) parse(in []byte, dst *UnmarshaledConfig) error { + decoder := json.NewDecoder(bytes.NewReader(in)) + decoder.DisallowUnknownFields() + return p.handleError(decoder.Decode(dst)) +} + +// handleError handle a json raw error, transforms it into a user-friendly +// standardized format and returns the resulting error. +func (p jsonParser) handleError(err error) error { + if err == nil { + return nil + } + + // handle syntax error + var errSyntax *json.SyntaxError + if errors.As(err, &errSyntax) { + return fmt.Errorf("syntax error near %d: %w", errSyntax.Offset, err) + } + + // handle type error + var errType *json.UnmarshalTypeError + if errors.As(err, &errType) { + return fmt.Errorf( + "wrong type for field %s: want %s, got %s", + errType.Field, errType.Type, errType.Value, + ) + } + + // handle unknown field error + if field := p.parseUnknownFieldError(err.Error()); field != "" { + return fmt.Errorf(`invalid field ("%s"): does not exist`, field) + } + + return err +} + +// parseUnknownFieldError parses the raw string as a json error +// from an unknown field and returns the field name. +// If the raw string is not an unknown field error, it returns "". +func (p jsonParser) parseUnknownFieldError(raw string) (field string) { + unknownFieldRgx := regexp.MustCompile( + // raw output example: + // json: unknown field "notafield" + `json: unknown field "(\S+)"`, + ) + if matches := unknownFieldRgx.FindStringSubmatch(raw); len(matches) >= 2 { + return matches[1] + } + return "" +} diff --git a/internal/configfile/parser_internal_test.go b/internal/configfile/parser_internal_test.go new file mode 100644 index 0000000..6407d77 --- /dev/null +++ b/internal/configfile/parser_internal_test.go @@ -0,0 +1,146 @@ +package configfile + +import ( + "errors" + "reflect" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestYAMLParser(t *testing.T) { + t.Run("return expected errors", func(t *testing.T) { + testcases := []struct { + label string + in []byte + expErr error + }{ + { + label: "unknown field", + in: []byte("notafield: 123\n"), + expErr: &yaml.TypeError{ + Errors: []string{ + `line 1: invalid field ("notafield"): does not exist`, + }, + }, + }, + { + label: "wrong type unknown value", + in: []byte("runner:\n requests: [123]\n"), + expErr: &yaml.TypeError{ + Errors: []string{ + `line 2: wrong type: want int`, + }, + }, + }, + { + label: "wrong type known value", + in: []byte("runner:\n requests: \"123\"\n"), + expErr: &yaml.TypeError{ + Errors: []string{ + `line 2: wrong type ("123"): want int`, + }, + }, + }, + { + label: "cumulate errors", + in: []byte("runner:\n requests: [123]\n concurrency: \"123\"\nnotafield: 123\n"), + expErr: &yaml.TypeError{ + Errors: []string{ + `line 2: wrong type: want int`, + `line 3: wrong type ("123"): want int`, + `line 4: invalid field ("notafield"): does not exist`, + }, + }, + }, + { + label: "no errors custom fields", + in: []byte("x-data: &count\n requests: 100\rrunner:\n <<: *count\n"), + expErr: nil, + }, + } + + for _, tc := range testcases { + t.Run(tc.label, func(t *testing.T) { + var ( + parser yamlParser + rawcfg UnmarshaledConfig + yamlErr *yaml.TypeError + ) + + gotErr := parser.parse(tc.in, &rawcfg) + + if tc.expErr == nil { + if gotErr != nil { + t.Fatalf("unexpected error: %v", gotErr) + } + return + } + + if !errors.As(gotErr, &yamlErr) && tc.expErr != nil { + t.Fatalf("unexpected error: %v", gotErr) + } + + if !reflect.DeepEqual(yamlErr, tc.expErr) { + t.Errorf("unexpected error messages:\nexp %v\ngot %v", tc.expErr, yamlErr) + } + }) + } + }) +} + +func TestJSONParser(t *testing.T) { + t.Run("return expected errors", func(t *testing.T) { + testcases := []struct { + label string + in []byte + exp string + }{ + { + label: "syntax error", + in: []byte("{\n \"runner\": {},\n}\n"), + exp: "syntax error near 19: invalid character '}' looking for beginning of object key string", + }, + { + label: "unknown field", + in: []byte("{\n \"notafield\": 123\n}\n"), + exp: `invalid field ("notafield"): does not exist`, + }, + { + label: "wrong type", + in: []byte("{\n \"runner\": {\n \"requests\": [123]\n }\n}\n"), + exp: "wrong type for field runner.requests: want int, got array", + }, + { + label: "valid config", + in: []byte("{\n \"runner\": {\n \"requests\": 123\n }\n}\n"), + exp: "", + }, + } + + for _, tc := range testcases { + t.Run(tc.label, func(t *testing.T) { + var ( + parser jsonParser + rawcfg UnmarshaledConfig + ) + + gotErr := parser.parse(tc.in, &rawcfg) + + if tc.exp == "" { + if gotErr != nil { + t.Fatalf("unexpected error: %v", gotErr) + } + return + } + + if gotErr.Error() != tc.exp { + t.Errorf( + "unexpected error messages:\nexp %s\ngot %v", + tc.exp, gotErr, + ) + } + }) + } + }) +} diff --git a/internal/configfile/testdata/extends/extends-circular-0.yml b/internal/configfile/testdata/extends/extends-circular-0.yml new file mode 100644 index 0000000..55c7ac7 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-circular-0.yml @@ -0,0 +1 @@ +extends: ./extends-circular-1.yml diff --git a/internal/configfile/testdata/extends/extends-circular-1.yml b/internal/configfile/testdata/extends/extends-circular-1.yml new file mode 100644 index 0000000..f451260 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-circular-1.yml @@ -0,0 +1 @@ +extends: ./extends-circular-2.yml diff --git a/internal/configfile/testdata/extends/extends-circular-2.yml b/internal/configfile/testdata/extends/extends-circular-2.yml new file mode 100644 index 0000000..b862fa4 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-circular-2.yml @@ -0,0 +1 @@ +extends: ./extends-circular-0.yml diff --git a/internal/configfile/testdata/extends/extends-circular-self.yml b/internal/configfile/testdata/extends/extends-circular-self.yml new file mode 100644 index 0000000..2fa66ac --- /dev/null +++ b/internal/configfile/testdata/extends/extends-circular-self.yml @@ -0,0 +1 @@ +extends: ./extends-circular-self.yml diff --git a/internal/configfile/testdata/extends/extends-valid-child.yml b/internal/configfile/testdata/extends/extends-valid-child.yml new file mode 100644 index 0000000..a344080 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-valid-child.yml @@ -0,0 +1,4 @@ +extends: ./extends-valid-parent.yml + +request: + url: http://child.config diff --git a/internal/configfile/testdata/extends/extends-valid-parent.yml b/internal/configfile/testdata/extends/extends-valid-parent.yml new file mode 100644 index 0000000..7f3b136 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-valid-parent.yml @@ -0,0 +1,3 @@ +request: + method: POST + url: http://parent.config diff --git a/internal/configfile/testdata/extends/nest-0/nest-1/extends-valid-nested.yml b/internal/configfile/testdata/extends/nest-0/nest-1/extends-valid-nested.yml new file mode 100644 index 0000000..7810890 --- /dev/null +++ b/internal/configfile/testdata/extends/nest-0/nest-1/extends-valid-nested.yml @@ -0,0 +1,4 @@ +extends: ../../extends-valid-parent.yml + +request: + url: http://nested.config diff --git a/internal/configfile/testdata/invalid/badext.yams b/internal/configfile/testdata/invalid/badext.yams new file mode 100644 index 0000000..f7aba57 --- /dev/null +++ b/internal/configfile/testdata/invalid/badext.yams @@ -0,0 +1,2 @@ +request: + url: https://benchttp.app diff --git a/internal/configfile/testdata/invalid/badfields.json b/internal/configfile/testdata/invalid/badfields.json new file mode 100644 index 0000000..9b45468 --- /dev/null +++ b/internal/configfile/testdata/invalid/badfields.json @@ -0,0 +1,7 @@ +{ + "runner": { + "requests": [123], + "concurrency": "123" + }, + "notafield": 123 +} diff --git a/internal/configfile/testdata/invalid/badfields.yml b/internal/configfile/testdata/invalid/badfields.yml new file mode 100644 index 0000000..6899cd8 --- /dev/null +++ b/internal/configfile/testdata/invalid/badfields.yml @@ -0,0 +1,4 @@ +runner: + requests: [123] # error: invalid type for field requests + concurrency: "123" # error: invalid type for field concurrency +notafield: 123 # error: field does not exist diff --git a/internal/configfile/testdata/valid/benchttp-zeros.yml b/internal/configfile/testdata/valid/benchttp-zeros.yml new file mode 100644 index 0000000..38b1bdb --- /dev/null +++ b/internal/configfile/testdata/valid/benchttp-zeros.yml @@ -0,0 +1,3 @@ +runner: + requests: 0 + globalTimeout: 42ms diff --git a/internal/configfile/testdata/valid/benchttp.json b/internal/configfile/testdata/valid/benchttp.json new file mode 100644 index 0000000..5ffb803 --- /dev/null +++ b/internal/configfile/testdata/valid/benchttp.json @@ -0,0 +1,47 @@ +{ + "request": { + "method": "POST", + "url": "http://localhost:9999?delay=200ms", + "queryParams": { + "fib": "30" + }, + "header": { + "key0": ["val0", "val1"], + "key1": ["val0"] + }, + "body": { + "type": "raw", + "content": "{\"key0\":\"val0\",\"key1\":\"val1\"}" + } + }, + "runner": { + "requests": 100, + "concurrency": 1, + "interval": "50ms", + "requestTimeout": "2s", + "globalTimeout": "60s" + }, + "output": { + "silent": true + }, + "tests": [ + { + "name": "minimum response time", + "field": "ResponseTimes.Min", + "predicate": "GT", + "target": "80ms" + }, + { + "name": "maximum response time", + "field": "ResponseTimes.Max", + "predicate": "LTE", + "target": "120ms" + }, + { + "name": "100% availability", + "field": "RequestFailureCount", + "predicate": "EQ", + "target": "0" + } + ] +} diff --git a/internal/configfile/testdata/valid/benchttp.yaml b/internal/configfile/testdata/valid/benchttp.yaml new file mode 100644 index 0000000..df1b091 --- /dev/null +++ b/internal/configfile/testdata/valid/benchttp.yaml @@ -0,0 +1,38 @@ +x-custom: &data + method: POST + url: http://localhost:9999?delay=200ms + +request: + <<: *data + queryParams: + fib: 30 + header: + key0: [val0, val1] + key1: [val0] + body: + type: raw + content: '{"key0":"val0","key1":"val1"}' + +runner: + requests: 100 + concurrency: 1 + interval: 50ms + requestTimeout: 2s + globalTimeout: 60s + +output: + silent: true + +tests: + - name: minimum response time + field: ResponseTimes.Min + predicate: GT + target: 80ms + - name: maximum response time + field: ResponseTimes.Max + predicate: LTE + target: 120ms + - name: 100% availability + field: RequestFailureCount + predicate: EQ + target: 0 diff --git a/internal/configfile/testdata/valid/benchttp.yml b/internal/configfile/testdata/valid/benchttp.yml new file mode 100644 index 0000000..13a2973 --- /dev/null +++ b/internal/configfile/testdata/valid/benchttp.yml @@ -0,0 +1,35 @@ +request: + method: POST + url: http://localhost:9999?delay=200ms + queryParams: + fib: 30 + header: + key0: [val0, val1] + key1: [val0] + body: + type: raw + content: '{"key0":"val0","key1":"val1"}' + +runner: + requests: 100 + concurrency: 1 + interval: 50ms + requestTimeout: 2s + globalTimeout: 60s + +output: + silent: true + +tests: + - name: minimum response time + field: ResponseTimes.Min + predicate: GT + target: 80ms + - name: maximum response time + field: ResponseTimes.Max + predicate: LTE + target: 120ms + - name: 100% availability + field: RequestFailureCount + predicate: EQ + target: 0 diff --git a/internal/configflag/bind.go b/internal/configflag/bind.go new file mode 100644 index 0000000..726a552 --- /dev/null +++ b/internal/configflag/bind.go @@ -0,0 +1,83 @@ +package configflag + +import ( + "flag" + "net/http" + "net/url" + + "github.com/benchttp/engine/runner" +) + +// Bind reads arguments provided to flagset as config.Fields and binds +// their value to the appropriate fields of given *config.Global. +// The provided *flag.Flagset must not have been parsed yet, otherwise +// bindings its values would fail. +func Bind(flagset *flag.FlagSet, dst *runner.Config) { + // avoid nil pointer dereferences + if dst.Request.URL == nil { + dst.Request.URL = &url.URL{} + } + if dst.Request.Header == nil { + dst.Request.Header = http.Header{} + } + + // request url + flagset.Var(urlValue{url: dst.Request.URL}, + runner.ConfigFieldURL, + runner.ConfigFieldsUsage[runner.ConfigFieldURL], + ) + // request method + flagset.StringVar(&dst.Request.Method, + runner.ConfigFieldMethod, + dst.Request.Method, + runner.ConfigFieldsUsage[runner.ConfigFieldMethod], + ) + // request header + flagset.Var(headerValue{header: &dst.Request.Header}, + runner.ConfigFieldHeader, + runner.ConfigFieldsUsage[runner.ConfigFieldHeader], + ) + // request body + flagset.Var(bodyValue{body: &dst.Request.Body}, + runner.ConfigFieldBody, + runner.ConfigFieldsUsage[runner.ConfigFieldBody], + ) + // requests number + flagset.IntVar(&dst.Runner.Requests, + runner.ConfigFieldRequests, + dst.Runner.Requests, + runner.ConfigFieldsUsage[runner.ConfigFieldRequests], + ) + + // concurrency + flagset.IntVar(&dst.Runner.Concurrency, + runner.ConfigFieldConcurrency, + dst.Runner.Concurrency, + runner.ConfigFieldsUsage[runner.ConfigFieldConcurrency], + ) + // non-conurrent requests interval + flagset.DurationVar(&dst.Runner.Interval, + runner.ConfigFieldInterval, + dst.Runner.Interval, + runner.ConfigFieldsUsage[runner.ConfigFieldInterval], + ) + // request timeout + flagset.DurationVar(&dst.Runner.RequestTimeout, + runner.ConfigFieldRequestTimeout, + dst.Runner.RequestTimeout, + runner.ConfigFieldsUsage[runner.ConfigFieldRequestTimeout], + ) + // global timeout + flagset.DurationVar(&dst.Runner.GlobalTimeout, + runner.ConfigFieldGlobalTimeout, + dst.Runner.GlobalTimeout, + runner.ConfigFieldsUsage[runner.ConfigFieldGlobalTimeout], + ) + + // silent mode + flagset.BoolVar(&dst.Output.Silent, + runner.ConfigFieldSilent, + dst.Output.Silent, + runner.ConfigFieldsUsage[runner.ConfigFieldSilent], + ) +} diff --git a/internal/configflag/bind_test.go b/internal/configflag/bind_test.go new file mode 100644 index 0000000..8ec835b --- /dev/null +++ b/internal/configflag/bind_test.go @@ -0,0 +1,74 @@ +package configflag_test + +import ( + "flag" + "net/http" + "reflect" + "testing" + "time" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/configflag" +) + +func TestBind(t *testing.T) { + t.Run("default to base config", func(t *testing.T) { + flagset := flag.NewFlagSet("run", flag.ExitOnError) + args := []string{} // no args + + cfg := runner.DefaultConfig() + configflag.Bind(flagset, &cfg) + if err := flagset.Parse(args); err != nil { + t.Fatal(err) // critical error, stop the test + } + + if exp := runner.DefaultConfig(); !reflect.DeepEqual(cfg, exp) { + t.Errorf("\nexp %#v\ngot %#v", exp, cfg) + } + }) + + t.Run("set config with flags values", func(t *testing.T) { + flagset := flag.NewFlagSet("run", flag.ExitOnError) + args := []string{ + "-method", "POST", + "-url", "https://benchttp.app?cool=yes", + "-header", "Content-Type:application/json", + "-body", "raw:hello", + "-requests", "1", + "-concurrency", "2", + "-interval", "3s", + "-requestTimeout", "4s", + "-globalTimeout", "5s", + "-silent", + } + + cfg := runner.Config{} + configflag.Bind(flagset, &cfg) + if err := flagset.Parse(args); err != nil { + t.Fatal(err) // critical error, stop the test + } + + exp := runner.Config{ + Request: runner.RequestConfig{ + Method: "POST", + Header: http.Header{"Content-Type": {"application/json"}}, + Body: runner.RequestBody{Type: "raw", Content: []byte("hello")}, + }.WithURL("https://benchttp.app?cool=yes"), + Runner: runner.RecorderConfig{ + Requests: 1, + Concurrency: 2, + Interval: 3 * time.Second, + RequestTimeout: 4 * time.Second, + GlobalTimeout: 5 * time.Second, + }, + Output: runner.OutputConfig{ + Silent: true, + }, + } + + if !reflect.DeepEqual(cfg, exp) { + t.Errorf("\nexp %#v\ngot %#v", exp, cfg) + } + }) +} diff --git a/internal/configflag/body.go b/internal/configflag/body.go new file mode 100644 index 0000000..f3f0dfb --- /dev/null +++ b/internal/configflag/body.go @@ -0,0 +1,50 @@ +package configflag + +import ( + "fmt" + "strings" + + "github.com/benchttp/engine/runner" +) + +// bodyValue implements flag.Value +type bodyValue struct { + body *runner.RequestBody +} + +// String returns a string representation of the referenced body. +func (v bodyValue) String() string { + return fmt.Sprint(v.body) +} + +// Set reads input string in format "type:content" and sets +// the referenced body accordingly. +// +// Note: only type "raw" is supported at the moment. +func (v bodyValue) Set(raw string) error { + errFormat := fmt.Errorf(`expect format ":", got "%s"`, raw) + + if raw == "" { + return errFormat + } + + split := strings.SplitN(raw, ":", 2) + if len(split) != 2 { + return errFormat + } + + btype, bcontent := split[0], split[1] + if bcontent == "" { + return errFormat + } + + switch btype { + case "raw": + *v.body = runner.NewRequestBody(btype, bcontent) + // case "file": + // // TODO + default: + return fmt.Errorf(`unsupported type: %s (only "raw" accepted)`, btype) + } + return nil +} diff --git a/internal/configflag/header.go b/internal/configflag/header.go new file mode 100644 index 0000000..232180e --- /dev/null +++ b/internal/configflag/header.go @@ -0,0 +1,30 @@ +package configflag + +import ( + "errors" + "fmt" + "net/http" + "strings" +) + +// headerValue implements flag.Value +type headerValue struct { + header *http.Header +} + +// String returns a string representation of the referenced header. +func (v headerValue) String() string { + return fmt.Sprint(v.header) +} + +// Set reads input string in format "key:value" and appends value +// to the key's values of the referenced header. +func (v headerValue) Set(raw string) error { + keyval := strings.SplitN(raw, ":", 2) + if len(keyval) != 2 { + return errors.New(`expect format ":"`) + } + key, val := keyval[0], keyval[1] + (*v.header)[key] = append((*v.header)[key], val) + return nil +} diff --git a/internal/configflag/url.go b/internal/configflag/url.go new file mode 100644 index 0000000..c1eb907 --- /dev/null +++ b/internal/configflag/url.go @@ -0,0 +1,29 @@ +package configflag + +import ( + "fmt" + "net/url" +) + +// urlValue implements flag.Value +type urlValue struct { + url *url.URL +} + +// String returns a string representation of urlValue.url. +func (v urlValue) String() string { + if v.url == nil { + return "" + } + return v.url.String() +} + +// Set parses input string as a URL and sets the referenced URL accordingly. +func (v urlValue) Set(in string) error { + urlURL, err := url.ParseRequestURI(in) + if err != nil { + return fmt.Errorf(`invalid url: "%s"`, in) + } + *v.url = *urlURL + return nil +} diff --git a/internal/configflag/which.go b/internal/configflag/which.go new file mode 100644 index 0000000..c084ecd --- /dev/null +++ b/internal/configflag/which.go @@ -0,0 +1,19 @@ +package configflag + +import ( + "flag" + + "github.com/benchttp/engine/runner" +) + +// Which returns a slice of all config fields set via the CLI +// for the given *flag.FlagSet. +func Which(flagset *flag.FlagSet) []string { + var fields []string + flagset.Visit(func(f *flag.Flag) { + if name := f.Name; runner.IsConfigField(name) { + fields = append(fields, name) + } + }) + return fields +} diff --git a/internal/configflag/which_test.go b/internal/configflag/which_test.go new file mode 100644 index 0000000..80cf4ae --- /dev/null +++ b/internal/configflag/which_test.go @@ -0,0 +1,52 @@ +package configflag_test + +import ( + "flag" + "reflect" + "testing" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/configflag" +) + +func TestWhich(t *testing.T) { + for _, tc := range []struct { + label string + args []string + exp []string + }{ + { + label: "return all config flags set", + args: []string{ + "-method", "POST", + "-url", "https://benchttp.app?cool=yes", + "-concurrency", "2", + "-requests", "3", + "-requestTimeout", "1s", + "-globalTimeout", "4s", + }, + exp: []string{ + "concurrency", "globalTimeout", "method", + "requestTimeout", "requests", "url", + }, + }, + { + label: "do not return config flags not set", + args: []string{"-requests", "3"}, + exp: []string{"requests"}, + }, + } { + flagset := flag.NewFlagSet("run", flag.ExitOnError) + + configflag.Bind(flagset, &runner.Config{}) + + if err := flagset.Parse(tc.args); err != nil { + t.Fatal(err) // critical error, stop the test + } + + if got := configflag.Which(flagset); !reflect.DeepEqual(got, tc.exp) { + t.Errorf("\nexp %v\ngot %v", tc.exp, got) + } + } +} diff --git a/internal/errorutil/errorutil.go b/internal/errorutil/errorutil.go new file mode 100644 index 0000000..d733a93 --- /dev/null +++ b/internal/errorutil/errorutil.go @@ -0,0 +1,24 @@ +package errorutil + +import ( + "fmt" + "strings" +) + +// WithDetails returns an error wrapping err, appended with a string +// representation of details separated by ": ". +// +// Example +// +// var ErrNotFound = errors.New("not found") +// err := WithDetails(ErrNotFound, "abc.jpg", "deleted yesterday") +// +// errors.Is(err, ErrNotFound) == true +// err.Error() == "not found: abc.jpg: deleted yesterday" +func WithDetails(base error, details ...interface{}) error { + detailsStr := make([]string, len(details)) + for i := range details { + detailsStr[i] = fmt.Sprint(details[i]) + } + return fmt.Errorf("%w: %s", base, strings.Join(detailsStr, ": ")) +} diff --git a/internal/signals/signals.go b/internal/signals/signals.go new file mode 100644 index 0000000..3425d45 --- /dev/null +++ b/internal/signals/signals.go @@ -0,0 +1,17 @@ +package signals + +import ( + "os" + "os/signal" + "syscall" +) + +// ListenOSInterrupt listens for OS interrupt signals and calls callback +// on receive. It should be called in a separate goroutine from the main +// program as it blocks the execution until a signal is received. +func ListenOSInterrupt(callback func()) { + sigC := make(chan os.Signal, 1) + signal.Notify(sigC, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) + <-sigC + callback() +} diff --git a/script/build b/script/build new file mode 100755 index 0000000..09dac45 --- /dev/null +++ b/script/build @@ -0,0 +1,35 @@ +#!/bin/bash + +platforms="darwin/amd64 darwin/arm64 linux/386 linux/amd64 windows/386 windows/amd64" + +version=$(git describe --tags --abbrev=0) +ldflags="-X main.benchttpVersion=$version" +tags="prod" + +cmddir="./cmd/benchttp" +bindir="./bin" + +# clear bin directory +rm -rf ./bin/* + +i=0 +for platform in ${platforms}; do + ((i++)) + + split=(${platform//// }) # split platform by sep "/" + goos="${split[0]}" + goarch="${split[1]}" + output="benchttp_${goos}_${goarch}" # e.g. benchttp_darwin_amd64 + + # add .exe to windows binaries + [[ "$goos" == "windows" ]] && output="$output.exe" + + output="$bindir/$output" + + # build binary + GOOS="$goos" GOARCH="$goarch" go build -tags "$tags" -ldflags "$ldflags" -o "$output" "$cmddir" + + echo "[$i/6] $output" +done + +echo -e "\033[1;32m✔︎\033[0m Build complete!" diff --git a/script/build-healthcheck b/script/build-healthcheck new file mode 100755 index 0000000..4f9b2e9 --- /dev/null +++ b/script/build-healthcheck @@ -0,0 +1,17 @@ +#!/bin/bash + +goos=$(go env GOOS) +goarch=$(go env GOARCH) +benchttp="./bin/benchttp_${goos}_${goarch}" + +expVersion="benchttp $(git describe --tags --abbrev=0)" +gotVersion=$(eval $benchttp version) + +if [[ "$gotVersion" != "$expVersion" ]]; then + echo -e "\033[1;31m✘\033[0m Error running ./bin/benchttp version" + echo " exp $expVersion" + echo " got $gotVersion" + exit 1 +fi + +echo -e "\033[1;32m✔︎\033[0m Build integrity OK!" From dfc14139372a6aedba9484139ef2f65c3dd4f956 Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Fri, 7 Oct 2022 22:52:30 +0200 Subject: [PATCH 02/12] temp: override benchttp/engine with local source --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 417262f..291e036 100644 --- a/go.mod +++ b/go.mod @@ -8,3 +8,5 @@ require ( ) require golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + +replace github.com/benchttp/engine => ../engine From 8459993cf57edbdc118b9bdd15d05432191c1376 Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 00:15:31 +0200 Subject: [PATCH 03/12] refactor: output rendering - move runner.Report rendering from engine to cli - gather rendering in new package internal/render - move package ansi into internal/render --- cmd/benchttp/run.go | 12 +- internal/{ => render}/ansi/style.go | 0 internal/{cli/state.go => render/progress.go} | 28 ++--- internal/render/renderer.go | 15 +++ internal/render/report.go | 105 ++++++++++++++++++ internal/render/report_test.go | 68 ++++++++++++ 6 files changed, 208 insertions(+), 20 deletions(-) rename internal/{ => render}/ansi/style.go (100%) rename internal/{cli/state.go => render/progress.go} (71%) create mode 100644 internal/render/renderer.go create mode 100644 internal/render/report.go create mode 100644 internal/render/report_test.go diff --git a/cmd/benchttp/run.go b/cmd/benchttp/run.go index d9fd1b0..f01b64f 100644 --- a/cmd/benchttp/run.go +++ b/cmd/benchttp/run.go @@ -9,9 +9,9 @@ import ( "github.com/benchttp/engine/runner" - "github.com/benchttp/cli/internal/cli" "github.com/benchttp/cli/internal/configfile" "github.com/benchttp/cli/internal/configflag" + "github.com/benchttp/cli/internal/render" "github.com/benchttp/cli/internal/signals" ) @@ -56,7 +56,7 @@ func (cmd *cmdRun) execute(args []string) error { go signals.ListenOSInterrupt(cancel) // Run the benchmark - out, err := runner. + report, err := runner. New(onRecordingProgress(cfg.Output.Silent)). Run(ctx, cfg) if err != nil { @@ -64,11 +64,11 @@ func (cmd *cmdRun) execute(args []string) error { } // Write results to stdout - if _, err := out.Write(os.Stdout); err != nil { + if _, err := render.Report(os.Stdout, report); err != nil { return err } - if !out.Tests.Pass { + if !report.Tests.Pass { return errors.New("test suite failed") } @@ -127,11 +127,11 @@ func onRecordingProgress(silent bool) func(runner.RecordingProgress) { return func(runner.RecordingProgress) {} } - // hack: write a blank line as cli.WriteRecordingProgress always + // hack: write a blank line as render.Progress always // erases the previous line fmt.Println() return func(progress runner.RecordingProgress) { - cli.WriteRecordingProgress(os.Stdout, progress) //nolint: errcheck + render.Progress(os.Stdout, progress) //nolint: errcheck } } diff --git a/internal/ansi/style.go b/internal/render/ansi/style.go similarity index 100% rename from internal/ansi/style.go rename to internal/render/ansi/style.go diff --git a/internal/cli/state.go b/internal/render/progress.go similarity index 71% rename from internal/cli/state.go rename to internal/render/progress.go index 255a3b5..2495e88 100644 --- a/internal/cli/state.go +++ b/internal/render/progress.go @@ -1,4 +1,4 @@ -package cli +package render import ( "fmt" @@ -8,24 +8,24 @@ import ( "github.com/benchttp/engine/runner" - "github.com/benchttp/cli/internal/ansi" + "github.com/benchttp/cli/internal/render/ansi" ) -// WriteRecordingProgress renders a fancy representation of p as a string +// Progress renders a fancy representation of a runner.RecordingProgress // and writes the result to w. -func WriteRecordingProgress(w io.Writer, p runner.RecordingProgress) (int, error) { - return fmt.Fprint(w, renderProgress(p)) +func Progress(w io.Writer, p runner.RecordingProgress) (int, error) { + return fmt.Fprint(w, progressString(p)) } -// renderProgress returns a string representation of runner.RecordingProgress +// progressString returns a string representation of a runner.RecordingProgress // for a fancy display in a CLI: // // RUNNING ◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎ 50% | 50/100 requests | 27s timeout -func renderProgress(s runner.RecordingProgress) string { +func progressString(p runner.RecordingProgress) string { var ( - countdown = s.Timeout - s.Elapsed - reqmax = strconv.Itoa(s.MaxCount) - pctdone = s.Percent() + countdown = p.Timeout - p.Elapsed + reqmax = strconv.Itoa(p.MaxCount) + pctdone = p.Percent() timeline = renderTimeline(pctdone) ) @@ -39,8 +39,8 @@ func renderProgress(s runner.RecordingProgress) string { return fmt.Sprintf( "%s%s %s %d%% | %d/%s requests | %.0fs timeout \n", ansi.Erase(1), // replace previous line - renderStatus(s.Status()), timeline, pctdone, // progress - s.DoneCount, reqmax, // requests + renderStatus(p.Status()), timeline, pctdone, // progress + p.DoneCount, reqmax, // requests countdown.Seconds(), // timeout ) } @@ -69,8 +69,8 @@ func renderTimeline(pctdone int) string { // depending on whether the run is done or not and the value // of its context error. func renderStatus(status runner.RecordingStatus) string { - color := statusStyle(status) - return color(string(status)) + styled := statusStyle(status) + return styled(string(status)) } func statusStyle(status runner.RecordingStatus) ansi.StyleFunc { diff --git a/internal/render/renderer.go b/internal/render/renderer.go new file mode 100644 index 0000000..7067332 --- /dev/null +++ b/internal/render/renderer.go @@ -0,0 +1,15 @@ +package render + +// type Renderer struct { +// Silent bool +// Writer io.Writer +// } + +// func NewRenderer(w io.Writer, silent bool) Renderer { +// return Renderer{ +// Writer: w, +// Silent: silent, +// } +// } + +// func (r Renderer) Progress diff --git a/internal/render/report.go b/internal/render/report.go new file mode 100644 index 0000000..978219f --- /dev/null +++ b/internal/render/report.go @@ -0,0 +1,105 @@ +package render + +import ( + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/render/ansi" +) + +func Report(w io.Writer, rep *runner.Report) (int, error) { + return w.Write([]byte(ReportString(rep))) +} + +// String returns a default summary of the Report as a string. +func ReportString(rep *runner.Report) string { + var b strings.Builder + writeDefaultSummary(&b, rep) + writeTestsResult(&b, rep) + return b.String() +} + +func writeDefaultSummary(w io.StringWriter, rep *runner.Report) { + line := func(name string, value interface{}) string { + const template = "%-18s %v\n" + return fmt.Sprintf(template, name, value) + } + + msString := func(d time.Duration) string { + return fmt.Sprintf("%dms", d.Milliseconds()) + } + + formatRequests := func(n, max int) string { + maxString := strconv.Itoa(max) + if maxString == "-1" { + maxString = "∞" + } + return fmt.Sprintf("%d/%s", n, maxString) + } + + var ( + m = rep.Metrics + cfg = rep.Metadata.Config + ) + + w.WriteString(ansi.Bold("→ Summary")) + w.WriteString("\n") + w.WriteString(line("Endpoint", cfg.Request.URL)) + w.WriteString(line("Requests", formatRequests(len(m.Records), cfg.Runner.Requests))) + w.WriteString(line("Errors", len(m.RequestFailures))) + w.WriteString(line("Min response time", msString(m.ResponseTimes.Min))) + w.WriteString(line("Max response time", msString(m.ResponseTimes.Max))) + w.WriteString(line("Mean response time", msString(m.ResponseTimes.Mean))) + w.WriteString(line("Total duration", msString(rep.Metadata.TotalDuration))) +} + +func writeTestsResult(w io.StringWriter, rep *runner.Report) { + sr := rep.Tests + if len(sr.Results) == 0 { + return + } + + w.WriteString("\n") + w.WriteString(ansi.Bold("→ Test suite")) + w.WriteString("\n") + + writeResultString(w, sr.Pass) + w.WriteString("\n") + + for _, tr := range sr.Results { + writeIndent(w, 1) + writeResultString(w, tr.Pass) + w.WriteString(" ") + w.WriteString(tr.Input.Name) + + if !tr.Pass { + w.WriteString("\n ") + writeIndent(w, 3) + w.WriteString(ansi.Bold("→ ")) + w.WriteString(tr.Summary) + } + + w.WriteString("\n") + } +} + +func writeResultString(w io.StringWriter, pass bool) { + if pass { + w.WriteString(ansi.Green("PASS")) + } else { + w.WriteString(ansi.Red("FAIL")) + } +} + +func writeIndent(w io.StringWriter, count int) { + if count <= 0 { + return + } + const baseIndent = " " + w.WriteString(strings.Repeat(baseIndent, count)) +} diff --git a/internal/render/report_test.go b/internal/render/report_test.go new file mode 100644 index 0000000..04d3ed4 --- /dev/null +++ b/internal/render/report_test.go @@ -0,0 +1,68 @@ +package render_test + +import ( + "testing" + "time" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/render" + "github.com/benchttp/cli/internal/render/ansi" +) + +func TestReport_String(t *testing.T) { + t.Run("returns metrics summary", func(t *testing.T) { + metrics, duration := metricsStub() + cfg := configStub() + + rep := &runner.Report{ + Metrics: metrics, + Metadata: runner.ReportMetadata{ + Config: cfg, + TotalDuration: duration, + }, + } + checkSummary(t, render.ReportString(rep)) + }) +} + +// helpers + +func metricsStub() (agg runner.MetricsAggregate, total time.Duration) { + return runner.MetricsAggregate{ + RequestFailures: make([]struct { + Reason string + }, 1), + Records: make([]struct{ ResponseTime time.Duration }, 3), + ResponseTimes: runner.MetricsTimeStats{ + Min: 4 * time.Second, + Max: 6 * time.Second, + Mean: 5 * time.Second, + }, + }, 15 * time.Second +} + +func configStub() runner.Config { + cfg := runner.Config{} + cfg.Request = cfg.Request.WithURL("https://a.b.com") + cfg.Runner.Requests = -1 + return cfg +} + +func checkSummary(t *testing.T, summary string) { + t.Helper() + + expSummary := ansi.Bold("→ Summary") + ` +Endpoint https://a.b.com +Requests 3/∞ +Errors 1 +Min response time 4000ms +Max response time 6000ms +Mean response time 5000ms +Total duration 15000ms +` + + if summary != expSummary { + t.Errorf("\nexp summary:\n%q\ngot summary:\n%q", expSummary, summary) + } +} From 5af761de8f673e0d9ed3884bd14ffd2e71e3afba Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 01:08:03 +0200 Subject: [PATCH 04/12] feat: implement output.ConditionalWriter --- internal/output/conditional.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 internal/output/conditional.go diff --git a/internal/output/conditional.go b/internal/output/conditional.go new file mode 100644 index 0000000..c651313 --- /dev/null +++ b/internal/output/conditional.go @@ -0,0 +1,33 @@ +package output + +import ( + "io" +) + +type ConditionalWriter struct { + Writer io.Writer + ok bool +} + +// Write writes b only if MuteableWriter.Mute is false, +// otherwise it is no-op. +func (w ConditionalWriter) Write(b []byte) (int, error) { + if !w.ok { + return 0, nil + } + return w.Writer.Write(b) +} + +func (w ConditionalWriter) If(ok bool) ConditionalWriter { + return ConditionalWriter{ + Writer: w.Writer, + ok: ok, + } +} + +func (w ConditionalWriter) Or(ok bool) ConditionalWriter { + return ConditionalWriter{ + Writer: w.Writer, + ok: w.ok || ok, + } +} From 303779e781c04063e06e63c83de1e50f02cb0c19 Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 01:10:56 +0200 Subject: [PATCH 05/12] fix: silent behavior - fix flag -silent that was ignored - always print test suite results, even if silent is enabled - necessary step: separate summary and test suite resutlts outputs --- cmd/benchttp/run.go | 16 ++++++- internal/render/report.go | 76 +++++++--------------------------- internal/render/report_test.go | 3 +- internal/render/testsuite.go | 63 ++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 65 deletions(-) create mode 100644 internal/render/testsuite.go diff --git a/cmd/benchttp/run.go b/cmd/benchttp/run.go index f01b64f..661b896 100644 --- a/cmd/benchttp/run.go +++ b/cmd/benchttp/run.go @@ -11,6 +11,7 @@ import ( "github.com/benchttp/cli/internal/configfile" "github.com/benchttp/cli/internal/configflag" + "github.com/benchttp/cli/internal/output" "github.com/benchttp/cli/internal/render" "github.com/benchttp/cli/internal/signals" ) @@ -24,6 +25,8 @@ type cmdRun struct { // config is the runner config resulting from parsing CLI flags. config runner.Config + + output output.ConditionalWriter } // init initializes cmdRun with default values. @@ -34,6 +37,7 @@ func (cmd *cmdRun) init() { "./.benchttp.yaml", "./.benchttp.json", }) + cmd.output = output.ConditionalWriter{Writer: os.Stdout} } // execute runs the benchttp runner: it parses CLI flags, loads config @@ -63,8 +67,16 @@ func (cmd *cmdRun) execute(args []string) error { return err } - // Write results to stdout - if _, err := render.Report(os.Stdout, report); err != nil { + cmd.output = cmd.output.If(!cfg.Output.Silent) + + // Print summary + if _, err := render.ReportSummary(cmd.output, report); err != nil { + return err + } + + if _, err := render.TestSuite( + cmd.output.Or(!report.Tests.Pass), report.Tests, + ); err != nil { return err } diff --git a/internal/render/report.go b/internal/render/report.go index 978219f..d5f996f 100644 --- a/internal/render/report.go +++ b/internal/render/report.go @@ -12,19 +12,14 @@ import ( "github.com/benchttp/cli/internal/render/ansi" ) -func Report(w io.Writer, rep *runner.Report) (int, error) { - return w.Write([]byte(ReportString(rep))) +func ReportSummary(w io.Writer, rep *runner.Report) (int, error) { + return w.Write([]byte(ReportSummaryString(rep))) } // String returns a default summary of the Report as a string. -func ReportString(rep *runner.Report) string { +func ReportSummaryString(rep *runner.Report) string { var b strings.Builder - writeDefaultSummary(&b, rep) - writeTestsResult(&b, rep) - return b.String() -} -func writeDefaultSummary(w io.StringWriter, rep *runner.Report) { line := func(name string, value interface{}) string { const template = "%-18s %v\n" return fmt.Sprintf(template, name, value) @@ -47,59 +42,16 @@ func writeDefaultSummary(w io.StringWriter, rep *runner.Report) { cfg = rep.Metadata.Config ) - w.WriteString(ansi.Bold("→ Summary")) - w.WriteString("\n") - w.WriteString(line("Endpoint", cfg.Request.URL)) - w.WriteString(line("Requests", formatRequests(len(m.Records), cfg.Runner.Requests))) - w.WriteString(line("Errors", len(m.RequestFailures))) - w.WriteString(line("Min response time", msString(m.ResponseTimes.Min))) - w.WriteString(line("Max response time", msString(m.ResponseTimes.Max))) - w.WriteString(line("Mean response time", msString(m.ResponseTimes.Mean))) - w.WriteString(line("Total duration", msString(rep.Metadata.TotalDuration))) -} - -func writeTestsResult(w io.StringWriter, rep *runner.Report) { - sr := rep.Tests - if len(sr.Results) == 0 { - return - } - - w.WriteString("\n") - w.WriteString(ansi.Bold("→ Test suite")) - w.WriteString("\n") - - writeResultString(w, sr.Pass) - w.WriteString("\n") - - for _, tr := range sr.Results { - writeIndent(w, 1) - writeResultString(w, tr.Pass) - w.WriteString(" ") - w.WriteString(tr.Input.Name) - - if !tr.Pass { - w.WriteString("\n ") - writeIndent(w, 3) - w.WriteString(ansi.Bold("→ ")) - w.WriteString(tr.Summary) - } + b.WriteString(ansi.Bold("→ Summary")) + b.WriteString("\n") + b.WriteString(line("Endpoint", cfg.Request.URL)) + b.WriteString(line("Requests", formatRequests(len(m.Records), cfg.Runner.Requests))) + b.WriteString(line("Errors", len(m.RequestFailures))) + b.WriteString(line("Min response time", msString(m.ResponseTimes.Min))) + b.WriteString(line("Max response time", msString(m.ResponseTimes.Max))) + b.WriteString(line("Mean response time", msString(m.ResponseTimes.Mean))) + b.WriteString(line("Total duration", msString(rep.Metadata.TotalDuration))) + b.WriteString("\n") - w.WriteString("\n") - } -} - -func writeResultString(w io.StringWriter, pass bool) { - if pass { - w.WriteString(ansi.Green("PASS")) - } else { - w.WriteString(ansi.Red("FAIL")) - } -} - -func writeIndent(w io.StringWriter, count int) { - if count <= 0 { - return - } - const baseIndent = " " - w.WriteString(strings.Repeat(baseIndent, count)) + return b.String() } diff --git a/internal/render/report_test.go b/internal/render/report_test.go index 04d3ed4..1fad710 100644 --- a/internal/render/report_test.go +++ b/internal/render/report_test.go @@ -22,7 +22,7 @@ func TestReport_String(t *testing.T) { TotalDuration: duration, }, } - checkSummary(t, render.ReportString(rep)) + checkSummary(t, render.ReportSummaryString(rep)) }) } @@ -60,6 +60,7 @@ Min response time 4000ms Max response time 6000ms Mean response time 5000ms Total duration 15000ms + ` if summary != expSummary { diff --git a/internal/render/testsuite.go b/internal/render/testsuite.go new file mode 100644 index 0000000..648ad17 --- /dev/null +++ b/internal/render/testsuite.go @@ -0,0 +1,63 @@ +package render + +import ( + "io" + "strings" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/render/ansi" +) + +func TestSuite(w io.Writer, suite runner.TestSuiteResults) (int, error) { + return w.Write([]byte(TestSuiteString(suite))) +} + +// String returns a default summary of the Report as a string. +func TestSuiteString(suite runner.TestSuiteResults) string { + if len(suite.Results) == 0 { + return "" + } + + var b strings.Builder + + b.WriteString(ansi.Bold("→ Test suite")) + b.WriteString("\n") + + writeResultString(&b, suite.Pass) + b.WriteString("\n") + + for _, tr := range suite.Results { + writeIndent(&b, 1) + writeResultString(&b, tr.Pass) + b.WriteString(" ") + b.WriteString(tr.Input.Name) + + if !tr.Pass { + b.WriteString("\n ") + writeIndent(&b, 3) + b.WriteString(ansi.Bold("→ ")) + b.WriteString(tr.Summary) + } + + b.WriteString("\n") + } + + return b.String() +} + +func writeResultString(w io.StringWriter, pass bool) { + if pass { + w.WriteString(ansi.Green("PASS")) + } else { + w.WriteString(ansi.Red("FAIL")) + } +} + +func writeIndent(w io.StringWriter, count int) { + if count <= 0 { + return + } + const baseIndent = " " + w.WriteString(strings.Repeat(baseIndent, count)) +} From 00cea9a19c66db0a2af177f5d9272dd85ae3392a Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 02:10:06 +0200 Subject: [PATCH 06/12] refactor: extract functions from main functions --- cmd/benchttp/main.go | 38 ++++++++++++++------- cmd/benchttp/run.go | 80 +++++++++++++++++++++++++------------------- 2 files changed, 70 insertions(+), 48 deletions(-) diff --git a/cmd/benchttp/main.go b/cmd/benchttp/main.go index e3c11b3..47dcf7c 100644 --- a/cmd/benchttp/main.go +++ b/cmd/benchttp/main.go @@ -21,26 +21,38 @@ func main() { } func run() error { - if len(os.Args) < 2 { - return fmt.Errorf("%w: no command specified", errUsage) + commandName, nextArgs, err := shiftArgs(os.Args[1:]) + if err != nil { + return err } - var cmd command - args := os.Args[1:] - - switch sub := args[0]; sub { - case "run": - cmd = &cmdRun{flagset: flag.NewFlagSet("run", flag.ExitOnError)} - case "version": - cmd = &cmdVersion{} - default: - return fmt.Errorf("%w: unknown command: %s", errUsage, sub) + cmd, err := commandOf(commandName) + if err != nil { + return err } - return cmd.execute(args) + return cmd.execute(nextArgs) +} + +func shiftArgs(args []string) (commandName string, nextArgs []string, err error) { + if len(args) < 1 { + return "", []string{}, fmt.Errorf("%w: no command specified", errUsage) + } + return args[0], args[1:], nil } // command is the interface that all benchttp subcommands must implement. type command interface { execute(args []string) error } + +func commandOf(name string) (command, error) { + switch name { + case "run": + return &cmdRun{flagset: flag.NewFlagSet("run", flag.ExitOnError)}, nil + case "version": + return &cmdVersion{}, nil + default: + return nil, fmt.Errorf("%w: unknown command: %s", errUsage, name) + } +} diff --git a/cmd/benchttp/run.go b/cmd/benchttp/run.go index 661b896..bd7cde4 100644 --- a/cmd/benchttp/run.go +++ b/cmd/benchttp/run.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "io" "os" "github.com/benchttp/engine/runner" @@ -25,8 +26,6 @@ type cmdRun struct { // config is the runner config resulting from parsing CLI flags. config runner.Config - - output output.ConditionalWriter } // init initializes cmdRun with default values. @@ -37,7 +36,6 @@ func (cmd *cmdRun) init() { "./.benchttp.yaml", "./.benchttp.json", }) - cmd.output = output.ConditionalWriter{Writer: os.Stdout} } // execute runs the benchttp runner: it parses CLI flags, loads config @@ -46,45 +44,18 @@ func (cmd *cmdRun) init() { func (cmd *cmdRun) execute(args []string) error { cmd.init() - // Set CLI config from flags and retrieve fields that were set - fieldsSet := cmd.parseArgs(args) - - // Generate merged config (defaults < config file < CLI flags) - cfg, err := cmd.makeConfig(fieldsSet) + // Generate merged config (default < config file < CLI flags) + cfg, err := cmd.makeConfig(args) if err != nil { return err } - // Prepare graceful shutdown in case of os.Interrupt (Ctrl+C) - ctx, cancel := context.WithCancel(context.Background()) - go signals.ListenOSInterrupt(cancel) - - // Run the benchmark - report, err := runner. - New(onRecordingProgress(cfg.Output.Silent)). - Run(ctx, cfg) + report, err := runBenchmark(cfg) if err != nil { return err } - cmd.output = cmd.output.If(!cfg.Output.Silent) - - // Print summary - if _, err := render.ReportSummary(cmd.output, report); err != nil { - return err - } - - if _, err := render.TestSuite( - cmd.output.Or(!report.Tests.Pass), report.Tests, - ); err != nil { - return err - } - - if !report.Tests.Pass { - return errors.New("test suite failed") - } - - return nil + return renderReport(os.Stdout, report, cfg.Output.Silent) } // parseArgs parses input args as config fields and returns @@ -115,7 +86,10 @@ func (cmd *cmdRun) parseArgs(args []string) []string { // makeConfig returns a runner.ConfigGlobal initialized with config file // options if found, overridden with CLI options listed in fields // slice param. -func (cmd *cmdRun) makeConfig(fields []string) (cfg runner.Config, err error) { +func (cmd *cmdRun) makeConfig(args []string) (cfg runner.Config, err error) { + // Set CLI config from flags and retrieve fields that were set + fields := cmd.parseArgs(args) + // configFile not set and default ones not found: // skip the merge and return the cli config if cmd.configFile == "" { @@ -147,3 +121,39 @@ func onRecordingProgress(silent bool) func(runner.RecordingProgress) { render.Progress(os.Stdout, progress) //nolint: errcheck } } + +func runBenchmark(cfg runner.Config) (*runner.Report, error) { + // Prepare graceful shutdown in case of os.Interrupt (Ctrl+C) + ctx, cancel := context.WithCancel(context.Background()) + go signals.ListenOSInterrupt(cancel) + + // Run the benchmark + report, err := runner. + New(onRecordingProgress(cfg.Output.Silent)). + Run(ctx, cfg) + if err != nil { + return report, err + } + + return report, nil +} + +func renderReport(w io.Writer, report *runner.Report, silent bool) error { + cw := output.ConditionalWriter{Writer: w}.If(!silent) + + if _, err := render.ReportSummary(cw, report); err != nil { + return err + } + + if _, err := render.TestSuite( + cw.Or(!report.Tests.Pass), report.Tests, + ); err != nil { + return err + } + + if !report.Tests.Pass { + return errors.New("test suite failed") + } + + return nil +} From e15b9a380888e34dc5ee76a39c28f3ba3faa0396 Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 02:21:33 +0200 Subject: [PATCH 07/12] docs: restore & adapt Readme --- README.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/README.md b/README.md index e69de29..eb32163 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,98 @@ +

benchttp/cli

+ +

+ + Github Worklow Status + + Code coverage + + Go Report Card +
+ + Go package Reference + + Latest version +

+ +## About + +`benchttp/cli` is a command-line interface that runs benchmarks on HTTP endpoints. +Highly configurable, it can be used as a CI step as well as +a testing tool at development time. + +## Installation + +1. Visit https://github.com/benchttp/cli/releases and download the asset + `benchttp__` matching your OS and CPU architecture. +1. Rename the downloaded asset to `benchttp`, add it to your `PATH`, + and refresh your terminal if necessary +1. Run `benchttp version` to check it works properly. + +## Usage + +### Run a benchmark + +```sh +benchttp run [options] +``` + +## Configuration + +In this section we dive into the many configuration options provided by the runner. + +First, a good way to get started with is via our [configuration generator](https://www.benchttp.app/setup). + +By default, the runner uses a default configuration that is valid for use without further tuning, except for `url` that must always be set. + +### Configuration Flow + +To determine the final configuration of a benchmark, the runner follows that flow: + +1. It starts with a [default configuration](./examples/config/default.yml) +1. Then it tries to find a config file and overrides the defaults with the values set in it + + - If flag `-configFile` is set, it resolves its value as a path + - Else, it tries to find a config file in the working directory, by priority order: + `.benchttp.yml` > `.benchttp.yaml` > `.benchttp.json` + + The config file is _optional_: if none is found, this step is ignored. + If a config file has an option `extends`, it resolves config file recursively until the root is reached and overrides the values from parent to child. + +1. Then it overrides the current config values with any value set via the CLI +1. Finally, it performs a validation on the resulting config (not before!). + This allows composed configurations for better granularity. + +### Specifications + +With rare exceptions, any option can be set either via CLI flags or config file, +and option names always match. + +A full config file example is available [here](./examples/config/full.yml). + +#### HTTP request options + +| CLI flag | File option | Description | Usage example | +| --------- | --------------------- | ------------------------- | ----------------------------------------- | +| `-url` | `request.url` | Target URL (**Required**) | `-url http://localhost:8080/users?page=3` | +| `-method` | `request.method` | HTTP Method | `-method POST` | +| - | `request.queryParams` | Added query params to URL | - | +| `-header` | `request.header` | Request headers | `-header 'key0:val0' -header 'key1:val1'` | +| `-body` | `request.body` | Raw request body | `-body 'raw:{"id":"abc"}'` | + +#### Benchmark runner options + +| CLI flag | File option | Description | Usage example | +| ----------------- | ----------------------- | -------------------------------------------------------------------- | -------------------- | +| `-requests` | `runner.requests` | Number of requests to run (-1 means infinite, stop on globalTimeout) | `-requests 100` | +| `-concurrency` | `runner.concurrency` | Maximum concurrent requests | `-concurrency 10` | +| `-interval` | `runner.interval` | Minimum duration between two non-concurrent requests | `-interval 200ms` | +| `-requestTimeout` | `runner.requestTimeout` | Timeout for every single request | `-requestTimeout 5s` | +| `-globalTimeout` | `runner.globalTimeout` | Timeout for the whole benchmark | `-globalTimeout 30s` | + +Note: the expected format for durations is ``, with `unit` being any of `ns`, `µs`, `ms`, `s`, `m`, `h`. + +#### Output options + +| CLI flag | File option | Description | Usage example | +| --------- | --------------- | ------------------------- | --------------------------- | +| `-silent` | `output.silent` | Remove convenience prints | `-silent` / `-silent=false` | From 671b419deabeee5d2af55514fadc7cbc3fccd379 Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 11:47:50 +0200 Subject: [PATCH 08/12] refactor: use package engine/configparse - remove json/yaml parsing logics - use engine/configparse instead - use new method config.WithField - apply inversion of Config.Override - run go mod tidy --- cmd/benchttp/run.go | 2 +- go.mod | 8 +- go.sum | 7 +- internal/configfile/parse.go | 305 ++------------------ internal/configfile/parser.go | 155 +--------- internal/configfile/parser_internal_test.go | 146 ---------- 6 files changed, 40 insertions(+), 583 deletions(-) delete mode 100644 internal/configfile/parser_internal_test.go diff --git a/cmd/benchttp/run.go b/cmd/benchttp/run.go index bd7cde4..93daff6 100644 --- a/cmd/benchttp/run.go +++ b/cmd/benchttp/run.go @@ -103,7 +103,7 @@ func (cmd *cmdRun) makeConfig(args []string) (cfg runner.Config, err error) { return } - mergedConfig := fileConfig.Override(cmd.config, fields...) + mergedConfig := cmd.config.WithFields(fields...).Override(fileConfig) return mergedConfig, mergedConfig.Validate() } diff --git a/go.mod b/go.mod index 291e036..d621b05 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,11 @@ module github.com/benchttp/cli go 1.17 +require github.com/benchttp/engine v0.0.0-20221006130541-30d09b451066 + require ( - github.com/benchttp/engine v0.0.0-20221006130541-30d09b451066 - gopkg.in/yaml.v3 v3.0.1 + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) -require golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - replace github.com/benchttp/engine => ../engine diff --git a/go.sum b/go.sum index da7311a..fe39a5e 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,10 @@ -github.com/benchttp/engine v0.0.0-20221006130541-30d09b451066 h1:epXygHg38XvHZ41VXf/mvDWKg33UcuJE7qKSVhwZwtg= -github.com/benchttp/engine v0.0.0-20221006130541-30d09b451066/go.mod h1:FRfUnUjoL1s0aHVGlrxB3pdPAEDLNCnWh6cVOur24hM= github.com/drykit-go/cond v0.1.0 h1:y7MNxREQLT83vGfcfSKjyFPLC/ZDjYBNp6KuaVVjOg4= +github.com/drykit-go/cond v0.1.0/go.mod h1:7MXBFjjaB5ZCEB8Q4w2euNOaWuTqf7NjOFZAyV1Jpfg= +github.com/drykit-go/strcase v0.2.0/go.mod h1:cWK0/az2f09UPIbJ42Sb8Iqdv01uENrFX+XXKGjPo+8= +github.com/drykit-go/testx v0.1.0/go.mod h1:qGXb49a8CzQ82crBeCVW8R3kGU1KRgWHnI+Q6CNVbz8= github.com/drykit-go/testx v1.2.0 h1:UsH+tFd24z3Xu+mwvwPY+9eBEg9CUyMsUeMYyUprG0o= +github.com/drykit-go/testx v1.2.0/go.mod h1:qTzXJgnAg8n31woklBzNTaWzLMJrnFk93x/aeaIpc20= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/internal/configfile/parse.go b/internal/configfile/parse.go index 0cfeb10..5217600 100644 --- a/internal/configfile/parse.go +++ b/internal/configfile/parse.go @@ -2,62 +2,20 @@ package configfile import ( "errors" - "fmt" - "net/http" - "net/url" "os" "path/filepath" - "strconv" - "time" + "github.com/benchttp/engine/configparse" "github.com/benchttp/engine/runner" "github.com/benchttp/cli/internal/errorutil" ) -// UnmarshaledConfig is a raw data model for runner config files. -// It serves as a receiver for unmarshaling processes and for that reason -// its types are kept simple (certain types are incompatible with certain -// unmarshalers). -type UnmarshaledConfig struct { - Extends *string `yaml:"extends" json:"extends"` - - Request struct { - Method *string `yaml:"method" json:"method"` - URL *string `yaml:"url" json:"url"` - QueryParams map[string]string `yaml:"queryParams" json:"queryParams"` - Header map[string][]string `yaml:"header" json:"header"` - Body *struct { - Type string `yaml:"type" json:"type"` - Content string `yaml:"content" json:"content"` - } `yaml:"body" json:"body"` - } `yaml:"request" json:"request"` - - Runner struct { - Requests *int `yaml:"requests" json:"requests"` - Concurrency *int `yaml:"concurrency" json:"concurrency"` - Interval *string `yaml:"interval" json:"interval"` - RequestTimeout *string `yaml:"requestTimeout" json:"requestTimeout"` - GlobalTimeout *string `yaml:"globalTimeout" json:"globalTimeout"` - } `yaml:"runner" json:"runner"` - - Output struct { - Silent *bool `yaml:"silent" json:"silent"` - } `yaml:"output" json:"output"` - - Tests []struct { - Name *string `yaml:"name" json:"name"` - Field *string `yaml:"field" json:"field"` - Predicate *string `yaml:"predicate" json:"predicate"` - Target interface{} `yaml:"target" json:"target"` - } `yaml:"tests" json:"tests"` -} - // Parse parses a benchttp runner config file into a runner.ConfigGlobal // and returns it or the first non-nil error occurring in the process, // which can be any of the values declared in the package. func Parse(filename string) (cfg runner.Config, err error) { - uconfs, err := parseFileRecursive(filename, []UnmarshaledConfig{}, set{}) + uconfs, err := parseFileRecursive(filename, []configparse.Representation{}, set{}) if err != nil { return } @@ -83,62 +41,62 @@ func (s set) add(v string) error { // occurring in the process. func parseFileRecursive( filename string, - uconfs []UnmarshaledConfig, + reprs []configparse.Representation, seen set, -) ([]UnmarshaledConfig, error) { +) ([]configparse.Representation, error) { // avoid infinite recursion caused by circular reference if err := seen.add(filename); err != nil { - return uconfs, ErrCircularExtends + return reprs, ErrCircularExtends } // parse current file, append parsed config - uconf, err := parseFile(filename) + repr, err := parseFile(filename) if err != nil { - return uconfs, err + return reprs, err } - uconfs = append(uconfs, uconf) + reprs = append(reprs, repr) // root config reached: stop now and return the parsed configs - if uconf.Extends == nil { - return uconfs, nil + if repr.Extends == nil { + return reprs, nil } // config has parent: resolve its path and parse it recursively - parentPath := filepath.Join(filepath.Dir(filename), *uconf.Extends) - return parseFileRecursive(parentPath, uconfs, seen) + parentPath := filepath.Join(filepath.Dir(filename), *repr.Extends) + return parseFileRecursive(parentPath, reprs, seen) } // parseFile parses a single config file and returns the result as an -// unmarshaledConfig and an appropriate error predeclared in the package. -func parseFile(filename string) (uconf UnmarshaledConfig, err error) { +// configparse.Representation and an appropriate error predeclared in the package. +func parseFile(filename string) (repr configparse.Representation, err error) { b, err := os.ReadFile(filename) switch { case err == nil: case errors.Is(err, os.ErrNotExist): - return uconf, errorutil.WithDetails(ErrFileNotFound, filename) + return repr, errorutil.WithDetails(ErrFileNotFound, filename) default: - return uconf, errorutil.WithDetails(ErrFileRead, filename, err) + return repr, errorutil.WithDetails(ErrFileRead, filename, err) } ext := extension(filepath.Ext(filename)) parser, err := newParser(ext) if err != nil { - return uconf, errorutil.WithDetails(ErrFileExt, ext, err) + return repr, errorutil.WithDetails(ErrFileExt, ext, err) } - if err = parser.parse(b, &uconf); err != nil { - return uconf, errorutil.WithDetails(ErrParse, filename, err) + if err = parser.Parse(b, &repr); err != nil { + return repr, errorutil.WithDetails(ErrParse, filename, err) } - return uconf, nil + return repr, nil } // parseAndMergeConfigs iterates backwards over uconfs, parsing them // as runner.ConfigGlobal and merging them into a single one. // It returns the merged result or the first non-nil error occurring in the // process. -func parseAndMergeConfigs(uconfs []UnmarshaledConfig) (cfg runner.Config, err error) { - if len(uconfs) == 0 { // supposedly catched upstream, should not occur +func parseAndMergeConfigs(reprs []configparse.Representation) (cfg runner.Config, err error) { + if len(reprs) == 0 { // supposedly catched upstream, should not occur return cfg, errors.New( "an unacceptable error occurred parsing the config file, " + "please visit https://github.com/benchttp/runner/issues/new " + @@ -148,225 +106,14 @@ func parseAndMergeConfigs(uconfs []UnmarshaledConfig) (cfg runner.Config, err er cfg = runner.DefaultConfig() - for i := len(uconfs) - 1; i >= 0; i-- { - uconf := uconfs[i] - pconf, err := newParsedConfig(uconf) + for i := len(reprs) - 1; i >= 0; i-- { + repr := reprs[i] + currentConfig, err := configparse.ParseRepresentation(repr) if err != nil { return cfg, errorutil.WithDetails(ErrParse, err) } - cfg = cfg.Override(pconf.config, pconf.fields...) + cfg = currentConfig.Override(cfg) } return cfg, nil } - -// parsedConfig embeds a parsed runner.ConfigGlobal and the list of its set fields. -type parsedConfig struct { - // TODO: do not embed, use field config - config runner.Config - fields []string -} - -// addField adds a field to the list of set fields. -func (pconf *parsedConfig) add(field string) { - pconf.fields = append(pconf.fields, field) -} - -// newParsedConfig parses an input raw config as a runner.ConfigGlobal and returns -// a parsedConfig or the first non-nil error occurring in the process. -func newParsedConfig(uconf UnmarshaledConfig) (parsedConfig, error) { //nolint:gocognit // acceptable complexity for a parsing func - maxFields := len(runner.ConfigFieldsUsage) - pconf := parsedConfig{fields: make([]string, 0, maxFields)} - cfg := &pconf.config - - abort := func(err error) (parsedConfig, error) { - return parsedConfig{}, err - } - - if method := uconf.Request.Method; method != nil { - cfg.Request.Method = *method - pconf.add(runner.ConfigFieldMethod) - } - - if rawURL := uconf.Request.URL; rawURL != nil { - parsedURL, err := parseAndBuildURL(*uconf.Request.URL, uconf.Request.QueryParams) - if err != nil { - return abort(err) - } - cfg.Request.URL = parsedURL - pconf.add(runner.ConfigFieldURL) - } - - if header := uconf.Request.Header; header != nil { - httpHeader := http.Header{} - for key, val := range header { - httpHeader[key] = val - } - cfg.Request.Header = httpHeader - pconf.add(runner.ConfigFieldHeader) - } - - if body := uconf.Request.Body; body != nil { - cfg.Request.Body = runner.RequestBody{ - Type: body.Type, - Content: []byte(body.Content), - } - pconf.add(runner.ConfigFieldBody) - } - - if requests := uconf.Runner.Requests; requests != nil { - cfg.Runner.Requests = *requests - pconf.add(runner.ConfigFieldRequests) - } - - if concurrency := uconf.Runner.Concurrency; concurrency != nil { - cfg.Runner.Concurrency = *concurrency - pconf.add(runner.ConfigFieldConcurrency) - } - - if interval := uconf.Runner.Interval; interval != nil { - parsedInterval, err := parseOptionalDuration(*interval) - if err != nil { - return abort(err) - } - cfg.Runner.Interval = parsedInterval - pconf.add(runner.ConfigFieldInterval) - } - - if requestTimeout := uconf.Runner.RequestTimeout; requestTimeout != nil { - parsedTimeout, err := parseOptionalDuration(*requestTimeout) - if err != nil { - return abort(err) - } - cfg.Runner.RequestTimeout = parsedTimeout - pconf.add(runner.ConfigFieldRequestTimeout) - } - - if globalTimeout := uconf.Runner.GlobalTimeout; globalTimeout != nil { - parsedGlobalTimeout, err := parseOptionalDuration(*globalTimeout) - if err != nil { - return abort(err) - } - cfg.Runner.GlobalTimeout = parsedGlobalTimeout - pconf.add(runner.ConfigFieldGlobalTimeout) - } - - if silent := uconf.Output.Silent; silent != nil { - cfg.Output.Silent = *silent - pconf.add(runner.ConfigFieldSilent) - } - - testSuite := uconf.Tests - if len(testSuite) == 0 { - return pconf, nil - } - - cases := make([]runner.TestCase, len(testSuite)) - for i, t := range testSuite { - fieldPath := func(caseField string) string { - return fmt.Sprintf("tests[%d].%s", i, caseField) - } - - if err := requireConfigFields(map[string]interface{}{ - fieldPath("name"): t.Name, - fieldPath("field"): t.Field, - fieldPath("predicate"): t.Predicate, - fieldPath("target"): t.Target, - }); err != nil { - return abort(err) - } - - field := runner.MetricsField(*t.Field) - if err := field.Validate(); err != nil { - return abort(fmt.Errorf("%s: %s", fieldPath("field"), err)) - } - - predicate := runner.TestPredicate(*t.Predicate) - if err := predicate.Validate(); err != nil { - return abort(fmt.Errorf("%s: %s", fieldPath("predicate"), err)) - } - - target, err := parseMetricValue(field, fmt.Sprint(t.Target)) - if err != nil { - return abort(fmt.Errorf("%s: %s", fieldPath("target"), err)) - } - - cases[i] = runner.TestCase{ - Name: *t.Name, - Field: field, - Predicate: predicate, - Target: target, - } - } - cfg.Tests = cases - pconf.add(runner.ConfigFieldTests) - - return pconf, nil -} - -// helpers - -// parseAndBuildURL parses a raw string as a *url.URL and adds any extra -// query parameters. It returns the first non-nil error occurring in the -// process. -func parseAndBuildURL(raw string, qp map[string]string) (*url.URL, error) { - u, err := url.ParseRequestURI(raw) - if err != nil { - return nil, err - } - - // retrieve url query, add extra params, re-attach to url - if qp != nil { - q := u.Query() - for k, v := range qp { - q.Add(k, v) - } - u.RawQuery = q.Encode() - } - - return u, nil -} - -// parseOptionalDuration parses the raw string as a time.Duration -// and returns the parsed value or a non-nil error. -// Contrary to time.ParseDuration, it does not return an error -// if raw == "". -func parseOptionalDuration(raw string) (time.Duration, error) { - if raw == "" { - return 0, nil - } - return time.ParseDuration(raw) -} - -func parseMetricValue( - field runner.MetricsField, - inputValue string, -) (runner.MetricsValue, error) { - fieldType := field.Type() - handleError := func(v interface{}, err error) (runner.MetricsValue, error) { - if err != nil { - return nil, fmt.Errorf( - "value %q is incompatible with field %s (want %s)", - inputValue, field, fieldType, - ) - } - return v, nil - } - switch fieldType { - case "int": - return handleError(strconv.Atoi(inputValue)) - case "time.Duration": - return handleError(time.ParseDuration(inputValue)) - default: - return nil, fmt.Errorf("unknown field: %s", field) - } -} - -func requireConfigFields(fields map[string]interface{}) error { - for name, value := range fields { - if value == nil { - return fmt.Errorf("%s: missing field", name) - } - } - return nil -} diff --git a/internal/configfile/parser.go b/internal/configfile/parser.go index 3103f13..c50876e 100644 --- a/internal/configfile/parser.go +++ b/internal/configfile/parser.go @@ -1,13 +1,9 @@ package configfile import ( - "bytes" - "encoding/json" "errors" - "fmt" - "regexp" - "gopkg.in/yaml.v3" + "github.com/benchttp/engine/configparse" ) type extension string @@ -22,7 +18,7 @@ const ( type configParser interface { // parse parses a raw bytes input as a raw config and stores // the resulting value into dst. - parse(in []byte, dst *UnmarshaledConfig) error + Parse(in []byte, dst *configparse.Representation) error } // newParser returns an appropriate parser according to ext, or a non-nil @@ -30,153 +26,10 @@ type configParser interface { func newParser(ext extension) (configParser, error) { switch ext { case extYML, extYAML: - return yamlParser{}, nil + return configparse.YAMLParser{}, nil case extJSON: - return jsonParser{}, nil + return configparse.JSONParser{}, nil default: return nil, errors.New("unsupported config format") } } - -// yamlParser implements configParser. -type yamlParser struct{} - -// parse decodes a raw yaml input in strict mode (unknown fields disallowed) -// and stores the resulting value into dst. -func (p yamlParser) parse(in []byte, dst *UnmarshaledConfig) error { - decoder := yaml.NewDecoder(bytes.NewReader(in)) - decoder.KnownFields(true) - return p.handleError(decoder.Decode(dst)) -} - -// handleError handles a raw yaml decoder.Decode error, filters it, -// and return the resulting error. -func (p yamlParser) handleError(err error) error { - // yaml.TypeError errors require special handling, other errors - // (nil included) can be returned as is. - var typeError *yaml.TypeError - if !errors.As(err, &typeError) { - return err - } - - // filter out unwanted errors - filtered := &yaml.TypeError{} - for _, msg := range typeError.Errors { - // With decoder.KnownFields set to true, Decode reports any field - // that do not match the destination structure as a non-nil error. - // It is a wanted behavior but prevents the usage of custom aliases. - // To work around this we allow an exception for that rule with fields - // starting with x- (inspired by docker compose api). - if p.isCustomFieldError(msg) { - continue - } - filtered.Errors = append(filtered.Errors, p.prettyErrorMessage(msg)) - } - - if len(filtered.Errors) != 0 { - return filtered - } - - return nil -} - -// isCustomFieldError returns true if the raw error message is due -// to an allowed custom field. -func (p yamlParser) isCustomFieldError(raw string) bool { - customFieldRgx := regexp.MustCompile( - // raw output example: - // line 9: field x-my-alias not found in type struct { ... } - `^line \d+: field (x-\S+) not found in type`, - ) - return customFieldRgx.MatchString(raw) -} - -// prettyErrorMessage transforms a raw Decode error message into a more -// user-friendly one by removing noisy information and returns the resulting -// value. -func (p yamlParser) prettyErrorMessage(raw string) string { - // field not found error - fieldNotFoundRgx := regexp.MustCompile( - // raw output example (type unmarshaledConfig is entirely exposed): - // line 11: field interval not found in type struct { ... } - `^line (\d+): field (\S+) not found in type`, - ) - if matches := fieldNotFoundRgx.FindStringSubmatch(raw); len(matches) >= 3 { - line, field := matches[1], matches[2] - return fmt.Sprintf(`line %s: invalid field ("%s"): does not exist`, line, field) - } - - // wrong field type error - fieldBadValueRgx := regexp.MustCompile( - // raw output examples: - // line 9: cannot unmarshal !!seq into int // unknown input value - // line 10: cannot unmarshal !!str `hello` into int // known input value - `^line (\d+): cannot unmarshal !!\w+(?: ` + "`" + `(\S+)` + "`" + `)? into (\S+)$`, - ) - if matches := fieldBadValueRgx.FindStringSubmatch(raw); len(matches) >= 3 { - line, value, exptype := matches[1], matches[2], matches[3] - if value == "" { - return fmt.Sprintf("line %s: wrong type: want %s", line, exptype) - } - return fmt.Sprintf(`line %s: wrong type ("%s"): want %s`, line, value, exptype) - } - - // we may not have covered all cases, return raw output in this case - return raw -} - -// jsonParser implements configParser. -type jsonParser struct{} - -// parse decodes a raw JSON input in strict mode (unknown fields disallowed) -// and stores the resulting value into dst. -func (p jsonParser) parse(in []byte, dst *UnmarshaledConfig) error { - decoder := json.NewDecoder(bytes.NewReader(in)) - decoder.DisallowUnknownFields() - return p.handleError(decoder.Decode(dst)) -} - -// handleError handle a json raw error, transforms it into a user-friendly -// standardized format and returns the resulting error. -func (p jsonParser) handleError(err error) error { - if err == nil { - return nil - } - - // handle syntax error - var errSyntax *json.SyntaxError - if errors.As(err, &errSyntax) { - return fmt.Errorf("syntax error near %d: %w", errSyntax.Offset, err) - } - - // handle type error - var errType *json.UnmarshalTypeError - if errors.As(err, &errType) { - return fmt.Errorf( - "wrong type for field %s: want %s, got %s", - errType.Field, errType.Type, errType.Value, - ) - } - - // handle unknown field error - if field := p.parseUnknownFieldError(err.Error()); field != "" { - return fmt.Errorf(`invalid field ("%s"): does not exist`, field) - } - - return err -} - -// parseUnknownFieldError parses the raw string as a json error -// from an unknown field and returns the field name. -// If the raw string is not an unknown field error, it returns "". -func (p jsonParser) parseUnknownFieldError(raw string) (field string) { - unknownFieldRgx := regexp.MustCompile( - // raw output example: - // json: unknown field "notafield" - `json: unknown field "(\S+)"`, - ) - if matches := unknownFieldRgx.FindStringSubmatch(raw); len(matches) >= 2 { - return matches[1] - } - return "" -} diff --git a/internal/configfile/parser_internal_test.go b/internal/configfile/parser_internal_test.go deleted file mode 100644 index 6407d77..0000000 --- a/internal/configfile/parser_internal_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package configfile - -import ( - "errors" - "reflect" - "testing" - - "gopkg.in/yaml.v3" -) - -func TestYAMLParser(t *testing.T) { - t.Run("return expected errors", func(t *testing.T) { - testcases := []struct { - label string - in []byte - expErr error - }{ - { - label: "unknown field", - in: []byte("notafield: 123\n"), - expErr: &yaml.TypeError{ - Errors: []string{ - `line 1: invalid field ("notafield"): does not exist`, - }, - }, - }, - { - label: "wrong type unknown value", - in: []byte("runner:\n requests: [123]\n"), - expErr: &yaml.TypeError{ - Errors: []string{ - `line 2: wrong type: want int`, - }, - }, - }, - { - label: "wrong type known value", - in: []byte("runner:\n requests: \"123\"\n"), - expErr: &yaml.TypeError{ - Errors: []string{ - `line 2: wrong type ("123"): want int`, - }, - }, - }, - { - label: "cumulate errors", - in: []byte("runner:\n requests: [123]\n concurrency: \"123\"\nnotafield: 123\n"), - expErr: &yaml.TypeError{ - Errors: []string{ - `line 2: wrong type: want int`, - `line 3: wrong type ("123"): want int`, - `line 4: invalid field ("notafield"): does not exist`, - }, - }, - }, - { - label: "no errors custom fields", - in: []byte("x-data: &count\n requests: 100\rrunner:\n <<: *count\n"), - expErr: nil, - }, - } - - for _, tc := range testcases { - t.Run(tc.label, func(t *testing.T) { - var ( - parser yamlParser - rawcfg UnmarshaledConfig - yamlErr *yaml.TypeError - ) - - gotErr := parser.parse(tc.in, &rawcfg) - - if tc.expErr == nil { - if gotErr != nil { - t.Fatalf("unexpected error: %v", gotErr) - } - return - } - - if !errors.As(gotErr, &yamlErr) && tc.expErr != nil { - t.Fatalf("unexpected error: %v", gotErr) - } - - if !reflect.DeepEqual(yamlErr, tc.expErr) { - t.Errorf("unexpected error messages:\nexp %v\ngot %v", tc.expErr, yamlErr) - } - }) - } - }) -} - -func TestJSONParser(t *testing.T) { - t.Run("return expected errors", func(t *testing.T) { - testcases := []struct { - label string - in []byte - exp string - }{ - { - label: "syntax error", - in: []byte("{\n \"runner\": {},\n}\n"), - exp: "syntax error near 19: invalid character '}' looking for beginning of object key string", - }, - { - label: "unknown field", - in: []byte("{\n \"notafield\": 123\n}\n"), - exp: `invalid field ("notafield"): does not exist`, - }, - { - label: "wrong type", - in: []byte("{\n \"runner\": {\n \"requests\": [123]\n }\n}\n"), - exp: "wrong type for field runner.requests: want int, got array", - }, - { - label: "valid config", - in: []byte("{\n \"runner\": {\n \"requests\": 123\n }\n}\n"), - exp: "", - }, - } - - for _, tc := range testcases { - t.Run(tc.label, func(t *testing.T) { - var ( - parser jsonParser - rawcfg UnmarshaledConfig - ) - - gotErr := parser.parse(tc.in, &rawcfg) - - if tc.exp == "" { - if gotErr != nil { - t.Fatalf("unexpected error: %v", gotErr) - } - return - } - - if gotErr.Error() != tc.exp { - t.Errorf( - "unexpected error messages:\nexp %s\ngot %v", - tc.exp, gotErr, - ) - } - }) - } - }) -} From 998bc7729e1eac4762abc5e337bb44d731205838 Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 14:27:59 +0200 Subject: [PATCH 09/12] refactor: remove silent from runner config - use -silent as a separate CLI-only flag (same as -configFile) - update tests & fixtures accordingly - update docs accordingly --- README.md | 9 +++++---- cmd/benchttp/run.go | 18 ++++++++++++++---- examples/config/default.yml | 3 --- examples/config/full.yml | 3 --- internal/configfile/parse_test.go | 7 ++----- .../configfile/testdata/valid/benchttp.json | 3 --- .../configfile/testdata/valid/benchttp.yaml | 3 --- .../configfile/testdata/valid/benchttp.yml | 3 --- internal/configflag/bind.go | 7 ------- internal/configflag/bind_test.go | 4 ---- 10 files changed, 21 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index eb32163..f2955da 100644 --- a/README.md +++ b/README.md @@ -91,8 +91,9 @@ A full config file example is available [here](./examples/config/full.yml). Note: the expected format for durations is ``, with `unit` being any of `ns`, `µs`, `ms`, `s`, `m`, `h`. -#### Output options +#### CLI-specific options -| CLI flag | File option | Description | Usage example | -| --------- | --------------- | ------------------------- | --------------------------- | -| `-silent` | `output.silent` | Remove convenience prints | `-silent` / `-silent=false` | +| CLI flag | Description | Usage example | +| ------------- | ---------------------------- | ---------------------------------- | +| `-silent` | Remove convenience prints | `-silent` / `-silent=false` | +| `-configFile` | Path to benchttp config file | `-configFile=path/to/benchttp.yml` | diff --git a/cmd/benchttp/run.go b/cmd/benchttp/run.go index 93daff6..9c62fcc 100644 --- a/cmd/benchttp/run.go +++ b/cmd/benchttp/run.go @@ -24,6 +24,9 @@ type cmdRun struct { // configFile is the parsed value for flag -configFile configFile string + // silent is the parsed value for flag -silent + silent bool + // config is the runner config resulting from parsing CLI flags. config runner.Config } @@ -50,12 +53,12 @@ func (cmd *cmdRun) execute(args []string) error { return err } - report, err := runBenchmark(cfg) + report, err := runBenchmark(cfg, cmd.silent) if err != nil { return err } - return renderReport(os.Stdout, report, cfg.Output.Silent) + return renderReport(os.Stdout, report, cmd.silent) } // parseArgs parses input args as config fields and returns @@ -74,6 +77,13 @@ func (cmd *cmdRun) parseArgs(args []string) []string { "Config file path", ) + // silent mode + cmd.flagset.BoolVar(&cmd.silent, + "silent", + false, + "Silent mode", + ) + // attach config options flags to the flagset // and bind their value to the config struct configflag.Bind(cmd.flagset, &cmd.config) @@ -122,14 +132,14 @@ func onRecordingProgress(silent bool) func(runner.RecordingProgress) { } } -func runBenchmark(cfg runner.Config) (*runner.Report, error) { +func runBenchmark(cfg runner.Config, silent bool) (*runner.Report, error) { // Prepare graceful shutdown in case of os.Interrupt (Ctrl+C) ctx, cancel := context.WithCancel(context.Background()) go signals.ListenOSInterrupt(cancel) // Run the benchmark report, err := runner. - New(onRecordingProgress(cfg.Output.Silent)). + New(onRecordingProgress(silent)). Run(ctx, cfg) if err != nil { return report, err diff --git a/examples/config/default.yml b/examples/config/default.yml index 07e6bea..53369ae 100644 --- a/examples/config/default.yml +++ b/examples/config/default.yml @@ -8,6 +8,3 @@ runner: interval: 0ms requestTimeout: 5s globalTimeout: 30s - -output: - silent: false diff --git a/examples/config/full.yml b/examples/config/full.yml index c20fb5e..c83d434 100644 --- a/examples/config/full.yml +++ b/examples/config/full.yml @@ -17,6 +17,3 @@ runner: interval: 50ms requestTimeout: 2s globalTimeout: 60s - -output: - silent: true diff --git a/internal/configfile/parse_test.go b/internal/configfile/parse_test.go index 7b910db..06f7de2 100644 --- a/internal/configfile/parse_test.go +++ b/internal/configfile/parse_test.go @@ -108,7 +108,7 @@ func TestParse(t *testing.T) { restoreGotCfg := setTempValue(&gotURL.RawQuery, "replaced by test") restoreExpCfg := setTempValue(&expURL.RawQuery, "replaced by test") - if !reflect.DeepEqual(gotCfg, expCfg) { + if !gotCfg.Equal(expCfg) { t.Errorf("unexpected parsed config for %s file:\nexp %v\ngot %v", ext, expCfg, gotCfg) } @@ -176,7 +176,7 @@ func TestParse(t *testing.T) { } if gotURL := cfg.Request.URL.String(); gotURL != expURL { - t.Errorf("method: exp %s, got %s", expURL, gotURL) + t.Errorf("url: exp %s, got %s", expURL, gotURL) } }) } @@ -206,9 +206,6 @@ func newExpConfig() runner.Config { RequestTimeout: 2 * time.Second, GlobalTimeout: 60 * time.Second, }, - Output: runner.OutputConfig{ - Silent: true, - }, Tests: []runner.TestCase{ { Name: "minimum response time", diff --git a/internal/configfile/testdata/valid/benchttp.json b/internal/configfile/testdata/valid/benchttp.json index 5ffb803..714506f 100644 --- a/internal/configfile/testdata/valid/benchttp.json +++ b/internal/configfile/testdata/valid/benchttp.json @@ -21,9 +21,6 @@ "requestTimeout": "2s", "globalTimeout": "60s" }, - "output": { - "silent": true - }, "tests": [ { "name": "minimum response time", diff --git a/internal/configfile/testdata/valid/benchttp.yaml b/internal/configfile/testdata/valid/benchttp.yaml index df1b091..2ee790c 100644 --- a/internal/configfile/testdata/valid/benchttp.yaml +++ b/internal/configfile/testdata/valid/benchttp.yaml @@ -20,9 +20,6 @@ runner: requestTimeout: 2s globalTimeout: 60s -output: - silent: true - tests: - name: minimum response time field: ResponseTimes.Min diff --git a/internal/configfile/testdata/valid/benchttp.yml b/internal/configfile/testdata/valid/benchttp.yml index 13a2973..27a2fc9 100644 --- a/internal/configfile/testdata/valid/benchttp.yml +++ b/internal/configfile/testdata/valid/benchttp.yml @@ -17,9 +17,6 @@ runner: requestTimeout: 2s globalTimeout: 60s -output: - silent: true - tests: - name: minimum response time field: ResponseTimes.Min diff --git a/internal/configflag/bind.go b/internal/configflag/bind.go index 726a552..359cae8 100644 --- a/internal/configflag/bind.go +++ b/internal/configflag/bind.go @@ -73,11 +73,4 @@ func Bind(flagset *flag.FlagSet, dst *runner.Config) { dst.Runner.GlobalTimeout, runner.ConfigFieldsUsage[runner.ConfigFieldGlobalTimeout], ) - - // silent mode - flagset.BoolVar(&dst.Output.Silent, - runner.ConfigFieldSilent, - dst.Output.Silent, - runner.ConfigFieldsUsage[runner.ConfigFieldSilent], - ) } diff --git a/internal/configflag/bind_test.go b/internal/configflag/bind_test.go index 8ec835b..5c54727 100644 --- a/internal/configflag/bind_test.go +++ b/internal/configflag/bind_test.go @@ -40,7 +40,6 @@ func TestBind(t *testing.T) { "-interval", "3s", "-requestTimeout", "4s", "-globalTimeout", "5s", - "-silent", } cfg := runner.Config{} @@ -62,9 +61,6 @@ func TestBind(t *testing.T) { RequestTimeout: 4 * time.Second, GlobalTimeout: 5 * time.Second, }, - Output: runner.OutputConfig{ - Silent: true, - }, } if !reflect.DeepEqual(cfg, exp) { From 1d1bc3fc5c4a94de1a13f5f498b43a6cee906c6e Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 18:55:33 +0200 Subject: [PATCH 10/12] refactor: renamings, docs --- cmd/benchttp/main.go | 4 ++-- cmd/benchttp/run.go | 7 ++++--- internal/output/conditional.go | 15 ++++++++++----- internal/render/renderer.go | 15 --------------- 4 files changed, 16 insertions(+), 25 deletions(-) delete mode 100644 internal/render/renderer.go diff --git a/cmd/benchttp/main.go b/cmd/benchttp/main.go index 47dcf7c..82e27c7 100644 --- a/cmd/benchttp/main.go +++ b/cmd/benchttp/main.go @@ -21,7 +21,7 @@ func main() { } func run() error { - commandName, nextArgs, err := shiftArgs(os.Args[1:]) + commandName, options, err := shiftArgs(os.Args[1:]) if err != nil { return err } @@ -31,7 +31,7 @@ func run() error { return err } - return cmd.execute(nextArgs) + return cmd.execute(options) } func shiftArgs(args []string) (commandName string, nextArgs []string, err error) { diff --git a/cmd/benchttp/run.go b/cmd/benchttp/run.go index 9c62fcc..e1269e6 100644 --- a/cmd/benchttp/run.go +++ b/cmd/benchttp/run.go @@ -149,14 +149,15 @@ func runBenchmark(cfg runner.Config, silent bool) (*runner.Report, error) { } func renderReport(w io.Writer, report *runner.Report, silent bool) error { - cw := output.ConditionalWriter{Writer: w}.If(!silent) + writeIfNotSilent := output.ConditionalWriter{Writer: w}.If(!silent) - if _, err := render.ReportSummary(cw, report); err != nil { + if _, err := render.ReportSummary(writeIfNotSilent, report); err != nil { return err } if _, err := render.TestSuite( - cw.Or(!report.Tests.Pass), report.Tests, + writeIfNotSilent.ElseIf(!report.Tests.Pass), + report.Tests, ); err != nil { return err } diff --git a/internal/output/conditional.go b/internal/output/conditional.go index c651313..ffe0f28 100644 --- a/internal/output/conditional.go +++ b/internal/output/conditional.go @@ -4,12 +4,14 @@ import ( "io" ) +// ConditionalWriter is an io.Writer that wraps an input writer +// and exposes methods to condition its action. type ConditionalWriter struct { Writer io.Writer ok bool } -// Write writes b only if MuteableWriter.Mute is false, +// Write writes b only if ConditionalWriter.Mute is false, // otherwise it is no-op. func (w ConditionalWriter) Write(b []byte) (int, error) { if !w.ok { @@ -18,16 +20,19 @@ func (w ConditionalWriter) Write(b []byte) (int, error) { return w.Writer.Write(b) } -func (w ConditionalWriter) If(ok bool) ConditionalWriter { +// If sets the write condition to v. +func (w ConditionalWriter) If(v bool) ConditionalWriter { return ConditionalWriter{ Writer: w.Writer, - ok: ok, + ok: v, } } -func (w ConditionalWriter) Or(ok bool) ConditionalWriter { +// ElseIf either keeps the previous write condition if it is true, +// else it sets it to v. +func (w ConditionalWriter) ElseIf(v bool) ConditionalWriter { return ConditionalWriter{ Writer: w.Writer, - ok: w.ok || ok, + ok: w.ok || v, } } diff --git a/internal/render/renderer.go b/internal/render/renderer.go deleted file mode 100644 index 7067332..0000000 --- a/internal/render/renderer.go +++ /dev/null @@ -1,15 +0,0 @@ -package render - -// type Renderer struct { -// Silent bool -// Writer io.Writer -// } - -// func NewRenderer(w io.Writer, silent bool) Renderer { -// return Renderer{ -// Writer: w, -// Silent: silent, -// } -// } - -// func (r Renderer) Progress From ce7124abe338f2832a0e2909d84f876421d23831 Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 20:34:35 +0200 Subject: [PATCH 11/12] Revert "temp: override benchttp/engine with local source" This reverts commit dfc14139372a6aedba9484139ef2f65c3dd4f956. --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index d621b05..175d3d0 100644 --- a/go.mod +++ b/go.mod @@ -8,5 +8,3 @@ require ( golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - -replace github.com/benchttp/engine => ../engine From 0ca7b8e3799396204273d63b8cc1626f66c31c7a Mon Sep 17 00:00:00 2001 From: Gregory Albouy Date: Sat, 8 Oct 2022 20:34:57 +0200 Subject: [PATCH 12/12] deps: upgrade --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 175d3d0..9d38e26 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/benchttp/cli go 1.17 -require github.com/benchttp/engine v0.0.0-20221006130541-30d09b451066 +require github.com/benchttp/engine v0.0.0-20221008174504-d1162e9ac007 require ( golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect diff --git a/go.sum b/go.sum index fe39a5e..90a8750 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/benchttp/engine v0.0.0-20221008174504-d1162e9ac007 h1:h6ON0tn3i/83eUMoXq0FHxBMLYIA/OIli3m9KYkcxPI= +github.com/benchttp/engine v0.0.0-20221008174504-d1162e9ac007/go.mod h1:FRfUnUjoL1s0aHVGlrxB3pdPAEDLNCnWh6cVOur24hM= github.com/drykit-go/cond v0.1.0 h1:y7MNxREQLT83vGfcfSKjyFPLC/ZDjYBNp6KuaVVjOg4= github.com/drykit-go/cond v0.1.0/go.mod h1:7MXBFjjaB5ZCEB8Q4w2euNOaWuTqf7NjOFZAyV1Jpfg= github.com/drykit-go/strcase v0.2.0/go.mod h1:cWK0/az2f09UPIbJ42Sb8Iqdv01uENrFX+XXKGjPo+8=