diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..5116628 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,18 @@ +## Description + + + +## Tasks + + + +## Suggestions + + + +## Notes + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..d359971 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ + + +## Description + + + +## Changes + + + +## Notes + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5f2bf95 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: Lint & Test & Build + +on: + pull_request: + push: + branches: + - main + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.17 + + - name: Install coverage tool + run: go get github.com/ory/go-acc + + # Check #1: Lint + - name: Lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.47.2 + + # Check #2: Test & generate coverage report + - name: Test & coverage + run: make test-cov + + # Check #3: Build binaries + - name: Build + run: make build + + - name: Upload coverage report + uses: codecov/codecov-action@v1.0.2 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.txt + flags: unittests diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a925db --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Binary files +/bin + +# IDE files +/.vscode +/.idea + +# Benchttp configs +/.benchttp.yml +/.benchttp.yaml +/.benchttp.json diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..64c7726 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,161 @@ +run: + timeout: 1m + uniq-by-line: false + +linters-settings: + dupl: + threshold: 100 + + errcheck: + exclude-functions: + - (net/http.ResponseWriter).Write + - (io.StringWriter).WriteString + - (io.Writer).Write + + gocognit: + min-complexity: 15 + + gocritic: + enabled-checks: + - appendAssign + - appendCombine + - argOrder + - assignOp + - badCall + - badCond + - badLock + - badRegexp + - boolExprSimplify + - builtinShadow + - builtinShadowDecl + - captLocal + - caseOrder + - codegenComment + - commentedOutCode + - commentedOutImport + - commentFormatting + - defaultCaseOrder + - deferUnlambda + - deprecatedComment + - docStub + - dupArg + - dupBranchBody + - dupCase + - dupImport + - dupSubExpr + - elseif + - emptyFallthrough + - emptyStringTest + - equalFold + - evalOrder + - exitAfterDefer + - filepathJoin + - flagDeref + - flagName + - hexLiteral + - hugeParam + - ifElseChain + - importShadow + - indexAlloc + - initClause + - mapKey + - methodExprCall + - nestingReduce + - newDeref + - nilValReturn + - octalLiteral + - offBy1 + - paramTypeCombine + - ptrToRefParam + - rangeExprCopy + - rangeValCopy + - regexpMust + - regexpPattern + - regexpSimplify + - ruleguard + - singleCaseSwitch + - sloppyLen + - sloppyReassign + - sloppyTypeAssert + - sortSlice + - sqlQuery + - stringXbytes + - switchTrue + - tooManyResultsChecker + - truncateCmp + - typeAssertChain + - typeDefFirst + - typeSwitchVar + - typeUnparen + - underef + # - unamedResult + - unlabelStmt + - unlambda + - unnecessaryBlock + - unnecessaryDefer + - unslice + - valSwap + - weakCond + # - whynolint + - wrapperFunc + - yodaStyleExpr + + settings: + hugeParam: + sizeThreshold: 256 + rangeValCopy: + sizeThreshold: 256 + + gofumpt: + lang-version: "1.17" + extra-rules: true + + goimports: + local-prefixes: github.com/benchttp/cli + + misspell: + locale: US + + revive: + enableAllRules: true + + staticcheck: + go: "1.17" + checks: [all] + + stylecheck: + go: "1.17" + checks: [all] + +linters: + disable-all: true + enable: + - bodyclose # enforce resp.Body.Close() + - deadcode + - dupl # duplicate code + - errcheck + - exportloopref + - gocognit # smart complexity analyzer + - gocritic # opinionated linter + - gofumpt # stricter gofmt + - goimports # imports order + - gosec # security checks + - govet + - misspell # typos in strings, comments + - prealloc # enforce capacity allocation when possible + - revive # golint enhancement + - staticcheck # go vet enhancement + - structcheck # unused struct fields + - testpackage # checks on tests (*_test) + - thelper # enforce t.Helper() + - varcheck # unused global var and const + - wastedassign + fast: false + +issues: + exclude-rules: + - path: _test\.go + linters: + - dupl + - gocognit + - gocyclo diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..73ec2c6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +MIT License + +Copyright (c) 2021 +Gregory Albouy, Clara Gonnon, Damien Mathieu +Alex Mongeot, Thomas Moreira, Kerian Pelat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a5bb555 --- /dev/null +++ b/Makefile @@ -0,0 +1,55 @@ +# Default command + +.PHONY: default +default: + @make check + +# Check code + +.PHONY: check +check: + @make lint + @make tests + +.PHONY: lint +lint: + @golangci-lint run + +.PHONY: tests +tests: + @go test -race -timeout 10s ./... + +TEST_FUNC=^.*$$ +ifdef t +TEST_FUNC=$(t) +endif +TEST_PKG=./... +ifdef p +TEST_PKG=./$(p) +endif + +.PHONY: test +test: + @go test -race -timeout 10s -run $(TEST_FUNC) $(TEST_PKG) + + +.PHONY: test-cov +test-cov: + @go-acc ./... + +# Build +.PHONY: build +build: + @./script/build + @./script/build-healthcheck + +.PHONY: clear +clear: + @rm -rf ./bin/* + +# Docs + +.PHONY: docs +docs: + @echo "\033[4mhttp://localhost:9995/pkg/github.com/benchttp/cli/\033[0m" + @godoc -http=localhost:9995 diff --git a/README.md b/README.md index 2c38811..f2955da 100644 --- a/README.md +++ b/README.md @@ -1 +1,99 @@ -# benchttp/cli +

benchttp/cli

+ +

+ + Github Worklow Status + + Code coverage + + Go Report Card +
+ + Go package Reference + + Latest version +

+ +## About + +`benchttp/cli` is a command-line interface that runs benchmarks on HTTP endpoints. +Highly configurable, it can be used as a CI step as well as +a testing tool at development time. + +## Installation + +1. Visit https://github.com/benchttp/cli/releases and download the asset + `benchttp__` matching your OS and CPU architecture. +1. Rename the downloaded asset to `benchttp`, add it to your `PATH`, + and refresh your terminal if necessary +1. Run `benchttp version` to check it works properly. + +## Usage + +### Run a benchmark + +```sh +benchttp run [options] +``` + +## Configuration + +In this section we dive into the many configuration options provided by the runner. + +First, a good way to get started with is via our [configuration generator](https://www.benchttp.app/setup). + +By default, the runner uses a default configuration that is valid for use without further tuning, except for `url` that must always be set. + +### Configuration Flow + +To determine the final configuration of a benchmark, the runner follows that flow: + +1. It starts with a [default configuration](./examples/config/default.yml) +1. Then it tries to find a config file and overrides the defaults with the values set in it + + - If flag `-configFile` is set, it resolves its value as a path + - Else, it tries to find a config file in the working directory, by priority order: + `.benchttp.yml` > `.benchttp.yaml` > `.benchttp.json` + + The config file is _optional_: if none is found, this step is ignored. + If a config file has an option `extends`, it resolves config file recursively until the root is reached and overrides the values from parent to child. + +1. Then it overrides the current config values with any value set via the CLI +1. Finally, it performs a validation on the resulting config (not before!). + This allows composed configurations for better granularity. + +### Specifications + +With rare exceptions, any option can be set either via CLI flags or config file, +and option names always match. + +A full config file example is available [here](./examples/config/full.yml). + +#### HTTP request options + +| CLI flag | File option | Description | Usage example | +| --------- | --------------------- | ------------------------- | ----------------------------------------- | +| `-url` | `request.url` | Target URL (**Required**) | `-url http://localhost:8080/users?page=3` | +| `-method` | `request.method` | HTTP Method | `-method POST` | +| - | `request.queryParams` | Added query params to URL | - | +| `-header` | `request.header` | Request headers | `-header 'key0:val0' -header 'key1:val1'` | +| `-body` | `request.body` | Raw request body | `-body 'raw:{"id":"abc"}'` | + +#### Benchmark runner options + +| CLI flag | File option | Description | Usage example | +| ----------------- | ----------------------- | -------------------------------------------------------------------- | -------------------- | +| `-requests` | `runner.requests` | Number of requests to run (-1 means infinite, stop on globalTimeout) | `-requests 100` | +| `-concurrency` | `runner.concurrency` | Maximum concurrent requests | `-concurrency 10` | +| `-interval` | `runner.interval` | Minimum duration between two non-concurrent requests | `-interval 200ms` | +| `-requestTimeout` | `runner.requestTimeout` | Timeout for every single request | `-requestTimeout 5s` | +| `-globalTimeout` | `runner.globalTimeout` | Timeout for the whole benchmark | `-globalTimeout 30s` | + +Note: the expected format for durations is ``, with `unit` being any of `ns`, `µs`, `ms`, `s`, `m`, `h`. + +#### CLI-specific options + +| CLI flag | Description | Usage example | +| ------------- | ---------------------------- | ---------------------------------- | +| `-silent` | Remove convenience prints | `-silent` / `-silent=false` | +| `-configFile` | Path to benchttp config file | `-configFile=path/to/benchttp.yml` | diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..44a72fb --- /dev/null +++ b/TODO.md @@ -0,0 +1,14 @@ +# TODO + +## `refactor/config` + +- `runner.ConfigField` -> `runner.Option` +- `Config.WithFelds` -> `Config.WithOptionsSet` +- commit `test(config): fix breaking` -> fixup `feat: implement config.SetFields` +- rename `interface configfiler` -> `interface configParser` (renaming error) +- remove dependency to `testx` +- embed token.txt? + +## `poc/test-suite` + +- move `configfile.requireConfigField` into `configfile.requireConfigFields` diff --git a/cmd/benchttp/main.go b/cmd/benchttp/main.go new file mode 100644 index 0000000..82e27c7 --- /dev/null +++ b/cmd/benchttp/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "os" +) + +// errUsage reports an incorrect usage of the benchttp command. +var errUsage = errors.New("usage") + +func main() { + if err := run(); err != nil { + fmt.Println(err) + if errors.Is(err, errUsage) { + flag.Usage() + } + os.Exit(1) + } +} + +func run() error { + commandName, options, err := shiftArgs(os.Args[1:]) + if err != nil { + return err + } + + cmd, err := commandOf(commandName) + if err != nil { + return err + } + + return cmd.execute(options) +} + +func shiftArgs(args []string) (commandName string, nextArgs []string, err error) { + if len(args) < 1 { + return "", []string{}, fmt.Errorf("%w: no command specified", errUsage) + } + return args[0], args[1:], nil +} + +// command is the interface that all benchttp subcommands must implement. +type command interface { + execute(args []string) error +} + +func commandOf(name string) (command, error) { + switch name { + case "run": + return &cmdRun{flagset: flag.NewFlagSet("run", flag.ExitOnError)}, nil + case "version": + return &cmdVersion{}, nil + default: + return nil, fmt.Errorf("%w: unknown command: %s", errUsage, name) + } +} diff --git a/cmd/benchttp/run.go b/cmd/benchttp/run.go new file mode 100644 index 0000000..e1269e6 --- /dev/null +++ b/cmd/benchttp/run.go @@ -0,0 +1,170 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/configfile" + "github.com/benchttp/cli/internal/configflag" + "github.com/benchttp/cli/internal/output" + "github.com/benchttp/cli/internal/render" + "github.com/benchttp/cli/internal/signals" +) + +// cmdRun handles subcommand "benchttp run [options]". +type cmdRun struct { + flagset *flag.FlagSet + + // configFile is the parsed value for flag -configFile + configFile string + + // silent is the parsed value for flag -silent + silent bool + + // config is the runner config resulting from parsing CLI flags. + config runner.Config +} + +// init initializes cmdRun with default values. +func (cmd *cmdRun) init() { + cmd.config = runner.DefaultConfig() + cmd.configFile = configfile.Find([]string{ + "./.benchttp.yml", + "./.benchttp.yaml", + "./.benchttp.json", + }) +} + +// execute runs the benchttp runner: it parses CLI flags, loads config +// from config file and parsed flags, then runs the benchmark and outputs +// it according to the config. +func (cmd *cmdRun) execute(args []string) error { + cmd.init() + + // Generate merged config (default < config file < CLI flags) + cfg, err := cmd.makeConfig(args) + if err != nil { + return err + } + + report, err := runBenchmark(cfg, cmd.silent) + if err != nil { + return err + } + + return renderReport(os.Stdout, report, cmd.silent) +} + +// parseArgs parses input args as config fields and returns +// a slice of fields that were set by the user. +func (cmd *cmdRun) parseArgs(args []string) []string { + // first arg is subcommand "run" + // skip parsing if no flags are provided + if len(args) <= 1 { + return []string{} + } + + // config file path + cmd.flagset.StringVar(&cmd.configFile, + "configFile", + cmd.configFile, + "Config file path", + ) + + // silent mode + cmd.flagset.BoolVar(&cmd.silent, + "silent", + false, + "Silent mode", + ) + + // attach config options flags to the flagset + // and bind their value to the config struct + configflag.Bind(cmd.flagset, &cmd.config) + + cmd.flagset.Parse(args[1:]) //nolint:errcheck // never occurs due to flag.ExitOnError + + return configflag.Which(cmd.flagset) +} + +// makeConfig returns a runner.ConfigGlobal initialized with config file +// options if found, overridden with CLI options listed in fields +// slice param. +func (cmd *cmdRun) makeConfig(args []string) (cfg runner.Config, err error) { + // Set CLI config from flags and retrieve fields that were set + fields := cmd.parseArgs(args) + + // configFile not set and default ones not found: + // skip the merge and return the cli config + if cmd.configFile == "" { + return cmd.config, cmd.config.Validate() + } + + fileConfig, err := configfile.Parse(cmd.configFile) + if err != nil && !errors.Is(err, configfile.ErrFileNotFound) { + // config file is not mandatory: discard ErrFileNotFound. + // other errors are critical + return + } + + mergedConfig := cmd.config.WithFields(fields...).Override(fileConfig) + + return mergedConfig, mergedConfig.Validate() +} + +func onRecordingProgress(silent bool) func(runner.RecordingProgress) { + if silent { + return func(runner.RecordingProgress) {} + } + + // hack: write a blank line as render.Progress always + // erases the previous line + fmt.Println() + + return func(progress runner.RecordingProgress) { + render.Progress(os.Stdout, progress) //nolint: errcheck + } +} + +func runBenchmark(cfg runner.Config, silent bool) (*runner.Report, error) { + // Prepare graceful shutdown in case of os.Interrupt (Ctrl+C) + ctx, cancel := context.WithCancel(context.Background()) + go signals.ListenOSInterrupt(cancel) + + // Run the benchmark + report, err := runner. + New(onRecordingProgress(silent)). + Run(ctx, cfg) + if err != nil { + return report, err + } + + return report, nil +} + +func renderReport(w io.Writer, report *runner.Report, silent bool) error { + writeIfNotSilent := output.ConditionalWriter{Writer: w}.If(!silent) + + if _, err := render.ReportSummary(writeIfNotSilent, report); err != nil { + return err + } + + if _, err := render.TestSuite( + writeIfNotSilent.ElseIf(!report.Tests.Pass), + report.Tests, + ); err != nil { + return err + } + + if !report.Tests.Pass { + return errors.New("test suite failed") + } + + return nil +} diff --git a/cmd/benchttp/version.go b/cmd/benchttp/version.go new file mode 100644 index 0000000..94d3527 --- /dev/null +++ b/cmd/benchttp/version.go @@ -0,0 +1,21 @@ +package main + +import "fmt" + +// benchttpVersion is the current version of benchttp +// as output by `benchttp version`. It is assumed to be set +// by `go build -ldflags "-X main.benchttpVersion="`, +// allowing us to set the value dynamically at build time +// using latest git tag. +// +// Its default value "development" is only used when the app +// is ran locally without a build (e.g. `go run ./cmd/benchttp`). +var benchttpVersion = "development" + +// cmdVersion handles subcommand "benchttp version". +type cmdVersion struct{} + +func (cmdVersion) execute([]string) error { + fmt.Println("benchttp", benchttpVersion) + return nil +} diff --git a/examples/config/default.yml b/examples/config/default.yml new file mode 100644 index 0000000..53369ae --- /dev/null +++ b/examples/config/default.yml @@ -0,0 +1,10 @@ +request: + method: GET + url: "" # empty + +runner: + requests: 100 + concurrency: 10 + interval: 0ms + requestTimeout: 5s + globalTimeout: 30s diff --git a/examples/config/full.yml b/examples/config/full.yml new file mode 100644 index 0000000..c83d434 --- /dev/null +++ b/examples/config/full.yml @@ -0,0 +1,19 @@ +request: + method: POST + url: http://localhost:8080/users + queryParams: + page: 3 + sort: asc + header: + key0: [val0, val1] + key1: [val0] + body: + type: raw # only "raw" accepted at the moment + content: '{"key0":"val0","key1":"val1"}' + +runner: + requests: 100 + concurrency: 1 + interval: 50ms + requestTimeout: 2s + globalTimeout: 60s diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9d38e26 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/benchttp/cli + +go 1.17 + +require github.com/benchttp/engine v0.0.0-20221008174504-d1162e9ac007 + +require ( + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..90a8750 --- /dev/null +++ b/go.sum @@ -0,0 +1,15 @@ +github.com/benchttp/engine v0.0.0-20221008174504-d1162e9ac007 h1:h6ON0tn3i/83eUMoXq0FHxBMLYIA/OIli3m9KYkcxPI= +github.com/benchttp/engine v0.0.0-20221008174504-d1162e9ac007/go.mod h1:FRfUnUjoL1s0aHVGlrxB3pdPAEDLNCnWh6cVOur24hM= +github.com/drykit-go/cond v0.1.0 h1:y7MNxREQLT83vGfcfSKjyFPLC/ZDjYBNp6KuaVVjOg4= +github.com/drykit-go/cond v0.1.0/go.mod h1:7MXBFjjaB5ZCEB8Q4w2euNOaWuTqf7NjOFZAyV1Jpfg= +github.com/drykit-go/strcase v0.2.0/go.mod h1:cWK0/az2f09UPIbJ42Sb8Iqdv01uENrFX+XXKGjPo+8= +github.com/drykit-go/testx v0.1.0/go.mod h1:qGXb49a8CzQ82crBeCVW8R3kGU1KRgWHnI+Q6CNVbz8= +github.com/drykit-go/testx v1.2.0 h1:UsH+tFd24z3Xu+mwvwPY+9eBEg9CUyMsUeMYyUprG0o= +github.com/drykit-go/testx v1.2.0/go.mod h1:qTzXJgnAg8n31woklBzNTaWzLMJrnFk93x/aeaIpc20= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/configfile/error.go b/internal/configfile/error.go new file mode 100644 index 0000000..61fdcf6 --- /dev/null +++ b/internal/configfile/error.go @@ -0,0 +1,26 @@ +package configfile + +import ( + "errors" +) + +var ( + // ErrFileNotFound signals a config file not found. + ErrFileNotFound = errors.New("file not found") + + // ErrFileRead signals an error trying to read a config file. + // It can be due to a corrupted file or an invalid permission + // for instance. + ErrFileRead = errors.New("invalid file") + + // ErrFileExt signals an unsupported extension for the config file. + ErrFileExt = errors.New("invalid extension") + + // ErrParse signals an error parsing a retrieved config file. + // It is returned if it contains an unexpected field or an unexpected + // value for a field. + ErrParse = errors.New("parsing error: invalid config file") + + // ErrCircularExtends signals a circular reference in the config file. + ErrCircularExtends = errors.New("circular reference detected") +) diff --git a/internal/configfile/find.go b/internal/configfile/find.go new file mode 100644 index 0000000..755e39a --- /dev/null +++ b/internal/configfile/find.go @@ -0,0 +1,14 @@ +package configfile + +import "os" + +// Find returns the first name tham matches a file path. +// If no match is found, it returns an empty string. +func Find(names []string) string { + for _, path := range names { + if _, err := os.Stat(path); err == nil { // err IS nil: file exists + return path + } + } + return "" +} diff --git a/internal/configfile/find_test.go b/internal/configfile/find_test.go new file mode 100644 index 0000000..df6490d --- /dev/null +++ b/internal/configfile/find_test.go @@ -0,0 +1,31 @@ +package configfile_test + +import ( + "testing" + + "github.com/benchttp/cli/internal/configfile" +) + +var ( + goodFileYML = configPath("valid/benchttp.yml") + goodFileJSON = configPath("valid/benchttp.json") + badFile = configPath("does-not-exist.json") +) + +func TestFind(t *testing.T) { + t.Run("return first existing file", func(t *testing.T) { + files := []string{badFile, goodFileYML, goodFileJSON} + + if got := configfile.Find(files); got != goodFileYML { + t.Errorf("did not retrieve good file: exp %s, got %s", goodFileYML, got) + } + }) + + t.Run("return empty string when no match", func(t *testing.T) { + files := []string{badFile} + + if got := configfile.Find(files); got != "" { + t.Errorf("retrieved unexpected file: %s", got) + } + }) +} diff --git a/internal/configfile/parse.go b/internal/configfile/parse.go new file mode 100644 index 0000000..5217600 --- /dev/null +++ b/internal/configfile/parse.go @@ -0,0 +1,119 @@ +package configfile + +import ( + "errors" + "os" + "path/filepath" + + "github.com/benchttp/engine/configparse" + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/errorutil" +) + +// Parse parses a benchttp runner config file into a runner.ConfigGlobal +// and returns it or the first non-nil error occurring in the process, +// which can be any of the values declared in the package. +func Parse(filename string) (cfg runner.Config, err error) { + uconfs, err := parseFileRecursive(filename, []configparse.Representation{}, set{}) + if err != nil { + return + } + return parseAndMergeConfigs(uconfs) +} + +// set is a collection of unique string values. +type set map[string]bool + +// add adds v to the receiver. If v is already set, it returns a non-nil +// error instead. +func (s set) add(v string) error { + if _, exists := s[v]; exists { + return errors.New("value already set") + } + s[v] = true + return nil +} + +// parseFileRecursive parses a config file and its parent found from key +// "extends" recursively until the root config file is reached. +// It returns the list of all parsed configs or the first non-nil error +// occurring in the process. +func parseFileRecursive( + filename string, + reprs []configparse.Representation, + seen set, +) ([]configparse.Representation, error) { + // avoid infinite recursion caused by circular reference + if err := seen.add(filename); err != nil { + return reprs, ErrCircularExtends + } + + // parse current file, append parsed config + repr, err := parseFile(filename) + if err != nil { + return reprs, err + } + reprs = append(reprs, repr) + + // root config reached: stop now and return the parsed configs + if repr.Extends == nil { + return reprs, nil + } + + // config has parent: resolve its path and parse it recursively + parentPath := filepath.Join(filepath.Dir(filename), *repr.Extends) + return parseFileRecursive(parentPath, reprs, seen) +} + +// parseFile parses a single config file and returns the result as an +// configparse.Representation and an appropriate error predeclared in the package. +func parseFile(filename string) (repr configparse.Representation, err error) { + b, err := os.ReadFile(filename) + switch { + case err == nil: + case errors.Is(err, os.ErrNotExist): + return repr, errorutil.WithDetails(ErrFileNotFound, filename) + default: + return repr, errorutil.WithDetails(ErrFileRead, filename, err) + } + + ext := extension(filepath.Ext(filename)) + parser, err := newParser(ext) + if err != nil { + return repr, errorutil.WithDetails(ErrFileExt, ext, err) + } + + if err = parser.Parse(b, &repr); err != nil { + return repr, errorutil.WithDetails(ErrParse, filename, err) + } + + return repr, nil +} + +// parseAndMergeConfigs iterates backwards over uconfs, parsing them +// as runner.ConfigGlobal and merging them into a single one. +// It returns the merged result or the first non-nil error occurring in the +// process. +func parseAndMergeConfigs(reprs []configparse.Representation) (cfg runner.Config, err error) { + if len(reprs) == 0 { // supposedly catched upstream, should not occur + return cfg, errors.New( + "an unacceptable error occurred parsing the config file, " + + "please visit https://github.com/benchttp/runner/issues/new " + + "and insult us properly", + ) + } + + cfg = runner.DefaultConfig() + + for i := len(reprs) - 1; i >= 0; i-- { + repr := reprs[i] + currentConfig, err := configparse.ParseRepresentation(repr) + if err != nil { + return cfg, errorutil.WithDetails(ErrParse, err) + } + cfg = currentConfig.Override(cfg) + } + + return cfg, nil +} diff --git a/internal/configfile/parse_test.go b/internal/configfile/parse_test.go new file mode 100644 index 0000000..06f7de2 --- /dev/null +++ b/internal/configfile/parse_test.go @@ -0,0 +1,261 @@ +package configfile_test + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/configfile" +) + +const ( + testdataConfigPath = "./testdata" + testURL = "http://localhost:9999?fib=30&delay=200ms" // value from testdata files +) + +var supportedExt = []string{ + ".yml", + ".yaml", + ".json", +} + +// TestParse ensures the config file is open, read, and correctly parsed. +func TestParse(t *testing.T) { + t.Run("return file errors early", func(t *testing.T) { + testcases := []struct { + label string + path string + expErr error + }{ + { + label: "not found", + path: configPath("invalid/bad path"), + expErr: configfile.ErrFileNotFound, + }, + { + label: "unsupported extension", + path: configPath("invalid/badext.yams"), + expErr: configfile.ErrFileExt, + }, + { + label: "yaml invalid fields", + path: configPath("invalid/badfields.yml"), + expErr: configfile.ErrParse, + }, + { + label: "json invalid fields", + path: configPath("invalid/badfields.json"), + expErr: configfile.ErrParse, + }, + { + label: "self reference", + path: configPath("extends/extends-circular-self.yml"), + expErr: configfile.ErrCircularExtends, + }, + { + label: "circular reference", + path: configPath("extends/extends-circular-0.yml"), + expErr: configfile.ErrCircularExtends, + }, + } + + for _, tc := range testcases { + t.Run(tc.label, func(t *testing.T) { + gotCfg, gotErr := configfile.Parse(tc.path) + + if gotErr == nil { + t.Fatal("exp non-nil error, got nil") + } + + if !errors.Is(gotErr, tc.expErr) { + t.Errorf("\nexp %v\ngot %v", tc.expErr, gotErr) + } + + if !reflect.DeepEqual(gotCfg, runner.Config{}) { + t.Errorf("\nexp empty config\ngot %v", gotCfg) + } + }) + } + }) + + t.Run("happy path for all extensions", func(t *testing.T) { + for _, ext := range supportedExt { + expCfg := newExpConfig() + fname := configPath("valid/benchttp" + ext) + + gotCfg, err := configfile.Parse(fname) + if err != nil { + // critical error, stop the test + t.Fatal(err) + } + + expURL, gotURL := expCfg.Request.URL, gotCfg.Request.URL + + // compare *url.URLs separately, as they contain unpredictable values + // they need special checks + if !sameURL(gotURL, expURL) { + t.Errorf("unexpected parsed URL:\nexp %v, got %v", expURL, gotURL) + } + + // replace unpredictable values (undetermined query params order) + restoreGotCfg := setTempValue(&gotURL.RawQuery, "replaced by test") + restoreExpCfg := setTempValue(&expURL.RawQuery, "replaced by test") + + if !gotCfg.Equal(expCfg) { + t.Errorf("unexpected parsed config for %s file:\nexp %v\ngot %v", ext, expCfg, gotCfg) + } + + restoreExpCfg() + restoreGotCfg() + } + }) + + t.Run("override default values", func(t *testing.T) { + const ( + expRequests = 0 // default is -1 + expGlobalTimeout = 42 * time.Millisecond + ) + + fname := configPath("valid/benchttp-zeros.yml") + + cfg, err := configfile.Parse(fname) + if err != nil { + t.Fatal(err) + } + + if gotRequests := cfg.Runner.Requests; gotRequests != expRequests { + t.Errorf("did not override Requests: exp %d, got %d", expRequests, gotRequests) + } + + if gotGlobalTimeout := cfg.Runner.GlobalTimeout; gotGlobalTimeout != expGlobalTimeout { + t.Errorf("did not override GlobalTimeout: exp %d, got %d", expGlobalTimeout, gotGlobalTimeout) + } + + t.Log(cfg) + }) + + t.Run("extend config files", func(t *testing.T) { + testcases := []struct { + label string + cfname string + cfpath string + }{ + { + label: "same directory", + cfname: "child", + cfpath: configPath("extends/extends-valid-child.yml"), + }, + { + label: "nested directory", + cfname: "nested", + cfpath: configPath("extends/nest-0/nest-1/extends-valid-nested.yml"), + }, + } + + for _, tc := range testcases { + t.Run(tc.label, func(t *testing.T) { + cfg, err := configfile.Parse(tc.cfpath) + if err != nil { + t.Fatal(err) + } + + var ( + expMethod = "POST" + expURL = fmt.Sprintf("http://%s.config", tc.cfname) + ) + + if gotMethod := cfg.Request.Method; gotMethod != expMethod { + t.Errorf("method: exp %s, got %s", expMethod, gotMethod) + } + + if gotURL := cfg.Request.URL.String(); gotURL != expURL { + t.Errorf("url: exp %s, got %s", expURL, gotURL) + } + }) + } + }) +} + +// helpers + +// newExpConfig returns the expected runner.ConfigConfig result after parsing +// one of the config files in testdataConfigPath. +func newExpConfig() runner.Config { + u, _ := url.ParseRequestURI(testURL) + return runner.Config{ + Request: runner.RequestConfig{ + Method: "POST", + URL: u, + Header: http.Header{ + "key0": []string{"val0", "val1"}, + "key1": []string{"val0"}, + }, + Body: runner.NewRequestBody("raw", `{"key0":"val0","key1":"val1"}`), + }, + Runner: runner.RecorderConfig{ + Requests: 100, + Concurrency: 1, + Interval: 50 * time.Millisecond, + RequestTimeout: 2 * time.Second, + GlobalTimeout: 60 * time.Second, + }, + Tests: []runner.TestCase{ + { + Name: "minimum response time", + Field: "ResponseTimes.Min", + Predicate: "GT", + Target: 80 * time.Millisecond, + }, + { + Name: "maximum response time", + Field: "ResponseTimes.Max", + Predicate: "LTE", + Target: 120 * time.Millisecond, + }, + { + Name: "100% availability", + Field: "RequestFailureCount", + Predicate: "EQ", + Target: 0, + }, + }, + } +} + +// sameURL returns true if a and b are the same *url.URL, taking into account +// the undeterministic nature of their RawQuery. +func sameURL(a, b *url.URL) bool { + // check query params equality via Query() rather than RawQuery + if !reflect.DeepEqual(a.Query(), b.Query()) { + return false + } + + // temporarily set RawQuery to a determined value + for _, u := range []*url.URL{a, b} { + defer setTempValue(&u.RawQuery, "replaced by test")() + } + + // we can now rely on deep equality check + return reflect.DeepEqual(a, b) +} + +// setTempValue sets *ptr to val and returns a restore func that sets *ptr +// back to its previous value. +func setTempValue(ptr *string, val string) (restore func()) { + previousValue := *ptr + *ptr = val + return func() { + *ptr = previousValue + } +} + +func configPath(name string) string { + return filepath.Join(testdataConfigPath, name) +} diff --git a/internal/configfile/parser.go b/internal/configfile/parser.go new file mode 100644 index 0000000..c50876e --- /dev/null +++ b/internal/configfile/parser.go @@ -0,0 +1,35 @@ +package configfile + +import ( + "errors" + + "github.com/benchttp/engine/configparse" +) + +type extension string + +const ( + extYML extension = ".yml" + extYAML extension = ".yaml" + extJSON extension = ".json" +) + +// configParser exposes a method parse to read bytes as a raw config. +type configParser interface { + // parse parses a raw bytes input as a raw config and stores + // the resulting value into dst. + Parse(in []byte, dst *configparse.Representation) error +} + +// newParser returns an appropriate parser according to ext, or a non-nil +// error if ext is not an expected extension. +func newParser(ext extension) (configParser, error) { + switch ext { + case extYML, extYAML: + return configparse.YAMLParser{}, nil + case extJSON: + return configparse.JSONParser{}, nil + default: + return nil, errors.New("unsupported config format") + } +} diff --git a/internal/configfile/testdata/extends/extends-circular-0.yml b/internal/configfile/testdata/extends/extends-circular-0.yml new file mode 100644 index 0000000..55c7ac7 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-circular-0.yml @@ -0,0 +1 @@ +extends: ./extends-circular-1.yml diff --git a/internal/configfile/testdata/extends/extends-circular-1.yml b/internal/configfile/testdata/extends/extends-circular-1.yml new file mode 100644 index 0000000..f451260 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-circular-1.yml @@ -0,0 +1 @@ +extends: ./extends-circular-2.yml diff --git a/internal/configfile/testdata/extends/extends-circular-2.yml b/internal/configfile/testdata/extends/extends-circular-2.yml new file mode 100644 index 0000000..b862fa4 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-circular-2.yml @@ -0,0 +1 @@ +extends: ./extends-circular-0.yml diff --git a/internal/configfile/testdata/extends/extends-circular-self.yml b/internal/configfile/testdata/extends/extends-circular-self.yml new file mode 100644 index 0000000..2fa66ac --- /dev/null +++ b/internal/configfile/testdata/extends/extends-circular-self.yml @@ -0,0 +1 @@ +extends: ./extends-circular-self.yml diff --git a/internal/configfile/testdata/extends/extends-valid-child.yml b/internal/configfile/testdata/extends/extends-valid-child.yml new file mode 100644 index 0000000..a344080 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-valid-child.yml @@ -0,0 +1,4 @@ +extends: ./extends-valid-parent.yml + +request: + url: http://child.config diff --git a/internal/configfile/testdata/extends/extends-valid-parent.yml b/internal/configfile/testdata/extends/extends-valid-parent.yml new file mode 100644 index 0000000..7f3b136 --- /dev/null +++ b/internal/configfile/testdata/extends/extends-valid-parent.yml @@ -0,0 +1,3 @@ +request: + method: POST + url: http://parent.config diff --git a/internal/configfile/testdata/extends/nest-0/nest-1/extends-valid-nested.yml b/internal/configfile/testdata/extends/nest-0/nest-1/extends-valid-nested.yml new file mode 100644 index 0000000..7810890 --- /dev/null +++ b/internal/configfile/testdata/extends/nest-0/nest-1/extends-valid-nested.yml @@ -0,0 +1,4 @@ +extends: ../../extends-valid-parent.yml + +request: + url: http://nested.config diff --git a/internal/configfile/testdata/invalid/badext.yams b/internal/configfile/testdata/invalid/badext.yams new file mode 100644 index 0000000..f7aba57 --- /dev/null +++ b/internal/configfile/testdata/invalid/badext.yams @@ -0,0 +1,2 @@ +request: + url: https://benchttp.app diff --git a/internal/configfile/testdata/invalid/badfields.json b/internal/configfile/testdata/invalid/badfields.json new file mode 100644 index 0000000..9b45468 --- /dev/null +++ b/internal/configfile/testdata/invalid/badfields.json @@ -0,0 +1,7 @@ +{ + "runner": { + "requests": [123], + "concurrency": "123" + }, + "notafield": 123 +} diff --git a/internal/configfile/testdata/invalid/badfields.yml b/internal/configfile/testdata/invalid/badfields.yml new file mode 100644 index 0000000..6899cd8 --- /dev/null +++ b/internal/configfile/testdata/invalid/badfields.yml @@ -0,0 +1,4 @@ +runner: + requests: [123] # error: invalid type for field requests + concurrency: "123" # error: invalid type for field concurrency +notafield: 123 # error: field does not exist diff --git a/internal/configfile/testdata/valid/benchttp-zeros.yml b/internal/configfile/testdata/valid/benchttp-zeros.yml new file mode 100644 index 0000000..38b1bdb --- /dev/null +++ b/internal/configfile/testdata/valid/benchttp-zeros.yml @@ -0,0 +1,3 @@ +runner: + requests: 0 + globalTimeout: 42ms diff --git a/internal/configfile/testdata/valid/benchttp.json b/internal/configfile/testdata/valid/benchttp.json new file mode 100644 index 0000000..714506f --- /dev/null +++ b/internal/configfile/testdata/valid/benchttp.json @@ -0,0 +1,44 @@ +{ + "request": { + "method": "POST", + "url": "http://localhost:9999?delay=200ms", + "queryParams": { + "fib": "30" + }, + "header": { + "key0": ["val0", "val1"], + "key1": ["val0"] + }, + "body": { + "type": "raw", + "content": "{\"key0\":\"val0\",\"key1\":\"val1\"}" + } + }, + "runner": { + "requests": 100, + "concurrency": 1, + "interval": "50ms", + "requestTimeout": "2s", + "globalTimeout": "60s" + }, + "tests": [ + { + "name": "minimum response time", + "field": "ResponseTimes.Min", + "predicate": "GT", + "target": "80ms" + }, + { + "name": "maximum response time", + "field": "ResponseTimes.Max", + "predicate": "LTE", + "target": "120ms" + }, + { + "name": "100% availability", + "field": "RequestFailureCount", + "predicate": "EQ", + "target": "0" + } + ] +} diff --git a/internal/configfile/testdata/valid/benchttp.yaml b/internal/configfile/testdata/valid/benchttp.yaml new file mode 100644 index 0000000..2ee790c --- /dev/null +++ b/internal/configfile/testdata/valid/benchttp.yaml @@ -0,0 +1,35 @@ +x-custom: &data + method: POST + url: http://localhost:9999?delay=200ms + +request: + <<: *data + queryParams: + fib: 30 + header: + key0: [val0, val1] + key1: [val0] + body: + type: raw + content: '{"key0":"val0","key1":"val1"}' + +runner: + requests: 100 + concurrency: 1 + interval: 50ms + requestTimeout: 2s + globalTimeout: 60s + +tests: + - name: minimum response time + field: ResponseTimes.Min + predicate: GT + target: 80ms + - name: maximum response time + field: ResponseTimes.Max + predicate: LTE + target: 120ms + - name: 100% availability + field: RequestFailureCount + predicate: EQ + target: 0 diff --git a/internal/configfile/testdata/valid/benchttp.yml b/internal/configfile/testdata/valid/benchttp.yml new file mode 100644 index 0000000..27a2fc9 --- /dev/null +++ b/internal/configfile/testdata/valid/benchttp.yml @@ -0,0 +1,32 @@ +request: + method: POST + url: http://localhost:9999?delay=200ms + queryParams: + fib: 30 + header: + key0: [val0, val1] + key1: [val0] + body: + type: raw + content: '{"key0":"val0","key1":"val1"}' + +runner: + requests: 100 + concurrency: 1 + interval: 50ms + requestTimeout: 2s + globalTimeout: 60s + +tests: + - name: minimum response time + field: ResponseTimes.Min + predicate: GT + target: 80ms + - name: maximum response time + field: ResponseTimes.Max + predicate: LTE + target: 120ms + - name: 100% availability + field: RequestFailureCount + predicate: EQ + target: 0 diff --git a/internal/configflag/bind.go b/internal/configflag/bind.go new file mode 100644 index 0000000..359cae8 --- /dev/null +++ b/internal/configflag/bind.go @@ -0,0 +1,76 @@ +package configflag + +import ( + "flag" + "net/http" + "net/url" + + "github.com/benchttp/engine/runner" +) + +// Bind reads arguments provided to flagset as config.Fields and binds +// their value to the appropriate fields of given *config.Global. +// The provided *flag.Flagset must not have been parsed yet, otherwise +// bindings its values would fail. +func Bind(flagset *flag.FlagSet, dst *runner.Config) { + // avoid nil pointer dereferences + if dst.Request.URL == nil { + dst.Request.URL = &url.URL{} + } + if dst.Request.Header == nil { + dst.Request.Header = http.Header{} + } + + // request url + flagset.Var(urlValue{url: dst.Request.URL}, + runner.ConfigFieldURL, + runner.ConfigFieldsUsage[runner.ConfigFieldURL], + ) + // request method + flagset.StringVar(&dst.Request.Method, + runner.ConfigFieldMethod, + dst.Request.Method, + runner.ConfigFieldsUsage[runner.ConfigFieldMethod], + ) + // request header + flagset.Var(headerValue{header: &dst.Request.Header}, + runner.ConfigFieldHeader, + runner.ConfigFieldsUsage[runner.ConfigFieldHeader], + ) + // request body + flagset.Var(bodyValue{body: &dst.Request.Body}, + runner.ConfigFieldBody, + runner.ConfigFieldsUsage[runner.ConfigFieldBody], + ) + // requests number + flagset.IntVar(&dst.Runner.Requests, + runner.ConfigFieldRequests, + dst.Runner.Requests, + runner.ConfigFieldsUsage[runner.ConfigFieldRequests], + ) + + // concurrency + flagset.IntVar(&dst.Runner.Concurrency, + runner.ConfigFieldConcurrency, + dst.Runner.Concurrency, + runner.ConfigFieldsUsage[runner.ConfigFieldConcurrency], + ) + // non-conurrent requests interval + flagset.DurationVar(&dst.Runner.Interval, + runner.ConfigFieldInterval, + dst.Runner.Interval, + runner.ConfigFieldsUsage[runner.ConfigFieldInterval], + ) + // request timeout + flagset.DurationVar(&dst.Runner.RequestTimeout, + runner.ConfigFieldRequestTimeout, + dst.Runner.RequestTimeout, + runner.ConfigFieldsUsage[runner.ConfigFieldRequestTimeout], + ) + // global timeout + flagset.DurationVar(&dst.Runner.GlobalTimeout, + runner.ConfigFieldGlobalTimeout, + dst.Runner.GlobalTimeout, + runner.ConfigFieldsUsage[runner.ConfigFieldGlobalTimeout], + ) +} diff --git a/internal/configflag/bind_test.go b/internal/configflag/bind_test.go new file mode 100644 index 0000000..5c54727 --- /dev/null +++ b/internal/configflag/bind_test.go @@ -0,0 +1,70 @@ +package configflag_test + +import ( + "flag" + "net/http" + "reflect" + "testing" + "time" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/configflag" +) + +func TestBind(t *testing.T) { + t.Run("default to base config", func(t *testing.T) { + flagset := flag.NewFlagSet("run", flag.ExitOnError) + args := []string{} // no args + + cfg := runner.DefaultConfig() + configflag.Bind(flagset, &cfg) + if err := flagset.Parse(args); err != nil { + t.Fatal(err) // critical error, stop the test + } + + if exp := runner.DefaultConfig(); !reflect.DeepEqual(cfg, exp) { + t.Errorf("\nexp %#v\ngot %#v", exp, cfg) + } + }) + + t.Run("set config with flags values", func(t *testing.T) { + flagset := flag.NewFlagSet("run", flag.ExitOnError) + args := []string{ + "-method", "POST", + "-url", "https://benchttp.app?cool=yes", + "-header", "Content-Type:application/json", + "-body", "raw:hello", + "-requests", "1", + "-concurrency", "2", + "-interval", "3s", + "-requestTimeout", "4s", + "-globalTimeout", "5s", + } + + cfg := runner.Config{} + configflag.Bind(flagset, &cfg) + if err := flagset.Parse(args); err != nil { + t.Fatal(err) // critical error, stop the test + } + + exp := runner.Config{ + Request: runner.RequestConfig{ + Method: "POST", + Header: http.Header{"Content-Type": {"application/json"}}, + Body: runner.RequestBody{Type: "raw", Content: []byte("hello")}, + }.WithURL("https://benchttp.app?cool=yes"), + Runner: runner.RecorderConfig{ + Requests: 1, + Concurrency: 2, + Interval: 3 * time.Second, + RequestTimeout: 4 * time.Second, + GlobalTimeout: 5 * time.Second, + }, + } + + if !reflect.DeepEqual(cfg, exp) { + t.Errorf("\nexp %#v\ngot %#v", exp, cfg) + } + }) +} diff --git a/internal/configflag/body.go b/internal/configflag/body.go new file mode 100644 index 0000000..f3f0dfb --- /dev/null +++ b/internal/configflag/body.go @@ -0,0 +1,50 @@ +package configflag + +import ( + "fmt" + "strings" + + "github.com/benchttp/engine/runner" +) + +// bodyValue implements flag.Value +type bodyValue struct { + body *runner.RequestBody +} + +// String returns a string representation of the referenced body. +func (v bodyValue) String() string { + return fmt.Sprint(v.body) +} + +// Set reads input string in format "type:content" and sets +// the referenced body accordingly. +// +// Note: only type "raw" is supported at the moment. +func (v bodyValue) Set(raw string) error { + errFormat := fmt.Errorf(`expect format ":", got "%s"`, raw) + + if raw == "" { + return errFormat + } + + split := strings.SplitN(raw, ":", 2) + if len(split) != 2 { + return errFormat + } + + btype, bcontent := split[0], split[1] + if bcontent == "" { + return errFormat + } + + switch btype { + case "raw": + *v.body = runner.NewRequestBody(btype, bcontent) + // case "file": + // // TODO + default: + return fmt.Errorf(`unsupported type: %s (only "raw" accepted)`, btype) + } + return nil +} diff --git a/internal/configflag/header.go b/internal/configflag/header.go new file mode 100644 index 0000000..232180e --- /dev/null +++ b/internal/configflag/header.go @@ -0,0 +1,30 @@ +package configflag + +import ( + "errors" + "fmt" + "net/http" + "strings" +) + +// headerValue implements flag.Value +type headerValue struct { + header *http.Header +} + +// String returns a string representation of the referenced header. +func (v headerValue) String() string { + return fmt.Sprint(v.header) +} + +// Set reads input string in format "key:value" and appends value +// to the key's values of the referenced header. +func (v headerValue) Set(raw string) error { + keyval := strings.SplitN(raw, ":", 2) + if len(keyval) != 2 { + return errors.New(`expect format ":"`) + } + key, val := keyval[0], keyval[1] + (*v.header)[key] = append((*v.header)[key], val) + return nil +} diff --git a/internal/configflag/url.go b/internal/configflag/url.go new file mode 100644 index 0000000..c1eb907 --- /dev/null +++ b/internal/configflag/url.go @@ -0,0 +1,29 @@ +package configflag + +import ( + "fmt" + "net/url" +) + +// urlValue implements flag.Value +type urlValue struct { + url *url.URL +} + +// String returns a string representation of urlValue.url. +func (v urlValue) String() string { + if v.url == nil { + return "" + } + return v.url.String() +} + +// Set parses input string as a URL and sets the referenced URL accordingly. +func (v urlValue) Set(in string) error { + urlURL, err := url.ParseRequestURI(in) + if err != nil { + return fmt.Errorf(`invalid url: "%s"`, in) + } + *v.url = *urlURL + return nil +} diff --git a/internal/configflag/which.go b/internal/configflag/which.go new file mode 100644 index 0000000..c084ecd --- /dev/null +++ b/internal/configflag/which.go @@ -0,0 +1,19 @@ +package configflag + +import ( + "flag" + + "github.com/benchttp/engine/runner" +) + +// Which returns a slice of all config fields set via the CLI +// for the given *flag.FlagSet. +func Which(flagset *flag.FlagSet) []string { + var fields []string + flagset.Visit(func(f *flag.Flag) { + if name := f.Name; runner.IsConfigField(name) { + fields = append(fields, name) + } + }) + return fields +} diff --git a/internal/configflag/which_test.go b/internal/configflag/which_test.go new file mode 100644 index 0000000..80cf4ae --- /dev/null +++ b/internal/configflag/which_test.go @@ -0,0 +1,52 @@ +package configflag_test + +import ( + "flag" + "reflect" + "testing" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/configflag" +) + +func TestWhich(t *testing.T) { + for _, tc := range []struct { + label string + args []string + exp []string + }{ + { + label: "return all config flags set", + args: []string{ + "-method", "POST", + "-url", "https://benchttp.app?cool=yes", + "-concurrency", "2", + "-requests", "3", + "-requestTimeout", "1s", + "-globalTimeout", "4s", + }, + exp: []string{ + "concurrency", "globalTimeout", "method", + "requestTimeout", "requests", "url", + }, + }, + { + label: "do not return config flags not set", + args: []string{"-requests", "3"}, + exp: []string{"requests"}, + }, + } { + flagset := flag.NewFlagSet("run", flag.ExitOnError) + + configflag.Bind(flagset, &runner.Config{}) + + if err := flagset.Parse(tc.args); err != nil { + t.Fatal(err) // critical error, stop the test + } + + if got := configflag.Which(flagset); !reflect.DeepEqual(got, tc.exp) { + t.Errorf("\nexp %v\ngot %v", tc.exp, got) + } + } +} diff --git a/internal/errorutil/errorutil.go b/internal/errorutil/errorutil.go new file mode 100644 index 0000000..d733a93 --- /dev/null +++ b/internal/errorutil/errorutil.go @@ -0,0 +1,24 @@ +package errorutil + +import ( + "fmt" + "strings" +) + +// WithDetails returns an error wrapping err, appended with a string +// representation of details separated by ": ". +// +// Example +// +// var ErrNotFound = errors.New("not found") +// err := WithDetails(ErrNotFound, "abc.jpg", "deleted yesterday") +// +// errors.Is(err, ErrNotFound) == true +// err.Error() == "not found: abc.jpg: deleted yesterday" +func WithDetails(base error, details ...interface{}) error { + detailsStr := make([]string, len(details)) + for i := range details { + detailsStr[i] = fmt.Sprint(details[i]) + } + return fmt.Errorf("%w: %s", base, strings.Join(detailsStr, ": ")) +} diff --git a/internal/output/conditional.go b/internal/output/conditional.go new file mode 100644 index 0000000..ffe0f28 --- /dev/null +++ b/internal/output/conditional.go @@ -0,0 +1,38 @@ +package output + +import ( + "io" +) + +// ConditionalWriter is an io.Writer that wraps an input writer +// and exposes methods to condition its action. +type ConditionalWriter struct { + Writer io.Writer + ok bool +} + +// Write writes b only if ConditionalWriter.Mute is false, +// otherwise it is no-op. +func (w ConditionalWriter) Write(b []byte) (int, error) { + if !w.ok { + return 0, nil + } + return w.Writer.Write(b) +} + +// If sets the write condition to v. +func (w ConditionalWriter) If(v bool) ConditionalWriter { + return ConditionalWriter{ + Writer: w.Writer, + ok: v, + } +} + +// ElseIf either keeps the previous write condition if it is true, +// else it sets it to v. +func (w ConditionalWriter) ElseIf(v bool) ConditionalWriter { + return ConditionalWriter{ + Writer: w.Writer, + ok: w.ok || v, + } +} diff --git a/internal/render/ansi/style.go b/internal/render/ansi/style.go new file mode 100644 index 0000000..a7c9227 --- /dev/null +++ b/internal/render/ansi/style.go @@ -0,0 +1,70 @@ +package ansi + +import ( + "fmt" + "strings" +) + +type StyleFunc func(in string) string + +type style string + +const ( + reset style = "\033[0m" + + bold style = "\033[1m" + + grey style = "\033[1;30m" + red style = "\033[1;31m" + green style = "\033[1;32m" + yellow style = "\033[1;33m" + cyan style = "\033[1;36m" + + erase style = "\033[1A" +) + +func withStyle(in string, s style) string { + return fmt.Sprintf("%s%s%s", s, in, reset) +} + +// Bold returns the bold version of the input string. +func Bold(in string) string { + return withStyle(in, bold) +} + +// Green returns the green version of the input string. +func Green(in string) string { + return withStyle(in, green) +} + +// Yellow returns the yellow version of the input string. +func Yellow(in string) string { + return withStyle(in, yellow) +} + +// Cyan returns the cyan version of the input string. +func Cyan(in string) string { + return withStyle(in, cyan) +} + +// Red returns the red version of the input string. +func Red(in string) string { + return withStyle(in, red) +} + +// Grey returns the grey version of the input string. +func Grey(in string) string { + return withStyle(in, grey) +} + +// Erase returns a string that erases the previous line n times. +func Erase(n int) string { + if n < 1 { + return "" + } + var b strings.Builder + for i := 0; i < n; i++ { + b.Write([]byte(erase)) + } + return b.String() +} diff --git a/internal/render/progress.go b/internal/render/progress.go new file mode 100644 index 0000000..2495e88 --- /dev/null +++ b/internal/render/progress.go @@ -0,0 +1,88 @@ +package render + +import ( + "fmt" + "io" + "strconv" + "strings" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/render/ansi" +) + +// Progress renders a fancy representation of a runner.RecordingProgress +// and writes the result to w. +func Progress(w io.Writer, p runner.RecordingProgress) (int, error) { + return fmt.Fprint(w, progressString(p)) +} + +// progressString returns a string representation of a runner.RecordingProgress +// for a fancy display in a CLI: +// +// RUNNING ◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎ 50% | 50/100 requests | 27s timeout +func progressString(p runner.RecordingProgress) string { + var ( + countdown = p.Timeout - p.Elapsed + reqmax = strconv.Itoa(p.MaxCount) + pctdone = p.Percent() + timeline = renderTimeline(pctdone) + ) + + if reqmax == "-1" { + reqmax = "∞" + } + if countdown < 0 { + countdown = 0 + } + + return fmt.Sprintf( + "%s%s %s %d%% | %d/%s requests | %.0fs timeout \n", + ansi.Erase(1), // replace previous line + renderStatus(p.Status()), timeline, pctdone, // progress + p.DoneCount, reqmax, // requests + countdown.Seconds(), // timeout + ) +} + +var ( + tlBlock = "◼︎" + tlBlockGrey = ansi.Grey(tlBlock) + tlBlockGreen = ansi.Green(tlBlock) + tlLen = 10 +) + +// renderTimeline returns a colored representation of the progress as a string: +// +// ◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎◼︎ +func renderTimeline(pctdone int) string { + tl := strings.Repeat(tlBlockGrey, tlLen) + for i := 0; i < tlLen; i++ { + if pctdone >= (tlLen * i) { + tl = strings.Replace(tl, tlBlockGrey, tlBlockGreen, 1) + } + } + return tl +} + +// renderStatus returns a string representing the status, +// depending on whether the run is done or not and the value +// of its context error. +func renderStatus(status runner.RecordingStatus) string { + styled := statusStyle(status) + return styled(string(status)) +} + +func statusStyle(status runner.RecordingStatus) ansi.StyleFunc { + switch status { + case runner.StatusRunning: + return ansi.Yellow + case runner.StatusDone: + return ansi.Green + case runner.StatusCanceled: + return ansi.Red + case runner.StatusTimeout: + return ansi.Cyan + } + return ansi.Grey // should not occur +} diff --git a/internal/render/report.go b/internal/render/report.go new file mode 100644 index 0000000..d5f996f --- /dev/null +++ b/internal/render/report.go @@ -0,0 +1,57 @@ +package render + +import ( + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/render/ansi" +) + +func ReportSummary(w io.Writer, rep *runner.Report) (int, error) { + return w.Write([]byte(ReportSummaryString(rep))) +} + +// String returns a default summary of the Report as a string. +func ReportSummaryString(rep *runner.Report) string { + var b strings.Builder + + line := func(name string, value interface{}) string { + const template = "%-18s %v\n" + return fmt.Sprintf(template, name, value) + } + + msString := func(d time.Duration) string { + return fmt.Sprintf("%dms", d.Milliseconds()) + } + + formatRequests := func(n, max int) string { + maxString := strconv.Itoa(max) + if maxString == "-1" { + maxString = "∞" + } + return fmt.Sprintf("%d/%s", n, maxString) + } + + var ( + m = rep.Metrics + cfg = rep.Metadata.Config + ) + + b.WriteString(ansi.Bold("→ Summary")) + b.WriteString("\n") + b.WriteString(line("Endpoint", cfg.Request.URL)) + b.WriteString(line("Requests", formatRequests(len(m.Records), cfg.Runner.Requests))) + b.WriteString(line("Errors", len(m.RequestFailures))) + b.WriteString(line("Min response time", msString(m.ResponseTimes.Min))) + b.WriteString(line("Max response time", msString(m.ResponseTimes.Max))) + b.WriteString(line("Mean response time", msString(m.ResponseTimes.Mean))) + b.WriteString(line("Total duration", msString(rep.Metadata.TotalDuration))) + b.WriteString("\n") + + return b.String() +} diff --git a/internal/render/report_test.go b/internal/render/report_test.go new file mode 100644 index 0000000..1fad710 --- /dev/null +++ b/internal/render/report_test.go @@ -0,0 +1,69 @@ +package render_test + +import ( + "testing" + "time" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/render" + "github.com/benchttp/cli/internal/render/ansi" +) + +func TestReport_String(t *testing.T) { + t.Run("returns metrics summary", func(t *testing.T) { + metrics, duration := metricsStub() + cfg := configStub() + + rep := &runner.Report{ + Metrics: metrics, + Metadata: runner.ReportMetadata{ + Config: cfg, + TotalDuration: duration, + }, + } + checkSummary(t, render.ReportSummaryString(rep)) + }) +} + +// helpers + +func metricsStub() (agg runner.MetricsAggregate, total time.Duration) { + return runner.MetricsAggregate{ + RequestFailures: make([]struct { + Reason string + }, 1), + Records: make([]struct{ ResponseTime time.Duration }, 3), + ResponseTimes: runner.MetricsTimeStats{ + Min: 4 * time.Second, + Max: 6 * time.Second, + Mean: 5 * time.Second, + }, + }, 15 * time.Second +} + +func configStub() runner.Config { + cfg := runner.Config{} + cfg.Request = cfg.Request.WithURL("https://a.b.com") + cfg.Runner.Requests = -1 + return cfg +} + +func checkSummary(t *testing.T, summary string) { + t.Helper() + + expSummary := ansi.Bold("→ Summary") + ` +Endpoint https://a.b.com +Requests 3/∞ +Errors 1 +Min response time 4000ms +Max response time 6000ms +Mean response time 5000ms +Total duration 15000ms + +` + + if summary != expSummary { + t.Errorf("\nexp summary:\n%q\ngot summary:\n%q", expSummary, summary) + } +} diff --git a/internal/render/testsuite.go b/internal/render/testsuite.go new file mode 100644 index 0000000..648ad17 --- /dev/null +++ b/internal/render/testsuite.go @@ -0,0 +1,63 @@ +package render + +import ( + "io" + "strings" + + "github.com/benchttp/engine/runner" + + "github.com/benchttp/cli/internal/render/ansi" +) + +func TestSuite(w io.Writer, suite runner.TestSuiteResults) (int, error) { + return w.Write([]byte(TestSuiteString(suite))) +} + +// String returns a default summary of the Report as a string. +func TestSuiteString(suite runner.TestSuiteResults) string { + if len(suite.Results) == 0 { + return "" + } + + var b strings.Builder + + b.WriteString(ansi.Bold("→ Test suite")) + b.WriteString("\n") + + writeResultString(&b, suite.Pass) + b.WriteString("\n") + + for _, tr := range suite.Results { + writeIndent(&b, 1) + writeResultString(&b, tr.Pass) + b.WriteString(" ") + b.WriteString(tr.Input.Name) + + if !tr.Pass { + b.WriteString("\n ") + writeIndent(&b, 3) + b.WriteString(ansi.Bold("→ ")) + b.WriteString(tr.Summary) + } + + b.WriteString("\n") + } + + return b.String() +} + +func writeResultString(w io.StringWriter, pass bool) { + if pass { + w.WriteString(ansi.Green("PASS")) + } else { + w.WriteString(ansi.Red("FAIL")) + } +} + +func writeIndent(w io.StringWriter, count int) { + if count <= 0 { + return + } + const baseIndent = " " + w.WriteString(strings.Repeat(baseIndent, count)) +} diff --git a/internal/signals/signals.go b/internal/signals/signals.go new file mode 100644 index 0000000..3425d45 --- /dev/null +++ b/internal/signals/signals.go @@ -0,0 +1,17 @@ +package signals + +import ( + "os" + "os/signal" + "syscall" +) + +// ListenOSInterrupt listens for OS interrupt signals and calls callback +// on receive. It should be called in a separate goroutine from the main +// program as it blocks the execution until a signal is received. +func ListenOSInterrupt(callback func()) { + sigC := make(chan os.Signal, 1) + signal.Notify(sigC, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) + <-sigC + callback() +} diff --git a/script/build b/script/build new file mode 100755 index 0000000..09dac45 --- /dev/null +++ b/script/build @@ -0,0 +1,35 @@ +#!/bin/bash + +platforms="darwin/amd64 darwin/arm64 linux/386 linux/amd64 windows/386 windows/amd64" + +version=$(git describe --tags --abbrev=0) +ldflags="-X main.benchttpVersion=$version" +tags="prod" + +cmddir="./cmd/benchttp" +bindir="./bin" + +# clear bin directory +rm -rf ./bin/* + +i=0 +for platform in ${platforms}; do + ((i++)) + + split=(${platform//// }) # split platform by sep "/" + goos="${split[0]}" + goarch="${split[1]}" + output="benchttp_${goos}_${goarch}" # e.g. benchttp_darwin_amd64 + + # add .exe to windows binaries + [[ "$goos" == "windows" ]] && output="$output.exe" + + output="$bindir/$output" + + # build binary + GOOS="$goos" GOARCH="$goarch" go build -tags "$tags" -ldflags "$ldflags" -o "$output" "$cmddir" + + echo "[$i/6] $output" +done + +echo -e "\033[1;32m✔︎\033[0m Build complete!" diff --git a/script/build-healthcheck b/script/build-healthcheck new file mode 100755 index 0000000..4f9b2e9 --- /dev/null +++ b/script/build-healthcheck @@ -0,0 +1,17 @@ +#!/bin/bash + +goos=$(go env GOOS) +goarch=$(go env GOARCH) +benchttp="./bin/benchttp_${goos}_${goarch}" + +expVersion="benchttp $(git describe --tags --abbrev=0)" +gotVersion=$(eval $benchttp version) + +if [[ "$gotVersion" != "$expVersion" ]]; then + echo -e "\033[1;31m✘\033[0m Error running ./bin/benchttp version" + echo " exp $expVersion" + echo " got $gotVersion" + exit 1 +fi + +echo -e "\033[1;32m✔︎\033[0m Build integrity OK!"