diff --git a/README.md b/README.md index 59b6e03..f64b918 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ ## About `benchttp/engine` runs benchmarks on specified endpoints. -Highly configurable, it can serve many purposes such as monitoring (paired with our [webapp](https://www.benchttp.app)), CI tool (with output templates) or as a simple testing tool at development time. +Highly configurable, it can serve many purposes such as monitoring (paired with our [webapp](https://www.benchttp.app)), CI integration (with test suites) or as a simple testing tool at development time. ## Installation @@ -90,11 +90,6 @@ Note: the expected format for durations is ``, with `unit` being any #### Output options -| CLI flag | File option | Description | Usage example | -| ----------- | ----------------- | ------------------------------- | -------------------------------- | -| `-silent` | `output.silent` | Remove convenience prints | `-silent` / `-silent=false` | -| `-template` | `output.template` | Custom output when using stdout | `-template '{{ .Metrics.Avg }}'` | - -Note: the template uses Go's powerful templating engine. -To take full advantage of it, see our [templating docs](./examples/output/templating.md) -for the available fields and functions, with usage examples. +| CLI flag | File option | Description | Usage example | +| --------- | --------------- | ------------------------- | --------------------------- | +| `-silent` | `output.silent` | Remove convenience prints | `-silent` / `-silent=false` | diff --git a/examples/config/full.yml b/examples/config/full.yml index a31d040..c20fb5e 100644 --- a/examples/config/full.yml +++ b/examples/config/full.yml @@ -20,4 +20,3 @@ runner: output: silent: true - template: "{{ .Metrics.Avg }}" diff --git a/examples/output/templating.md b/examples/output/templating.md deleted file mode 100644 index eb78728..0000000 --- a/examples/output/templating.md +++ /dev/null @@ -1,67 +0,0 @@ -# Output templating - -## Go template syntax - -See Go's [text/template package documentation](https://pkg.go.dev/text/template) - -## Report structure reference for usage in templates - -See [IO Structures](https://github.com/benchttp/engine/wiki/IO-Structures#go-1) in our wiki. - -### Additionnal template functions - -- `fail`: - - - `{{ fail }}`: Fails the test and exit 1 (better used in a condition!) - - `{{ fail "Too long!" }}`: Same with error message - -## Some examples - -- Custom summary - - ```yml - template: | - {{ .Metrics.TotalCount }}/{{ .Metadata.Config.Runner.Requests }} requests - {{ .Metrics.FailureCount }} errors - ✔︎ Done in {{ .Metadata.TotalDuration.Milliseconds }}ms. - ``` - - ```txt - 100/100 requests - 0 errors - ✔︎ Done in 2034ms. - ``` - -- Display only the average request time - - ```yml - template: "{{ .Metrics.Avg }}" - ``` - - ```txt - 237ms - ``` - -- Fail the test if any request exceeds 200ms - - ```yml - template: | - {{- if ge stats.Max.Milliseconds 200 -}} - {{ fail "TOO SLOW" }} - {{- else -}} - OK - {{- end -}} - ``` - - if max >= 200ms: - - ```txt - test failed: TOO SLOW - exit status 1 - ``` - - else: - - ```txt - OK - ``` diff --git a/internal/cli/configflags/bind.go b/internal/cli/configflags/bind.go index 718cb87..a3ee44f 100644 --- a/internal/cli/configflags/bind.go +++ b/internal/cli/configflags/bind.go @@ -80,10 +80,4 @@ func Bind(flagset *flag.FlagSet, dst *runner.Config) { dst.Output.Silent, runner.ConfigFieldsUsage[runner.ConfigFieldSilent], ) - // output template - flagset.StringVar(&dst.Output.Template, - runner.ConfigFieldTemplate, - dst.Output.Template, - runner.ConfigFieldsUsage[runner.ConfigFieldTemplate], - ) } diff --git a/internal/cli/configflags/bind_test.go b/internal/cli/configflags/bind_test.go index 4cdee5e..c48645b 100644 --- a/internal/cli/configflags/bind_test.go +++ b/internal/cli/configflags/bind_test.go @@ -40,7 +40,6 @@ func TestBind(t *testing.T) { "-requestTimeout", "4s", "-globalTimeout", "5s", "-silent", - "-template", "{{ .Report.Length }}", } cfg := runner.Config{} @@ -63,8 +62,7 @@ func TestBind(t *testing.T) { GlobalTimeout: 5 * time.Second, }, Output: runner.OutputConfig{ - Silent: true, - Template: "{{ .Report.Length }}", + Silent: true, }, } diff --git a/internal/configparse/parse.go b/internal/configparse/parse.go index 84e324d..eef7570 100644 --- a/internal/configparse/parse.go +++ b/internal/configparse/parse.go @@ -41,8 +41,7 @@ type UnmarshaledConfig struct { } `yaml:"runner" json:"runner"` Output struct { - Silent *bool `yaml:"silent" json:"silent"` - Template *string `yaml:"template" json:"template"` + Silent *bool `yaml:"silent" json:"silent"` } `yaml:"output" json:"output"` Tests []struct { @@ -256,11 +255,6 @@ func newParsedConfig(uconf UnmarshaledConfig) (parsedConfig, error) { //nolint:g pconf.add(runner.ConfigFieldSilent) } - if template := uconf.Output.Template; template != nil { - cfg.Output.Template = *template - pconf.add(runner.ConfigFieldTemplate) - } - testSuite := uconf.Tests if len(testSuite) == 0 { return pconf, nil diff --git a/internal/configparse/parse_test.go b/internal/configparse/parse_test.go index f6cc9b3..e189b58 100644 --- a/internal/configparse/parse_test.go +++ b/internal/configparse/parse_test.go @@ -206,8 +206,7 @@ func newExpConfig() runner.Config { GlobalTimeout: 60 * time.Second, }, Output: runner.OutputConfig{ - Silent: true, - Template: "{{ .Metrics.Avg }}", + Silent: true, }, Tests: []runner.TestCase{ { diff --git a/internal/configparse/testdata/valid/benchttp.json b/internal/configparse/testdata/valid/benchttp.json index f68438e..cb3efdd 100644 --- a/internal/configparse/testdata/valid/benchttp.json +++ b/internal/configparse/testdata/valid/benchttp.json @@ -22,8 +22,7 @@ "globalTimeout": "60s" }, "output": { - "silent": true, - "template": "{{ .Metrics.Avg }}" + "silent": true }, "tests": [ { diff --git a/internal/configparse/testdata/valid/benchttp.yaml b/internal/configparse/testdata/valid/benchttp.yaml index 2871ec0..cc3240e 100644 --- a/internal/configparse/testdata/valid/benchttp.yaml +++ b/internal/configparse/testdata/valid/benchttp.yaml @@ -22,7 +22,6 @@ runner: output: silent: true - template: "{{ .Metrics.Avg }}" tests: - name: minimum response time diff --git a/internal/configparse/testdata/valid/benchttp.yml b/internal/configparse/testdata/valid/benchttp.yml index 4968772..da9d048 100644 --- a/internal/configparse/testdata/valid/benchttp.yml +++ b/internal/configparse/testdata/valid/benchttp.yml @@ -19,7 +19,6 @@ runner: output: silent: true - template: "{{ .Metrics.Avg }}" tests: - name: minimum response time diff --git a/runner/internal/config/config.go b/runner/internal/config/config.go index 3cfde53..debcdd0 100644 --- a/runner/internal/config/config.go +++ b/runner/internal/config/config.go @@ -80,8 +80,7 @@ type Runner struct { // Output contains options relative to the output. type Output struct { - Silent bool - Template string + Silent bool } // Global represents the global configuration of the runner. @@ -126,8 +125,6 @@ func (cfg Global) Override(c Global, fields ...string) Global { cfg.Runner.GlobalTimeout = c.Runner.GlobalTimeout case FieldSilent: cfg.Output.Silent = c.Output.Silent - case FieldTemplate: - cfg.Output.Template = c.Output.Template case FieldTests: cfg.Tests = c.Tests } diff --git a/runner/internal/config/default.go b/runner/internal/config/default.go index 022eda2..b07e90d 100644 --- a/runner/internal/config/default.go +++ b/runner/internal/config/default.go @@ -21,8 +21,7 @@ var defaultConfig = Global{ GlobalTimeout: 30 * time.Second, }, Output: Output{ - Silent: false, - Template: "", + Silent: false, }, } diff --git a/runner/internal/config/field.go b/runner/internal/config/field.go index 8124264..693ae33 100644 --- a/runner/internal/config/field.go +++ b/runner/internal/config/field.go @@ -11,7 +11,6 @@ const ( FieldRequestTimeout = "requestTimeout" FieldGlobalTimeout = "globalTimeout" FieldSilent = "silent" - FieldTemplate = "template" FieldTests = "tests" ) @@ -27,7 +26,6 @@ var FieldsUsage = map[string]string{ FieldRequestTimeout: "Timeout for each HTTP request", FieldGlobalTimeout: "Max duration of test", FieldSilent: "Silent mode (no write to stdout)", - FieldTemplate: "Output template", FieldTests: "Test suite", } diff --git a/runner/internal/config/field_test.go b/runner/internal/config/field_test.go index f140c86..2ba02c9 100644 --- a/runner/internal/config/field_test.go +++ b/runner/internal/config/field_test.go @@ -20,7 +20,6 @@ func TestIsField(t *testing.T) { {In: config.FieldRequestTimeout, Exp: true}, {In: config.FieldGlobalTimeout, Exp: true}, {In: config.FieldSilent, Exp: true}, - {In: config.FieldTemplate, Exp: true}, {In: "notafield", Exp: false}, }).Run(t) } diff --git a/runner/internal/report/report.go b/runner/internal/report/report.go index cc069cd..601829f 100644 --- a/runner/internal/report/report.go +++ b/runner/internal/report/report.go @@ -2,7 +2,6 @@ package report import ( "encoding/json" - "errors" "fmt" "io" "strconv" @@ -20,8 +19,6 @@ type Report struct { Metrics metrics.Aggregate Metadata Metadata Tests tests.SuiteResult - - errTemplateFailTriggered error } // Metadata contains contextual information about a run. @@ -52,26 +49,8 @@ func New( // String returns a default summary of the Report as a string. func (rep *Report) String() string { var b strings.Builder - - s, err := rep.applyTemplate(rep.Metadata.Config.Output.Template) - switch { - case err == nil: - // template is non-empty and correctly executed, - // return its result instead of default summary. - rep.writeTestsResult(&b) - return s - case errors.Is(err, errTemplateSyntax): - // template is non-empty but has syntax errors, - // inform the user about it and fallback to default summary. - b.WriteString(err.Error()) - b.WriteString("\nFalling back to default summary:\n") - case errors.Is(err, errTemplateEmpty): - // template is empty, use default summary. - } - rep.writeDefaultSummary(&b) rep.writeTestsResult(&b) - return b.String() } diff --git a/runner/internal/report/report_test.go b/runner/internal/report/report_test.go index a02800c..d5cbf33 100644 --- a/runner/internal/report/report_test.go +++ b/runner/internal/report/report_test.go @@ -1,9 +1,6 @@ package report_test import ( - "net/url" - "strconv" - "strings" "testing" "time" @@ -15,49 +12,18 @@ import ( ) func TestReport_String(t *testing.T) { - const d = 15 * time.Second + t.Run("returns metrics summary", func(t *testing.T) { + agg, d := metricsStub() + cfg := configStub() - t.Run("return default summary if template is empty", func(t *testing.T) { - const tpl = "" - - rep := report.New(newMetrics(), newConfigWithTemplate(tpl), d, tests.SuiteResult{}) + rep := report.New(agg, cfg, d, tests.SuiteResult{}) checkSummary(t, rep.String()) }) - - t.Run("return executed template if valid", func(t *testing.T) { - const tpl = "{{ .Metrics.TotalCount }}" - - m := newMetrics() - rep := report.New(m, newConfigWithTemplate(tpl), d, tests.SuiteResult{}) - - if got, exp := rep.String(), strconv.Itoa(m.TotalCount); got != exp { - t.Errorf("\nunexpected output\nexp %s\ngot %s", exp, got) - } - }) - - t.Run("fallback to default summary if template is invalid", func(t *testing.T) { - const tpl = "{{ .Marcel.Patulacci }}" - - rep := report.New(newMetrics(), newConfigWithTemplate(tpl), d, tests.SuiteResult{}) - got := rep.String() - split := strings.Split(got, "Falling back to default summary:\n") - - if len(split) != 2 { - t.Fatalf("\nunexpected output:\n%s", got) - } - - errMsg, summary := split[0], split[1] - if !strings.Contains(errMsg, "template syntax error") { - t.Errorf("\nexp template syntax error\ngot %s", errMsg) - } - - checkSummary(t, summary) - }) } // helpers -func newMetrics() metrics.Aggregate { +func metricsStub() (agg metrics.Aggregate, total time.Duration) { return metrics.Aggregate{ FailureCount: 1, SuccessCount: 2, @@ -65,16 +31,14 @@ func newMetrics() metrics.Aggregate { Min: 4 * time.Second, Max: 6 * time.Second, Avg: 5 * time.Second, - } + }, 15 * time.Second } -func newConfigWithTemplate(tpl string) config.Global { - urlURL, _ := url.ParseRequestURI("https://a.b.com") - return config.Global{ - Request: config.Request{URL: urlURL}, - Runner: config.Runner{Requests: -1}, - Output: config.Output{Template: tpl}, - } +func configStub() config.Global { + cfg := config.Global{} + cfg.Request = cfg.Request.WithURL("https://a.b.com") + cfg.Runner.Requests = -1 + return cfg } func checkSummary(t *testing.T, summary string) { diff --git a/runner/internal/report/template.go b/runner/internal/report/template.go deleted file mode 100644 index 68942dc..0000000 --- a/runner/internal/report/template.go +++ /dev/null @@ -1,60 +0,0 @@ -package report - -import ( - "errors" - "fmt" - "strings" - "text/template" -) - -var ( - // ErrTemplateFailTriggered a fail triggered by a user - // using the function {{ fail }} in an output template. - ErrTemplateFailTriggered = errors.New("test failed") - - errTemplateEmpty = errors.New("empty template") - errTemplateSyntax = errors.New("template syntax error") -) - -// applyTemplate applies Report to a template using given pattern and returns -// the result as a string. If pattern == "", it returns errTemplateEmpty. -// If an error occurs parsing the pattern or executing the template, -// it returns errTemplateSyntax. -func (rep *Report) applyTemplate(pattern string) (string, error) { - if pattern == "" { - return "", errTemplateEmpty - } - - t, err := template. - New("report"). - Funcs(rep.templateFuncs()). - Parse(pattern) - if err != nil { - return "", fmt.Errorf("%w: %s", errTemplateSyntax, err) - } - - var b strings.Builder - if err := t.Execute(&b, rep); err != nil { - return "", fmt.Errorf("%w: %s", errTemplateSyntax, err) - } - - return b.String(), nil -} - -// templateFuncs returns a template.FuncMap defining template functions -// that are specific to the Report: stats, event, fail. -func (rep *Report) templateFuncs() template.FuncMap { - return template.FuncMap{ - // fail sets rep.errTplFailTriggered to the given error, causing - // the test to fail - "fail": func(a ...interface{}) string { - if rep.errTemplateFailTriggered == nil { - rep.errTemplateFailTriggered = fmt.Errorf( - "%w: %s", - ErrTemplateFailTriggered, fmt.Sprint(a...), - ) - } - return "" - }, - } -} diff --git a/runner/internal/report/template_internal_test.go b/runner/internal/report/template_internal_test.go deleted file mode 100644 index eebdfdf..0000000 --- a/runner/internal/report/template_internal_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package report - -import ( - "errors" - "testing" -) - -func TestReport_applyTemplate(t *testing.T) { - testcases := []struct { - label string - pattern string - expStr string - expErr error - }{ - { - label: "return errTemplateEmpty if pattern is empty", - pattern: "", - expStr: "", - expErr: errTemplateEmpty, - }, - { - label: "return errTemplateSyntaxt if pattern has syntax error", - pattern: "{{ else }}", - expStr: "", - expErr: errTemplateSyntax, - }, - { - label: "return errTemplateSyntaxt if pattern doesn't match report values", - pattern: "{{ .Foo }}", // Report.Foo doesn't exist - expStr: "", - expErr: errTemplateSyntax, - }, - } - - for _, tc := range testcases { - t.Run(tc.label, func(t *testing.T) { - r := &Report{} - gotStr, gotErr := r.applyTemplate(tc.pattern) - if !errors.Is(gotErr, tc.expErr) { - t.Errorf("unexpected error: %v", gotErr) - } - if gotStr != tc.expStr { - t.Errorf("unexpected string: %q", gotStr) - } - }) - } -} - -func TestReport_templateFuncs(t *testing.T) { - t.Run("fail", func(t *testing.T) { - rep := Report{} - - untypedFailFunc := retrieveTemplateFuncOrFatal(t, &rep, "fail") - - failFunc, ok := untypedFailFunc.(func(...interface{}) string) - if !ok { - t.Fatalf("wrong type:\nexp func(...interface{}) string\ngot %T", untypedFailFunc) - } - - if got := failFunc("a", "b", "c"); got != "" { - t.Errorf("unexpected output: exp always %q, got %q", "", got) - } - - gotErr := rep.errTemplateFailTriggered - if !errors.Is(gotErr, ErrTemplateFailTriggered) { - t.Fatalf("unexpected error:\nexp ErrTemplateFailTriggered\ngot %v", gotErr) - } - if gotMsg, expMsg := gotErr.Error(), "test failed: abc"; gotMsg != expMsg { - t.Errorf("unexpected error message:\nexp %q\ngot %q", expMsg, gotMsg) - } - }) -} - -// helpers - -func retrieveTemplateFuncOrFatal(t *testing.T, r *Report, name string) interface{} { - t.Helper() - v, exists := r.templateFuncs()[name] - if !exists { - t.Fatalf("template func %q does not exist", name) - } - return v -} diff --git a/runner/runner.go b/runner/runner.go index 096e8a4..00d9f9a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -47,7 +47,6 @@ const ( ConfigFieldRequestTimeout = config.FieldRequestTimeout ConfigFieldGlobalTimeout = config.FieldGlobalTimeout ConfigFieldSilent = config.FieldSilent - ConfigFieldTemplate = config.FieldTemplate ConfigFieldTests = config.FieldTests )