feat: implement test suites on computed metrics

cmd/benchttp/run.go

@@ -67,6 +67,10 @@ func (cmd *cmdRun) execute(args []string) error {
 		return err
 	}
 
+	if !out.Tests.Pass {
+		return errors.New("test suite failed")
+	}
+
 	return nil
 }
 

internal/configparse/error.go

@@ -2,8 +2,6 @@ package configparse
 
 import (
 	"errors"
-	"fmt"
-	"strings"
 )
 
 var (
@@ -26,13 +24,3 @@ var (
 	// ErrCircularExtends signals a circular reference in the config file.
 	ErrCircularExtends = errors.New("circular reference detected")
 )
-
-// errWithDetails returns an error wrapping err, appended with a string
-// representation of details separated by ": ".
-func errWithDetails(err error, details ...interface{}) error {
-	detailsStr := make([]string, len(details))
-	for i := range details {
-		detailsStr[i] = fmt.Sprint(details[i])
-	}
-	return fmt.Errorf("%w: %s", err, strings.Join(detailsStr, ": "))
-}

internal/configparse/parse.go

@@ -2,12 +2,15 @@ package configparse
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"time"
 
+	"github.com/benchttp/engine/internal/errorutil"
 	"github.com/benchttp/engine/runner"
 )
 
@@ -41,6 +44,13 @@ type UnmarshaledConfig struct {
 		Silent   *bool   `yaml:"silent" json:"silent"`
 		Template *string `yaml:"template" json:"template"`
 	} `yaml:"output" json:"output"`
+
+	Tests []struct {
+		Name      *string     `yaml:"name" json:"name"`
+		Field     *string     `yaml:"field" json:"field"`
+		Predicate *string     `yaml:"predicate" json:"predicate"`
+		Target    interface{} `yaml:"target" json:"target"`
+	} `yaml:"tests" json:"tests"`
 }
 
 // Parse parses a benchttp runner config file into a runner.ConfigGlobal
@@ -105,19 +115,19 @@ func parseFile(filename string) (uconf UnmarshaledConfig, err error) {
 	switch {
 	case err == nil:
 	case errors.Is(err, os.ErrNotExist):
-		return uconf, errWithDetails(ErrFileNotFound, filename)
+		return uconf, errorutil.WithDetails(ErrFileNotFound, filename)
 	default:
-		return uconf, errWithDetails(ErrFileRead, filename, err)
+		return uconf, errorutil.WithDetails(ErrFileRead, filename, err)
 	}
 
 	ext := extension(filepath.Ext(filename))
 	parser, err := newParser(ext)
 	if err != nil {
-		return uconf, errWithDetails(ErrFileExt, ext, err)
+		return uconf, errorutil.WithDetails(ErrFileExt, ext, err)
 	}
 
 	if err = parser.parse(b, &uconf); err != nil {
-		return uconf, errWithDetails(ErrParse, filename, err)
+		return uconf, errorutil.WithDetails(ErrParse, filename, err)
 	}
 
 	return uconf, nil
@@ -142,7 +152,7 @@ func parseAndMergeConfigs(uconfs []UnmarshaledConfig) (cfg runner.Config, err er
 		uconf := uconfs[i]
 		pconf, err := newParsedConfig(uconf)
 		if err != nil {
-			return cfg, errWithDetails(ErrParse, err)
+			return cfg, errorutil.WithDetails(ErrParse, err)
 		}
 		cfg = cfg.Override(pconf.config, pconf.fields...)
 	}
@@ -165,12 +175,13 @@ func (pconf *parsedConfig) add(field string) {
 // newParsedConfig parses an input raw config as a runner.ConfigGlobal and returns
 // a parsedConfig or the first non-nil error occurring in the process.
 func newParsedConfig(uconf UnmarshaledConfig) (parsedConfig, error) { //nolint:gocognit // acceptable complexity for a parsing func
-	const numField = 12 // should match the number of config Fields (not critical)
+	maxFields := len(runner.ConfigFieldsUsage)
+	pconf := parsedConfig{fields: make([]string, 0, maxFields)}
+	cfg := &pconf.config
 
-	pconf := parsedConfig{
-		fields: make([]string, 0, numField),
+	abort := func(err error) (parsedConfig, error) {
+		return parsedConfig{}, err
 	}
-	cfg := &pconf.config
 
 	if method := uconf.Request.Method; method != nil {
 		cfg.Request.Method = *method
@@ -180,7 +191,7 @@ func newParsedConfig(uconf UnmarshaledConfig) (parsedConfig, error) { //nolint:g
 	if rawURL := uconf.Request.URL; rawURL != nil {
 		parsedURL, err := parseAndBuildURL(*uconf.Request.URL, uconf.Request.QueryParams)
 		if err != nil {
-			return parsedConfig{}, err
+			return abort(err)
 		}
 		cfg.Request.URL = parsedURL
 		pconf.add(runner.ConfigFieldURL)
@@ -216,7 +227,7 @@ func newParsedConfig(uconf UnmarshaledConfig) (parsedConfig, error) { //nolint:g
 	if interval := uconf.Runner.Interval; interval != nil {
 		parsedInterval, err := parseOptionalDuration(*interval)
 		if err != nil {
-			return parsedConfig{}, err
+			return abort(err)
 		}
 		cfg.Runner.Interval = parsedInterval
 		pconf.add(runner.ConfigFieldInterval)
@@ -225,7 +236,7 @@ func newParsedConfig(uconf UnmarshaledConfig) (parsedConfig, error) { //nolint:g
 	if requestTimeout := uconf.Runner.RequestTimeout; requestTimeout != nil {
 		parsedTimeout, err := parseOptionalDuration(*requestTimeout)
 		if err != nil {
-			return parsedConfig{}, err
+			return abort(err)
 		}
 		cfg.Runner.RequestTimeout = parsedTimeout
 		pconf.add(runner.ConfigFieldRequestTimeout)
@@ -234,7 +245,7 @@ func newParsedConfig(uconf UnmarshaledConfig) (parsedConfig, error) { //nolint:g
 	if globalTimeout := uconf.Runner.GlobalTimeout; globalTimeout != nil {
 		parsedGlobalTimeout, err := parseOptionalDuration(*globalTimeout)
 		if err != nil {
-			return parsedConfig{}, err
+			return abort(err)
 		}
 		cfg.Runner.GlobalTimeout = parsedGlobalTimeout
 		pconf.add(runner.ConfigFieldGlobalTimeout)
@@ -250,6 +261,51 @@ func newParsedConfig(uconf UnmarshaledConfig) (parsedConfig, error) { //nolint:g
 		pconf.add(runner.ConfigFieldTemplate)
 	}
 
+	testSuite := uconf.Tests
+	if len(testSuite) == 0 {
+		return pconf, nil
+	}
+
+	cases := make([]runner.TestCase, len(testSuite))
+	for i, t := range testSuite {
+		fieldPath := func(caseField string) string {
+			return fmt.Sprintf("tests[%d].%s", i, caseField)
+		}
+
+		if err := requireConfigFields(map[string]interface{}{
+			fieldPath("name"):      t.Name,
+			fieldPath("field"):     t.Field,
+			fieldPath("predicate"): t.Predicate,
+			fieldPath("target"):    t.Target,
+		}); err != nil {
+			return abort(err)
+		}
+
+		field := runner.MetricsField(*t.Field)
+		if err := field.Validate(); err != nil {
+			return abort(fmt.Errorf("%s: %s", fieldPath("field"), err))
+		}
+
+		predicate := runner.TestPredicate(*t.Predicate)
+		if err := predicate.Validate(); err != nil {
+			return abort(fmt.Errorf("%s: %s", fieldPath("predicate"), err))
+		}
+
+		target, err := parseMetricValue(field, fmt.Sprint(t.Target))
+		if err != nil {
+			return abort(fmt.Errorf("%s: %s", fieldPath("target"), err))
+		}
+
+		cases[i] = runner.TestCase{
+			Name:      *t.Name,
+			Field:     field,
+			Predicate: predicate,
+			Target:    target,
+		}
+	}
+	cfg.Tests = cases
+	pconf.add(runner.ConfigFieldTests)
+
 	return pconf, nil
 }
 
@@ -286,3 +342,32 @@ func parseOptionalDuration(raw string) (time.Duration, error) {
 	}
 	return time.ParseDuration(raw)
 }
+
+func parseMetricValue(field runner.MetricsField, in string) (runner.MetricsValue, error) {
+	handleError := func(v interface{}, err error) (runner.MetricsValue, error) {
+		if err != nil {
+			return nil, fmt.Errorf(
+				"value %q is incompatible with field %s (want %s)",
+				in, field, field.Type(),
+			)
+		}
+		return v, nil
+	}
+	switch typ := field.Type(); typ {
+	case runner.MetricsTypeInt:
+		return handleError(strconv.Atoi(in))
+	case runner.MetricsTypeDuration:
+		return handleError(time.ParseDuration(in))
+	default:
+		return nil, fmt.Errorf("unknown field: %v", field)
+	}
+}
+
+func requireConfigFields(fields map[string]interface{}) error {
+	for name, value := range fields {
+		if value == nil {
+			return fmt.Errorf("%s: missing field", name)
+		}
+	}
+	return nil
+}

internal/configparse/parse_test.go

@@ -209,6 +209,26 @@ func newExpConfig() runner.Config {
 			Silent:   true,
 			Template: "{{ .Metrics.Avg }}",
 		},
+		Tests: []runner.TestCase{
+			{
+				Name:      "minimum response time",
+				Field:     "MIN",
+				Predicate: "GT",
+				Target:    80 * time.Millisecond,
+			},
+			{
+				Name:      "maximum response time",
+				Field:     "MAX",
+				Predicate: "LTE",
+				Target:    120 * time.Millisecond,
+			},
+			{
+				Name:      "100% availability",
+				Field:     "FAILURE_COUNT",
+				Predicate: "EQ",
+				Target:    0,
+			},
+		},
 	}
 }
 

internal/configparse/testdata/valid/benchttp.json

@@ -24,5 +24,25 @@
   "output": {
     "silent": true,
     "template": "{{ .Metrics.Avg }}"
-  }
+  },
+  "tests": [
+    {
+      "name": "minimum response time",
+      "field": "MIN",
+      "predicate": "GT",
+      "target": "80ms"
+    },
+    {
+      "name": "maximum response time",
+      "field": "MAX",
+      "predicate": "LTE",
+      "target": "120ms"
+    },
+    {
+      "name": "100% availability",
+      "field": "FAILURE_COUNT",
+      "predicate": "EQ",
+      "target": "0"
+    }
+  ]
 }

internal/configparse/testdata/valid/benchttp.yaml

@@ -23,3 +23,17 @@ runner:
 output:
   silent: true
   template: "{{ .Metrics.Avg }}"
+
+tests:
+  - name: minimum response time
+    field: MIN
+    predicate: GT
+    target: 80ms
+  - name: maximum response time
+    field: MAX
+    predicate: LTE
+    target: 120ms
+  - name: 100% availability
+    field: FAILURE_COUNT
+    predicate: EQ
+    target: 0

internal/configparse/testdata/valid/benchttp.yml

@@ -20,3 +20,17 @@ runner:
 output:
   silent: true
   template: "{{ .Metrics.Avg }}"
+
+tests:
+  - name: minimum response time
+    field: MIN
+    predicate: GT
+    target: 80ms
+  - name: maximum response time
+    field: MAX
+    predicate: LTE
+    target: 120ms
+  - name: 100% availability
+    field: FAILURE_COUNT
+    predicate: EQ
+    target: 0

internal/errorutil/errorutil.go

@@ -0,0 +1,23 @@
+package errorutil
+
+import (
+	"fmt"
+	"strings"
+)
+
+// WithDetails returns an error wrapping err, appended with a string
+// representation of details separated by ": ".
+//
+// Example
+// 	var ErrNotFound = errors.New("not found")
+// 	err := WithDetails(ErrNotFound, "abc.jpg", "deleted yesterday")
+//
+// 	errors.Is(err, ErrNotFound) == true
+// 	err.Error() == "not found: abc.jpg: deleted yesterday"
+func WithDetails(base error, details ...interface{}) error {
+	detailsStr := make([]string, len(details))
+	for i := range details {
+		detailsStr[i] = fmt.Sprint(details[i])
+	}
+	return fmt.Errorf("%w: %s", base, strings.Join(detailsStr, ": "))
+}

runner/internal/config/config.go

@@ -8,6 +8,8 @@ import (
 	"net/http"
 	"net/url"
 	"time"
+
+	"github.com/benchttp/engine/runner/internal/tests"
 )
 
 // RequestBody represents a request body associated with a type.
@@ -88,6 +90,7 @@ type Global struct {
 	Request Request
 	Runner  Runner
 	Output  Output
+	Tests   []tests.Case
 }
 
 // String returns an indented JSON representation of Config
@@ -125,6 +128,8 @@ func (cfg Global) Override(c Global, fields ...string) Global {
 			cfg.Output.Silent = c.Output.Silent
 		case FieldTemplate:
 			cfg.Output.Template = c.Output.Template
+		case FieldTests:
+			cfg.Tests = c.Tests
 		}
 	}
 	return cfg

runner/internal/config/field.go

@@ -12,6 +12,7 @@ const (
 	FieldGlobalTimeout  = "globalTimeout"
 	FieldSilent         = "silent"
 	FieldTemplate       = "template"
+	FieldTests          = "tests"
 )
 
 // FieldsUsage is a record of all available config fields and their usage.
@@ -27,6 +28,7 @@ var FieldsUsage = map[string]string{
 	FieldGlobalTimeout:  "Max duration of test",
 	FieldSilent:         "Silent mode (no write to stdout)",
 	FieldTemplate:       "Output template",
+	FieldTests:          "Test suite",
 }
 
 func IsField(v string) bool {