Add threshold for statistics

examples/CSharpDev/HttpTests/SimpleHttpTest.cs

@@ -1,7 +1,5 @@
 using System;
 using System.Net.Http;
-using HdrHistogram;
-using NBomber;
 using NBomber.Contracts;
 using NBomber.CSharp;
 using static NBomber.Time;
@@ -28,11 +26,32 @@ public static void Run()
                 .WithWarmUpDuration(Seconds(5))
                 .WithLoadSimulations(
                     Simulation.InjectPerSec(rate: 1, during: TimeSpan.FromSeconds(30))
+                )
+                .WithThresholds(
+                    Threshold.RequestAllCount(x => x > 200, "request all count > 200"),
+                    Threshold.RequestOkCount(x => x > 190),
+                    Threshold.RequestFailedCount(x => x <= 10),
+                    Threshold.RequestFailedRate(x => x < 0.1),
+                    Threshold.RPS(GetRpsConfig),
+                    Threshold.LatencyMin(x => x < 100),
+                    Threshold.LatencyMean(x => x < 400),
+                    Threshold.LatencyMax(x => x < 500),
+                    Threshold.LatencyStdDev(x => x is > 100 and < 200, "latency standard deviation > 50 and < 100"),
+                    Threshold.LatencyPercent50(x => x < 300),
+                    Threshold.LatencyPercent75(x => x < 320),
+                    Threshold.LatencyPercent95(x => x < 400),
+                    Threshold.LatencyPercent99(x => x < 500),
+                    Threshold.DataTransferAllBytes(x => x < 10000)
                 );
 
             NBomberRunner
                 .RegisterScenarios(scenario)
                 .Run();
         }
+
+        private static bool GetRpsConfig(double x)
+        {
+            return x > 20.0;
+        }
     }
 }

examples/FSharpDev/HttpTests/SimpleHttpTest.fs

@@ -3,6 +3,7 @@ module FSharpDev.HttpTests.SimpleHttpTest
 open System.Net.Http
 open NBomber
 open NBomber.Contracts
+open NBomber.Contracts.Thresholds
 open NBomber.FSharp
 
 let run () =
@@ -22,7 +23,29 @@ let run () =
 
     Scenario.create "simple_http" [step]
     |> Scenario.withWarmUpDuration(seconds 5)
-    |> Scenario.withLoadSimulations [InjectPerSec(rate = 20, during = seconds 30)]
+    |> Scenario.withLoadSimulations [ InjectPerSec(rate = 20, during = seconds 30) ]
+    |> Scenario.withThresholds(
+        thresholds {
+            request_all_count (fun x -> x > 1290) "request all count > 290"
+            request_ok_count (fun x -> x > 180)
+            request_failed_count (fun x -> x <= 10)
+            request_failed_rate (fun x -> x > 0.1)
+            rps (fun x -> x > 15.0)
+            latency_min (fun x -> x < 100)
+            latency_mean (fun x -> x < 200)
+            latency_max (fun x -> x < 700)
+            latency_std_dev (fun x -> x > 50 && x < 100) "latency standard deviation > 50 and < 100"
+            latency_p50 (fun x -> x < 200)
+            latency_p75 (fun x -> x < 250)
+            latency_p95 (fun x -> x < 400)
+            latency_p99 (fun x -> x < 400)
+            data_transfer_all_bytes (fun x -> x < 10000)
+        }
+    )
+//    |> Scenario.withThresholds [
+//        RequestAllCount((fun x -> x > 1290), Some "request all count > 290")
+//        RequestOkCount((fun x -> x > 180), None)
+//    ]
     |> NBomberRunner.registerScenario
     |> NBomberRunner.run
     |> ignore

examples/FSharpDev/Program.fs

@@ -12,8 +12,8 @@ let main argv =
     //HelloWorldExample.run()
     //CustomSettingsExample.run()
     //DataFeedTest.run()
-    //SimpleHttpTest.run()
-    HttpClientFactoryExample.run()
+    SimpleHttpTest.run()
+    //HttpClientFactoryExample.run()
     //MqttScenario.run()
 
     0 // return an integer exit code

src/NBomber.Contracts/Contracts.fs

@@ -11,6 +11,7 @@ open Serilog
 open Microsoft.Extensions.Configuration
 
 open NBomber.Contracts.Stats
+open NBomber.Contracts.Thresholds
 
 type Response = {
     StatusCode: Nullable<int>
@@ -140,6 +141,7 @@ type Scenario = {
     LoadSimulations: LoadSimulation list
     CustomStepOrder: (unit -> string[]) option
     CustomStepExecControl: (IStepExecControlContext voption -> string voption) option
+    Thresholds: Threshold list option
 }
 
 type IReportingSink =

src/NBomber.Contracts/NBomber.Contracts.fsproj

@@ -16,6 +16,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <Compile Include="Thresholds.fs" />
         <Compile Include="Stats.fs" />
         <Compile Include="Contracts.fs" />
     </ItemGroup>

src/NBomber.Contracts/Stats.fs

@@ -6,6 +6,8 @@ open System.Data
 open Newtonsoft.Json
 open Newtonsoft.Json.Converters
 
+open NBomber.Contracts.Thresholds
+
 type ReportFormat =
     | Txt = 0
     | Html = 1
@@ -110,6 +112,18 @@ type LoadSimulationStats = {
     Value: int
 }
 
+type ThresholdStatus =
+    | Passed
+    | Failed
+with
+    static member map value =
+        if value then Passed else Failed
+
+type ThresholdStats = {
+    Threshold: Threshold
+    Status: ThresholdStatus
+}
+
 type ScenarioStats = {
     ScenarioName: string
     RequestCount: int
@@ -122,6 +136,7 @@ type ScenarioStats = {
     StatusCodes: StatusCodeStats[]
     CurrentOperation: OperationType
     Duration: TimeSpan
+    ThresholdStats: ThresholdStats[] option
 } with
 
     member this.GetStepStats(stepName: string) = ScenarioStats.getStepStats stepName this

src/NBomber.Contracts/Thresholds.fs

@@ -0,0 +1,84 @@
+namespace NBomber.Contracts.Thresholds
+
+module private ThresholdDefaultDescriptions =
+
+    let [<Literal>] RequestAllCountDefaultDescription = "request count - all"
+    let [<Literal>] RequestOkCountDefaultDescription = "request count - ok"
+    let [<Literal>] RequestFailedCountDefaultDescription = "request count - failed"
+    let [<Literal>] RequestFailedRateDefaultDescription = "request rate - failed"
+    let [<Literal>] RPSDefaultDescription = "request count - RPS"
+    let [<Literal>] LatencyMinDefaultDescription = "latency - min"
+    let [<Literal>] LatencyMeanDefaultDescription = "latency - mean"
+    let [<Literal>] LatencyMaxDefaultDescription = "latency - max"
+    let [<Literal>] LatencyStdDevDefaultDescription = "latency - StdDev"
+    let [<Literal>] LatencyPercent50DefaultDescription = "latency percentile - 50%"
+    let [<Literal>] LatencyPercent75DefaultDescription = "latency percentile - 75%"
+    let [<Literal>] LatencyPercent95DefaultDescription = "latency percentile - 95%"
+    let [<Literal>] LatencyPercent99DefaultDescription = "latency percentile - 99%"
+    let [<Literal>] DataTransferMinBytesDefaultDescription = "data transfer bytes - min"
+    let [<Literal>] DataTransferMeanBytesDefaultDescription = "data transfer bytes - mean"
+    let [<Literal>] DataTransferMaxBytesDefaultDescription = "data transfer bytes - max"
+    let [<Literal>] DataTransferAllBytesDefaultDescription = "data transfer bytes - all"
+    let [<Literal>] DataTransferStdDevDefaultDescription = "data transfer - StdDev"
+    let [<Literal>] DataTransferPercent50DefaultDescription = "data transfer percentile - 50%"
+    let [<Literal>] DataTransferPercent75DefaultDescription = "data transfer percentile - 75%"
+    let [<Literal>] DataTransferPercent95DefaultDescription = "data transfer percentile - 95%"
+    let [<Literal>] DataTransferPercent99DefaultDescription = "data transfer percentile - 99%"
+
+open ThresholdDefaultDescriptions
+
+type ThresholdBody<'a> = ('a -> bool) * string option
+
+type Threshold =
+    | RequestAllCount of ThresholdBody<int>
+    | RequestOkCount of ThresholdBody<int>
+    | RequestFailedCount of ThresholdBody<int>
+    | RequestFailedRate of ThresholdBody<float>
+    | RPS of ThresholdBody<float>
+    | LatencyMin of ThresholdBody<float>
+    | LatencyMean of ThresholdBody<float>
+    | LatencyMax of ThresholdBody<float>
+    | LatencyStdDev of ThresholdBody<float>
+    | LatencyPercent50 of ThresholdBody<float>
+    | LatencyPercent75 of ThresholdBody<float>
+    | LatencyPercent95 of ThresholdBody<float>
+    | LatencyPercent99 of ThresholdBody<float>
+    | DataTransferMinBytes of ThresholdBody<int>
+    | DataTransferMeanBytes of ThresholdBody<int>
+    | DataTransferMaxBytes of ThresholdBody<int>
+    | DataTransferPercent50 of ThresholdBody<int>
+    | DataTransferPercent75 of ThresholdBody<int>
+    | DataTransferPercent95 of ThresholdBody<int>
+    | DataTransferPercent99 of ThresholdBody<int>
+    | DataTransferStdDev of ThresholdBody<float>
+    | DataTransferAllBytes of ThresholdBody<int64>
+with
+    member this.Description =
+        match this with
+        | RequestAllCount (_, desc) -> desc, RequestAllCountDefaultDescription
+        | RequestOkCount (_, desc) -> desc, RequestOkCountDefaultDescription
+        | RequestFailedCount (_, desc) -> desc, RequestFailedCountDefaultDescription
+        | RequestFailedRate (_, desc) -> desc, RequestFailedRateDefaultDescription
+        | RPS (_, desc) -> desc, RPSDefaultDescription
+        | LatencyMin (_, desc) -> desc, LatencyMinDefaultDescription
+        | LatencyMean (_, desc) -> desc, LatencyMeanDefaultDescription
+        | LatencyMax (_, desc) -> desc, LatencyMaxDefaultDescription
+        | LatencyStdDev (_, desc) -> desc, LatencyStdDevDefaultDescription
+        | LatencyPercent50 (_, desc) -> desc, LatencyPercent50DefaultDescription
+        | LatencyPercent75 (_, desc) -> desc, LatencyPercent75DefaultDescription
+        | LatencyPercent95 (_, desc) -> desc, LatencyPercent95DefaultDescription
+        | LatencyPercent99 (_, desc) -> desc, LatencyPercent99DefaultDescription
+        | DataTransferMinBytes (_, desc) -> desc, DataTransferMinBytesDefaultDescription
+        | DataTransferMeanBytes (_, desc) -> desc, DataTransferMeanBytesDefaultDescription
+        | DataTransferMaxBytes (_, desc) -> desc, DataTransferMaxBytesDefaultDescription
+        | DataTransferPercent50 (_, desc) -> desc, DataTransferPercent50DefaultDescription
+        | DataTransferPercent75 (_, desc) -> desc, DataTransferPercent75DefaultDescription
+        | DataTransferPercent95 (_, desc) -> desc, DataTransferPercent95DefaultDescription
+        | DataTransferPercent99 (_, desc) -> desc, DataTransferPercent99DefaultDescription
+        | DataTransferStdDev (_, desc) -> desc, DataTransferStdDevDefaultDescription
+        | DataTransferAllBytes (_, desc) -> desc, DataTransferAllBytesDefaultDescription
+        |> fun x ->
+            match x with
+            | desc, defaultDesc -> desc |> Option.defaultValue defaultDesc
+
+    override this.ToString () = this.Description

src/NBomber/Api/CSharp.fs

@@ -12,6 +12,7 @@ open Serilog
 open NBomber
 open NBomber.Contracts
 open NBomber.Contracts.Stats
+open NBomber.Contracts.Thresholds
 
 /// ClientFactory helps you create and initialize API clients to work with specific API or protocol (HTTP, WebSockets, gRPC, GraphQL).
 type ClientFactory =
@@ -209,6 +210,10 @@ type ScenarioBuilder =
     static member WithCustomStepExecControl(scenario: Scenario, execControl: Func<IStepExecControlContext voption, string voption>) =
         scenario |> FSharp.Scenario.withCustomStepExecControl(execControl.Invoke)
 
+    [<Extension>]
+    static member WithThresholds(scenario: Scenario, [<ParamArray>]thresholds: Threshold[]) =
+        scenario |> FSharp.Scenario.withThresholds(Seq.toList thresholds)
+
 [<Extension>]
 type NBomberRunner =
 
@@ -360,3 +365,70 @@ type ValueOption =
     static member Some(value: 'T) = ValueSome value
     static member None() = ValueNone
 
+type Threshold =
+
+    static member RequestAllCount(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        RequestAllCount(predicate.Invoke, Option.ofObj description)
+
+    static member RequestOkCount(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        RequestOkCount(predicate.Invoke, Option.ofObj description)
+
+    static member RequestFailedCount(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        RequestFailedCount(predicate.Invoke, Option.ofObj description)
+
+    static member RequestFailedRate(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        RequestFailedRate(predicate.Invoke, Option.ofObj description)
+
+    static member RPS(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        RPS(predicate.Invoke, Option.ofObj description)
+
+    static member LatencyMin(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        LatencyMin(predicate.Invoke, Option.ofObj description)
+
+    static member LatencyMean(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        LatencyMean(predicate.Invoke, Option.ofObj description)
+
+    static member LatencyMax(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        LatencyMax(predicate.Invoke, Option.ofObj description)
+
+    static member LatencyStdDev(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        LatencyStdDev(predicate.Invoke, Option.ofObj description)
+
+    static member LatencyPercent50(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        LatencyPercent50(predicate.Invoke, Option.ofObj description)
+
+    static member LatencyPercent75(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        LatencyPercent75(predicate.Invoke, Option.ofObj description)
+
+    static member LatencyPercent95(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        LatencyPercent95(predicate.Invoke, Option.ofObj description)
+
+    static member LatencyPercent99(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        LatencyPercent99(predicate.Invoke, Option.ofObj description)
+
+    static member DataTransferMinBytes(predicate: Func<int, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        DataTransferMinBytes(predicate.Invoke, Option.ofObj description)
+
+    static member DataTransferMeanBytes(predicate: Func<int, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        DataTransferMeanBytes(predicate.Invoke, Option.ofObj description)
+
+    static member DataTransferMaxBytes(predicate: Func<int, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        DataTransferMaxBytes(predicate.Invoke, Option.ofObj description)
+
+    static member DataTransferPercent50(predicate: Func<int, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        DataTransferPercent50(predicate.Invoke, Option.ofObj description)
+
+    static member DataTransferPercent75(predicate: Func<int, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        DataTransferPercent75(predicate.Invoke, Option.ofObj description)
+
+    static member DataTransferPercent95(predicate: Func<int, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        DataTransferPercent95(predicate.Invoke, Option.ofObj description)
+
+    static member DataTransferPercent99(predicate: Func<int, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        DataTransferPercent99(predicate.Invoke, Option.ofObj description)
+
+    static member DataTransferStdDev(predicate: Func<float, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        DataTransferStdDev(predicate.Invoke, Option.ofObj description)
+
+    static member DataTransferAllBytes(predicate: Func<int64, bool>, [<Optional;DefaultParameterValue(null)>] description: string) =
+        DataTransferAllBytes(predicate.Invoke, Option.ofObj description)