diff --git a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs index 31f66564..6f1e7048 100644 --- a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs +++ b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs @@ -1,7 +1,5 @@ using System; using System.Net.Http; -using HdrHistogram; -using NBomber; using NBomber.Contracts; using NBomber.CSharp; using static NBomber.Time; @@ -28,11 +26,32 @@ public static void Run() .WithWarmUpDuration(Seconds(5)) .WithLoadSimulations( Simulation.InjectPerSec(rate: 1, during: TimeSpan.FromSeconds(30)) + ) + .WithThresholds( + Threshold.RequestAllCount(x => x > 200, "request all count > 200"), + Threshold.RequestOkCount(x => x > 190), + Threshold.RequestFailedCount(x => x <= 10), + Threshold.RequestFailedRate(x => x < 0.1), + Threshold.RPS(GetRpsConfig), + Threshold.LatencyMin(x => x < 100), + Threshold.LatencyMean(x => x < 400), + Threshold.LatencyMax(x => x < 500), + Threshold.LatencyStdDev(x => x is > 100 and < 200, "latency standard deviation > 50 and < 100"), + Threshold.LatencyPercent50(x => x < 300), + Threshold.LatencyPercent75(x => x < 320), + Threshold.LatencyPercent95(x => x < 400), + Threshold.LatencyPercent99(x => x < 500), + Threshold.DataTransferAllBytes(x => x < 10000) ); NBomberRunner .RegisterScenarios(scenario) .Run(); } + + private static bool GetRpsConfig(double x) + { + return x > 20.0; + } } } diff --git a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs index 980ab008..e4cda5f6 100644 --- a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs +++ b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs @@ -3,6 +3,7 @@ module FSharpDev.HttpTests.SimpleHttpTest open System.Net.Http open NBomber open NBomber.Contracts +open NBomber.Contracts.Thresholds open NBomber.FSharp let run () = @@ -22,7 +23,29 @@ let run () = Scenario.create "simple_http" [step] |> Scenario.withWarmUpDuration(seconds 5) - |> Scenario.withLoadSimulations [InjectPerSec(rate = 20, during = seconds 30)] + |> Scenario.withLoadSimulations [ InjectPerSec(rate = 20, during = seconds 30) ] + |> Scenario.withThresholds( + thresholds { + request_all_count (fun x -> x > 1290) "request all count > 290" + request_ok_count (fun x -> x > 180) + request_failed_count (fun x -> x <= 10) + request_failed_rate (fun x -> x > 0.1) + rps (fun x -> x > 15.0) + latency_min (fun x -> x < 100) + latency_mean (fun x -> x < 200) + latency_max (fun x -> x < 700) + latency_std_dev (fun x -> x > 50 && x < 100) "latency standard deviation > 50 and < 100" + latency_p50 (fun x -> x < 200) + latency_p75 (fun x -> x < 250) + latency_p95 (fun x -> x < 400) + latency_p99 (fun x -> x < 400) + data_transfer_all_bytes (fun x -> x < 10000) + } + ) +// |> Scenario.withThresholds [ +// RequestAllCount((fun x -> x > 1290), Some "request all count > 290") +// RequestOkCount((fun x -> x > 180), None) +// ] |> NBomberRunner.registerScenario |> NBomberRunner.run |> ignore diff --git a/examples/FSharpDev/Program.fs b/examples/FSharpDev/Program.fs index 72ce247d..8d2d33bf 100644 --- a/examples/FSharpDev/Program.fs +++ b/examples/FSharpDev/Program.fs @@ -12,8 +12,8 @@ let main argv = //HelloWorldExample.run() //CustomSettingsExample.run() //DataFeedTest.run() - //SimpleHttpTest.run() - HttpClientFactoryExample.run() + SimpleHttpTest.run() + //HttpClientFactoryExample.run() //MqttScenario.run() 0 // return an integer exit code diff --git a/src/NBomber.Contracts/Contracts.fs b/src/NBomber.Contracts/Contracts.fs index b699d092..b44e2672 100644 --- a/src/NBomber.Contracts/Contracts.fs +++ b/src/NBomber.Contracts/Contracts.fs @@ -11,6 +11,7 @@ open Serilog open Microsoft.Extensions.Configuration open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds type Response = { StatusCode: Nullable @@ -140,6 +141,7 @@ type Scenario = { LoadSimulations: LoadSimulation list CustomStepOrder: (unit -> string[]) option CustomStepExecControl: (IStepExecControlContext voption -> string voption) option + Thresholds: Threshold list option } type IReportingSink = diff --git a/src/NBomber.Contracts/NBomber.Contracts.fsproj b/src/NBomber.Contracts/NBomber.Contracts.fsproj index 31e7976c..26d1373e 100644 --- a/src/NBomber.Contracts/NBomber.Contracts.fsproj +++ b/src/NBomber.Contracts/NBomber.Contracts.fsproj @@ -16,6 +16,7 @@ + diff --git a/src/NBomber.Contracts/Stats.fs b/src/NBomber.Contracts/Stats.fs index 20d1e1ef..d07f4952 100644 --- a/src/NBomber.Contracts/Stats.fs +++ b/src/NBomber.Contracts/Stats.fs @@ -6,6 +6,8 @@ open System.Data open Newtonsoft.Json open Newtonsoft.Json.Converters +open NBomber.Contracts.Thresholds + type ReportFormat = | Txt = 0 | Html = 1 @@ -110,6 +112,18 @@ type LoadSimulationStats = { Value: int } +type ThresholdStatus = + | Passed + | Failed +with + static member map value = + if value then Passed else Failed + +type ThresholdStats = { + Threshold: Threshold + Status: ThresholdStatus +} + type ScenarioStats = { ScenarioName: string RequestCount: int @@ -122,6 +136,7 @@ type ScenarioStats = { StatusCodes: StatusCodeStats[] CurrentOperation: OperationType Duration: TimeSpan + ThresholdStats: ThresholdStats[] option } with member this.GetStepStats(stepName: string) = ScenarioStats.getStepStats stepName this diff --git a/src/NBomber.Contracts/Thresholds.fs b/src/NBomber.Contracts/Thresholds.fs new file mode 100644 index 00000000..50e5492c --- /dev/null +++ b/src/NBomber.Contracts/Thresholds.fs @@ -0,0 +1,84 @@ +namespace NBomber.Contracts.Thresholds + +module private ThresholdDefaultDescriptions = + + let [] RequestAllCountDefaultDescription = "request count - all" + let [] RequestOkCountDefaultDescription = "request count - ok" + let [] RequestFailedCountDefaultDescription = "request count - failed" + let [] RequestFailedRateDefaultDescription = "request rate - failed" + let [] RPSDefaultDescription = "request count - RPS" + let [] LatencyMinDefaultDescription = "latency - min" + let [] LatencyMeanDefaultDescription = "latency - mean" + let [] LatencyMaxDefaultDescription = "latency - max" + let [] LatencyStdDevDefaultDescription = "latency - StdDev" + let [] LatencyPercent50DefaultDescription = "latency percentile - 50%" + let [] LatencyPercent75DefaultDescription = "latency percentile - 75%" + let [] LatencyPercent95DefaultDescription = "latency percentile - 95%" + let [] LatencyPercent99DefaultDescription = "latency percentile - 99%" + let [] DataTransferMinBytesDefaultDescription = "data transfer bytes - min" + let [] DataTransferMeanBytesDefaultDescription = "data transfer bytes - mean" + let [] DataTransferMaxBytesDefaultDescription = "data transfer bytes - max" + let [] DataTransferAllBytesDefaultDescription = "data transfer bytes - all" + let [] DataTransferStdDevDefaultDescription = "data transfer - StdDev" + let [] DataTransferPercent50DefaultDescription = "data transfer percentile - 50%" + let [] DataTransferPercent75DefaultDescription = "data transfer percentile - 75%" + let [] DataTransferPercent95DefaultDescription = "data transfer percentile - 95%" + let [] DataTransferPercent99DefaultDescription = "data transfer percentile - 99%" + +open ThresholdDefaultDescriptions + +type ThresholdBody<'a> = ('a -> bool) * string option + +type Threshold = + | RequestAllCount of ThresholdBody + | RequestOkCount of ThresholdBody + | RequestFailedCount of ThresholdBody + | RequestFailedRate of ThresholdBody + | RPS of ThresholdBody + | LatencyMin of ThresholdBody + | LatencyMean of ThresholdBody + | LatencyMax of ThresholdBody + | LatencyStdDev of ThresholdBody + | LatencyPercent50 of ThresholdBody + | LatencyPercent75 of ThresholdBody + | LatencyPercent95 of ThresholdBody + | LatencyPercent99 of ThresholdBody + | DataTransferMinBytes of ThresholdBody + | DataTransferMeanBytes of ThresholdBody + | DataTransferMaxBytes of ThresholdBody + | DataTransferPercent50 of ThresholdBody + | DataTransferPercent75 of ThresholdBody + | DataTransferPercent95 of ThresholdBody + | DataTransferPercent99 of ThresholdBody + | DataTransferStdDev of ThresholdBody + | DataTransferAllBytes of ThresholdBody +with + member this.Description = + match this with + | RequestAllCount (_, desc) -> desc, RequestAllCountDefaultDescription + | RequestOkCount (_, desc) -> desc, RequestOkCountDefaultDescription + | RequestFailedCount (_, desc) -> desc, RequestFailedCountDefaultDescription + | RequestFailedRate (_, desc) -> desc, RequestFailedRateDefaultDescription + | RPS (_, desc) -> desc, RPSDefaultDescription + | LatencyMin (_, desc) -> desc, LatencyMinDefaultDescription + | LatencyMean (_, desc) -> desc, LatencyMeanDefaultDescription + | LatencyMax (_, desc) -> desc, LatencyMaxDefaultDescription + | LatencyStdDev (_, desc) -> desc, LatencyStdDevDefaultDescription + | LatencyPercent50 (_, desc) -> desc, LatencyPercent50DefaultDescription + | LatencyPercent75 (_, desc) -> desc, LatencyPercent75DefaultDescription + | LatencyPercent95 (_, desc) -> desc, LatencyPercent95DefaultDescription + | LatencyPercent99 (_, desc) -> desc, LatencyPercent99DefaultDescription + | DataTransferMinBytes (_, desc) -> desc, DataTransferMinBytesDefaultDescription + | DataTransferMeanBytes (_, desc) -> desc, DataTransferMeanBytesDefaultDescription + | DataTransferMaxBytes (_, desc) -> desc, DataTransferMaxBytesDefaultDescription + | DataTransferPercent50 (_, desc) -> desc, DataTransferPercent50DefaultDescription + | DataTransferPercent75 (_, desc) -> desc, DataTransferPercent75DefaultDescription + | DataTransferPercent95 (_, desc) -> desc, DataTransferPercent95DefaultDescription + | DataTransferPercent99 (_, desc) -> desc, DataTransferPercent99DefaultDescription + | DataTransferStdDev (_, desc) -> desc, DataTransferStdDevDefaultDescription + | DataTransferAllBytes (_, desc) -> desc, DataTransferAllBytesDefaultDescription + |> fun x -> + match x with + | desc, defaultDesc -> desc |> Option.defaultValue defaultDesc + + override this.ToString () = this.Description diff --git a/src/NBomber/Api/CSharp.fs b/src/NBomber/Api/CSharp.fs index d5c6cbca..86f72751 100644 --- a/src/NBomber/Api/CSharp.fs +++ b/src/NBomber/Api/CSharp.fs @@ -12,6 +12,7 @@ open Serilog open NBomber open NBomber.Contracts open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds /// ClientFactory helps you create and initialize API clients to work with specific API or protocol (HTTP, WebSockets, gRPC, GraphQL). type ClientFactory = @@ -209,6 +210,10 @@ type ScenarioBuilder = static member WithCustomStepExecControl(scenario: Scenario, execControl: Func) = scenario |> FSharp.Scenario.withCustomStepExecControl(execControl.Invoke) + [] + static member WithThresholds(scenario: Scenario, []thresholds: Threshold[]) = + scenario |> FSharp.Scenario.withThresholds(Seq.toList thresholds) + [] type NBomberRunner = @@ -360,3 +365,70 @@ type ValueOption = static member Some(value: 'T) = ValueSome value static member None() = ValueNone +type Threshold = + + static member RequestAllCount(predicate: Func, [] description: string) = + RequestAllCount(predicate.Invoke, Option.ofObj description) + + static member RequestOkCount(predicate: Func, [] description: string) = + RequestOkCount(predicate.Invoke, Option.ofObj description) + + static member RequestFailedCount(predicate: Func, [] description: string) = + RequestFailedCount(predicate.Invoke, Option.ofObj description) + + static member RequestFailedRate(predicate: Func, [] description: string) = + RequestFailedRate(predicate.Invoke, Option.ofObj description) + + static member RPS(predicate: Func, [] description: string) = + RPS(predicate.Invoke, Option.ofObj description) + + static member LatencyMin(predicate: Func, [] description: string) = + LatencyMin(predicate.Invoke, Option.ofObj description) + + static member LatencyMean(predicate: Func, [] description: string) = + LatencyMean(predicate.Invoke, Option.ofObj description) + + static member LatencyMax(predicate: Func, [] description: string) = + LatencyMax(predicate.Invoke, Option.ofObj description) + + static member LatencyStdDev(predicate: Func, [] description: string) = + LatencyStdDev(predicate.Invoke, Option.ofObj description) + + static member LatencyPercent50(predicate: Func, [] description: string) = + LatencyPercent50(predicate.Invoke, Option.ofObj description) + + static member LatencyPercent75(predicate: Func, [] description: string) = + LatencyPercent75(predicate.Invoke, Option.ofObj description) + + static member LatencyPercent95(predicate: Func, [] description: string) = + LatencyPercent95(predicate.Invoke, Option.ofObj description) + + static member LatencyPercent99(predicate: Func, [] description: string) = + LatencyPercent99(predicate.Invoke, Option.ofObj description) + + static member DataTransferMinBytes(predicate: Func, [] description: string) = + DataTransferMinBytes(predicate.Invoke, Option.ofObj description) + + static member DataTransferMeanBytes(predicate: Func, [] description: string) = + DataTransferMeanBytes(predicate.Invoke, Option.ofObj description) + + static member DataTransferMaxBytes(predicate: Func, [] description: string) = + DataTransferMaxBytes(predicate.Invoke, Option.ofObj description) + + static member DataTransferPercent50(predicate: Func, [] description: string) = + DataTransferPercent50(predicate.Invoke, Option.ofObj description) + + static member DataTransferPercent75(predicate: Func, [] description: string) = + DataTransferPercent75(predicate.Invoke, Option.ofObj description) + + static member DataTransferPercent95(predicate: Func, [] description: string) = + DataTransferPercent95(predicate.Invoke, Option.ofObj description) + + static member DataTransferPercent99(predicate: Func, [] description: string) = + DataTransferPercent99(predicate.Invoke, Option.ofObj description) + + static member DataTransferStdDev(predicate: Func, [] description: string) = + DataTransferStdDev(predicate.Invoke, Option.ofObj description) + + static member DataTransferAllBytes(predicate: Func, [] description: string) = + DataTransferAllBytes(predicate.Invoke, Option.ofObj description) diff --git a/src/NBomber/Api/FSharp.fs b/src/NBomber/Api/FSharp.fs index 033dde39..ee88fcba 100644 --- a/src/NBomber/Api/FSharp.fs +++ b/src/NBomber/Api/FSharp.fs @@ -13,8 +13,9 @@ open Microsoft.Extensions.Configuration open NBomber open NBomber.Contracts -open NBomber.Contracts.Stats open NBomber.Contracts.Internal +open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds open NBomber.Configuration open NBomber.Errors open NBomber.Domain @@ -185,7 +186,8 @@ module Scenario = WarmUpDuration = Constants.DefaultWarmUpDuration LoadSimulations = [LoadSimulation.KeepConstant(copies = Constants.DefaultCopiesCount, during = Constants.DefaultSimulationDuration)] CustomStepOrder = None - CustomStepExecControl = None } + CustomStepExecControl = None + Thresholds = None } /// Initializes scenario. /// You can use it to for example to prepare your target system or to parse and apply configuration. @@ -226,6 +228,9 @@ module Scenario = let withCustomStepExecControl (execControl: IStepExecControlContext voption -> string voption) (scenario: Contracts.Scenario) = { scenario with CustomStepExecControl = Some execControl } + let withThresholds (thresholds: Threshold list) (scenario: Contracts.Scenario) = + { scenario with Thresholds = Some thresholds } + /// NBomberRunner is responsible for registering and running scenarios. /// Also it provides configuration points related to infrastructure, reporting, loading plugins. [] @@ -388,3 +393,69 @@ module NBomberRunner = |> runWithResult args |> Result.map(fun x -> x.FinalStats) |> Result.mapError AppError.toString + +type ThresholdsBuilder () = + member _.Yield _ = [ ] + + member _.Run state = state + + [] + member _.RequestAllCount(state, predicate, ?description: string) = + state @ [ RequestAllCount(predicate, description) ] + + [] + member _.RequestOkCount(state, predicate, ?description: string) = + state @ [ RequestOkCount(predicate, description) ] + + [] + member _.RequestFailedCount(state, predicate, ?description: string) = + state @ [ RequestFailedCount(predicate, description) ] + + [] + member _.RequestFailedRate(state, predicate, ?description: string) = + state @ [ RequestFailedRate(predicate, description) ] + + [] + member _.RPS(state, predicate, ?description: string) = + state @ [ RPS(predicate, description) ] + + [] + member _.LatencyMin(state, predicate, ?description: string) = + state @ [ LatencyMin(predicate, description) ] + + [] + member _.LatencyMean(state, predicate, ?description: string) = + state @ [ LatencyMean(predicate, description) ] + + [] + member _.LatencyMax(state, predicate, ?description: string) = + state @ [ LatencyMax(predicate, description) ] + + [] + member _.LatencyStdDev(state, predicate, ?description: string) = + state @ [ LatencyStdDev(predicate, description) ] + + [] + member _.LatencyPercent50(state, predicate, ?description: string) = + state @ [ LatencyPercent50(predicate, description) ] + + [] + member _.LatencyPercent75(state, predicate, ?description: string) = + state @ [ LatencyPercent75(predicate, description) ] + + [] + member _.LatencyPercent95(state, predicate, ?description: string) = + state @ [ LatencyPercent95(predicate, description) ] + + [] + member _.LatencyPercent99(state, predicate, ?description: string) = + state @ [ LatencyPercent99(predicate, description) ] + + [] + member _.DataTransferAllBytes(state, predicate, ?description: string) = + state @ [ DataTransferAllBytes(predicate, description) ] + +[] +module ComputationExpressions = + + let thresholds = ThresholdsBuilder() diff --git a/src/NBomber/Domain/DomainTypes.fs b/src/NBomber/Domain/DomainTypes.fs index 3954ba69..c8708bbc 100644 --- a/src/NBomber/Domain/DomainTypes.fs +++ b/src/NBomber/Domain/DomainTypes.fs @@ -10,6 +10,7 @@ open Serilog open NBomber.Contracts open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds open NBomber.Domain.ClientFactory open NBomber.Domain.ClientPool @@ -102,4 +103,5 @@ type Scenario = { CustomStepOrder: (unit -> string[]) option CustomStepExecControl: (IStepExecControlContext voption -> string voption) option IsEnabled: bool // used for stats in the cluster mode + Thresholds: Threshold list option } diff --git a/src/NBomber/Domain/Errors.fs b/src/NBomber/Domain/Errors.fs index a3bcdca6..3ca961e5 100644 --- a/src/NBomber/Domain/Errors.fs +++ b/src/NBomber/Domain/Errors.fs @@ -29,6 +29,7 @@ type ValidationError = | InvalidClientFactoryName of factoryName:string | DuplicateClientFactoryName of scenarioName:string * factoryName:string | DuplicateStepNameButDiffImpl of scenarioName:string * stepName:string + | EmptyThresholds of scenarioName:string // ScenarioSettings | CustomStepOrderContainsNotFoundStepName of scenarioName:string * stepName:string @@ -128,6 +129,9 @@ type AppError = | EnterpriseOnlyFeature message -> message + | EmptyThresholds scenarioName -> + $"Scenario: '{scenarioName}' has no thresholds" + static member toString (error: AppError) = match error with | Domain e -> AppError.toString(e) diff --git a/src/NBomber/Domain/Scenario.fs b/src/NBomber/Domain/Scenario.fs index a91b2536..a22f297b 100644 --- a/src/NBomber/Domain/Scenario.fs +++ b/src/NBomber/Domain/Scenario.fs @@ -10,6 +10,7 @@ open FsToolkit.ErrorHandling open Microsoft.Extensions.Configuration open NBomber +open NBomber.Contracts.Thresholds open NBomber.Extensions.Internal open NBomber.Extensions.Operator.Result open NBomber.Configuration @@ -74,6 +75,15 @@ module Validation = | Ok _ -> Ok scenario | Error errors -> errors |> List.head |> AppError.createResult + let checkEmptyThresholds (scenario: Contracts.Scenario) = + scenario.Thresholds + |> Option.map (fun thresholds -> + match thresholds with + | [] -> AppError.createResult(EmptyThresholds scenario.ScenarioName) + | _ -> Ok scenario + ) + |> Option.defaultValue (Ok scenario) + let validate = checkEmptyScenarioName >=> checkInitOnlyScenario @@ -81,6 +91,7 @@ module Validation = >=> checkDuplicateStepNameButDiffImpl >=> checkClientFactoryName >=> checkDuplicateClientFactories + >=> checkEmptyThresholds module ClientFactory = @@ -222,7 +233,8 @@ let createScenario (scn: Contracts.Scenario) = result { StepOrderIndex = stepOrderIndex CustomStepOrder = scenario.CustomStepOrder CustomStepExecControl = scenario.CustomStepExecControl - IsEnabled = true } + IsEnabled = true + Thresholds = scenario.Thresholds } } let createScenarios (scenarios: Contracts.Scenario list) = result { diff --git a/src/NBomber/Domain/Stats/Statistics.fs b/src/NBomber/Domain/Stats/Statistics.fs index 179cd743..cb5afc0b 100644 --- a/src/NBomber/Domain/Stats/Statistics.fs +++ b/src/NBomber/Domain/Stats/Statistics.fs @@ -8,6 +8,7 @@ open FSharp.UMX open NBomber open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds open NBomber.Domain open NBomber.Domain.DomainTypes @@ -151,6 +152,52 @@ module StepStats = let getAllRequestCount (stats: StepStats) = stats.Ok.Request.Count + stats.Fail.Request.Count +module private ThresholdStats = + + let private applySepsStats stats f = + (true, stats) + ||> Array.fold (fun status stats -> + status + && (f stats.Ok || stats.Ok.Request.Count = 0) + && (f stats.Fail || stats.Fail.Request.Count = 0) + ) + + let applyThresholds stepStats thresholds = + let sum by = stepStats |> Array.sumBy by + let okCount = sum (fun x -> x.Ok.Request.Count) + let failedCount = sum (fun x -> x.Fail.Request.Count) + let allCount = okCount + failedCount + thresholds + |> List.map (fun threshold -> + let status = + match threshold with + | RequestAllCount (f, _) -> f allCount + | RequestOkCount (f, _) -> f okCount + | RequestFailedCount (f, _) -> f failedCount + | RequestFailedRate (f, _) -> f (float failedCount * 100. / float allCount) + | RPS (f, _) -> applySepsStats stepStats (fun s -> f s.Request.RPS) + | LatencyMin (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.MinMs) + | LatencyMean (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.MeanMs) + | LatencyMax (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.MaxMs) + | LatencyStdDev (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.StdDev) + | LatencyPercent50 (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.Percent50) + | LatencyPercent75 (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.Percent75) + | LatencyPercent95 (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.Percent95) + | LatencyPercent99 (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.Percent99) + | DataTransferMinBytes (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.MinBytes) + | DataTransferMeanBytes (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.MeanBytes) + | DataTransferMaxBytes (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.MaxBytes) + | DataTransferPercent50 (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent50) + | DataTransferPercent75 (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent75) + | DataTransferPercent95 (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent95) + | DataTransferPercent99 (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent99) + | DataTransferStdDev (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.StdDev) + | DataTransferAllBytes (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.AllBytes) + |> ThresholdStatus.map + + { Threshold = threshold; Status = status } + ) + module ScenarioStats = let create (scenario: Scenario) @@ -184,6 +231,12 @@ module ScenarioStats = let failCodes = allStepsData |> Array.collect(fun x -> StatusCodeStats.create x.FailStats.StatusCodes) let statusCodes = StatusCodeStats.merge(okCodes |> Array.append(failCodes)) + let thresholdStats = + scenario.Thresholds + |> Option.map ( + ThresholdStats.applyThresholds stepStats >> Array.ofList + ) + { ScenarioName = scenario.ScenarioName RequestCount = okCount + failCount OkCount = okCount @@ -194,7 +247,8 @@ module ScenarioStats = LoadSimulationStats = simulationStats StatusCodes = statusCodes CurrentOperation = currentOperation - Duration = %duration } + Duration = %duration + ThresholdStats = thresholdStats } let round (stats: ScenarioStats) = { stats with StepStats = stats.StepStats |> Array.map(StepStats.round) @@ -203,6 +257,17 @@ module ScenarioStats = let failStepStatsExist (stats: ScenarioStats) = stats.StepStats |> Array.exists(fun stats -> stats.Fail.Request.Count > 0) + let failThresholdStatsExist stats = + stats.ThresholdStats + |> Option.map ( + Array.exists (fun stats -> + match stats.Status with + | Passed -> false + | Failed -> true + ) + ) + |> Option.defaultValue false + module NodeStats = let create (testInfo: TestInfo) (nodeInfo: NodeInfo) (scnStats: ScenarioStats[]) = diff --git a/src/NBomber/DomainServices/Reports/ConsoleReport.fs b/src/NBomber/DomainServices/Reports/ConsoleReport.fs index 30d1f489..7b8664c8 100644 --- a/src/NBomber/DomainServices/Reports/ConsoleReport.fs +++ b/src/NBomber/DomainServices/Reports/ConsoleReport.fs @@ -4,11 +4,11 @@ open System open System.Collections.Generic open System.Data +open NBomber.Domain.Stats.Statistics open Serilog open NBomber.Contracts open NBomber.Contracts.Stats -open NBomber.Domain.Stats open NBomber.Extensions.Data open NBomber.Extensions.Internal open NBomber.Infra @@ -51,7 +51,7 @@ module ConsoleNodeStats = Console.addLine $" - duration: {Console.okEscColor scnStats.Duration}" ] let private printStepStatsHeader (stepStats: StepStats[]) = - let print (stats) = seq { + let print stats = seq { $"step: {Console.blueEscColor stats.StepName}" $" - timeout: {Console.okEscColor stats.StepInfo.Timeout.TotalMilliseconds} ms" $" - client factory: {Console.okEscColor stats.StepInfo.ClientFactoryName}, clients: {Console.okEscColor stats.StepInfo.ClientFactoryClientCount}" @@ -70,6 +70,16 @@ module ConsoleNodeStats = [ ConsoleStatusCodesStats.printScenarioHeader scnStats.ScenarioName ConsoleStatusCodesStats.printStatusCodeTable scnStats ] + let private printThresholdStats (scnStats: ScenarioStats) = + let headers = ["threshold"; "status"] + let rows = + scnStats.ThresholdStats + |> Option.map (ReportHelper.ThresholdStats.createTableRows Console.okEscColor Console.errorEscColor) + |> Option.defaultValue List.empty + + [ Console.addLine $"thresholds for scenario: {Console.okColor scnStats.ScenarioName}" + Console.addTable headers rows ] + let private printScenarioStats (scnStats: ScenarioStats) (simulations: LoadSimulation list) = [ yield! printScenarioHeader scnStats Console.addLine String.Empty @@ -82,12 +92,16 @@ module ConsoleNodeStats = printStepStatsTable true scnStats.StepStats - if Statistics.ScenarioStats.failStepStatsExist scnStats then + if ScenarioStats.failStepStatsExist scnStats then printStepStatsTable false scnStats.StepStats if scnStats.StatusCodes.Length > 0 then Console.addLine String.Empty - yield! printScenarioStatusCodes scnStats ] + yield! printScenarioStatusCodes scnStats + + if ScenarioStats.failThresholdStatsExist scnStats then + Console.addLine String.Empty + yield! printThresholdStats scnStats ] let printNodeStats (stats: NodeStats) (loadSimulations: IDictionary) = let scenarioStats = diff --git a/src/NBomber/DomainServices/Reports/ReportHelper.fs b/src/NBomber/DomainServices/Reports/ReportHelper.fs index b19c36ea..706b3c6d 100644 --- a/src/NBomber/DomainServices/Reports/ReportHelper.fs +++ b/src/NBomber/DomainServices/Reports/ReportHelper.fs @@ -123,3 +123,32 @@ module StatusCodesStats = // all status codes okNotAvailableStatusCodes @ okStatusCodes @ failNotAvailableStatusCodes @ failStatusCodes + +module ThresholdStats = + + let createTableRows (okColor: obj -> string) + (errorColor: obj -> string) + (thresholdStats: ThresholdStats[]) = + + let createThresholdRows f thresholdStats = + thresholdStats + |> Seq.choose f + |> Seq.toList + + let okThresholds = + thresholdStats + |> createThresholdRows(fun x -> + match x.Status with + | Passed -> Some [$"{x.Threshold}"; okColor "passed"] + | Failed -> None + ) + + let failThresholds = + thresholdStats + |> createThresholdRows(fun x -> + match x.Status with + | Failed -> Some [$"{x.Threshold}"; errorColor "failed"] + | Passed -> None + ) + + okThresholds @ failThresholds diff --git a/src/NBomber/NBomber.fsproj b/src/NBomber/NBomber.fsproj index fc234be5..3f2d2306 100644 --- a/src/NBomber/NBomber.fsproj +++ b/src/NBomber/NBomber.fsproj @@ -110,7 +110,7 @@ - - - + + + diff --git a/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs b/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs index 2915d1bb..c8d27893 100644 --- a/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs +++ b/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs @@ -34,6 +34,7 @@ let baseScnStats = { AllBytes = 0; StepStats = Array.empty; LatencyCount = { LessOrEq800 = 0; More800Less1200 = 0; MoreOrEq1200 = 0 } LoadSimulationStats = { SimulationName = ""; Value = 0 } StatusCodes = Array.empty; CurrentOperation = OperationType.None; Duration = TimeSpan.MinValue + ThresholdStats = None } let baseStepStats = { diff --git a/tests/NBomber.IntegrationTests/StatisticsTests.fs b/tests/NBomber.IntegrationTests/StatisticsTests.fs index 2042bb1a..f8d14645 100644 --- a/tests/NBomber.IntegrationTests/StatisticsTests.fs +++ b/tests/NBomber.IntegrationTests/StatisticsTests.fs @@ -53,6 +53,7 @@ module ScenarioStatsTests = CustomStepOrder = None CustomStepExecControl = None IsEnabled = false + Thresholds = None } let internal baseRawStepStats ={ @@ -275,6 +276,7 @@ module NodeStatsTests = StatusCodes = Array.empty CurrentOperation = OperationType.Complete Duration = TimeSpan.Zero + ThresholdStats = None } []