From 289771ffaaf5579e3d78e849dba17cfdb92dff3f Mon Sep 17 00:00:00 2001 From: pavlogrushetsky Date: Sat, 14 May 2022 23:32:10 +0300 Subject: [PATCH 1/8] Add thresholds for statistics - prototype --- .../CSharpDev/HttpTests/SimpleHttpTest.cs | 28 +++++++++- .../FSharpDev/HttpTests/SimpleHttpTest.fs | 24 +++++++- src/NBomber.Contracts/Contracts.fs | 2 + src/NBomber.Contracts/Metrics.fs | 25 +++++++++ .../NBomber.Contracts.fsproj | 1 + src/NBomber.Contracts/Stats.fs | 12 ++++ src/NBomber/Api/CSharp.fs | 56 +++++++++++++++++++ src/NBomber/Api/FSharp.fs | 7 ++- src/NBomber/Domain/DomainTypes.fs | 2 + src/NBomber/Domain/Scenario.fs | 3 +- src/NBomber/Domain/Stats/Statistics.fs | 29 +++++++++- .../HintsAnalyzerTests.fs | 1 + .../StatisticsTests.fs | 2 + 13 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 src/NBomber.Contracts/Metrics.fs diff --git a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs index 31f66564..67bb42c5 100644 --- a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs +++ b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs @@ -1,7 +1,5 @@ using System; using System.Net.Http; -using HdrHistogram; -using NBomber; using NBomber.Contracts; using NBomber.CSharp; using static NBomber.Time; @@ -28,11 +26,37 @@ public static void Run() .WithWarmUpDuration(Seconds(5)) .WithLoadSimulations( Simulation.InjectPerSec(rate: 1, during: TimeSpan.FromSeconds(30)) + ) + .WithThresholds( + Metric.RequestCount( + Threshold.AllCount(x => x > 200), + Threshold.OkCount(x => x > 190), + Threshold.FailedCount(x => x <= 10), + Threshold.FailedRate(x => x < 0.1), + Threshold.RPS(GetRpsConfig) + ), + Metric.Latency( + Threshold.Min(x => x < 100), + Threshold.Mean(x => x < 400), + Threshold.Max(x => x < 500), + Threshold.StdDev(x => x is > 100 and < 200) + ), + Metric.LatencyPercentile( + Threshold.P50(x => x < 300), + Threshold.P75(x => x < 320), + Threshold.P95(x => x < 400), + Threshold.P99(x => x < 500) + ) ); NBomberRunner .RegisterScenarios(scenario) .Run(); } + + private static bool GetRpsConfig(double x) + { + return x > 20.0; + } } } diff --git a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs index 980ab008..3e8888fb 100644 --- a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs +++ b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs @@ -3,6 +3,7 @@ module FSharpDev.HttpTests.SimpleHttpTest open System.Net.Http open NBomber open NBomber.Contracts +open NBomber.Contracts.Metrics open NBomber.FSharp let run () = @@ -22,7 +23,28 @@ let run () = Scenario.create "simple_http" [step] |> Scenario.withWarmUpDuration(seconds 5) - |> Scenario.withLoadSimulations [InjectPerSec(rate = 20, during = seconds 30)] + |> Scenario.withLoadSimulations [ InjectPerSec(rate = 20, during = seconds 30) ] + |> Scenario.withThresholds [ + RequestCount [ + AllCount(fun x -> x > 200) + OkCount(fun x -> x > 190) + FailedCount(fun x -> x <= 10) + FailedRate(fun x -> x < 0.1) + RPS(fun x -> x > 20.0) + ] + Latency [ + Min(fun x -> x < 100) + Mean(fun x -> x < 400) + Max(fun x -> x < 500) + StdDev(fun x -> x > 100 && x < 200) + ] + LatencyPercentile [ + P50(fun x -> x < 300) + P75(fun x -> x < 320) + P95(fun x -> x < 400) + P99(fun x -> x < 500) + ] + ] |> NBomberRunner.registerScenario |> NBomberRunner.run |> ignore diff --git a/src/NBomber.Contracts/Contracts.fs b/src/NBomber.Contracts/Contracts.fs index b699d092..f149efca 100644 --- a/src/NBomber.Contracts/Contracts.fs +++ b/src/NBomber.Contracts/Contracts.fs @@ -10,6 +10,7 @@ open System.Threading.Tasks open Serilog open Microsoft.Extensions.Configuration +open NBomber.Contracts.Metrics open NBomber.Contracts.Stats type Response = { @@ -140,6 +141,7 @@ type Scenario = { LoadSimulations: LoadSimulation list CustomStepOrder: (unit -> string[]) option CustomStepExecControl: (IStepExecControlContext voption -> string voption) option + Thresholds: Metric list option } type IReportingSink = diff --git a/src/NBomber.Contracts/Metrics.fs b/src/NBomber.Contracts/Metrics.fs new file mode 100644 index 00000000..1c82bd63 --- /dev/null +++ b/src/NBomber.Contracts/Metrics.fs @@ -0,0 +1,25 @@ +namespace NBomber.Contracts.Metrics + +type RequestCountThreshold = + | AllCount of (float -> bool) + | OkCount of (float -> bool) + | FailedCount of (float -> bool) + | FailedRate of (float -> bool) + | RPS of (float -> bool) + +type LatencyThreshold = + | Min of (float -> bool) + | Mean of (float -> bool) + | Max of (float -> bool) + | StdDev of (float -> bool) + +type LatencyPercentileThreshold = + | P50 of (float -> bool) + | P75 of (float -> bool) + | P95 of (float -> bool) + | P99 of (float -> bool) + +type Metric = + | RequestCount of RequestCountThreshold list + | Latency of LatencyThreshold list + | LatencyPercentile of LatencyPercentileThreshold list diff --git a/src/NBomber.Contracts/NBomber.Contracts.fsproj b/src/NBomber.Contracts/NBomber.Contracts.fsproj index 01530b60..9a2eb64e 100644 --- a/src/NBomber.Contracts/NBomber.Contracts.fsproj +++ b/src/NBomber.Contracts/NBomber.Contracts.fsproj @@ -16,6 +16,7 @@ + diff --git a/src/NBomber.Contracts/Stats.fs b/src/NBomber.Contracts/Stats.fs index 20d1e1ef..a31fca4f 100644 --- a/src/NBomber.Contracts/Stats.fs +++ b/src/NBomber.Contracts/Stats.fs @@ -6,6 +6,8 @@ open System.Data open Newtonsoft.Json open Newtonsoft.Json.Converters +open NBomber.Contracts.Metrics + type ReportFormat = | Txt = 0 | Html = 1 @@ -110,6 +112,15 @@ type LoadSimulationStats = { Value: int } +type ThresholdStatus = + | Passed + | Failed + +type MetricStats = + | RequestCountStats of (RequestCountThreshold * ThresholdStatus) list + | LatencyStats of (LatencyThreshold * ThresholdStatus) list + | LatencyPercentileStats of (LatencyPercentileThreshold * ThresholdStatus) list + type ScenarioStats = { ScenarioName: string RequestCount: int @@ -122,6 +133,7 @@ type ScenarioStats = { StatusCodes: StatusCodeStats[] CurrentOperation: OperationType Duration: TimeSpan + MetricStats: MetricStats[] option } with member this.GetStepStats(stepName: string) = ScenarioStats.getStepStats stepName this diff --git a/src/NBomber/Api/CSharp.fs b/src/NBomber/Api/CSharp.fs index 7def9729..a36ba5ef 100644 --- a/src/NBomber/Api/CSharp.fs +++ b/src/NBomber/Api/CSharp.fs @@ -11,6 +11,7 @@ open Serilog open NBomber open NBomber.Contracts +open NBomber.Contracts.Metrics open NBomber.Contracts.Stats /// ClientFactory helps you create and initialize API clients to work with specific API or protocol (HTTP, WebSockets, gRPC, GraphQL). @@ -209,6 +210,10 @@ type ScenarioBuilder = static member WithCustomStepExecControl(scenario: Scenario, execControl: Func) = scenario |> FSharp.Scenario.withCustomStepExecControl(execControl.Invoke) + [] + static member WithThresholds(scenario: Scenario, []thresholds: Metric[]) = + scenario |> FSharp.Scenario.withThresholds(Seq.toList thresholds) + [] type NBomberRunner = @@ -352,3 +357,54 @@ type ValueOption = static member Some(value: 'T) = ValueSome value static member None() = ValueNone +type Metric = + + static member RequestCount([]thresholds: RequestCountThreshold[]) = + RequestCount(Seq.toList thresholds) + + static member Latency([]thresholds: LatencyThreshold[]) = + Latency(Seq.toList thresholds) + + static member LatencyPercentile([]thresholds: LatencyPercentileThreshold[]) = + LatencyPercentile(Seq.toList thresholds) + +type Threshold = + + static member AllCount(predicate: Func) = + AllCount(predicate.Invoke) + + static member OkCount(predicate: Func) = + OkCount(predicate.Invoke) + + static member FailedCount(predicate: Func) = + FailedCount(predicate.Invoke) + + static member FailedRate(predicate: Func) = + FailedRate(predicate.Invoke) + + static member RPS(predicate: Func) = + RPS(predicate.Invoke) + + static member Min(predicate: Func) = + Min(predicate.Invoke) + + static member Mean(predicate: Func) = + Mean(predicate.Invoke) + + static member Max(predicate: Func) = + Max(predicate.Invoke) + + static member StdDev(predicate: Func) = + StdDev(predicate.Invoke) + + static member P50(predicate: Func) = + P50(predicate.Invoke) + + static member P75(predicate: Func) = + P75(predicate.Invoke) + + static member P95(predicate: Func) = + P95(predicate.Invoke) + + static member P99(predicate: Func) = + P99(predicate.Invoke) diff --git a/src/NBomber/Api/FSharp.fs b/src/NBomber/Api/FSharp.fs index 7998d346..497e92f7 100644 --- a/src/NBomber/Api/FSharp.fs +++ b/src/NBomber/Api/FSharp.fs @@ -13,6 +13,7 @@ open Microsoft.Extensions.Configuration open NBomber open NBomber.Contracts +open NBomber.Contracts.Metrics open NBomber.Contracts.Stats open NBomber.Contracts.Internal open NBomber.Configuration @@ -185,7 +186,8 @@ module Scenario = WarmUpDuration = Constants.DefaultWarmUpDuration LoadSimulations = [LoadSimulation.KeepConstant(copies = Constants.DefaultCopiesCount, during = Constants.DefaultSimulationDuration)] CustomStepOrder = None - CustomStepExecControl = None } + CustomStepExecControl = None + Thresholds = None } /// Initializes scenario. /// You can use it to for example to prepare your target system or to parse and apply configuration. @@ -226,6 +228,9 @@ module Scenario = let withCustomStepExecControl (execControl: IStepExecControlContext voption -> string voption) (scenario: Contracts.Scenario) = { scenario with CustomStepExecControl = Some execControl } + let withThresholds (thresholds: Metric list) (scenario: Contracts.Scenario) = + { scenario with Thresholds = Some thresholds } + /// NBomberRunner is responsible for registering and running scenarios. /// Also it provides configuration points related to infrastructure, reporting, loading plugins. [] diff --git a/src/NBomber/Domain/DomainTypes.fs b/src/NBomber/Domain/DomainTypes.fs index 3954ba69..d19d6a73 100644 --- a/src/NBomber/Domain/DomainTypes.fs +++ b/src/NBomber/Domain/DomainTypes.fs @@ -6,6 +6,7 @@ open System.Threading open System.Threading.Tasks open HdrHistogram +open NBomber.Contracts.Metrics open Serilog open NBomber.Contracts @@ -102,4 +103,5 @@ type Scenario = { CustomStepOrder: (unit -> string[]) option CustomStepExecControl: (IStepExecControlContext voption -> string voption) option IsEnabled: bool // used for stats in the cluster mode + Thresholds: Metric list option } diff --git a/src/NBomber/Domain/Scenario.fs b/src/NBomber/Domain/Scenario.fs index aebb1a93..9db3f545 100644 --- a/src/NBomber/Domain/Scenario.fs +++ b/src/NBomber/Domain/Scenario.fs @@ -217,7 +217,8 @@ let createScenario (scn: Contracts.Scenario) = result { StepOrderIndex = stepOrderIndex CustomStepOrder = scenario.CustomStepOrder CustomStepExecControl = scenario.CustomStepExecControl - IsEnabled = true } + IsEnabled = true + Thresholds = scenario.Thresholds } } let createScenarios (scenarios: Contracts.Scenario list) = result { diff --git a/src/NBomber/Domain/Stats/Statistics.fs b/src/NBomber/Domain/Stats/Statistics.fs index 179cd743..003cdbd0 100644 --- a/src/NBomber/Domain/Stats/Statistics.fs +++ b/src/NBomber/Domain/Stats/Statistics.fs @@ -151,6 +151,26 @@ module StepStats = let getAllRequestCount (stats: StepStats) = stats.Ok.Request.Count + stats.Fail.Request.Count +module MetricStats = + + let failStatsExist stats = + let failed stats = + stats + |> List.map snd + |> List.exists (fun status -> + match status with + | Passed -> false + | Failed -> true + ) + + match stats with + | RequestCountStats stats -> stats |> failed + | LatencyStats stats -> stats |> failed + | LatencyPercentileStats stats -> stats |> failed + + let applyMetricThresholds metric stepStats = + Array.empty + module ScenarioStats = let create (scenario: Scenario) @@ -184,6 +204,8 @@ module ScenarioStats = let failCodes = allStepsData |> Array.collect(fun x -> StatusCodeStats.create x.FailStats.StatusCodes) let statusCodes = StatusCodeStats.merge(okCodes |> Array.append(failCodes)) + let metricStats = scenario.Thresholds |> Option.map (MetricStats.applyMetricThresholds stepStats) + { ScenarioName = scenario.ScenarioName RequestCount = okCount + failCount OkCount = okCount @@ -194,14 +216,17 @@ module ScenarioStats = LoadSimulationStats = simulationStats StatusCodes = statusCodes CurrentOperation = currentOperation - Duration = %duration } + Duration = duration + MetricStats = metricStats } let round (stats: ScenarioStats) = { stats with StepStats = stats.StepStats |> Array.map(StepStats.round) Duration = TimeSpan(stats.Duration.Days, stats.Duration.Hours, stats.Duration.Minutes, stats.Duration.Seconds) } let failStepStatsExist (stats: ScenarioStats) = - stats.StepStats |> Array.exists(fun stats -> stats.Fail.Request.Count > 0) + stats.MetricStats + |> Option.map (Array.exists MetricStats.failStatsExist) + |> Option.defaultValue (stats.StepStats |> Array.exists(fun stats -> stats.Fail.Request.Count > 0)) module NodeStats = diff --git a/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs b/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs index 43fd4d52..bc7d6871 100644 --- a/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs +++ b/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs @@ -34,6 +34,7 @@ let baseScnStats = { AllBytes = 0; StepStats = Array.empty; LatencyCount = { LessOrEq800 = 0; More800Less1200 = 0; MoreOrEq1200 = 0 } LoadSimulationStats = { SimulationName = ""; Value = 0 } StatusCodes = Array.empty; CurrentOperation = OperationType.None; Duration = TimeSpan.MinValue + MetricStats = None } let baseStepStats = { diff --git a/tests/NBomber.IntegrationTests/StatisticsTests.fs b/tests/NBomber.IntegrationTests/StatisticsTests.fs index 2042bb1a..28251dc7 100644 --- a/tests/NBomber.IntegrationTests/StatisticsTests.fs +++ b/tests/NBomber.IntegrationTests/StatisticsTests.fs @@ -53,6 +53,7 @@ module ScenarioStatsTests = CustomStepOrder = None CustomStepExecControl = None IsEnabled = false + Thresholds = None } let internal baseRawStepStats ={ @@ -275,6 +276,7 @@ module NodeStatsTests = StatusCodes = Array.empty CurrentOperation = OperationType.Complete Duration = TimeSpan.Zero + MetricStats = None } [] From 44db7277e1071f326f1df623f0e61da226a9ff44 Mon Sep 17 00:00:00 2001 From: pavlogrushetsky Date: Sun, 15 May 2022 21:00:39 +0300 Subject: [PATCH 2/8] Implement RequestCountStats calculation --- .../FSharpDev/HttpTests/SimpleHttpTest.fs | 18 ++--- examples/FSharpDev/Program.fs | 4 +- src/NBomber.Contracts/Metrics.fs | 6 +- src/NBomber.Contracts/Stats.fs | 3 + src/NBomber/Domain/Stats/Statistics.fs | 69 +++++++++++++++++-- 5 files changed, 82 insertions(+), 18 deletions(-) diff --git a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs index 3e8888fb..f8a9fa88 100644 --- a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs +++ b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs @@ -26,23 +26,23 @@ let run () = |> Scenario.withLoadSimulations [ InjectPerSec(rate = 20, during = seconds 30) ] |> Scenario.withThresholds [ RequestCount [ - AllCount(fun x -> x > 200) - OkCount(fun x -> x > 190) + AllCount(fun x -> x > 290) + OkCount(fun x -> x > 180) FailedCount(fun x -> x <= 10) FailedRate(fun x -> x < 0.1) - RPS(fun x -> x > 20.0) + RPS(fun x -> x > 15.0) ] Latency [ Min(fun x -> x < 100) - Mean(fun x -> x < 400) - Max(fun x -> x < 500) - StdDev(fun x -> x > 100 && x < 200) + Mean(fun x -> x < 200) + Max(fun x -> x < 700) + StdDev(fun x -> x > 50 && x < 100) ] LatencyPercentile [ - P50(fun x -> x < 300) - P75(fun x -> x < 320) + P50(fun x -> x < 200) + P75(fun x -> x < 250) P95(fun x -> x < 400) - P99(fun x -> x < 500) + P99(fun x -> x < 400) ] ] |> NBomberRunner.registerScenario diff --git a/examples/FSharpDev/Program.fs b/examples/FSharpDev/Program.fs index 72ce247d..8d2d33bf 100644 --- a/examples/FSharpDev/Program.fs +++ b/examples/FSharpDev/Program.fs @@ -12,8 +12,8 @@ let main argv = //HelloWorldExample.run() //CustomSettingsExample.run() //DataFeedTest.run() - //SimpleHttpTest.run() - HttpClientFactoryExample.run() + SimpleHttpTest.run() + //HttpClientFactoryExample.run() //MqttScenario.run() 0 // return an integer exit code diff --git a/src/NBomber.Contracts/Metrics.fs b/src/NBomber.Contracts/Metrics.fs index 1c82bd63..e40b9bd9 100644 --- a/src/NBomber.Contracts/Metrics.fs +++ b/src/NBomber.Contracts/Metrics.fs @@ -1,9 +1,9 @@ namespace NBomber.Contracts.Metrics type RequestCountThreshold = - | AllCount of (float -> bool) - | OkCount of (float -> bool) - | FailedCount of (float -> bool) + | AllCount of (int -> bool) + | OkCount of (int -> bool) + | FailedCount of (int -> bool) | FailedRate of (float -> bool) | RPS of (float -> bool) diff --git a/src/NBomber.Contracts/Stats.fs b/src/NBomber.Contracts/Stats.fs index a31fca4f..9dfdd50f 100644 --- a/src/NBomber.Contracts/Stats.fs +++ b/src/NBomber.Contracts/Stats.fs @@ -115,6 +115,9 @@ type LoadSimulationStats = { type ThresholdStatus = | Passed | Failed +with + static member map value = + if value then Passed else Failed type MetricStats = | RequestCountStats of (RequestCountThreshold * ThresholdStatus) list diff --git a/src/NBomber/Domain/Stats/Statistics.fs b/src/NBomber/Domain/Stats/Statistics.fs index 003cdbd0..df93de97 100644 --- a/src/NBomber/Domain/Stats/Statistics.fs +++ b/src/NBomber/Domain/Stats/Statistics.fs @@ -7,6 +7,7 @@ open HdrHistogram open FSharp.UMX open NBomber +open NBomber.Contracts.Metrics open NBomber.Contracts.Stats open NBomber.Domain open NBomber.Domain.DomainTypes @@ -162,14 +163,68 @@ module MetricStats = | Passed -> false | Failed -> true ) - match stats with | RequestCountStats stats -> stats |> failed | LatencyStats stats -> stats |> failed | LatencyPercentileStats stats -> stats |> failed - let applyMetricThresholds metric stepStats = - Array.empty + let applyRequestCountThresholds stepStats thresholds = + let sum by = stepStats |> Array.sumBy by + let okCount = sum (fun x -> x.Ok.Request.Count) + let failedCount = sum (fun x -> x.Fail.Request.Count) + let allCount = okCount + failedCount + let rps f = + (true, stepStats) + ||> Array.fold (fun status stats -> + status + && (f stats.Ok.Request.RPS || stats.Ok.Request.Count = 0) + && (f stats.Fail.Request.RPS || stats.Fail.Request.Count = 0) + ) + let applyThreshold threshold = + match threshold with + | AllCount f -> f allCount + | OkCount f -> f okCount + | FailedCount f -> f failedCount + | FailedRate f -> f (float failedCount * 100. / float allCount) + | RPS f -> rps f + |> ThresholdStatus.map + thresholds + |> (List.map (fun threshold -> threshold, applyThreshold threshold)) + |> RequestCountStats + + let applyLatencyThresholds (stepStats: StepStats[]) (thresholds: LatencyThreshold list) : MetricStats = + let applyThreshold threshold = + match threshold with + | Min f -> + threshold, Passed + | Mean f -> + threshold, Passed + | Max f -> + threshold, Passed + | StdDev f -> + threshold, Passed + + thresholds |> (List.map applyThreshold) |> LatencyStats + + let applyLatencyPercentileThresholds (stepStats: StepStats[]) (thresholds: LatencyPercentileThreshold list) : MetricStats = + let applyThreshold threshold = + match threshold with + | P50 f -> + threshold, Passed + | P75 f -> + threshold, Passed + | P95 f -> + threshold, Passed + | P99 f -> + threshold, Passed + + thresholds |> (List.map applyThreshold) |> LatencyPercentileStats + + let applyMetricThresholds (stepStats: StepStats[]) (metric: Metric) : MetricStats = + match metric with + | RequestCount thresholds -> applyRequestCountThresholds stepStats thresholds + | Latency thresholds -> applyLatencyThresholds stepStats thresholds + | LatencyPercentile thresholds -> applyLatencyPercentileThresholds stepStats thresholds module ScenarioStats = @@ -204,7 +259,13 @@ module ScenarioStats = let failCodes = allStepsData |> Array.collect(fun x -> StatusCodeStats.create x.FailStats.StatusCodes) let statusCodes = StatusCodeStats.merge(okCodes |> Array.append(failCodes)) - let metricStats = scenario.Thresholds |> Option.map (MetricStats.applyMetricThresholds stepStats) + let metricStats = + scenario.Thresholds + |> Option.map ( + MetricStats.applyMetricThresholds stepStats + |> List.map + >> Array.ofList + ) { ScenarioName = scenario.ScenarioName RequestCount = okCount + failCount From 3ab1f6de5150af8c0dca87f14b01a4da842eb370 Mon Sep 17 00:00:00 2001 From: pavlogrushetsky Date: Sun, 15 May 2022 21:52:00 +0300 Subject: [PATCH 3/8] Implement LatencyStats and LatencyPercentileStats calculation --- src/NBomber/Domain/Stats/Statistics.fs | 118 +++++++++++++------------ 1 file changed, 60 insertions(+), 58 deletions(-) diff --git a/src/NBomber/Domain/Stats/Statistics.fs b/src/NBomber/Domain/Stats/Statistics.fs index df93de97..5d2415be 100644 --- a/src/NBomber/Domain/Stats/Statistics.fs +++ b/src/NBomber/Domain/Stats/Statistics.fs @@ -154,6 +154,66 @@ module StepStats = module MetricStats = + let private applySepsStats stats f = + (true, stats) + ||> Array.fold (fun status stats -> + status + && (f stats.Ok || stats.Ok.Request.Count = 0) + && (f stats.Fail || stats.Fail.Request.Count = 0) + ) + + let private applyRequestCountThresholds stepStats thresholds = + let sum by = stepStats |> Array.sumBy by + let okCount = sum (fun x -> x.Ok.Request.Count) + let failedCount = sum (fun x -> x.Fail.Request.Count) + let allCount = okCount + failedCount + thresholds + |> List.map (fun threshold -> + let status = + match threshold with + | AllCount f -> f allCount + | OkCount f -> f okCount + | FailedCount f -> f failedCount + | FailedRate f -> f (float failedCount * 100. / float allCount) + | RPS f -> applySepsStats stepStats (fun s -> f s.Request.RPS) + |> ThresholdStatus.map + threshold, status) + |> RequestCountStats + + let private applyLatencyThresholds stepStats thresholds = + thresholds + |> List.map (fun threshold -> + let status = + match threshold with + | Min f -> (fun s -> f s.Latency.MinMs) + | Mean f -> (fun s -> f s.Latency.MeanMs) + | Max f -> (fun s -> f s.Latency.MaxMs) + | StdDev f -> (fun s -> f s.Latency.StdDev) + |> applySepsStats stepStats + |> ThresholdStatus.map + threshold, status) + |> LatencyStats + + let private applyLatencyPercentileThresholds stepStats thresholds = + thresholds + |> List.map (fun threshold -> + let status = + match threshold with + | P50 f -> (fun s -> f s.Latency.Percent50) + | P75 f -> (fun s -> f s.Latency.Percent75) + | P95 f -> (fun s -> f s.Latency.Percent95) + | P99 f -> (fun s -> f s.Latency.Percent99) + |> applySepsStats stepStats + |> ThresholdStatus.map + threshold, status) + |> LatencyPercentileStats + + let applyMetricThresholds stepStats metric = + match metric with + | RequestCount thresholds -> applyRequestCountThresholds stepStats thresholds + | Latency thresholds -> applyLatencyThresholds stepStats thresholds + | LatencyPercentile thresholds -> applyLatencyPercentileThresholds stepStats thresholds + let failStatsExist stats = let failed stats = stats @@ -168,64 +228,6 @@ module MetricStats = | LatencyStats stats -> stats |> failed | LatencyPercentileStats stats -> stats |> failed - let applyRequestCountThresholds stepStats thresholds = - let sum by = stepStats |> Array.sumBy by - let okCount = sum (fun x -> x.Ok.Request.Count) - let failedCount = sum (fun x -> x.Fail.Request.Count) - let allCount = okCount + failedCount - let rps f = - (true, stepStats) - ||> Array.fold (fun status stats -> - status - && (f stats.Ok.Request.RPS || stats.Ok.Request.Count = 0) - && (f stats.Fail.Request.RPS || stats.Fail.Request.Count = 0) - ) - let applyThreshold threshold = - match threshold with - | AllCount f -> f allCount - | OkCount f -> f okCount - | FailedCount f -> f failedCount - | FailedRate f -> f (float failedCount * 100. / float allCount) - | RPS f -> rps f - |> ThresholdStatus.map - thresholds - |> (List.map (fun threshold -> threshold, applyThreshold threshold)) - |> RequestCountStats - - let applyLatencyThresholds (stepStats: StepStats[]) (thresholds: LatencyThreshold list) : MetricStats = - let applyThreshold threshold = - match threshold with - | Min f -> - threshold, Passed - | Mean f -> - threshold, Passed - | Max f -> - threshold, Passed - | StdDev f -> - threshold, Passed - - thresholds |> (List.map applyThreshold) |> LatencyStats - - let applyLatencyPercentileThresholds (stepStats: StepStats[]) (thresholds: LatencyPercentileThreshold list) : MetricStats = - let applyThreshold threshold = - match threshold with - | P50 f -> - threshold, Passed - | P75 f -> - threshold, Passed - | P95 f -> - threshold, Passed - | P99 f -> - threshold, Passed - - thresholds |> (List.map applyThreshold) |> LatencyPercentileStats - - let applyMetricThresholds (stepStats: StepStats[]) (metric: Metric) : MetricStats = - match metric with - | RequestCount thresholds -> applyRequestCountThresholds stepStats thresholds - | Latency thresholds -> applyLatencyThresholds stepStats thresholds - | LatencyPercentile thresholds -> applyLatencyPercentileThresholds stepStats thresholds - module ScenarioStats = let create (scenario: Scenario) From 1a79f955fd6c93e65166b0f21e26f8fed4344f53 Mon Sep 17 00:00:00 2001 From: pavlogrushetsky Date: Tue, 17 May 2022 20:51:49 +0300 Subject: [PATCH 4/8] Rename Metric to Threshold --- examples/CSharpDev/HttpTests/SimpleHttpTest.cs | 6 +++--- examples/FSharpDev/HttpTests/SimpleHttpTest.fs | 2 +- src/NBomber.Contracts/Contracts.fs | 4 ++-- src/NBomber.Contracts/NBomber.Contracts.fsproj | 2 +- src/NBomber.Contracts/Stats.fs | 6 +++--- .../{Metrics.fs => Thresholds.fs} | 4 ++-- src/NBomber/Api/CSharp.fs | 8 +++----- src/NBomber/Api/FSharp.fs | 6 +++--- src/NBomber/Domain/DomainTypes.fs | 4 ++-- src/NBomber/Domain/Stats/Statistics.fs | 18 +++++++++--------- .../HintsAnalyzerTests.fs | 2 +- .../StatisticsTests.fs | 2 +- 12 files changed, 31 insertions(+), 33 deletions(-) rename src/NBomber.Contracts/{Metrics.fs => Thresholds.fs} (91%) diff --git a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs index 67bb42c5..9ab5d483 100644 --- a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs +++ b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs @@ -28,20 +28,20 @@ public static void Run() Simulation.InjectPerSec(rate: 1, during: TimeSpan.FromSeconds(30)) ) .WithThresholds( - Metric.RequestCount( + Threshold.RequestCount( Threshold.AllCount(x => x > 200), Threshold.OkCount(x => x > 190), Threshold.FailedCount(x => x <= 10), Threshold.FailedRate(x => x < 0.1), Threshold.RPS(GetRpsConfig) ), - Metric.Latency( + Threshold.Latency( Threshold.Min(x => x < 100), Threshold.Mean(x => x < 400), Threshold.Max(x => x < 500), Threshold.StdDev(x => x is > 100 and < 200) ), - Metric.LatencyPercentile( + Threshold.LatencyPercentile( Threshold.P50(x => x < 300), Threshold.P75(x => x < 320), Threshold.P95(x => x < 400), diff --git a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs index f8a9fa88..3a97eade 100644 --- a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs +++ b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs @@ -3,7 +3,7 @@ module FSharpDev.HttpTests.SimpleHttpTest open System.Net.Http open NBomber open NBomber.Contracts -open NBomber.Contracts.Metrics +open NBomber.Contracts.Thresholds open NBomber.FSharp let run () = diff --git a/src/NBomber.Contracts/Contracts.fs b/src/NBomber.Contracts/Contracts.fs index f149efca..b44e2672 100644 --- a/src/NBomber.Contracts/Contracts.fs +++ b/src/NBomber.Contracts/Contracts.fs @@ -10,8 +10,8 @@ open System.Threading.Tasks open Serilog open Microsoft.Extensions.Configuration -open NBomber.Contracts.Metrics open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds type Response = { StatusCode: Nullable @@ -141,7 +141,7 @@ type Scenario = { LoadSimulations: LoadSimulation list CustomStepOrder: (unit -> string[]) option CustomStepExecControl: (IStepExecControlContext voption -> string voption) option - Thresholds: Metric list option + Thresholds: Threshold list option } type IReportingSink = diff --git a/src/NBomber.Contracts/NBomber.Contracts.fsproj b/src/NBomber.Contracts/NBomber.Contracts.fsproj index 9a2eb64e..568156c2 100644 --- a/src/NBomber.Contracts/NBomber.Contracts.fsproj +++ b/src/NBomber.Contracts/NBomber.Contracts.fsproj @@ -16,7 +16,7 @@ - + diff --git a/src/NBomber.Contracts/Stats.fs b/src/NBomber.Contracts/Stats.fs index 9dfdd50f..47b48167 100644 --- a/src/NBomber.Contracts/Stats.fs +++ b/src/NBomber.Contracts/Stats.fs @@ -6,7 +6,7 @@ open System.Data open Newtonsoft.Json open Newtonsoft.Json.Converters -open NBomber.Contracts.Metrics +open NBomber.Contracts.Thresholds type ReportFormat = | Txt = 0 @@ -119,7 +119,7 @@ with static member map value = if value then Passed else Failed -type MetricStats = +type ThresholdStats = | RequestCountStats of (RequestCountThreshold * ThresholdStatus) list | LatencyStats of (LatencyThreshold * ThresholdStatus) list | LatencyPercentileStats of (LatencyPercentileThreshold * ThresholdStatus) list @@ -136,7 +136,7 @@ type ScenarioStats = { StatusCodes: StatusCodeStats[] CurrentOperation: OperationType Duration: TimeSpan - MetricStats: MetricStats[] option + ThresholdStats: ThresholdStats[] option } with member this.GetStepStats(stepName: string) = ScenarioStats.getStepStats stepName this diff --git a/src/NBomber.Contracts/Metrics.fs b/src/NBomber.Contracts/Thresholds.fs similarity index 91% rename from src/NBomber.Contracts/Metrics.fs rename to src/NBomber.Contracts/Thresholds.fs index e40b9bd9..5660534b 100644 --- a/src/NBomber.Contracts/Metrics.fs +++ b/src/NBomber.Contracts/Thresholds.fs @@ -1,4 +1,4 @@ -namespace NBomber.Contracts.Metrics +namespace NBomber.Contracts.Thresholds type RequestCountThreshold = | AllCount of (int -> bool) @@ -19,7 +19,7 @@ type LatencyPercentileThreshold = | P95 of (float -> bool) | P99 of (float -> bool) -type Metric = +type Threshold = | RequestCount of RequestCountThreshold list | Latency of LatencyThreshold list | LatencyPercentile of LatencyPercentileThreshold list diff --git a/src/NBomber/Api/CSharp.fs b/src/NBomber/Api/CSharp.fs index a36ba5ef..d92c828a 100644 --- a/src/NBomber/Api/CSharp.fs +++ b/src/NBomber/Api/CSharp.fs @@ -11,8 +11,8 @@ open Serilog open NBomber open NBomber.Contracts -open NBomber.Contracts.Metrics open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds /// ClientFactory helps you create and initialize API clients to work with specific API or protocol (HTTP, WebSockets, gRPC, GraphQL). type ClientFactory = @@ -211,7 +211,7 @@ type ScenarioBuilder = scenario |> FSharp.Scenario.withCustomStepExecControl(execControl.Invoke) [] - static member WithThresholds(scenario: Scenario, []thresholds: Metric[]) = + static member WithThresholds(scenario: Scenario, []thresholds: Threshold[]) = scenario |> FSharp.Scenario.withThresholds(Seq.toList thresholds) [] @@ -357,7 +357,7 @@ type ValueOption = static member Some(value: 'T) = ValueSome value static member None() = ValueNone -type Metric = +type Threshold = static member RequestCount([]thresholds: RequestCountThreshold[]) = RequestCount(Seq.toList thresholds) @@ -368,8 +368,6 @@ type Metric = static member LatencyPercentile([]thresholds: LatencyPercentileThreshold[]) = LatencyPercentile(Seq.toList thresholds) -type Threshold = - static member AllCount(predicate: Func) = AllCount(predicate.Invoke) diff --git a/src/NBomber/Api/FSharp.fs b/src/NBomber/Api/FSharp.fs index 497e92f7..53c90c67 100644 --- a/src/NBomber/Api/FSharp.fs +++ b/src/NBomber/Api/FSharp.fs @@ -13,9 +13,9 @@ open Microsoft.Extensions.Configuration open NBomber open NBomber.Contracts -open NBomber.Contracts.Metrics -open NBomber.Contracts.Stats open NBomber.Contracts.Internal +open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds open NBomber.Configuration open NBomber.Errors open NBomber.Domain @@ -228,7 +228,7 @@ module Scenario = let withCustomStepExecControl (execControl: IStepExecControlContext voption -> string voption) (scenario: Contracts.Scenario) = { scenario with CustomStepExecControl = Some execControl } - let withThresholds (thresholds: Metric list) (scenario: Contracts.Scenario) = + let withThresholds (thresholds: Threshold list) (scenario: Contracts.Scenario) = { scenario with Thresholds = Some thresholds } /// NBomberRunner is responsible for registering and running scenarios. diff --git a/src/NBomber/Domain/DomainTypes.fs b/src/NBomber/Domain/DomainTypes.fs index d19d6a73..c8708bbc 100644 --- a/src/NBomber/Domain/DomainTypes.fs +++ b/src/NBomber/Domain/DomainTypes.fs @@ -6,11 +6,11 @@ open System.Threading open System.Threading.Tasks open HdrHistogram -open NBomber.Contracts.Metrics open Serilog open NBomber.Contracts open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds open NBomber.Domain.ClientFactory open NBomber.Domain.ClientPool @@ -103,5 +103,5 @@ type Scenario = { CustomStepOrder: (unit -> string[]) option CustomStepExecControl: (IStepExecControlContext voption -> string voption) option IsEnabled: bool // used for stats in the cluster mode - Thresholds: Metric list option + Thresholds: Threshold list option } diff --git a/src/NBomber/Domain/Stats/Statistics.fs b/src/NBomber/Domain/Stats/Statistics.fs index 5d2415be..2875019c 100644 --- a/src/NBomber/Domain/Stats/Statistics.fs +++ b/src/NBomber/Domain/Stats/Statistics.fs @@ -7,8 +7,8 @@ open HdrHistogram open FSharp.UMX open NBomber -open NBomber.Contracts.Metrics open NBomber.Contracts.Stats +open NBomber.Contracts.Thresholds open NBomber.Domain open NBomber.Domain.DomainTypes @@ -152,7 +152,7 @@ module StepStats = let getAllRequestCount (stats: StepStats) = stats.Ok.Request.Count + stats.Fail.Request.Count -module MetricStats = +module ThresholdStats = let private applySepsStats stats f = (true, stats) @@ -208,8 +208,8 @@ module MetricStats = threshold, status) |> LatencyPercentileStats - let applyMetricThresholds stepStats metric = - match metric with + let applyThresholds stepStats threshold = + match threshold with | RequestCount thresholds -> applyRequestCountThresholds stepStats thresholds | Latency thresholds -> applyLatencyThresholds stepStats thresholds | LatencyPercentile thresholds -> applyLatencyPercentileThresholds stepStats thresholds @@ -261,10 +261,10 @@ module ScenarioStats = let failCodes = allStepsData |> Array.collect(fun x -> StatusCodeStats.create x.FailStats.StatusCodes) let statusCodes = StatusCodeStats.merge(okCodes |> Array.append(failCodes)) - let metricStats = + let thresholdStats = scenario.Thresholds |> Option.map ( - MetricStats.applyMetricThresholds stepStats + ThresholdStats.applyThresholds stepStats |> List.map >> Array.ofList ) @@ -280,15 +280,15 @@ module ScenarioStats = StatusCodes = statusCodes CurrentOperation = currentOperation Duration = duration - MetricStats = metricStats } + ThresholdStats = thresholdStats } let round (stats: ScenarioStats) = { stats with StepStats = stats.StepStats |> Array.map(StepStats.round) Duration = TimeSpan(stats.Duration.Days, stats.Duration.Hours, stats.Duration.Minutes, stats.Duration.Seconds) } let failStepStatsExist (stats: ScenarioStats) = - stats.MetricStats - |> Option.map (Array.exists MetricStats.failStatsExist) + stats.ThresholdStats + |> Option.map (Array.exists ThresholdStats.failStatsExist) |> Option.defaultValue (stats.StepStats |> Array.exists(fun stats -> stats.Fail.Request.Count > 0)) module NodeStats = diff --git a/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs b/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs index bc7d6871..83b58aec 100644 --- a/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs +++ b/tests/NBomber.IntegrationTests/HintsAnalyzerTests.fs @@ -34,7 +34,7 @@ let baseScnStats = { AllBytes = 0; StepStats = Array.empty; LatencyCount = { LessOrEq800 = 0; More800Less1200 = 0; MoreOrEq1200 = 0 } LoadSimulationStats = { SimulationName = ""; Value = 0 } StatusCodes = Array.empty; CurrentOperation = OperationType.None; Duration = TimeSpan.MinValue - MetricStats = None + ThresholdStats = None } let baseStepStats = { diff --git a/tests/NBomber.IntegrationTests/StatisticsTests.fs b/tests/NBomber.IntegrationTests/StatisticsTests.fs index 28251dc7..f8d14645 100644 --- a/tests/NBomber.IntegrationTests/StatisticsTests.fs +++ b/tests/NBomber.IntegrationTests/StatisticsTests.fs @@ -276,7 +276,7 @@ module NodeStatsTests = StatusCodes = Array.empty CurrentOperation = OperationType.Complete Duration = TimeSpan.Zero - MetricStats = None + ThresholdStats = None } [] From 9e8ad15df2643cd407992b9e33db7ea05a8677e7 Mon Sep 17 00:00:00 2001 From: pavlogrushetsky Date: Tue, 17 May 2022 21:22:13 +0300 Subject: [PATCH 5/8] Flatten thresholds model --- .../CSharpDev/HttpTests/SimpleHttpTest.cs | 32 +++----- .../FSharpDev/HttpTests/SimpleHttpTest.fs | 32 +++----- src/NBomber.Contracts/Stats.fs | 8 +- src/NBomber.Contracts/Thresholds.fs | 35 +++----- src/NBomber/Api/CSharp.fs | 57 ++++++------- src/NBomber/Domain/Stats/Statistics.fs | 79 ++++++------------- 6 files changed, 90 insertions(+), 153 deletions(-) diff --git a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs index 9ab5d483..6d12392d 100644 --- a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs +++ b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs @@ -28,25 +28,19 @@ public static void Run() Simulation.InjectPerSec(rate: 1, during: TimeSpan.FromSeconds(30)) ) .WithThresholds( - Threshold.RequestCount( - Threshold.AllCount(x => x > 200), - Threshold.OkCount(x => x > 190), - Threshold.FailedCount(x => x <= 10), - Threshold.FailedRate(x => x < 0.1), - Threshold.RPS(GetRpsConfig) - ), - Threshold.Latency( - Threshold.Min(x => x < 100), - Threshold.Mean(x => x < 400), - Threshold.Max(x => x < 500), - Threshold.StdDev(x => x is > 100 and < 200) - ), - Threshold.LatencyPercentile( - Threshold.P50(x => x < 300), - Threshold.P75(x => x < 320), - Threshold.P95(x => x < 400), - Threshold.P99(x => x < 500) - ) + Threshold.RequestAllCount(x => x > 200), + Threshold.RequestOkCount(x => x > 190), + Threshold.RequestFailedCount(x => x <= 10), + Threshold.RequestFailedRate(x => x < 0.1), + Threshold.RPS(GetRpsConfig), + Threshold.LatencyMin(x => x < 100), + Threshold.LatencyMean(x => x < 400), + Threshold.LatencyMax(x => x < 500), + Threshold.LatencyStdDev(x => x is > 100 and < 200), + Threshold.LatencyP50(x => x < 300), + Threshold.LatencyP75(x => x < 320), + Threshold.LatencyP95(x => x < 400), + Threshold.LatencyP99(x => x < 500) ); NBomberRunner diff --git a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs index 3a97eade..cec6761f 100644 --- a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs +++ b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs @@ -25,25 +25,19 @@ let run () = |> Scenario.withWarmUpDuration(seconds 5) |> Scenario.withLoadSimulations [ InjectPerSec(rate = 20, during = seconds 30) ] |> Scenario.withThresholds [ - RequestCount [ - AllCount(fun x -> x > 290) - OkCount(fun x -> x > 180) - FailedCount(fun x -> x <= 10) - FailedRate(fun x -> x < 0.1) - RPS(fun x -> x > 15.0) - ] - Latency [ - Min(fun x -> x < 100) - Mean(fun x -> x < 200) - Max(fun x -> x < 700) - StdDev(fun x -> x > 50 && x < 100) - ] - LatencyPercentile [ - P50(fun x -> x < 200) - P75(fun x -> x < 250) - P95(fun x -> x < 400) - P99(fun x -> x < 400) - ] + RequestAllCount(fun x -> x > 290) + RequestOkCount(fun x -> x > 180) + RequestFailedCount(fun x -> x <= 10) + RequestFailedRate(fun x -> x < 0.1) + RPS(fun x -> x > 15.0) + LatencyMin(fun x -> x < 100) + LatencyMean(fun x -> x < 200) + LatencyMax(fun x -> x < 700) + LatencyStdDev(fun x -> x > 50 && x < 100) + LatencyP50(fun x -> x < 200) + LatencyP75(fun x -> x < 250) + LatencyP95(fun x -> x < 400) + LatencyP99(fun x -> x < 400) ] |> NBomberRunner.registerScenario |> NBomberRunner.run diff --git a/src/NBomber.Contracts/Stats.fs b/src/NBomber.Contracts/Stats.fs index 47b48167..d07f4952 100644 --- a/src/NBomber.Contracts/Stats.fs +++ b/src/NBomber.Contracts/Stats.fs @@ -119,10 +119,10 @@ with static member map value = if value then Passed else Failed -type ThresholdStats = - | RequestCountStats of (RequestCountThreshold * ThresholdStatus) list - | LatencyStats of (LatencyThreshold * ThresholdStatus) list - | LatencyPercentileStats of (LatencyPercentileThreshold * ThresholdStatus) list +type ThresholdStats = { + Threshold: Threshold + Status: ThresholdStatus +} type ScenarioStats = { ScenarioName: string diff --git a/src/NBomber.Contracts/Thresholds.fs b/src/NBomber.Contracts/Thresholds.fs index 5660534b..cd3773d5 100644 --- a/src/NBomber.Contracts/Thresholds.fs +++ b/src/NBomber.Contracts/Thresholds.fs @@ -1,25 +1,16 @@ namespace NBomber.Contracts.Thresholds -type RequestCountThreshold = - | AllCount of (int -> bool) - | OkCount of (int -> bool) - | FailedCount of (int -> bool) - | FailedRate of (float -> bool) - | RPS of (float -> bool) - -type LatencyThreshold = - | Min of (float -> bool) - | Mean of (float -> bool) - | Max of (float -> bool) - | StdDev of (float -> bool) - -type LatencyPercentileThreshold = - | P50 of (float -> bool) - | P75 of (float -> bool) - | P95 of (float -> bool) - | P99 of (float -> bool) - type Threshold = - | RequestCount of RequestCountThreshold list - | Latency of LatencyThreshold list - | LatencyPercentile of LatencyPercentileThreshold list + | RequestAllCount of (int -> bool) + | RequestOkCount of (int -> bool) + | RequestFailedCount of (int -> bool) + | RequestFailedRate of (float -> bool) + | RPS of (float -> bool) + | LatencyMin of (float -> bool) + | LatencyMean of (float -> bool) + | LatencyMax of (float -> bool) + | LatencyStdDev of (float -> bool) + | LatencyP50 of (float -> bool) + | LatencyP75 of (float -> bool) + | LatencyP95 of (float -> bool) + | LatencyP99 of (float -> bool) diff --git a/src/NBomber/Api/CSharp.fs b/src/NBomber/Api/CSharp.fs index d92c828a..2f31d032 100644 --- a/src/NBomber/Api/CSharp.fs +++ b/src/NBomber/Api/CSharp.fs @@ -359,50 +359,41 @@ type ValueOption = type Threshold = - static member RequestCount([]thresholds: RequestCountThreshold[]) = - RequestCount(Seq.toList thresholds) + static member RequestAllCount(predicate: Func) = + RequestAllCount(predicate.Invoke) - static member Latency([]thresholds: LatencyThreshold[]) = - Latency(Seq.toList thresholds) + static member RequestOkCount(predicate: Func) = + RequestOkCount(predicate.Invoke) - static member LatencyPercentile([]thresholds: LatencyPercentileThreshold[]) = - LatencyPercentile(Seq.toList thresholds) + static member RequestFailedCount(predicate: Func) = + RequestFailedCount(predicate.Invoke) - static member AllCount(predicate: Func) = - AllCount(predicate.Invoke) - - static member OkCount(predicate: Func) = - OkCount(predicate.Invoke) - - static member FailedCount(predicate: Func) = - FailedCount(predicate.Invoke) - - static member FailedRate(predicate: Func) = - FailedRate(predicate.Invoke) + static member RequestFailedRate(predicate: Func) = + RequestFailedRate(predicate.Invoke) static member RPS(predicate: Func) = RPS(predicate.Invoke) - static member Min(predicate: Func) = - Min(predicate.Invoke) + static member LatencyMin(predicate: Func) = + LatencyMin(predicate.Invoke) - static member Mean(predicate: Func) = - Mean(predicate.Invoke) + static member LatencyMean(predicate: Func) = + LatencyMean(predicate.Invoke) - static member Max(predicate: Func) = - Max(predicate.Invoke) + static member LatencyMax(predicate: Func) = + LatencyMax(predicate.Invoke) - static member StdDev(predicate: Func) = - StdDev(predicate.Invoke) + static member LatencyStdDev(predicate: Func) = + LatencyStdDev(predicate.Invoke) - static member P50(predicate: Func) = - P50(predicate.Invoke) + static member LatencyP50(predicate: Func) = + LatencyP50(predicate.Invoke) - static member P75(predicate: Func) = - P75(predicate.Invoke) + static member LatencyP75(predicate: Func) = + LatencyP75(predicate.Invoke) - static member P95(predicate: Func) = - P95(predicate.Invoke) + static member LatencyP95(predicate: Func) = + LatencyP95(predicate.Invoke) - static member P99(predicate: Func) = - P99(predicate.Invoke) + static member LatencyP99(predicate: Func) = + LatencyP99(predicate.Invoke) diff --git a/src/NBomber/Domain/Stats/Statistics.fs b/src/NBomber/Domain/Stats/Statistics.fs index 2875019c..2c2d4861 100644 --- a/src/NBomber/Domain/Stats/Statistics.fs +++ b/src/NBomber/Domain/Stats/Statistics.fs @@ -162,7 +162,7 @@ module ThresholdStats = && (f stats.Fail || stats.Fail.Request.Count = 0) ) - let private applyRequestCountThresholds stepStats thresholds = + let applyThresholds stepStats thresholds = let sum by = stepStats |> Array.sumBy by let okCount = sum (fun x -> x.Ok.Request.Count) let failedCount = sum (fun x -> x.Fail.Request.Count) @@ -171,62 +171,31 @@ module ThresholdStats = |> List.map (fun threshold -> let status = match threshold with - | AllCount f -> f allCount - | OkCount f -> f okCount - | FailedCount f -> f failedCount - | FailedRate f -> f (float failedCount * 100. / float allCount) + | RequestAllCount f -> f allCount + | RequestOkCount f -> f okCount + | RequestFailedCount f -> f failedCount + | RequestFailedRate f -> f (float failedCount * 100. / float allCount) | RPS f -> applySepsStats stepStats (fun s -> f s.Request.RPS) + | LatencyMin f -> applySepsStats stepStats (fun s -> f s.Latency.MinMs) + | LatencyMean f -> applySepsStats stepStats (fun s -> f s.Latency.MeanMs) + | LatencyMax f -> applySepsStats stepStats (fun s -> f s.Latency.MaxMs) + | LatencyStdDev f -> applySepsStats stepStats (fun s -> f s.Latency.StdDev) + | LatencyP50 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent50) + | LatencyP75 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent75) + | LatencyP95 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent95) + | LatencyP99 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent99) |> ThresholdStatus.map - threshold, status) - |> RequestCountStats - let private applyLatencyThresholds stepStats thresholds = - thresholds - |> List.map (fun threshold -> - let status = - match threshold with - | Min f -> (fun s -> f s.Latency.MinMs) - | Mean f -> (fun s -> f s.Latency.MeanMs) - | Max f -> (fun s -> f s.Latency.MaxMs) - | StdDev f -> (fun s -> f s.Latency.StdDev) - |> applySepsStats stepStats - |> ThresholdStatus.map - threshold, status) - |> LatencyStats - - let private applyLatencyPercentileThresholds stepStats thresholds = - thresholds - |> List.map (fun threshold -> - let status = - match threshold with - | P50 f -> (fun s -> f s.Latency.Percent50) - | P75 f -> (fun s -> f s.Latency.Percent75) - | P95 f -> (fun s -> f s.Latency.Percent95) - | P99 f -> (fun s -> f s.Latency.Percent99) - |> applySepsStats stepStats - |> ThresholdStatus.map - threshold, status) - |> LatencyPercentileStats - - let applyThresholds stepStats threshold = - match threshold with - | RequestCount thresholds -> applyRequestCountThresholds stepStats thresholds - | Latency thresholds -> applyLatencyThresholds stepStats thresholds - | LatencyPercentile thresholds -> applyLatencyPercentileThresholds stepStats thresholds + { Threshold = threshold; Status = status } + ) let failStatsExist stats = - let failed stats = - stats - |> List.map snd - |> List.exists (fun status -> - match status with - | Passed -> false - | Failed -> true - ) - match stats with - | RequestCountStats stats -> stats |> failed - | LatencyStats stats -> stats |> failed - | LatencyPercentileStats stats -> stats |> failed + stats + |> Array.exists (fun stats -> + match stats.Status with + | Passed -> false + | Failed -> true + ) module ScenarioStats = @@ -264,9 +233,7 @@ module ScenarioStats = let thresholdStats = scenario.Thresholds |> Option.map ( - ThresholdStats.applyThresholds stepStats - |> List.map - >> Array.ofList + ThresholdStats.applyThresholds stepStats >> Array.ofList ) { ScenarioName = scenario.ScenarioName @@ -288,7 +255,7 @@ module ScenarioStats = let failStepStatsExist (stats: ScenarioStats) = stats.ThresholdStats - |> Option.map (Array.exists ThresholdStats.failStatsExist) + |> Option.map ThresholdStats.failStatsExist |> Option.defaultValue (stats.StepStats |> Array.exists(fun stats -> stats.Fail.Request.Count > 0)) module NodeStats = From cbda778b8b01364a23f14c9d9fafa9b0cd9f9529 Mon Sep 17 00:00:00 2001 From: pavlogrushetsky Date: Thu, 19 May 2022 23:12:00 +0300 Subject: [PATCH 6/8] Implement DataTransfer thresholds and ConsoleReport --- .../CSharpDev/HttpTests/SimpleHttpTest.cs | 9 ++-- .../FSharpDev/HttpTests/SimpleHttpTest.fs | 11 ++--- src/NBomber.Contracts/Thresholds.fs | 23 ++++++++-- src/NBomber/Api/CSharp.fs | 43 +++++++++++++++---- src/NBomber/Domain/Stats/Statistics.fs | 40 ++++++++++------- .../DomainServices/Reports/ConsoleReport.fs | 22 ++++++++-- .../DomainServices/Reports/ReportHelper.fs | 29 +++++++++++++ 7 files changed, 137 insertions(+), 40 deletions(-) diff --git a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs index 6d12392d..2fb2300d 100644 --- a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs +++ b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs @@ -37,10 +37,11 @@ public static void Run() Threshold.LatencyMean(x => x < 400), Threshold.LatencyMax(x => x < 500), Threshold.LatencyStdDev(x => x is > 100 and < 200), - Threshold.LatencyP50(x => x < 300), - Threshold.LatencyP75(x => x < 320), - Threshold.LatencyP95(x => x < 400), - Threshold.LatencyP99(x => x < 500) + Threshold.LatencyPercent50(x => x < 300), + Threshold.LatencyPercent75(x => x < 320), + Threshold.LatencyPercent95(x => x < 400), + Threshold.LatencyPercent99(x => x < 500), + Threshold.DataTransferAllBytes(x => x < 10000) ); NBomberRunner diff --git a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs index cec6761f..45c64de3 100644 --- a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs +++ b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs @@ -28,16 +28,17 @@ let run () = RequestAllCount(fun x -> x > 290) RequestOkCount(fun x -> x > 180) RequestFailedCount(fun x -> x <= 10) - RequestFailedRate(fun x -> x < 0.1) + RequestFailedRate(fun x -> x > 0.1) RPS(fun x -> x > 15.0) LatencyMin(fun x -> x < 100) LatencyMean(fun x -> x < 200) LatencyMax(fun x -> x < 700) LatencyStdDev(fun x -> x > 50 && x < 100) - LatencyP50(fun x -> x < 200) - LatencyP75(fun x -> x < 250) - LatencyP95(fun x -> x < 400) - LatencyP99(fun x -> x < 400) + LatencyPercent50(fun x -> x < 200) + LatencyPercent75(fun x -> x < 250) + LatencyPercent95(fun x -> x < 400) + LatencyPercent99(fun x -> x < 400) + DataTransferAllBytes(fun x -> x < 10000) ] |> NBomberRunner.registerScenario |> NBomberRunner.run diff --git a/src/NBomber.Contracts/Thresholds.fs b/src/NBomber.Contracts/Thresholds.fs index cd3773d5..2accf8d0 100644 --- a/src/NBomber.Contracts/Thresholds.fs +++ b/src/NBomber.Contracts/Thresholds.fs @@ -1,5 +1,7 @@ namespace NBomber.Contracts.Thresholds +open Microsoft.FSharp.Reflection + type Threshold = | RequestAllCount of (int -> bool) | RequestOkCount of (int -> bool) @@ -10,7 +12,20 @@ type Threshold = | LatencyMean of (float -> bool) | LatencyMax of (float -> bool) | LatencyStdDev of (float -> bool) - | LatencyP50 of (float -> bool) - | LatencyP75 of (float -> bool) - | LatencyP95 of (float -> bool) - | LatencyP99 of (float -> bool) + | LatencyPercent50 of (float -> bool) + | LatencyPercent75 of (float -> bool) + | LatencyPercent95 of (float -> bool) + | LatencyPercent99 of (float -> bool) + | DataTransferMinBytes of (int -> bool) + | DataTransferMeanBytes of (int -> bool) + | DataTransferMaxBytes of (int -> bool) + | DataTransferPercent50 of (int -> bool) + | DataTransferPercent75 of (int -> bool) + | DataTransferPercent95 of (int -> bool) + | DataTransferPercent99 of (int -> bool) + | DataTransferStdDev of (float -> bool) + | DataTransferAllBytes of (int64 -> bool) +with + override this.ToString () = + match FSharpValue.GetUnionFields(this, typeof) with + | case, _ -> case.Name diff --git a/src/NBomber/Api/CSharp.fs b/src/NBomber/Api/CSharp.fs index 2f31d032..e1ffc7bf 100644 --- a/src/NBomber/Api/CSharp.fs +++ b/src/NBomber/Api/CSharp.fs @@ -386,14 +386,41 @@ type Threshold = static member LatencyStdDev(predicate: Func) = LatencyStdDev(predicate.Invoke) - static member LatencyP50(predicate: Func) = - LatencyP50(predicate.Invoke) + static member LatencyPercent50(predicate: Func) = + LatencyPercent50(predicate.Invoke) - static member LatencyP75(predicate: Func) = - LatencyP75(predicate.Invoke) + static member LatencyPercent75(predicate: Func) = + LatencyPercent75(predicate.Invoke) - static member LatencyP95(predicate: Func) = - LatencyP95(predicate.Invoke) + static member LatencyPercent95(predicate: Func) = + LatencyPercent95(predicate.Invoke) - static member LatencyP99(predicate: Func) = - LatencyP99(predicate.Invoke) + static member LatencyPercent99(predicate: Func) = + LatencyPercent99(predicate.Invoke) + + static member DataTransferMinBytes(predicate: Func) = + DataTransferMinBytes(predicate.Invoke) + + static member DataTransferMeanBytes(predicate: Func) = + DataTransferMeanBytes(predicate.Invoke) + + static member DataTransferMaxBytes(predicate: Func) = + DataTransferMaxBytes(predicate.Invoke) + + static member DataTransferPercent50(predicate: Func) = + DataTransferPercent50(predicate.Invoke) + + static member DataTransferPercent75(predicate: Func) = + DataTransferPercent75(predicate.Invoke) + + static member DataTransferPercent95(predicate: Func) = + DataTransferPercent95(predicate.Invoke) + + static member DataTransferPercent99(predicate: Func) = + DataTransferPercent99(predicate.Invoke) + + static member DataTransferStdDev(predicate: Func) = + DataTransferStdDev(predicate.Invoke) + + static member DataTransferAllBytes(predicate: Func) = + DataTransferAllBytes(predicate.Invoke) diff --git a/src/NBomber/Domain/Stats/Statistics.fs b/src/NBomber/Domain/Stats/Statistics.fs index 2c2d4861..cdc834a3 100644 --- a/src/NBomber/Domain/Stats/Statistics.fs +++ b/src/NBomber/Domain/Stats/Statistics.fs @@ -152,7 +152,7 @@ module StepStats = let getAllRequestCount (stats: StepStats) = stats.Ok.Request.Count + stats.Fail.Request.Count -module ThresholdStats = +module private ThresholdStats = let private applySepsStats stats f = (true, stats) @@ -180,23 +180,24 @@ module ThresholdStats = | LatencyMean f -> applySepsStats stepStats (fun s -> f s.Latency.MeanMs) | LatencyMax f -> applySepsStats stepStats (fun s -> f s.Latency.MaxMs) | LatencyStdDev f -> applySepsStats stepStats (fun s -> f s.Latency.StdDev) - | LatencyP50 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent50) - | LatencyP75 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent75) - | LatencyP95 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent95) - | LatencyP99 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent99) + | LatencyPercent50 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent50) + | LatencyPercent75 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent75) + | LatencyPercent95 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent95) + | LatencyPercent99 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent99) + | DataTransferMinBytes f -> applySepsStats stepStats (fun s -> f s.DataTransfer.MinBytes) + | DataTransferMeanBytes f -> applySepsStats stepStats (fun s -> f s.DataTransfer.MeanBytes) + | DataTransferMaxBytes f -> applySepsStats stepStats (fun s -> f s.DataTransfer.MaxBytes) + | DataTransferPercent50 f -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent50) + | DataTransferPercent75 f -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent75) + | DataTransferPercent95 f -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent95) + | DataTransferPercent99 f -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent99) + | DataTransferStdDev f -> applySepsStats stepStats (fun s -> f s.DataTransfer.StdDev) + | DataTransferAllBytes f -> applySepsStats stepStats (fun s -> f s.DataTransfer.AllBytes) |> ThresholdStatus.map { Threshold = threshold; Status = status } ) - let failStatsExist stats = - stats - |> Array.exists (fun stats -> - match stats.Status with - | Passed -> false - | Failed -> true - ) - module ScenarioStats = let create (scenario: Scenario) @@ -254,9 +255,18 @@ module ScenarioStats = Duration = TimeSpan(stats.Duration.Days, stats.Duration.Hours, stats.Duration.Minutes, stats.Duration.Seconds) } let failStepStatsExist (stats: ScenarioStats) = + stats.StepStats |> Array.exists(fun stats -> stats.Fail.Request.Count > 0) + + let failThresholdStatsExist stats = stats.ThresholdStats - |> Option.map ThresholdStats.failStatsExist - |> Option.defaultValue (stats.StepStats |> Array.exists(fun stats -> stats.Fail.Request.Count > 0)) + |> Option.map ( + Array.exists (fun stats -> + match stats.Status with + | Passed -> false + | Failed -> true + ) + ) + |> Option.defaultValue false module NodeStats = diff --git a/src/NBomber/DomainServices/Reports/ConsoleReport.fs b/src/NBomber/DomainServices/Reports/ConsoleReport.fs index 30d1f489..7b8664c8 100644 --- a/src/NBomber/DomainServices/Reports/ConsoleReport.fs +++ b/src/NBomber/DomainServices/Reports/ConsoleReport.fs @@ -4,11 +4,11 @@ open System open System.Collections.Generic open System.Data +open NBomber.Domain.Stats.Statistics open Serilog open NBomber.Contracts open NBomber.Contracts.Stats -open NBomber.Domain.Stats open NBomber.Extensions.Data open NBomber.Extensions.Internal open NBomber.Infra @@ -51,7 +51,7 @@ module ConsoleNodeStats = Console.addLine $" - duration: {Console.okEscColor scnStats.Duration}" ] let private printStepStatsHeader (stepStats: StepStats[]) = - let print (stats) = seq { + let print stats = seq { $"step: {Console.blueEscColor stats.StepName}" $" - timeout: {Console.okEscColor stats.StepInfo.Timeout.TotalMilliseconds} ms" $" - client factory: {Console.okEscColor stats.StepInfo.ClientFactoryName}, clients: {Console.okEscColor stats.StepInfo.ClientFactoryClientCount}" @@ -70,6 +70,16 @@ module ConsoleNodeStats = [ ConsoleStatusCodesStats.printScenarioHeader scnStats.ScenarioName ConsoleStatusCodesStats.printStatusCodeTable scnStats ] + let private printThresholdStats (scnStats: ScenarioStats) = + let headers = ["threshold"; "status"] + let rows = + scnStats.ThresholdStats + |> Option.map (ReportHelper.ThresholdStats.createTableRows Console.okEscColor Console.errorEscColor) + |> Option.defaultValue List.empty + + [ Console.addLine $"thresholds for scenario: {Console.okColor scnStats.ScenarioName}" + Console.addTable headers rows ] + let private printScenarioStats (scnStats: ScenarioStats) (simulations: LoadSimulation list) = [ yield! printScenarioHeader scnStats Console.addLine String.Empty @@ -82,12 +92,16 @@ module ConsoleNodeStats = printStepStatsTable true scnStats.StepStats - if Statistics.ScenarioStats.failStepStatsExist scnStats then + if ScenarioStats.failStepStatsExist scnStats then printStepStatsTable false scnStats.StepStats if scnStats.StatusCodes.Length > 0 then Console.addLine String.Empty - yield! printScenarioStatusCodes scnStats ] + yield! printScenarioStatusCodes scnStats + + if ScenarioStats.failThresholdStatsExist scnStats then + Console.addLine String.Empty + yield! printThresholdStats scnStats ] let printNodeStats (stats: NodeStats) (loadSimulations: IDictionary) = let scenarioStats = diff --git a/src/NBomber/DomainServices/Reports/ReportHelper.fs b/src/NBomber/DomainServices/Reports/ReportHelper.fs index b19c36ea..706b3c6d 100644 --- a/src/NBomber/DomainServices/Reports/ReportHelper.fs +++ b/src/NBomber/DomainServices/Reports/ReportHelper.fs @@ -123,3 +123,32 @@ module StatusCodesStats = // all status codes okNotAvailableStatusCodes @ okStatusCodes @ failNotAvailableStatusCodes @ failStatusCodes + +module ThresholdStats = + + let createTableRows (okColor: obj -> string) + (errorColor: obj -> string) + (thresholdStats: ThresholdStats[]) = + + let createThresholdRows f thresholdStats = + thresholdStats + |> Seq.choose f + |> Seq.toList + + let okThresholds = + thresholdStats + |> createThresholdRows(fun x -> + match x.Status with + | Passed -> Some [$"{x.Threshold}"; okColor "passed"] + | Failed -> None + ) + + let failThresholds = + thresholdStats + |> createThresholdRows(fun x -> + match x.Status with + | Failed -> Some [$"{x.Threshold}"; errorColor "failed"] + | Passed -> None + ) + + okThresholds @ failThresholds From 1fe31219bc66f72d41b974dd30c72fc8c678a3d3 Mon Sep 17 00:00:00 2001 From: pavlogrushetsky Date: Fri, 20 May 2022 21:14:39 +0300 Subject: [PATCH 7/8] Implement thresholds validation --- src/NBomber.Contracts/Thresholds.fs | 4 +++- src/NBomber/Domain/Errors.fs | 8 ++++++++ src/NBomber/Domain/Scenario.fs | 20 ++++++++++++++++++++ src/NBomber/Domain/Stats/Statistics.fs | 2 +- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/NBomber.Contracts/Thresholds.fs b/src/NBomber.Contracts/Thresholds.fs index 2accf8d0..1b6338df 100644 --- a/src/NBomber.Contracts/Thresholds.fs +++ b/src/NBomber.Contracts/Thresholds.fs @@ -26,6 +26,8 @@ type Threshold = | DataTransferStdDev of (float -> bool) | DataTransferAllBytes of (int64 -> bool) with - override this.ToString () = + member this.Name = match FSharpValue.GetUnionFields(this, typeof) with | case, _ -> case.Name + + override this.ToString () = this.Name diff --git a/src/NBomber/Domain/Errors.fs b/src/NBomber/Domain/Errors.fs index a3bcdca6..4e96ecae 100644 --- a/src/NBomber/Domain/Errors.fs +++ b/src/NBomber/Domain/Errors.fs @@ -29,6 +29,8 @@ type ValidationError = | InvalidClientFactoryName of factoryName:string | DuplicateClientFactoryName of scenarioName:string * factoryName:string | DuplicateStepNameButDiffImpl of scenarioName:string * stepName:string + | EmptyThresholds of scenarioName:string + | DuplicateThresholdName of scenarioName:string * thresholdName:string // ScenarioSettings | CustomStepOrderContainsNotFoundStepName of scenarioName:string * stepName:string @@ -128,6 +130,12 @@ type AppError = | EnterpriseOnlyFeature message -> message + | EmptyThresholds scenarioName -> + $"Scenario: '{scenarioName}' has no thresholds" + + | DuplicateThresholdName(scenarioName, thresholdName) -> + $"Scenario: '{scenarioName}' contains thresholds with duplicated name: '{thresholdName}'" + static member toString (error: AppError) = match error with | Domain e -> AppError.toString(e) diff --git a/src/NBomber/Domain/Scenario.fs b/src/NBomber/Domain/Scenario.fs index 9db3f545..dd44be8f 100644 --- a/src/NBomber/Domain/Scenario.fs +++ b/src/NBomber/Domain/Scenario.fs @@ -10,6 +10,7 @@ open FsToolkit.ErrorHandling open Microsoft.Extensions.Configuration open NBomber +open NBomber.Contracts.Thresholds open NBomber.Extensions.Internal open NBomber.Extensions.Operator.Result open NBomber.Configuration @@ -74,6 +75,24 @@ module Validation = | Ok _ -> Ok scenario | Error errors -> errors |> List.head |> AppError.createResult + let checkEmptyOrDuplicatedThresholds (scenario: Contracts.Scenario) = + scenario.Thresholds + |> Option.map (fun thresholds -> + match thresholds with + | [] -> AppError.createResult(EmptyThresholds scenario.ScenarioName) + | _ -> + thresholds + |> List.groupBy (fun x -> x.Name) + |> List.choose (fun (name, thresholds) -> + if List.length(thresholds) > 1 then Some name + else None + ) + |> function + | [] -> Ok scenario + | thresholdName::_ -> AppError.createResult(DuplicateThresholdName(scenario.ScenarioName, thresholdName)) + ) + |> Option.defaultValue (Ok scenario) + let validate = checkEmptyScenarioName >=> checkInitOnlyScenario @@ -81,6 +100,7 @@ module Validation = >=> checkDuplicateStepNameButDiffImpl >=> checkClientFactoryName >=> checkDuplicateClientFactories + >=> checkEmptyOrDuplicatedThresholds module ClientFactory = diff --git a/src/NBomber/Domain/Stats/Statistics.fs b/src/NBomber/Domain/Stats/Statistics.fs index cdc834a3..be414318 100644 --- a/src/NBomber/Domain/Stats/Statistics.fs +++ b/src/NBomber/Domain/Stats/Statistics.fs @@ -247,7 +247,7 @@ module ScenarioStats = LoadSimulationStats = simulationStats StatusCodes = statusCodes CurrentOperation = currentOperation - Duration = duration + Duration = %duration ThresholdStats = thresholdStats } let round (stats: ScenarioStats) = From a56c012853a3a2b895e33f74bc12105bda8ffc19 Mon Sep 17 00:00:00 2001 From: pavlogrushetsky Date: Mon, 30 May 2022 22:55:45 +0300 Subject: [PATCH 8/8] Implement thresholds CE --- .../CSharpDev/HttpTests/SimpleHttpTest.cs | 4 +- .../FSharpDev/HttpTests/SimpleHttpTest.fs | 38 ++++--- src/NBomber.Contracts/Thresholds.fs | 105 +++++++++++++----- src/NBomber/Api/CSharp.fs | 88 +++++++-------- src/NBomber/Api/FSharp.fs | 66 +++++++++++ src/NBomber/Domain/Errors.fs | 4 - src/NBomber/Domain/Scenario.fs | 15 +-- src/NBomber/Domain/Stats/Statistics.fs | 44 ++++---- src/NBomber/NBomber.fsproj | 6 +- 9 files changed, 240 insertions(+), 130 deletions(-) diff --git a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs index 2fb2300d..6f1e7048 100644 --- a/examples/CSharpDev/HttpTests/SimpleHttpTest.cs +++ b/examples/CSharpDev/HttpTests/SimpleHttpTest.cs @@ -28,7 +28,7 @@ public static void Run() Simulation.InjectPerSec(rate: 1, during: TimeSpan.FromSeconds(30)) ) .WithThresholds( - Threshold.RequestAllCount(x => x > 200), + Threshold.RequestAllCount(x => x > 200, "request all count > 200"), Threshold.RequestOkCount(x => x > 190), Threshold.RequestFailedCount(x => x <= 10), Threshold.RequestFailedRate(x => x < 0.1), @@ -36,7 +36,7 @@ public static void Run() Threshold.LatencyMin(x => x < 100), Threshold.LatencyMean(x => x < 400), Threshold.LatencyMax(x => x < 500), - Threshold.LatencyStdDev(x => x is > 100 and < 200), + Threshold.LatencyStdDev(x => x is > 100 and < 200, "latency standard deviation > 50 and < 100"), Threshold.LatencyPercent50(x => x < 300), Threshold.LatencyPercent75(x => x < 320), Threshold.LatencyPercent95(x => x < 400), diff --git a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs index 45c64de3..e4cda5f6 100644 --- a/examples/FSharpDev/HttpTests/SimpleHttpTest.fs +++ b/examples/FSharpDev/HttpTests/SimpleHttpTest.fs @@ -24,22 +24,28 @@ let run () = Scenario.create "simple_http" [step] |> Scenario.withWarmUpDuration(seconds 5) |> Scenario.withLoadSimulations [ InjectPerSec(rate = 20, during = seconds 30) ] - |> Scenario.withThresholds [ - RequestAllCount(fun x -> x > 290) - RequestOkCount(fun x -> x > 180) - RequestFailedCount(fun x -> x <= 10) - RequestFailedRate(fun x -> x > 0.1) - RPS(fun x -> x > 15.0) - LatencyMin(fun x -> x < 100) - LatencyMean(fun x -> x < 200) - LatencyMax(fun x -> x < 700) - LatencyStdDev(fun x -> x > 50 && x < 100) - LatencyPercent50(fun x -> x < 200) - LatencyPercent75(fun x -> x < 250) - LatencyPercent95(fun x -> x < 400) - LatencyPercent99(fun x -> x < 400) - DataTransferAllBytes(fun x -> x < 10000) - ] + |> Scenario.withThresholds( + thresholds { + request_all_count (fun x -> x > 1290) "request all count > 290" + request_ok_count (fun x -> x > 180) + request_failed_count (fun x -> x <= 10) + request_failed_rate (fun x -> x > 0.1) + rps (fun x -> x > 15.0) + latency_min (fun x -> x < 100) + latency_mean (fun x -> x < 200) + latency_max (fun x -> x < 700) + latency_std_dev (fun x -> x > 50 && x < 100) "latency standard deviation > 50 and < 100" + latency_p50 (fun x -> x < 200) + latency_p75 (fun x -> x < 250) + latency_p95 (fun x -> x < 400) + latency_p99 (fun x -> x < 400) + data_transfer_all_bytes (fun x -> x < 10000) + } + ) +// |> Scenario.withThresholds [ +// RequestAllCount((fun x -> x > 1290), Some "request all count > 290") +// RequestOkCount((fun x -> x > 180), None) +// ] |> NBomberRunner.registerScenario |> NBomberRunner.run |> ignore diff --git a/src/NBomber.Contracts/Thresholds.fs b/src/NBomber.Contracts/Thresholds.fs index 1b6338df..50e5492c 100644 --- a/src/NBomber.Contracts/Thresholds.fs +++ b/src/NBomber.Contracts/Thresholds.fs @@ -1,33 +1,84 @@ namespace NBomber.Contracts.Thresholds -open Microsoft.FSharp.Reflection +module private ThresholdDefaultDescriptions = + + let [] RequestAllCountDefaultDescription = "request count - all" + let [] RequestOkCountDefaultDescription = "request count - ok" + let [] RequestFailedCountDefaultDescription = "request count - failed" + let [] RequestFailedRateDefaultDescription = "request rate - failed" + let [] RPSDefaultDescription = "request count - RPS" + let [] LatencyMinDefaultDescription = "latency - min" + let [] LatencyMeanDefaultDescription = "latency - mean" + let [] LatencyMaxDefaultDescription = "latency - max" + let [] LatencyStdDevDefaultDescription = "latency - StdDev" + let [] LatencyPercent50DefaultDescription = "latency percentile - 50%" + let [] LatencyPercent75DefaultDescription = "latency percentile - 75%" + let [] LatencyPercent95DefaultDescription = "latency percentile - 95%" + let [] LatencyPercent99DefaultDescription = "latency percentile - 99%" + let [] DataTransferMinBytesDefaultDescription = "data transfer bytes - min" + let [] DataTransferMeanBytesDefaultDescription = "data transfer bytes - mean" + let [] DataTransferMaxBytesDefaultDescription = "data transfer bytes - max" + let [] DataTransferAllBytesDefaultDescription = "data transfer bytes - all" + let [] DataTransferStdDevDefaultDescription = "data transfer - StdDev" + let [] DataTransferPercent50DefaultDescription = "data transfer percentile - 50%" + let [] DataTransferPercent75DefaultDescription = "data transfer percentile - 75%" + let [] DataTransferPercent95DefaultDescription = "data transfer percentile - 95%" + let [] DataTransferPercent99DefaultDescription = "data transfer percentile - 99%" + +open ThresholdDefaultDescriptions + +type ThresholdBody<'a> = ('a -> bool) * string option type Threshold = - | RequestAllCount of (int -> bool) - | RequestOkCount of (int -> bool) - | RequestFailedCount of (int -> bool) - | RequestFailedRate of (float -> bool) - | RPS of (float -> bool) - | LatencyMin of (float -> bool) - | LatencyMean of (float -> bool) - | LatencyMax of (float -> bool) - | LatencyStdDev of (float -> bool) - | LatencyPercent50 of (float -> bool) - | LatencyPercent75 of (float -> bool) - | LatencyPercent95 of (float -> bool) - | LatencyPercent99 of (float -> bool) - | DataTransferMinBytes of (int -> bool) - | DataTransferMeanBytes of (int -> bool) - | DataTransferMaxBytes of (int -> bool) - | DataTransferPercent50 of (int -> bool) - | DataTransferPercent75 of (int -> bool) - | DataTransferPercent95 of (int -> bool) - | DataTransferPercent99 of (int -> bool) - | DataTransferStdDev of (float -> bool) - | DataTransferAllBytes of (int64 -> bool) + | RequestAllCount of ThresholdBody + | RequestOkCount of ThresholdBody + | RequestFailedCount of ThresholdBody + | RequestFailedRate of ThresholdBody + | RPS of ThresholdBody + | LatencyMin of ThresholdBody + | LatencyMean of ThresholdBody + | LatencyMax of ThresholdBody + | LatencyStdDev of ThresholdBody + | LatencyPercent50 of ThresholdBody + | LatencyPercent75 of ThresholdBody + | LatencyPercent95 of ThresholdBody + | LatencyPercent99 of ThresholdBody + | DataTransferMinBytes of ThresholdBody + | DataTransferMeanBytes of ThresholdBody + | DataTransferMaxBytes of ThresholdBody + | DataTransferPercent50 of ThresholdBody + | DataTransferPercent75 of ThresholdBody + | DataTransferPercent95 of ThresholdBody + | DataTransferPercent99 of ThresholdBody + | DataTransferStdDev of ThresholdBody + | DataTransferAllBytes of ThresholdBody with - member this.Name = - match FSharpValue.GetUnionFields(this, typeof) with - | case, _ -> case.Name + member this.Description = + match this with + | RequestAllCount (_, desc) -> desc, RequestAllCountDefaultDescription + | RequestOkCount (_, desc) -> desc, RequestOkCountDefaultDescription + | RequestFailedCount (_, desc) -> desc, RequestFailedCountDefaultDescription + | RequestFailedRate (_, desc) -> desc, RequestFailedRateDefaultDescription + | RPS (_, desc) -> desc, RPSDefaultDescription + | LatencyMin (_, desc) -> desc, LatencyMinDefaultDescription + | LatencyMean (_, desc) -> desc, LatencyMeanDefaultDescription + | LatencyMax (_, desc) -> desc, LatencyMaxDefaultDescription + | LatencyStdDev (_, desc) -> desc, LatencyStdDevDefaultDescription + | LatencyPercent50 (_, desc) -> desc, LatencyPercent50DefaultDescription + | LatencyPercent75 (_, desc) -> desc, LatencyPercent75DefaultDescription + | LatencyPercent95 (_, desc) -> desc, LatencyPercent95DefaultDescription + | LatencyPercent99 (_, desc) -> desc, LatencyPercent99DefaultDescription + | DataTransferMinBytes (_, desc) -> desc, DataTransferMinBytesDefaultDescription + | DataTransferMeanBytes (_, desc) -> desc, DataTransferMeanBytesDefaultDescription + | DataTransferMaxBytes (_, desc) -> desc, DataTransferMaxBytesDefaultDescription + | DataTransferPercent50 (_, desc) -> desc, DataTransferPercent50DefaultDescription + | DataTransferPercent75 (_, desc) -> desc, DataTransferPercent75DefaultDescription + | DataTransferPercent95 (_, desc) -> desc, DataTransferPercent95DefaultDescription + | DataTransferPercent99 (_, desc) -> desc, DataTransferPercent99DefaultDescription + | DataTransferStdDev (_, desc) -> desc, DataTransferStdDevDefaultDescription + | DataTransferAllBytes (_, desc) -> desc, DataTransferAllBytesDefaultDescription + |> fun x -> + match x with + | desc, defaultDesc -> desc |> Option.defaultValue defaultDesc - override this.ToString () = this.Name + override this.ToString () = this.Description diff --git a/src/NBomber/Api/CSharp.fs b/src/NBomber/Api/CSharp.fs index 1dc34f1d..86f72751 100644 --- a/src/NBomber/Api/CSharp.fs +++ b/src/NBomber/Api/CSharp.fs @@ -367,68 +367,68 @@ type ValueOption = type Threshold = - static member RequestAllCount(predicate: Func) = - RequestAllCount(predicate.Invoke) + static member RequestAllCount(predicate: Func, [] description: string) = + RequestAllCount(predicate.Invoke, Option.ofObj description) - static member RequestOkCount(predicate: Func) = - RequestOkCount(predicate.Invoke) + static member RequestOkCount(predicate: Func, [] description: string) = + RequestOkCount(predicate.Invoke, Option.ofObj description) - static member RequestFailedCount(predicate: Func) = - RequestFailedCount(predicate.Invoke) + static member RequestFailedCount(predicate: Func, [] description: string) = + RequestFailedCount(predicate.Invoke, Option.ofObj description) - static member RequestFailedRate(predicate: Func) = - RequestFailedRate(predicate.Invoke) + static member RequestFailedRate(predicate: Func, [] description: string) = + RequestFailedRate(predicate.Invoke, Option.ofObj description) - static member RPS(predicate: Func) = - RPS(predicate.Invoke) + static member RPS(predicate: Func, [] description: string) = + RPS(predicate.Invoke, Option.ofObj description) - static member LatencyMin(predicate: Func) = - LatencyMin(predicate.Invoke) + static member LatencyMin(predicate: Func, [] description: string) = + LatencyMin(predicate.Invoke, Option.ofObj description) - static member LatencyMean(predicate: Func) = - LatencyMean(predicate.Invoke) + static member LatencyMean(predicate: Func, [] description: string) = + LatencyMean(predicate.Invoke, Option.ofObj description) - static member LatencyMax(predicate: Func) = - LatencyMax(predicate.Invoke) + static member LatencyMax(predicate: Func, [] description: string) = + LatencyMax(predicate.Invoke, Option.ofObj description) - static member LatencyStdDev(predicate: Func) = - LatencyStdDev(predicate.Invoke) + static member LatencyStdDev(predicate: Func, [] description: string) = + LatencyStdDev(predicate.Invoke, Option.ofObj description) - static member LatencyPercent50(predicate: Func) = - LatencyPercent50(predicate.Invoke) + static member LatencyPercent50(predicate: Func, [] description: string) = + LatencyPercent50(predicate.Invoke, Option.ofObj description) - static member LatencyPercent75(predicate: Func) = - LatencyPercent75(predicate.Invoke) + static member LatencyPercent75(predicate: Func, [] description: string) = + LatencyPercent75(predicate.Invoke, Option.ofObj description) - static member LatencyPercent95(predicate: Func) = - LatencyPercent95(predicate.Invoke) + static member LatencyPercent95(predicate: Func, [] description: string) = + LatencyPercent95(predicate.Invoke, Option.ofObj description) - static member LatencyPercent99(predicate: Func) = - LatencyPercent99(predicate.Invoke) + static member LatencyPercent99(predicate: Func, [] description: string) = + LatencyPercent99(predicate.Invoke, Option.ofObj description) - static member DataTransferMinBytes(predicate: Func) = - DataTransferMinBytes(predicate.Invoke) + static member DataTransferMinBytes(predicate: Func, [] description: string) = + DataTransferMinBytes(predicate.Invoke, Option.ofObj description) - static member DataTransferMeanBytes(predicate: Func) = - DataTransferMeanBytes(predicate.Invoke) + static member DataTransferMeanBytes(predicate: Func, [] description: string) = + DataTransferMeanBytes(predicate.Invoke, Option.ofObj description) - static member DataTransferMaxBytes(predicate: Func) = - DataTransferMaxBytes(predicate.Invoke) + static member DataTransferMaxBytes(predicate: Func, [] description: string) = + DataTransferMaxBytes(predicate.Invoke, Option.ofObj description) - static member DataTransferPercent50(predicate: Func) = - DataTransferPercent50(predicate.Invoke) + static member DataTransferPercent50(predicate: Func, [] description: string) = + DataTransferPercent50(predicate.Invoke, Option.ofObj description) - static member DataTransferPercent75(predicate: Func) = - DataTransferPercent75(predicate.Invoke) + static member DataTransferPercent75(predicate: Func, [] description: string) = + DataTransferPercent75(predicate.Invoke, Option.ofObj description) - static member DataTransferPercent95(predicate: Func) = - DataTransferPercent95(predicate.Invoke) + static member DataTransferPercent95(predicate: Func, [] description: string) = + DataTransferPercent95(predicate.Invoke, Option.ofObj description) - static member DataTransferPercent99(predicate: Func) = - DataTransferPercent99(predicate.Invoke) + static member DataTransferPercent99(predicate: Func, [] description: string) = + DataTransferPercent99(predicate.Invoke, Option.ofObj description) - static member DataTransferStdDev(predicate: Func) = - DataTransferStdDev(predicate.Invoke) + static member DataTransferStdDev(predicate: Func, [] description: string) = + DataTransferStdDev(predicate.Invoke, Option.ofObj description) - static member DataTransferAllBytes(predicate: Func) = - DataTransferAllBytes(predicate.Invoke) + static member DataTransferAllBytes(predicate: Func, [] description: string) = + DataTransferAllBytes(predicate.Invoke, Option.ofObj description) diff --git a/src/NBomber/Api/FSharp.fs b/src/NBomber/Api/FSharp.fs index 5771c3c2..ee88fcba 100644 --- a/src/NBomber/Api/FSharp.fs +++ b/src/NBomber/Api/FSharp.fs @@ -393,3 +393,69 @@ module NBomberRunner = |> runWithResult args |> Result.map(fun x -> x.FinalStats) |> Result.mapError AppError.toString + +type ThresholdsBuilder () = + member _.Yield _ = [ ] + + member _.Run state = state + + [] + member _.RequestAllCount(state, predicate, ?description: string) = + state @ [ RequestAllCount(predicate, description) ] + + [] + member _.RequestOkCount(state, predicate, ?description: string) = + state @ [ RequestOkCount(predicate, description) ] + + [] + member _.RequestFailedCount(state, predicate, ?description: string) = + state @ [ RequestFailedCount(predicate, description) ] + + [] + member _.RequestFailedRate(state, predicate, ?description: string) = + state @ [ RequestFailedRate(predicate, description) ] + + [] + member _.RPS(state, predicate, ?description: string) = + state @ [ RPS(predicate, description) ] + + [] + member _.LatencyMin(state, predicate, ?description: string) = + state @ [ LatencyMin(predicate, description) ] + + [] + member _.LatencyMean(state, predicate, ?description: string) = + state @ [ LatencyMean(predicate, description) ] + + [] + member _.LatencyMax(state, predicate, ?description: string) = + state @ [ LatencyMax(predicate, description) ] + + [] + member _.LatencyStdDev(state, predicate, ?description: string) = + state @ [ LatencyStdDev(predicate, description) ] + + [] + member _.LatencyPercent50(state, predicate, ?description: string) = + state @ [ LatencyPercent50(predicate, description) ] + + [] + member _.LatencyPercent75(state, predicate, ?description: string) = + state @ [ LatencyPercent75(predicate, description) ] + + [] + member _.LatencyPercent95(state, predicate, ?description: string) = + state @ [ LatencyPercent95(predicate, description) ] + + [] + member _.LatencyPercent99(state, predicate, ?description: string) = + state @ [ LatencyPercent99(predicate, description) ] + + [] + member _.DataTransferAllBytes(state, predicate, ?description: string) = + state @ [ DataTransferAllBytes(predicate, description) ] + +[] +module ComputationExpressions = + + let thresholds = ThresholdsBuilder() diff --git a/src/NBomber/Domain/Errors.fs b/src/NBomber/Domain/Errors.fs index 4e96ecae..3ca961e5 100644 --- a/src/NBomber/Domain/Errors.fs +++ b/src/NBomber/Domain/Errors.fs @@ -30,7 +30,6 @@ type ValidationError = | DuplicateClientFactoryName of scenarioName:string * factoryName:string | DuplicateStepNameButDiffImpl of scenarioName:string * stepName:string | EmptyThresholds of scenarioName:string - | DuplicateThresholdName of scenarioName:string * thresholdName:string // ScenarioSettings | CustomStepOrderContainsNotFoundStepName of scenarioName:string * stepName:string @@ -133,9 +132,6 @@ type AppError = | EmptyThresholds scenarioName -> $"Scenario: '{scenarioName}' has no thresholds" - | DuplicateThresholdName(scenarioName, thresholdName) -> - $"Scenario: '{scenarioName}' contains thresholds with duplicated name: '{thresholdName}'" - static member toString (error: AppError) = match error with | Domain e -> AppError.toString(e) diff --git a/src/NBomber/Domain/Scenario.fs b/src/NBomber/Domain/Scenario.fs index 36155602..a22f297b 100644 --- a/src/NBomber/Domain/Scenario.fs +++ b/src/NBomber/Domain/Scenario.fs @@ -75,21 +75,12 @@ module Validation = | Ok _ -> Ok scenario | Error errors -> errors |> List.head |> AppError.createResult - let checkEmptyOrDuplicatedThresholds (scenario: Contracts.Scenario) = + let checkEmptyThresholds (scenario: Contracts.Scenario) = scenario.Thresholds |> Option.map (fun thresholds -> match thresholds with | [] -> AppError.createResult(EmptyThresholds scenario.ScenarioName) - | _ -> - thresholds - |> List.groupBy (fun x -> x.Name) - |> List.choose (fun (name, thresholds) -> - if List.length(thresholds) > 1 then Some name - else None - ) - |> function - | [] -> Ok scenario - | thresholdName::_ -> AppError.createResult(DuplicateThresholdName(scenario.ScenarioName, thresholdName)) + | _ -> Ok scenario ) |> Option.defaultValue (Ok scenario) @@ -100,7 +91,7 @@ module Validation = >=> checkDuplicateStepNameButDiffImpl >=> checkClientFactoryName >=> checkDuplicateClientFactories - >=> checkEmptyOrDuplicatedThresholds + >=> checkEmptyThresholds module ClientFactory = diff --git a/src/NBomber/Domain/Stats/Statistics.fs b/src/NBomber/Domain/Stats/Statistics.fs index be414318..cb5afc0b 100644 --- a/src/NBomber/Domain/Stats/Statistics.fs +++ b/src/NBomber/Domain/Stats/Statistics.fs @@ -171,28 +171,28 @@ module private ThresholdStats = |> List.map (fun threshold -> let status = match threshold with - | RequestAllCount f -> f allCount - | RequestOkCount f -> f okCount - | RequestFailedCount f -> f failedCount - | RequestFailedRate f -> f (float failedCount * 100. / float allCount) - | RPS f -> applySepsStats stepStats (fun s -> f s.Request.RPS) - | LatencyMin f -> applySepsStats stepStats (fun s -> f s.Latency.MinMs) - | LatencyMean f -> applySepsStats stepStats (fun s -> f s.Latency.MeanMs) - | LatencyMax f -> applySepsStats stepStats (fun s -> f s.Latency.MaxMs) - | LatencyStdDev f -> applySepsStats stepStats (fun s -> f s.Latency.StdDev) - | LatencyPercent50 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent50) - | LatencyPercent75 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent75) - | LatencyPercent95 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent95) - | LatencyPercent99 f -> applySepsStats stepStats (fun s -> f s.Latency.Percent99) - | DataTransferMinBytes f -> applySepsStats stepStats (fun s -> f s.DataTransfer.MinBytes) - | DataTransferMeanBytes f -> applySepsStats stepStats (fun s -> f s.DataTransfer.MeanBytes) - | DataTransferMaxBytes f -> applySepsStats stepStats (fun s -> f s.DataTransfer.MaxBytes) - | DataTransferPercent50 f -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent50) - | DataTransferPercent75 f -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent75) - | DataTransferPercent95 f -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent95) - | DataTransferPercent99 f -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent99) - | DataTransferStdDev f -> applySepsStats stepStats (fun s -> f s.DataTransfer.StdDev) - | DataTransferAllBytes f -> applySepsStats stepStats (fun s -> f s.DataTransfer.AllBytes) + | RequestAllCount (f, _) -> f allCount + | RequestOkCount (f, _) -> f okCount + | RequestFailedCount (f, _) -> f failedCount + | RequestFailedRate (f, _) -> f (float failedCount * 100. / float allCount) + | RPS (f, _) -> applySepsStats stepStats (fun s -> f s.Request.RPS) + | LatencyMin (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.MinMs) + | LatencyMean (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.MeanMs) + | LatencyMax (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.MaxMs) + | LatencyStdDev (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.StdDev) + | LatencyPercent50 (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.Percent50) + | LatencyPercent75 (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.Percent75) + | LatencyPercent95 (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.Percent95) + | LatencyPercent99 (f, _) -> applySepsStats stepStats (fun s -> f s.Latency.Percent99) + | DataTransferMinBytes (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.MinBytes) + | DataTransferMeanBytes (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.MeanBytes) + | DataTransferMaxBytes (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.MaxBytes) + | DataTransferPercent50 (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent50) + | DataTransferPercent75 (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent75) + | DataTransferPercent95 (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent95) + | DataTransferPercent99 (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.Percent99) + | DataTransferStdDev (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.StdDev) + | DataTransferAllBytes (f, _) -> applySepsStats stepStats (fun s -> f s.DataTransfer.AllBytes) |> ThresholdStatus.map { Threshold = threshold; Status = status } diff --git a/src/NBomber/NBomber.fsproj b/src/NBomber/NBomber.fsproj index fc234be5..3f2d2306 100644 --- a/src/NBomber/NBomber.fsproj +++ b/src/NBomber/NBomber.fsproj @@ -110,7 +110,7 @@ - - - + + +