From cd117e2a5f7cec9c0c3adc6df4e5c7f8ededc6f6 Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Fri, 17 Jul 2026 14:10:21 +0200 Subject: [PATCH 01/11] Initial implementation of MaxFileCount --- .../tools/testing/loadtest/lib/request.h | 58 +++++++++++++++++-- .../testing/loadtest/lib/request_data.cpp | 18 +++++- .../loadtest/lib/request_datashard_like.cpp | 18 +++++- .../loadtest/lib/request_fastshard.cpp | 17 +++++- .../loadtest/lib/request_fastshard_stub.cpp | 4 +- .../testing/loadtest/lib/request_index.cpp | 20 ++++++- .../testing/loadtest/lib/request_replay.cpp | 4 +- .../testing/loadtest/lib/request_replay.h | 7 ++- .../loadtest/lib/request_replay_fs.cpp | 18 ++++-- .../loadtest/lib/request_replay_grpc.cpp | 28 +++++++-- .../tools/testing/loadtest/lib/test.cpp | 20 +++++-- .../testing/loadtest/protos/loadtest.proto | 3 + 12 files changed, 180 insertions(+), 35 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index 5765fac69ef..e16de005b74 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -16,8 +16,48 @@ #include #include +#include +#include + namespace NCloud::NFileStore::NLoadTest { +class TFileCreationLimiter +{ +private: + const ui64 MaxFileCount; + std::atomic ReservedFileCount = 0; + +public: + explicit TFileCreationLimiter(ui64 maxFileCount) + : MaxFileCount(maxFileCount) + {} + + bool TryReserve() + { + if (!MaxFileCount) { + return true; + } + + auto count = ReservedFileCount.load(std::memory_order_relaxed); + while (count < MaxFileCount && + !ReservedFileCount.compare_exchange_weak( + count, + count + 1, + std::memory_order_relaxed)) + {} + return count < MaxFileCount; + } + + void Release() + { + if (MaxFileCount) { + ReservedFileCount.fetch_sub(1, std::memory_order_relaxed); + } + } +}; + +using TFileCreationLimiterPtr = std::shared_ptr; + //////////////////////////////////////////////////////////////////////////////// struct TCompletedRequest @@ -72,33 +112,38 @@ IRequestGeneratorPtr CreateIndexRequestGenerator( IFileStoreServicePtr client, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers); + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter); IRequestGeneratorPtr CreateDataRequestGenerator( NProto::TDataLoadSpec spec, ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers); + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter); IRequestGeneratorPtr CreateReplayRequestGeneratorFs( NProto::TReplaySpec spec, ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers); + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter); IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( NProto::TReplaySpec spec, ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers); + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter); IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, - ILoggingServicePtr logging); + ILoggingServicePtr logging, + TFileCreationLimiterPtr fileCreationLimiter); IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( NProto::TDatashardLikeLoadSpec spec, @@ -106,6 +151,7 @@ IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( NClient::ISessionPtr session, IShmDataClientPtr dataClient, TString filesystemId, - NProto::THeaders headers); + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter); } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp index 85672ace397..b777ed7af31 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp @@ -106,6 +106,7 @@ class TDataRequestGenerator final const NProto::TDataLoadSpec Spec; const TString FileSystemId; const NProto::THeaders Headers; + const TFileCreationLimiterPtr FileCreationLimiter; TLog Log; @@ -134,10 +135,12 @@ class TDataRequestGenerator final ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) : Spec(std::move(spec)) , FileSystemId(std::move(filesystemId)) , Headers(std::move(headers)) + , FileCreationLimiter(std::move(fileCreationLimiter)) , Session(std::move(session)) { Log = logging->CreateLog(Headers.GetClientId()); @@ -238,6 +241,12 @@ class TDataRequestGenerator final auto started = TInstant::Now(); TGuard guard(StateLock); + if (!FileCreationLimiter->TryReserve()) { + return MakeFuture({ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(S_FALSE)}); + } auto name = GenerateNodeName(); auto request = CreateRequest(); @@ -301,6 +310,7 @@ class TDataRequestGenerator final return HandleResizeAfterCreateHandle(f, name, started); }); } catch (const TServiceError& e) { + FileCreationLimiter->Release(); auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR("create handle for %s has failed: %s", name.Quote().c_str(), @@ -646,14 +656,16 @@ IRequestGeneratorPtr CreateDataRequestGenerator( ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), std::move(logging), std::move(session), std::move(filesystemId), - std::move(headers)); + std::move(headers), + std::move(fileCreationLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp index 16abf5207fe..d3bffb58069 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp @@ -50,6 +50,7 @@ class TDatashardLikeRequestGenerator final const NProto::TDatashardLikeLoadSpec Spec; const TString FileSystemId; const NProto::THeaders Headers; + const TFileCreationLimiterPtr FileCreationLimiter; TLog Log; @@ -77,10 +78,12 @@ class TDatashardLikeRequestGenerator final ISessionPtr session, IShmDataClientPtr dataClient, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) : Spec(std::move(spec)) , FileSystemId(std::move(filesystemId)) , Headers(std::move(headers)) + , FileCreationLimiter(std::move(fileCreationLimiter)) , Session(std::move(session)) , DataClient(std::move(dataClient)) { @@ -174,6 +177,12 @@ class TDatashardLikeRequestGenerator final TFuture DoCreateNode() { auto started = TInstant::Now(); + if (!FileCreationLimiter->TryReserve()) { + return MakeFuture({ + NProto::ACTION_CREATE_NODE, + started, + MakeError(S_FALSE)}); + } auto request = CreateRequest(); request->SetNodeId(RootNodeId); @@ -231,6 +240,7 @@ class TDatashardLikeRequestGenerator final [=, this](const TFuture& f) { return HandleResizeAfterCreate(f, info, started); }); } catch (const TServiceError& e) { + FileCreationLimiter->Release(); auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR( "create node has failed: %s", @@ -496,7 +506,8 @@ IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( NClient::ISessionPtr session, IShmDataClientPtr dataClient, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), @@ -504,7 +515,8 @@ IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( std::move(session), std::move(dataClient), std::move(filesystemId), - std::move(headers)); + std::move(headers), + std::move(fileCreationLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp index ca3a18fb2f6..f9911de5d2d 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp @@ -73,6 +73,7 @@ class TFastShardRequestGenerator final const ui64 ReadBytes; const ui64 WriteBytes; const ui64 InitialFileSize; + TFileCreationLimiterPtr FileCreationLimiter; TClient Client; TDeque> Endpoints; @@ -88,12 +89,14 @@ class TFastShardRequestGenerator final TFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, - ILoggingServicePtr /*logging*/) + ILoggingServicePtr /*logging*/, + TFileCreationLimiterPtr fileCreationLimiter) : Spec(std::move(spec)) , ShardFileSystemId(Spec.GetShardFileSystemId()) , ReadBytes(Spec.GetReadBytes() ? Spec.GetReadBytes() : DefaultIoSize) , WriteBytes(Spec.GetWriteBytes() ? Spec.GetWriteBytes() : DefaultIoSize) , InitialFileSize(Spec.GetInitialFileSize()) + , FileCreationLimiter(std::move(fileCreationLimiter)) { Y_ENSURE(!ShardFileSystemId.empty(), "ShardFileSystemId must be set"); Y_ENSURE(maxParallelism > 0); @@ -199,6 +202,11 @@ class TFastShardRequestGenerator final // Create a file if the pool had nothing to offer. if (s->File.Handle == 0) { + if (!ptr->FileCreationLimiter->TryReserve()) { + releaseEndpoint(); + s->Promise.SetValue({s->Action, s->Started, MakeError(S_FALSE)}); + return; + } if (!ptr) { releaseEndpoint(); s->Promise.SetValue({s->Action, s->Started, @@ -220,6 +228,7 @@ class TFastShardRequestGenerator final ? resp.GetError() : resp.GetCreateHandle().GetError(); if (HasError(createErr)) { + ptr->FileCreationLimiter->Release(); releaseEndpoint(); s->Promise.SetValue({s->Action, s->Started, createErr}); return; @@ -349,12 +358,14 @@ class TFastShardRequestGenerator final IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, - ILoggingServicePtr logging) + ILoggingServicePtr logging, + TFileCreationLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), maxParallelism, - std::move(logging)); + std::move(logging), + std::move(fileCreationLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp index ff2b71eeafc..34d4319b8d0 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp @@ -31,11 +31,13 @@ class TFastShardRequestGenerator final IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, - ILoggingServicePtr logging) + ILoggingServicePtr logging, + TFileCreationLimiterPtr fileCreationLimiter) { Y_UNUSED(spec); Y_UNUSED(maxParallelism); Y_UNUSED(logging); + Y_UNUSED(fileCreationLimiter); return std::make_shared(); } diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp index 65a9a6fd7d0..4a8aac09811 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp @@ -48,6 +48,7 @@ class TIndexRequestGenerator final const NProto::TIndexLoadSpec Spec; const TString FileSystemId; const NProto::THeaders Headers; + const TFileCreationLimiterPtr FileCreationLimiter; const ui64 OwnerId = RandomNumber(100500u); TLog Log; @@ -76,10 +77,12 @@ class TIndexRequestGenerator final IFileStoreServicePtr client, ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) : Spec(std::move(spec)) , FileSystemId(std::move(filesystemId)) , Headers(std::move(headers)) + , FileCreationLimiter(std::move(fileCreationLimiter)) , Client(std::move(client)) , Session(std::move(session)) { @@ -148,7 +151,15 @@ class TIndexRequestGenerator final TGuard guard(StateLock); auto started = TInstant::Now(); + if (!FileCreationLimiter->TryReserve()) { + return MakeFuture({ + NProto::ACTION_CREATE_NODE, + started, + MakeError(S_FALSE)}); + } + if (Spec.GetMaxNodes() && Nodes.size() >= Spec.GetMaxNodes()) { + FileCreationLimiter->Release(); return MakeFuture({ NProto::ACTION_CREATE_NODE, started, @@ -197,6 +208,7 @@ class TIndexRequestGenerator final name.c_str(), FormatError(error).c_str()); + FileCreationLimiter->Release(); return {NProto::ACTION_CREATE_NODE, started, error}; } } @@ -777,7 +789,8 @@ IRequestGeneratorPtr CreateIndexRequestGenerator( IFileStoreServicePtr client, ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), @@ -785,7 +798,8 @@ IRequestGeneratorPtr CreateIndexRequestGenerator( std::move(client), std::move(session), std::move(filesystemId), - std::move(headers)); + std::move(headers), + std::move(fileCreationLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index c5f06892900..678f92c0230 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -20,10 +20,12 @@ IReplayRequestGenerator::IReplayRequestGenerator( ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) : Spec(std::move(spec)) , TargetFilesystemId(std::move(filesystemId)) , Headers(std::move(headers)) + , FileCreationLimiter(std::move(fileCreationLimiter)) , Session(std::move(session)) { Log = logging->CreateLog(Headers.GetClientId()); diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index e66c6e7660f..cef9ed60a2f 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -22,6 +22,7 @@ class IReplayRequestGenerator: public IRequestGenerator TString TargetFilesystemId; // Only for GRPC TString FileSystemIdFilter; const ::NCloud::NFileStore::NProto::THeaders Headers; + const TFileCreationLimiterPtr FileCreationLimiter; NClient::ISessionPtr Session; int EventMessageNumber = 0; i64 TimestampMicroSeconds = 0; @@ -54,7 +55,11 @@ class IReplayRequestGenerator: public IRequestGenerator ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers); + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter); + + bool TryReserveFile() { return FileCreationLimiter->TryReserve(); } + void ReleaseFile() { FileCreationLimiter->Release(); } bool ShouldImmediatelyProcessQueue() override; diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index e82db31917a..a8e39ff9f5e 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -93,13 +93,15 @@ class TReplayRequestGeneratorFs final ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) : IReplayRequestGenerator( std::move(spec), std::move(logging), std::move(session), std::move(filesystemId), - std::move(headers)) + std::move(headers), + std::move(fileCreationLimiter)) { if (Spec.GetReplayRoot().empty()) { ythrow yexception() << "ReplayRoot is not defined"; @@ -724,6 +726,12 @@ class TReplayRequestGeneratorFs final ui64 nodeid = 0; switch (logRequest.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: { + if (!TryReserveFile()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + Started, + MakeError(S_FALSE)}); + } // TODO(proller): transform r.GetNodeInfo().GetMode() to correct // open mode TFileHandle fh(fullName, OpenAlways | RdWr); @@ -1126,14 +1134,16 @@ IRequestGeneratorPtr CreateReplayRequestGeneratorFs( ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), std::move(logging), std::move(session), std::move(filesystemId), - std::move(headers)); + std::move(headers), + std::move(fileCreationLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp index 8806f4284b6..2aace23ce70 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp @@ -84,13 +84,15 @@ class TReplayRequestGeneratorGRPC final ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) : IReplayRequestGenerator( std::move(spec), std::move(logging), std::move(session), std::move(filesystemId), - std::move(headers)) + std::move(headers), + std::move(fileCreationLimiter)) { if (!Session) { ythrow yexception() << "Session not created. Missing FileSystemId?"; @@ -233,6 +235,10 @@ class TReplayRequestGeneratorGRPC final const auto request = CreateRequest(); auto name = logRequest.GetNodeInfo().GetNodeName(); + if (!TryReserveFile()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, Started, MakeError(S_FALSE)}); + } const auto nodeId = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); @@ -282,6 +288,9 @@ class TReplayRequestGeneratorGRPC final MakeError(E_INVALID_STATE, "cancelled")}); }); const auto& response = future.GetValueSync(); + if (HasError(response.Error)) { + ReleaseFile(); + } return MakeFuture( TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, @@ -562,6 +571,10 @@ class TReplayRequestGeneratorGRPC final switch (logRequest.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: + if (!TryReserveFile()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_NODE, Started, MakeError(S_FALSE)}); + } request->MutableFile()->SetMode( logRequest.GetNodeInfo().GetMode()); break; @@ -627,6 +640,11 @@ class TReplayRequestGeneratorGRPC final }); const auto& response = future.GetValueSync(); + if (HasError(response.Error) && + logRequest.GetNodeInfo().GetType() == NProto::E_REGULAR_NODE) + { + ReleaseFile(); + } return MakeFuture( TCompletedRequest{ NProto::ACTION_CREATE_NODE, @@ -1288,14 +1306,16 @@ IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers, + TFileCreationLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), std::move(logging), std::move(session), std::move(filesystemId), - std::move(headers)); + std::move(headers), + std::move(fileCreationLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index 762e9f02508..01451d00285 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -534,6 +534,8 @@ class TLoadTest final NProto::THeaders headers; headers.SetClientId(Config.GetClientId()); headers.SetSessionId(SessionId); + auto fileCreationLimiter = std::make_shared( + Config.GetMaxFileCount()); switch (Config.GetSpecsCase()) { case NProto::TLoadTest::kIndexLoadSpec: @@ -543,7 +545,8 @@ class TLoadTest final Client, Session, FileSystemId, - headers); + headers, + fileCreationLimiter); break; case NProto::TLoadTest::kDataLoadSpec: RequestGenerator = CreateDataRequestGenerator( @@ -551,7 +554,8 @@ class TLoadTest final Logging, Session, FileSystemId, - headers); + headers, + fileCreationLimiter); break; case NProto::TLoadTest::kReplayFsSpec: RequestGenerator = CreateReplayRequestGeneratorFs( @@ -559,7 +563,8 @@ class TLoadTest final Logging, Session, FileSystemId, - headers); + headers, + fileCreationLimiter); break; case NProto::TLoadTest::kReplayGrpcSpec: RequestGenerator = CreateReplayRequestGeneratorGRPC( @@ -567,7 +572,8 @@ class TLoadTest final Logging, Session, FileSystemId, - headers); + headers, + fileCreationLimiter); break; case NProto::TLoadTest::kDatashardLikeLoadSpec: { const auto& spec = Config.GetDatashardLikeLoadSpec(); @@ -594,14 +600,16 @@ class TLoadTest final Session, ShmClient, FileSystemId, - headers); + headers, + fileCreationLimiter); break; } case NProto::TLoadTest::kFastShardLoadSpec: RequestGenerator = CreateFastShardRequestGenerator( Config.GetFastShardLoadSpec(), Config.GetIODepth(), - Logging); + Logging, + fileCreationLimiter); break; default: ythrow yexception() diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index f8547575631..523e3a59cd6 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -210,6 +210,9 @@ message TLoadTest // disables destruction of the filestore created via CreateFileStoreRequest bool KeepFileStore = 16; string ClientId = 19; + + // Maximum number of regular files created by this test run. Zero means unlimited. + uint64 MaxFileCount = 22; } //////////////////////////////////////////////////////////////////////////////// From f0fe4ceb03dedee92245f44167a9b60ab38d6ddb Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Fri, 17 Jul 2026 15:58:22 +0200 Subject: [PATCH 02/11] Add an integration test for MaxFileCount --- .../service-kikimr-test/max-file-count.txt | 25 +++++++++ .../loadtest/service-kikimr-test/test.py | 33 ++++++++++++ .../loadtest/service-kikimr-test/ya.make | 1 + .../loadtest/lib/file_creation_limiter.h | 52 +++++++++++++++++++ .../tools/testing/loadtest/lib/request.h | 41 +-------------- 5 files changed, 112 insertions(+), 40 deletions(-) create mode 100644 cloud/filestore/tests/loadtest/service-kikimr-test/max-file-count.txt create mode 100644 cloud/filestore/tools/testing/loadtest/lib/file_creation_limiter.h diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/max-file-count.txt b/cloud/filestore/tests/loadtest/service-kikimr-test/max-file-count.txt new file mode 100644 index 00000000000..f50efe6ecc3 --- /dev/null +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/max-file-count.txt @@ -0,0 +1,25 @@ +Tests { + LoadTest { + Name: "max-file-count" + CreateFileStoreRequest { + FileSystemId: "max-file-count" + FolderId: "folder" + CloudId: "cloud" + BlocksCount: 10241024 + BlockSize: 4096 + } + KeepFileStore: true + MaxFileCount: 3 + RequestsCount: 30 + IODepth: 16 + DataLoadSpec { + Actions { + Action: ACTION_READ + Rate: 1 + } + ReadBytes: 4096 + WriteBytes: 4096 + MinFileCount: 3 + } + } +} diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/test.py b/cloud/filestore/tests/loadtest/service-kikimr-test/test.py index 9efd2f8f09f..ceb28890c69 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/test.py +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/test.py @@ -2,6 +2,7 @@ import pytest from cloud.filestore.tests.python.lib.loadtest import run_load_test +from cloud.filestore.tests.python.lib.client import FilestoreCliClient import yatest.common as common @@ -51,3 +52,35 @@ def test_load(test_case): ) return None + + +def test_max_file_count(): + filesystem_id = "max-file-count" + config_path = common.source_path( + "cloud/filestore/tests/loadtest/service-kikimr-test/max-file-count.txt" + ) + + try: + run_load_test( + filesystem_id, + config_path, + os.getenv("NFS_SERVER_PORT"), + ) + + client = FilestoreCliClient( + common.binary_path("cloud/filestore/apps/client/filestore-client"), + os.getenv("NFS_SERVER_PORT"), + cwd=common.output_path(), + ) + entries = client.find(filesystem_id, depth=1).decode().splitlines() + entries = [entry for entry in entries if entry] + + assert len(entries) == 3 + finally: + client = FilestoreCliClient( + common.binary_path("cloud/filestore/apps/client/filestore-client"), + os.getenv("NFS_SERVER_PORT"), + cwd=common.output_path(), + check_exit_code=False, + ) + client.destroy(filesystem_id) diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/ya.make b/cloud/filestore/tests/loadtest/service-kikimr-test/ya.make index 772ceb3820a..b0c7c5bb2d2 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/ya.make +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/ya.make @@ -7,6 +7,7 @@ TEST_SRCS( ) DEPENDS( + cloud/filestore/apps/client cloud/filestore/tools/testing/loadtest/bin ) diff --git a/cloud/filestore/tools/testing/loadtest/lib/file_creation_limiter.h b/cloud/filestore/tools/testing/loadtest/lib/file_creation_limiter.h new file mode 100644 index 00000000000..c52e71dc146 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/lib/file_creation_limiter.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include +#include + +namespace NCloud::NFileStore::NLoadTest { + +//////////////////////////////////////////////////////////////////////////////// + +class TFileCreationLimiter +{ +private: + const ui64 MaxFileCount; + std::atomic ReservedFileCount = 0; + +public: + explicit TFileCreationLimiter(ui64 maxFileCount) + : MaxFileCount(maxFileCount) + {} + + bool TryReserve() + { + if (!MaxFileCount) { + return true; + } + + auto count = ReservedFileCount.load(std::memory_order_relaxed); + while (count < MaxFileCount){ + if (ReservedFileCount.compare_exchange_weak( + count, + count + 1, + std::memory_order_relaxed)) + { + return true; + } + } + return count < MaxFileCount; + } + + void Release() + { + if (MaxFileCount) { + ReservedFileCount.fetch_sub(1, std::memory_order_relaxed); + } + } +}; + +using TFileCreationLimiterPtr = std::shared_ptr; + +} // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index e16de005b74..7aa115e8ae6 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -4,6 +4,7 @@ #include "shm_client.h" #include +#include #include #include @@ -16,48 +17,8 @@ #include #include -#include -#include - namespace NCloud::NFileStore::NLoadTest { -class TFileCreationLimiter -{ -private: - const ui64 MaxFileCount; - std::atomic ReservedFileCount = 0; - -public: - explicit TFileCreationLimiter(ui64 maxFileCount) - : MaxFileCount(maxFileCount) - {} - - bool TryReserve() - { - if (!MaxFileCount) { - return true; - } - - auto count = ReservedFileCount.load(std::memory_order_relaxed); - while (count < MaxFileCount && - !ReservedFileCount.compare_exchange_weak( - count, - count + 1, - std::memory_order_relaxed)) - {} - return count < MaxFileCount; - } - - void Release() - { - if (MaxFileCount) { - ReservedFileCount.fetch_sub(1, std::memory_order_relaxed); - } - } -}; - -using TFileCreationLimiterPtr = std::shared_ptr; - //////////////////////////////////////////////////////////////////////////////// struct TCompletedRequest From 53159ef9d1aaac1fde9843b6682afcd5e036a9a4 Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Fri, 17 Jul 2026 18:08:33 +0200 Subject: [PATCH 03/11] Move test to separate dir, use canonical output --- .../max-file-count-test/canondata/result.json | 5 +++ .../test.test_max_file_count/results.txt | 3 ++ .../max-file-count.txt | 1 + .../loadtest/max-file-count-test/test.py | 43 +++++++++++++++++++ .../loadtest/max-file-count-test/ya.make | 25 +++++++++++ .../loadtest/service-kikimr-test/test.py | 33 -------------- 6 files changed, 77 insertions(+), 33 deletions(-) create mode 100644 cloud/filestore/tests/loadtest/max-file-count-test/canondata/result.json create mode 100644 cloud/filestore/tests/loadtest/max-file-count-test/canondata/test.test_max_file_count/results.txt rename cloud/filestore/tests/loadtest/{service-kikimr-test => max-file-count-test}/max-file-count.txt (92%) create mode 100644 cloud/filestore/tests/loadtest/max-file-count-test/test.py create mode 100644 cloud/filestore/tests/loadtest/max-file-count-test/ya.make diff --git a/cloud/filestore/tests/loadtest/max-file-count-test/canondata/result.json b/cloud/filestore/tests/loadtest/max-file-count-test/canondata/result.json new file mode 100644 index 00000000000..eb1c01ff9a8 --- /dev/null +++ b/cloud/filestore/tests/loadtest/max-file-count-test/canondata/result.json @@ -0,0 +1,5 @@ +{ + "test.test_max_file_count": { + "uri": "file://test.test_max_file_count/results.txt" + } +} diff --git a/cloud/filestore/tests/loadtest/max-file-count-test/canondata/test.test_max_file_count/results.txt b/cloud/filestore/tests/loadtest/max-file-count-test/canondata/test.test_max_file_count/results.txt new file mode 100644 index 00000000000..0bb0339289e --- /dev/null +++ b/cloud/filestore/tests/loadtest/max-file-count-test/canondata/test.test_max_file_count/results.txt @@ -0,0 +1,3 @@ +/ max-file-count-loadtest: +/ max-file-count-loadtest: +/ max-file-count-loadtest: \ No newline at end of file diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/max-file-count.txt b/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt similarity index 92% rename from cloud/filestore/tests/loadtest/service-kikimr-test/max-file-count.txt rename to cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt index f50efe6ecc3..d9bbb37c6ae 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/max-file-count.txt +++ b/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt @@ -1,6 +1,7 @@ Tests { LoadTest { Name: "max-file-count" + ClientId: "max-file-count-loadtest" CreateFileStoreRequest { FileSystemId: "max-file-count" FolderId: "folder" diff --git a/cloud/filestore/tests/loadtest/max-file-count-test/test.py b/cloud/filestore/tests/loadtest/max-file-count-test/test.py new file mode 100644 index 00000000000..aec51bf1736 --- /dev/null +++ b/cloud/filestore/tests/loadtest/max-file-count-test/test.py @@ -0,0 +1,43 @@ +import os +import re +import yatest.common as common + +from cloud.filestore.tests.python.lib.client import FilestoreCliClient +from cloud.filestore.tests.python.lib.loadtest import run_load_test + + +def test_max_file_count(): + filesystem_id = "max-file-count" + results_path = common.output_path("results.txt") + config_path = common.source_path( + "cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt" + ) + client = FilestoreCliClient( + common.binary_path("cloud/filestore/apps/client/filestore-client"), + os.getenv("NFS_SERVER_PORT"), + cwd=common.output_path()) + + try: + run_load_test( + filesystem_id, + config_path, + os.getenv("NFS_SERVER_PORT"), + ) + + files = client.find(filesystem_id, depth=1).decode().splitlines() + normalized_entries = [ + re.sub( + r"(?<=max-file-count-loadtest:)[0-9a-f-]+$", + "", + file.rsplit("\t", 1)[0], + ) + for file in files + ] + + with open(results_path, "w") as results: + results.write("\n".join(normalized_entries)) + + finally: + client.destroy(filesystem_id) + + return common.canonical_file(results_path, local=True) diff --git a/cloud/filestore/tests/loadtest/max-file-count-test/ya.make b/cloud/filestore/tests/loadtest/max-file-count-test/ya.make new file mode 100644 index 00000000000..3c49bd8d1b1 --- /dev/null +++ b/cloud/filestore/tests/loadtest/max-file-count-test/ya.make @@ -0,0 +1,25 @@ +PY3TEST() + +INCLUDE(${ARCADIA_ROOT}/cloud/filestore/tests/recipes/medium.inc) + +TEST_SRCS( + test.py +) + +DEPENDS( + cloud/filestore/apps/client + cloud/filestore/tools/testing/loadtest/bin +) + +PEERDIR( + cloud/filestore/tests/python/lib +) + +SET( + NFS_STORAGE_CONFIG_PATCH + cloud/filestore/tests/common_configs/nfs-storage-migration-patch.txt +) + +INCLUDE(${ARCADIA_ROOT}/cloud/filestore/tests/recipes/service-kikimr.inc) + +END() diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/test.py b/cloud/filestore/tests/loadtest/service-kikimr-test/test.py index ceb28890c69..9efd2f8f09f 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/test.py +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/test.py @@ -2,7 +2,6 @@ import pytest from cloud.filestore.tests.python.lib.loadtest import run_load_test -from cloud.filestore.tests.python.lib.client import FilestoreCliClient import yatest.common as common @@ -52,35 +51,3 @@ def test_load(test_case): ) return None - - -def test_max_file_count(): - filesystem_id = "max-file-count" - config_path = common.source_path( - "cloud/filestore/tests/loadtest/service-kikimr-test/max-file-count.txt" - ) - - try: - run_load_test( - filesystem_id, - config_path, - os.getenv("NFS_SERVER_PORT"), - ) - - client = FilestoreCliClient( - common.binary_path("cloud/filestore/apps/client/filestore-client"), - os.getenv("NFS_SERVER_PORT"), - cwd=common.output_path(), - ) - entries = client.find(filesystem_id, depth=1).decode().splitlines() - entries = [entry for entry in entries if entry] - - assert len(entries) == 3 - finally: - client = FilestoreCliClient( - common.binary_path("cloud/filestore/apps/client/filestore-client"), - os.getenv("NFS_SERVER_PORT"), - cwd=common.output_path(), - check_exit_code=False, - ) - client.destroy(filesystem_id) From bf1f829ec9f31f631f420ae8a6965869851dec98 Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Fri, 17 Jul 2026 18:22:54 +0200 Subject: [PATCH 04/11] Do not apply MaxFileCount to request replays --- .../tools/testing/loadtest/lib/request.h | 6 ++-- .../testing/loadtest/lib/request_replay.cpp | 4 +-- .../testing/loadtest/lib/request_replay.h | 7 +---- .../loadtest/lib/request_replay_fs.cpp | 18 +++--------- .../loadtest/lib/request_replay_grpc.cpp | 29 +++---------------- .../tools/testing/loadtest/lib/test.cpp | 17 ++++------- .../testing/loadtest/protos/loadtest.proto | 3 +- 7 files changed, 20 insertions(+), 64 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index 7aa115e8ae6..8beae24908b 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -89,16 +89,14 @@ IRequestGeneratorPtr CreateReplayRequestGeneratorFs( ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter); + NProto::THeaders headers); IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( NProto::TReplaySpec spec, ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter); + NProto::THeaders headers); IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index 678f92c0230..c5f06892900 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -20,12 +20,10 @@ IReplayRequestGenerator::IReplayRequestGenerator( ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + NProto::THeaders headers) : Spec(std::move(spec)) , TargetFilesystemId(std::move(filesystemId)) , Headers(std::move(headers)) - , FileCreationLimiter(std::move(fileCreationLimiter)) , Session(std::move(session)) { Log = logging->CreateLog(Headers.GetClientId()); diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index cef9ed60a2f..e66c6e7660f 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -22,7 +22,6 @@ class IReplayRequestGenerator: public IRequestGenerator TString TargetFilesystemId; // Only for GRPC TString FileSystemIdFilter; const ::NCloud::NFileStore::NProto::THeaders Headers; - const TFileCreationLimiterPtr FileCreationLimiter; NClient::ISessionPtr Session; int EventMessageNumber = 0; i64 TimestampMicroSeconds = 0; @@ -55,11 +54,7 @@ class IReplayRequestGenerator: public IRequestGenerator ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter); - - bool TryReserveFile() { return FileCreationLimiter->TryReserve(); } - void ReleaseFile() { FileCreationLimiter->Release(); } + NProto::THeaders headers); bool ShouldImmediatelyProcessQueue() override; diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index a8e39ff9f5e..e82db31917a 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -93,15 +93,13 @@ class TReplayRequestGeneratorFs final ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + NProto::THeaders headers) : IReplayRequestGenerator( std::move(spec), std::move(logging), std::move(session), std::move(filesystemId), - std::move(headers), - std::move(fileCreationLimiter)) + std::move(headers)) { if (Spec.GetReplayRoot().empty()) { ythrow yexception() << "ReplayRoot is not defined"; @@ -726,12 +724,6 @@ class TReplayRequestGeneratorFs final ui64 nodeid = 0; switch (logRequest.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: { - if (!TryReserveFile()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_NODE, - Started, - MakeError(S_FALSE)}); - } // TODO(proller): transform r.GetNodeInfo().GetMode() to correct // open mode TFileHandle fh(fullName, OpenAlways | RdWr); @@ -1134,16 +1126,14 @@ IRequestGeneratorPtr CreateReplayRequestGeneratorFs( ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + NProto::THeaders headers) { return std::make_shared( std::move(spec), std::move(logging), std::move(session), std::move(filesystemId), - std::move(headers), - std::move(fileCreationLimiter)); + std::move(headers)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp index 2aace23ce70..75dc8fd3bd6 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp @@ -84,15 +84,13 @@ class TReplayRequestGeneratorGRPC final ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + NProto::THeaders headers) : IReplayRequestGenerator( std::move(spec), std::move(logging), std::move(session), std::move(filesystemId), - std::move(headers), - std::move(fileCreationLimiter)) + std::move(headers)) { if (!Session) { ythrow yexception() << "Session not created. Missing FileSystemId?"; @@ -235,11 +233,6 @@ class TReplayRequestGeneratorGRPC final const auto request = CreateRequest(); auto name = logRequest.GetNodeInfo().GetNodeName(); - if (!TryReserveFile()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, Started, MakeError(S_FALSE)}); - } - const auto nodeId = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (nodeId == InvalidNodeId) { @@ -288,9 +281,6 @@ class TReplayRequestGeneratorGRPC final MakeError(E_INVALID_STATE, "cancelled")}); }); const auto& response = future.GetValueSync(); - if (HasError(response.Error)) { - ReleaseFile(); - } return MakeFuture( TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, @@ -571,10 +561,6 @@ class TReplayRequestGeneratorGRPC final switch (logRequest.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: - if (!TryReserveFile()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_NODE, Started, MakeError(S_FALSE)}); - } request->MutableFile()->SetMode( logRequest.GetNodeInfo().GetMode()); break; @@ -640,11 +626,6 @@ class TReplayRequestGeneratorGRPC final }); const auto& response = future.GetValueSync(); - if (HasError(response.Error) && - logRequest.GetNodeInfo().GetType() == NProto::E_REGULAR_NODE) - { - ReleaseFile(); - } return MakeFuture( TCompletedRequest{ NProto::ACTION_CREATE_NODE, @@ -1306,16 +1287,14 @@ IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + NProto::THeaders headers) { return std::make_shared( std::move(spec), std::move(logging), std::move(session), std::move(filesystemId), - std::move(headers), - std::move(fileCreationLimiter)); + std::move(headers)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index 01451d00285..2e8ce525758 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -534,9 +534,6 @@ class TLoadTest final NProto::THeaders headers; headers.SetClientId(Config.GetClientId()); headers.SetSessionId(SessionId); - auto fileCreationLimiter = std::make_shared( - Config.GetMaxFileCount()); - switch (Config.GetSpecsCase()) { case NProto::TLoadTest::kIndexLoadSpec: RequestGenerator = CreateIndexRequestGenerator( @@ -546,7 +543,7 @@ class TLoadTest final Session, FileSystemId, headers, - fileCreationLimiter); + std::make_shared(Config.GetMaxFileCount())); break; case NProto::TLoadTest::kDataLoadSpec: RequestGenerator = CreateDataRequestGenerator( @@ -555,7 +552,7 @@ class TLoadTest final Session, FileSystemId, headers, - fileCreationLimiter); + std::make_shared(Config.GetMaxFileCount())); break; case NProto::TLoadTest::kReplayFsSpec: RequestGenerator = CreateReplayRequestGeneratorFs( @@ -563,8 +560,7 @@ class TLoadTest final Logging, Session, FileSystemId, - headers, - fileCreationLimiter); + headers); break; case NProto::TLoadTest::kReplayGrpcSpec: RequestGenerator = CreateReplayRequestGeneratorGRPC( @@ -572,8 +568,7 @@ class TLoadTest final Logging, Session, FileSystemId, - headers, - fileCreationLimiter); + headers); break; case NProto::TLoadTest::kDatashardLikeLoadSpec: { const auto& spec = Config.GetDatashardLikeLoadSpec(); @@ -601,7 +596,7 @@ class TLoadTest final ShmClient, FileSystemId, headers, - fileCreationLimiter); + std::make_shared(Config.GetMaxFileCount())); break; } case NProto::TLoadTest::kFastShardLoadSpec: @@ -609,7 +604,7 @@ class TLoadTest final Config.GetFastShardLoadSpec(), Config.GetIODepth(), Logging, - fileCreationLimiter); + std::make_shared(Config.GetMaxFileCount())); break; default: ythrow yexception() diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index 523e3a59cd6..b1cfd1f39f2 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -211,7 +211,8 @@ message TLoadTest bool KeepFileStore = 16; string ClientId = 19; - // Maximum number of regular files created by this test run. Zero means unlimited. + // Maximum number of regular files created by this synthetic load test. + // Zero means unlimited. Ignored by replay specs. uint64 MaxFileCount = 22; } From 54751bcd8ede46fefb40dbafe2358b6f9efcd6fb Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Fri, 17 Jul 2026 18:24:32 +0200 Subject: [PATCH 05/11] Remove unused dependency --- cloud/filestore/tests/loadtest/service-kikimr-test/ya.make | 1 - 1 file changed, 1 deletion(-) diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/ya.make b/cloud/filestore/tests/loadtest/service-kikimr-test/ya.make index b0c7c5bb2d2..772ceb3820a 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/ya.make +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/ya.make @@ -7,7 +7,6 @@ TEST_SRCS( ) DEPENDS( - cloud/filestore/apps/client cloud/filestore/tools/testing/loadtest/bin ) From d23c1c585309a8743a70284fbcfd931ee7f89ee0 Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Mon, 20 Jul 2026 15:54:42 +0200 Subject: [PATCH 06/11] Rename the limiter to CountLimiter --- .../{file_creation_limiter.h => count_limiter.h} | 6 +++--- .../filestore/tools/testing/loadtest/lib/request.h | 10 +++++----- .../tools/testing/loadtest/lib/request_data.cpp | 12 ++++++------ .../loadtest/lib/request_datashard_like.cpp | 12 ++++++------ .../testing/loadtest/lib/request_fastshard.cpp | 12 ++++++------ .../loadtest/lib/request_fastshard_stub.cpp | 2 +- .../tools/testing/loadtest/lib/request_index.cpp | 14 +++++++------- .../filestore/tools/testing/loadtest/lib/test.cpp | 8 ++++---- 8 files changed, 38 insertions(+), 38 deletions(-) rename cloud/filestore/tools/testing/loadtest/lib/{file_creation_limiter.h => count_limiter.h} (87%) diff --git a/cloud/filestore/tools/testing/loadtest/lib/file_creation_limiter.h b/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h similarity index 87% rename from cloud/filestore/tools/testing/loadtest/lib/file_creation_limiter.h rename to cloud/filestore/tools/testing/loadtest/lib/count_limiter.h index c52e71dc146..72c3c150f7c 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/file_creation_limiter.h +++ b/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h @@ -9,14 +9,14 @@ namespace NCloud::NFileStore::NLoadTest { //////////////////////////////////////////////////////////////////////////////// -class TFileCreationLimiter +class TCountLimiter { private: const ui64 MaxFileCount; std::atomic ReservedFileCount = 0; public: - explicit TFileCreationLimiter(ui64 maxFileCount) + explicit TCountLimiter(ui64 maxFileCount) : MaxFileCount(maxFileCount) {} @@ -47,6 +47,6 @@ class TFileCreationLimiter } }; -using TFileCreationLimiterPtr = std::shared_ptr; +using TCountLimiterPtr = std::shared_ptr; } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index 8beae24908b..9d53b9cec4b 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -4,7 +4,7 @@ #include "shm_client.h" #include -#include +#include #include #include @@ -74,7 +74,7 @@ IRequestGeneratorPtr CreateIndexRequestGenerator( NClient::ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter); + TCountLimiterPtr fileCreationLimiter); IRequestGeneratorPtr CreateDataRequestGenerator( NProto::TDataLoadSpec spec, @@ -82,7 +82,7 @@ IRequestGeneratorPtr CreateDataRequestGenerator( NClient::ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter); + TCountLimiterPtr fileCreationLimiter); IRequestGeneratorPtr CreateReplayRequestGeneratorFs( NProto::TReplaySpec spec, @@ -102,7 +102,7 @@ IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, ILoggingServicePtr logging, - TFileCreationLimiterPtr fileCreationLimiter); + TCountLimiterPtr fileCreationLimiter); IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( NProto::TDatashardLikeLoadSpec spec, @@ -111,6 +111,6 @@ IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( IShmDataClientPtr dataClient, TString filesystemId, NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter); + TCountLimiterPtr fileCreationLimiter); } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp index b777ed7af31..119af5b3e55 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp @@ -106,7 +106,7 @@ class TDataRequestGenerator final const NProto::TDataLoadSpec Spec; const TString FileSystemId; const NProto::THeaders Headers; - const TFileCreationLimiterPtr FileCreationLimiter; + const TCountLimiterPtr CountLimiter; TLog Log; @@ -136,11 +136,11 @@ class TDataRequestGenerator final ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + TCountLimiterPtr fileCreationLimiter) : Spec(std::move(spec)) , FileSystemId(std::move(filesystemId)) , Headers(std::move(headers)) - , FileCreationLimiter(std::move(fileCreationLimiter)) + , CountLimiter(std::move(fileCreationLimiter)) , Session(std::move(session)) { Log = logging->CreateLog(Headers.GetClientId()); @@ -241,7 +241,7 @@ class TDataRequestGenerator final auto started = TInstant::Now(); TGuard guard(StateLock); - if (!FileCreationLimiter->TryReserve()) { + if (!CountLimiter->TryReserve()) { return MakeFuture({ NProto::ACTION_CREATE_HANDLE, started, @@ -310,7 +310,7 @@ class TDataRequestGenerator final return HandleResizeAfterCreateHandle(f, name, started); }); } catch (const TServiceError& e) { - FileCreationLimiter->Release(); + CountLimiter->Release(); auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR("create handle for %s has failed: %s", name.Quote().c_str(), @@ -657,7 +657,7 @@ IRequestGeneratorPtr CreateDataRequestGenerator( ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + TCountLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp index d3bffb58069..fd14c3571b1 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp @@ -50,7 +50,7 @@ class TDatashardLikeRequestGenerator final const NProto::TDatashardLikeLoadSpec Spec; const TString FileSystemId; const NProto::THeaders Headers; - const TFileCreationLimiterPtr FileCreationLimiter; + const TCountLimiterPtr CountLimiter; TLog Log; @@ -79,11 +79,11 @@ class TDatashardLikeRequestGenerator final IShmDataClientPtr dataClient, TString filesystemId, NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + TCountLimiterPtr fileCreationLimiter) : Spec(std::move(spec)) , FileSystemId(std::move(filesystemId)) , Headers(std::move(headers)) - , FileCreationLimiter(std::move(fileCreationLimiter)) + , CountLimiter(std::move(fileCreationLimiter)) , Session(std::move(session)) , DataClient(std::move(dataClient)) { @@ -177,7 +177,7 @@ class TDatashardLikeRequestGenerator final TFuture DoCreateNode() { auto started = TInstant::Now(); - if (!FileCreationLimiter->TryReserve()) { + if (!CountLimiter->TryReserve()) { return MakeFuture({ NProto::ACTION_CREATE_NODE, started, @@ -240,7 +240,7 @@ class TDatashardLikeRequestGenerator final [=, this](const TFuture& f) { return HandleResizeAfterCreate(f, info, started); }); } catch (const TServiceError& e) { - FileCreationLimiter->Release(); + CountLimiter->Release(); auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR( "create node has failed: %s", @@ -507,7 +507,7 @@ IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( IShmDataClientPtr dataClient, TString filesystemId, NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + TCountLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp index f9911de5d2d..01d0b00b996 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp @@ -73,7 +73,7 @@ class TFastShardRequestGenerator final const ui64 ReadBytes; const ui64 WriteBytes; const ui64 InitialFileSize; - TFileCreationLimiterPtr FileCreationLimiter; + TCountLimiterPtr CountLimiter; TClient Client; TDeque> Endpoints; @@ -90,13 +90,13 @@ class TFastShardRequestGenerator final NProto::TFastShardLoadSpec spec, ui32 maxParallelism, ILoggingServicePtr /*logging*/, - TFileCreationLimiterPtr fileCreationLimiter) + TCountLimiterPtr fileCreationLimiter) : Spec(std::move(spec)) , ShardFileSystemId(Spec.GetShardFileSystemId()) , ReadBytes(Spec.GetReadBytes() ? Spec.GetReadBytes() : DefaultIoSize) , WriteBytes(Spec.GetWriteBytes() ? Spec.GetWriteBytes() : DefaultIoSize) , InitialFileSize(Spec.GetInitialFileSize()) - , FileCreationLimiter(std::move(fileCreationLimiter)) + , CountLimiter(std::move(fileCreationLimiter)) { Y_ENSURE(!ShardFileSystemId.empty(), "ShardFileSystemId must be set"); Y_ENSURE(maxParallelism > 0); @@ -202,7 +202,7 @@ class TFastShardRequestGenerator final // Create a file if the pool had nothing to offer. if (s->File.Handle == 0) { - if (!ptr->FileCreationLimiter->TryReserve()) { + if (!ptr->CountLimiter->TryReserve()) { releaseEndpoint(); s->Promise.SetValue({s->Action, s->Started, MakeError(S_FALSE)}); return; @@ -228,7 +228,7 @@ class TFastShardRequestGenerator final ? resp.GetError() : resp.GetCreateHandle().GetError(); if (HasError(createErr)) { - ptr->FileCreationLimiter->Release(); + ptr->CountLimiter->Release(); releaseEndpoint(); s->Promise.SetValue({s->Action, s->Started, createErr}); return; @@ -359,7 +359,7 @@ IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, ILoggingServicePtr logging, - TFileCreationLimiterPtr fileCreationLimiter) + TCountLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp index 34d4319b8d0..adc538b855e 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp @@ -32,7 +32,7 @@ IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, ILoggingServicePtr logging, - TFileCreationLimiterPtr fileCreationLimiter) + TCountLimiterPtr fileCreationLimiter) { Y_UNUSED(spec); Y_UNUSED(maxParallelism); diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp index 4a8aac09811..1a98b560448 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp @@ -48,7 +48,7 @@ class TIndexRequestGenerator final const NProto::TIndexLoadSpec Spec; const TString FileSystemId; const NProto::THeaders Headers; - const TFileCreationLimiterPtr FileCreationLimiter; + const TCountLimiterPtr CountLimiter; const ui64 OwnerId = RandomNumber(100500u); TLog Log; @@ -78,11 +78,11 @@ class TIndexRequestGenerator final ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + TCountLimiterPtr fileCreationLimiter) : Spec(std::move(spec)) , FileSystemId(std::move(filesystemId)) , Headers(std::move(headers)) - , FileCreationLimiter(std::move(fileCreationLimiter)) + , CountLimiter(std::move(fileCreationLimiter)) , Client(std::move(client)) , Session(std::move(session)) { @@ -151,7 +151,7 @@ class TIndexRequestGenerator final TGuard guard(StateLock); auto started = TInstant::Now(); - if (!FileCreationLimiter->TryReserve()) { + if (!CountLimiter->TryReserve()) { return MakeFuture({ NProto::ACTION_CREATE_NODE, started, @@ -159,7 +159,7 @@ class TIndexRequestGenerator final } if (Spec.GetMaxNodes() && Nodes.size() >= Spec.GetMaxNodes()) { - FileCreationLimiter->Release(); + CountLimiter->Release(); return MakeFuture({ NProto::ACTION_CREATE_NODE, started, @@ -208,7 +208,7 @@ class TIndexRequestGenerator final name.c_str(), FormatError(error).c_str()); - FileCreationLimiter->Release(); + CountLimiter->Release(); return {NProto::ACTION_CREATE_NODE, started, error}; } } @@ -790,7 +790,7 @@ IRequestGeneratorPtr CreateIndexRequestGenerator( ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TFileCreationLimiterPtr fileCreationLimiter) + TCountLimiterPtr fileCreationLimiter) { return std::make_shared( std::move(spec), diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index 2e8ce525758..aadefc60bcd 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -543,7 +543,7 @@ class TLoadTest final Session, FileSystemId, headers, - std::make_shared(Config.GetMaxFileCount())); + std::make_shared(Config.GetMaxFileCount())); break; case NProto::TLoadTest::kDataLoadSpec: RequestGenerator = CreateDataRequestGenerator( @@ -552,7 +552,7 @@ class TLoadTest final Session, FileSystemId, headers, - std::make_shared(Config.GetMaxFileCount())); + std::make_shared(Config.GetMaxFileCount())); break; case NProto::TLoadTest::kReplayFsSpec: RequestGenerator = CreateReplayRequestGeneratorFs( @@ -596,7 +596,7 @@ class TLoadTest final ShmClient, FileSystemId, headers, - std::make_shared(Config.GetMaxFileCount())); + std::make_shared(Config.GetMaxFileCount())); break; } case NProto::TLoadTest::kFastShardLoadSpec: @@ -604,7 +604,7 @@ class TLoadTest final Config.GetFastShardLoadSpec(), Config.GetIODepth(), Logging, - std::make_shared(Config.GetMaxFileCount())); + std::make_shared(Config.GetMaxFileCount())); break; default: ythrow yexception() From 2385bba6e66dd2a21281cdca3dff337cac90f2e5 Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Mon, 20 Jul 2026 16:24:56 +0200 Subject: [PATCH 07/11] Linting --- cloud/filestore/apps/client/lib/diagnose_filesystem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/filestore/apps/client/lib/diagnose_filesystem.cpp b/cloud/filestore/apps/client/lib/diagnose_filesystem.cpp index 9735905f453..acedd84c519 100644 --- a/cloud/filestore/apps/client/lib/diagnose_filesystem.cpp +++ b/cloud/filestore/apps/client/lib/diagnose_filesystem.cpp @@ -179,5 +179,5 @@ TCommandPtr NewDiagnoseFilesystemCommand() { return std::make_shared(); } - + } // namespace NCloud::NFileStore::NClient From 1957b8c94779df8eaad0395ecf49ba4106ab2e7a Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Mon, 20 Jul 2026 17:00:56 +0200 Subject: [PATCH 08/11] Applying requested changes --- cloud/filestore/tools/testing/loadtest/lib/count_limiter.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h b/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h index 72c3c150f7c..268ad0ce5ba 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h +++ b/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h @@ -23,10 +23,11 @@ class TCountLimiter bool TryReserve() { if (!MaxFileCount) { + // MaxFileCount == 0 means unlimited. return true; } - auto count = ReservedFileCount.load(std::memory_order_relaxed); + ui64 count = ReservedFileCount.load(std::memory_order_relaxed); while (count < MaxFileCount){ if (ReservedFileCount.compare_exchange_weak( count, From ca5b820300327d282422405f1f5da84d615963ec Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Tue, 21 Jul 2026 13:49:29 +0200 Subject: [PATCH 09/11] Applying requested changes --- .../testing/loadtest/lib/count_limiter.h | 40 ++++++++++++------- .../tools/testing/loadtest/lib/request.h | 8 ++-- .../testing/loadtest/lib/request_data.cpp | 10 ++--- .../loadtest/lib/request_datashard_like.cpp | 10 ++--- .../loadtest/lib/request_fastshard.cpp | 10 ++--- .../loadtest/lib/request_fastshard_stub.cpp | 4 +- .../testing/loadtest/lib/request_index.cpp | 10 ++--- .../loadtest/lib/request_replay_grpc.cpp | 1 + 8 files changed, 52 insertions(+), 41 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h b/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h index 268ad0ce5ba..738983c627d 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h +++ b/cloud/filestore/tools/testing/loadtest/lib/count_limiter.h @@ -12,24 +12,19 @@ namespace NCloud::NFileStore::NLoadTest { class TCountLimiter { private: - const ui64 MaxFileCount; - std::atomic ReservedFileCount = 0; - -public: - explicit TCountLimiter(ui64 maxFileCount) - : MaxFileCount(maxFileCount) - {} + const ui64 MaxCount; + std::atomic ReservedCount = 0; bool TryReserve() { - if (!MaxFileCount) { - // MaxFileCount == 0 means unlimited. + if (!MaxCount) { + // MaxCount == 0 means unlimited. return true; } - ui64 count = ReservedFileCount.load(std::memory_order_relaxed); - while (count < MaxFileCount){ - if (ReservedFileCount.compare_exchange_weak( + ui64 count = ReservedCount.load(std::memory_order_relaxed); + while (count < MaxCount){ + if (ReservedCount.compare_exchange_weak( count, count + 1, std::memory_order_relaxed)) @@ -37,13 +32,28 @@ class TCountLimiter return true; } } - return count < MaxFileCount; + return count < MaxCount; + } + +public: + explicit TCountLimiter(ui64 maxCount) + : MaxCount(maxCount) + {} + + bool TryReserveHandle() + { + return TryReserve(); + } + + bool TryReserveNode() + { + return TryReserve(); } void Release() { - if (MaxFileCount) { - ReservedFileCount.fetch_sub(1, std::memory_order_relaxed); + if (MaxCount) { + ReservedCount.fetch_sub(1, std::memory_order_relaxed); } } }; diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index 9d53b9cec4b..507e5e25400 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -74,7 +74,7 @@ IRequestGeneratorPtr CreateIndexRequestGenerator( NClient::ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TCountLimiterPtr fileCreationLimiter); + TCountLimiterPtr countLimiter); IRequestGeneratorPtr CreateDataRequestGenerator( NProto::TDataLoadSpec spec, @@ -82,7 +82,7 @@ IRequestGeneratorPtr CreateDataRequestGenerator( NClient::ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TCountLimiterPtr fileCreationLimiter); + TCountLimiterPtr countLimiter); IRequestGeneratorPtr CreateReplayRequestGeneratorFs( NProto::TReplaySpec spec, @@ -102,7 +102,7 @@ IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, ILoggingServicePtr logging, - TCountLimiterPtr fileCreationLimiter); + TCountLimiterPtr countLimiter); IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( NProto::TDatashardLikeLoadSpec spec, @@ -111,6 +111,6 @@ IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( IShmDataClientPtr dataClient, TString filesystemId, NProto::THeaders headers, - TCountLimiterPtr fileCreationLimiter); + TCountLimiterPtr countLimiter); } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp index 119af5b3e55..13052956247 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp @@ -136,11 +136,11 @@ class TDataRequestGenerator final ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TCountLimiterPtr fileCreationLimiter) + TCountLimiterPtr countLimiter) : Spec(std::move(spec)) , FileSystemId(std::move(filesystemId)) , Headers(std::move(headers)) - , CountLimiter(std::move(fileCreationLimiter)) + , CountLimiter(std::move(countLimiter)) , Session(std::move(session)) { Log = logging->CreateLog(Headers.GetClientId()); @@ -241,7 +241,7 @@ class TDataRequestGenerator final auto started = TInstant::Now(); TGuard guard(StateLock); - if (!CountLimiter->TryReserve()) { + if (!CountLimiter->TryReserveHandle()) { return MakeFuture({ NProto::ACTION_CREATE_HANDLE, started, @@ -657,7 +657,7 @@ IRequestGeneratorPtr CreateDataRequestGenerator( ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TCountLimiterPtr fileCreationLimiter) + TCountLimiterPtr countLimiter) { return std::make_shared( std::move(spec), @@ -665,7 +665,7 @@ IRequestGeneratorPtr CreateDataRequestGenerator( std::move(session), std::move(filesystemId), std::move(headers), - std::move(fileCreationLimiter)); + std::move(countLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp index fd14c3571b1..c0f7997932a 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_datashard_like.cpp @@ -79,11 +79,11 @@ class TDatashardLikeRequestGenerator final IShmDataClientPtr dataClient, TString filesystemId, NProto::THeaders headers, - TCountLimiterPtr fileCreationLimiter) + TCountLimiterPtr countLimiter) : Spec(std::move(spec)) , FileSystemId(std::move(filesystemId)) , Headers(std::move(headers)) - , CountLimiter(std::move(fileCreationLimiter)) + , CountLimiter(std::move(countLimiter)) , Session(std::move(session)) , DataClient(std::move(dataClient)) { @@ -177,7 +177,7 @@ class TDatashardLikeRequestGenerator final TFuture DoCreateNode() { auto started = TInstant::Now(); - if (!CountLimiter->TryReserve()) { + if (!CountLimiter->TryReserveNode()) { return MakeFuture({ NProto::ACTION_CREATE_NODE, started, @@ -507,7 +507,7 @@ IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( IShmDataClientPtr dataClient, TString filesystemId, NProto::THeaders headers, - TCountLimiterPtr fileCreationLimiter) + TCountLimiterPtr countLimiter) { return std::make_shared( std::move(spec), @@ -516,7 +516,7 @@ IRequestGeneratorPtr CreateDatashardLikeRequestGenerator( std::move(dataClient), std::move(filesystemId), std::move(headers), - std::move(fileCreationLimiter)); + std::move(countLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp index 01d0b00b996..ef95b165869 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard.cpp @@ -90,13 +90,13 @@ class TFastShardRequestGenerator final NProto::TFastShardLoadSpec spec, ui32 maxParallelism, ILoggingServicePtr /*logging*/, - TCountLimiterPtr fileCreationLimiter) + TCountLimiterPtr countLimiter) : Spec(std::move(spec)) , ShardFileSystemId(Spec.GetShardFileSystemId()) , ReadBytes(Spec.GetReadBytes() ? Spec.GetReadBytes() : DefaultIoSize) , WriteBytes(Spec.GetWriteBytes() ? Spec.GetWriteBytes() : DefaultIoSize) , InitialFileSize(Spec.GetInitialFileSize()) - , CountLimiter(std::move(fileCreationLimiter)) + , CountLimiter(std::move(countLimiter)) { Y_ENSURE(!ShardFileSystemId.empty(), "ShardFileSystemId must be set"); Y_ENSURE(maxParallelism > 0); @@ -202,7 +202,7 @@ class TFastShardRequestGenerator final // Create a file if the pool had nothing to offer. if (s->File.Handle == 0) { - if (!ptr->CountLimiter->TryReserve()) { + if (!ptr->CountLimiter->TryReserveHandle()) { releaseEndpoint(); s->Promise.SetValue({s->Action, s->Started, MakeError(S_FALSE)}); return; @@ -359,13 +359,13 @@ IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, ILoggingServicePtr logging, - TCountLimiterPtr fileCreationLimiter) + TCountLimiterPtr countLimiter) { return std::make_shared( std::move(spec), maxParallelism, std::move(logging), - std::move(fileCreationLimiter)); + std::move(countLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp index adc538b855e..89b3b0d9cff 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_fastshard_stub.cpp @@ -32,12 +32,12 @@ IRequestGeneratorPtr CreateFastShardRequestGenerator( NProto::TFastShardLoadSpec spec, ui32 maxParallelism, ILoggingServicePtr logging, - TCountLimiterPtr fileCreationLimiter) + TCountLimiterPtr countLimiter) { Y_UNUSED(spec); Y_UNUSED(maxParallelism); Y_UNUSED(logging); - Y_UNUSED(fileCreationLimiter); + Y_UNUSED(countLimiter); return std::make_shared(); } diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp index 1a98b560448..e7e2e2784a0 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp @@ -78,11 +78,11 @@ class TIndexRequestGenerator final ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TCountLimiterPtr fileCreationLimiter) + TCountLimiterPtr countLimiter) : Spec(std::move(spec)) , FileSystemId(std::move(filesystemId)) , Headers(std::move(headers)) - , CountLimiter(std::move(fileCreationLimiter)) + , CountLimiter(std::move(countLimiter)) , Client(std::move(client)) , Session(std::move(session)) { @@ -151,7 +151,7 @@ class TIndexRequestGenerator final TGuard guard(StateLock); auto started = TInstant::Now(); - if (!CountLimiter->TryReserve()) { + if (!CountLimiter->TryReserveNode()) { return MakeFuture({ NProto::ACTION_CREATE_NODE, started, @@ -790,7 +790,7 @@ IRequestGeneratorPtr CreateIndexRequestGenerator( ISessionPtr session, TString filesystemId, NProto::THeaders headers, - TCountLimiterPtr fileCreationLimiter) + TCountLimiterPtr countLimiter) { return std::make_shared( std::move(spec), @@ -799,7 +799,7 @@ IRequestGeneratorPtr CreateIndexRequestGenerator( std::move(session), std::move(filesystemId), std::move(headers), - std::move(fileCreationLimiter)); + std::move(countLimiter)); } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp index 75dc8fd3bd6..8806f4284b6 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp @@ -233,6 +233,7 @@ class TReplayRequestGeneratorGRPC final const auto request = CreateRequest(); auto name = logRequest.GetNodeInfo().GetNodeName(); + const auto nodeId = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (nodeId == InvalidNodeId) { From 6c7072d2d55de80f890dad7f7963840b73f7f0ad Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Tue, 21 Jul 2026 14:04:49 +0200 Subject: [PATCH 10/11] Format test result --- .../canondata/test.test_max_file_count/results.txt | 2 +- cloud/filestore/tests/loadtest/max-file-count-test/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud/filestore/tests/loadtest/max-file-count-test/canondata/test.test_max_file_count/results.txt b/cloud/filestore/tests/loadtest/max-file-count-test/canondata/test.test_max_file_count/results.txt index 0bb0339289e..35eed5ae6a3 100644 --- a/cloud/filestore/tests/loadtest/max-file-count-test/canondata/test.test_max_file_count/results.txt +++ b/cloud/filestore/tests/loadtest/max-file-count-test/canondata/test.test_max_file_count/results.txt @@ -1,3 +1,3 @@ / max-file-count-loadtest: / max-file-count-loadtest: -/ max-file-count-loadtest: \ No newline at end of file +/ max-file-count-loadtest: diff --git a/cloud/filestore/tests/loadtest/max-file-count-test/test.py b/cloud/filestore/tests/loadtest/max-file-count-test/test.py index aec51bf1736..4482e42dc43 100644 --- a/cloud/filestore/tests/loadtest/max-file-count-test/test.py +++ b/cloud/filestore/tests/loadtest/max-file-count-test/test.py @@ -35,7 +35,7 @@ def test_max_file_count(): ] with open(results_path, "w") as results: - results.write("\n".join(normalized_entries)) + results.write("\n".join(normalized_entries) + "\n") finally: client.destroy(filesystem_id) From fcc1885441183c6885ef9f268fb7f8d9ff07fa2e Mon Sep 17 00:00:00 2001 From: Jan Szumski Date: Tue, 21 Jul 2026 17:47:33 +0200 Subject: [PATCH 11/11] Reuse the MaxNode parameter for all bounded loadtests --- .../tests/loadtest/max-file-count-test/max-file-count.txt | 2 +- .../tools/testing/loadtest/lib/request_index.cpp | 8 -------- cloud/filestore/tools/testing/loadtest/lib/test.cpp | 8 ++++---- .../tools/testing/loadtest/protos/loadtest.proto | 5 ++--- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt b/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt index d9bbb37c6ae..009092fa46d 100644 --- a/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt +++ b/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt @@ -10,7 +10,7 @@ Tests { BlockSize: 4096 } KeepFileStore: true - MaxFileCount: 3 + MaxNodes: 3 RequestsCount: 30 IODepth: 16 DataLoadSpec { diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp index e7e2e2784a0..6ebb9a53160 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp @@ -158,14 +158,6 @@ class TIndexRequestGenerator final MakeError(S_FALSE)}); } - if (Spec.GetMaxNodes() && Nodes.size() >= Spec.GetMaxNodes()) { - CountLimiter->Release(); - return MakeFuture({ - NProto::ACTION_CREATE_NODE, - started, - MakeError(S_FALSE)}); - } - auto name = GenerateNodeName(); StagedNodes[name] = {}; diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index aadefc60bcd..0aa937900f5 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -543,7 +543,7 @@ class TLoadTest final Session, FileSystemId, headers, - std::make_shared(Config.GetMaxFileCount())); + std::make_shared(Config.GetMaxNodes())); break; case NProto::TLoadTest::kDataLoadSpec: RequestGenerator = CreateDataRequestGenerator( @@ -552,7 +552,7 @@ class TLoadTest final Session, FileSystemId, headers, - std::make_shared(Config.GetMaxFileCount())); + std::make_shared(Config.GetMaxNodes())); break; case NProto::TLoadTest::kReplayFsSpec: RequestGenerator = CreateReplayRequestGeneratorFs( @@ -596,7 +596,7 @@ class TLoadTest final ShmClient, FileSystemId, headers, - std::make_shared(Config.GetMaxFileCount())); + std::make_shared(Config.GetMaxNodes())); break; } case NProto::TLoadTest::kFastShardLoadSpec: @@ -604,7 +604,7 @@ class TLoadTest final Config.GetFastShardLoadSpec(), Config.GetIODepth(), Logging, - std::make_shared(Config.GetMaxFileCount())); + std::make_shared(Config.GetMaxNodes())); break; default: ythrow yexception() diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index b1cfd1f39f2..31a3b3a33f7 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -37,7 +37,6 @@ message TIndexLoadSpec } repeated TAction Actions = 1; - uint64 MaxNodes = 2; } message TDataLoadSpec @@ -211,9 +210,9 @@ message TLoadTest bool KeepFileStore = 16; string ClientId = 19; - // Maximum number of regular files created by this synthetic load test. + // Maximum number of nodes created by this synthetic load test. // Zero means unlimited. Ignored by replay specs. - uint64 MaxFileCount = 22; + uint64 MaxNodes = 22; } ////////////////////////////////////////////////////////////////////////////////