From 654489d1ea965fabcc45e74add795d239d2909ab Mon Sep 17 00:00:00 2001 From: Lobsinger Sebastian Date: Tue, 11 Aug 2026 06:57:37 +0000 Subject: [PATCH 1/4] Added IPC pocs and docs --- MODULE.bazel | 3 + MODULE.bazel.lock | 1 + score/crypto/docs/index.rst | 1 + score/crypto/src/ipc/docs/index.rst | 330 +++ score/crypto/src/ipc/docs/ipc_comparison.md | 135 ++ score/tests/ipc_poc/.gitignore | 2 + score/tests/ipc_poc/BUILD | 78 + score/tests/ipc_poc/async_control_interface.h | 57 + score/tests/ipc_poc/ipc_buffer.h | 92 + .../tests/ipc_poc/message_passing_overview.md | 234 ++ .../ipc_poc/mw_com_message_passing.drawio.svg | 2114 +++++++++++++++++ score/tests/ipc_poc/poc_async.cpp | 1321 ++++++++++ score/tests/ipc_poc/poc_control.fbs | 131 + score/tests/ipc_poc/poc_grpc.cpp | 515 ++++ score/tests/ipc_poc/poc_low_level.cpp | 1393 +++++++++++ third_party/grpc/grpc_symbol_reference.cpp | 5 +- 16 files changed, 6410 insertions(+), 2 deletions(-) create mode 100644 score/crypto/src/ipc/docs/index.rst create mode 100644 score/crypto/src/ipc/docs/ipc_comparison.md create mode 100644 score/tests/ipc_poc/.gitignore create mode 100644 score/tests/ipc_poc/BUILD create mode 100644 score/tests/ipc_poc/async_control_interface.h create mode 100644 score/tests/ipc_poc/ipc_buffer.h create mode 100644 score/tests/ipc_poc/message_passing_overview.md create mode 100644 score/tests/ipc_poc/mw_com_message_passing.drawio.svg create mode 100644 score/tests/ipc_poc/poc_async.cpp create mode 100644 score/tests/ipc_poc/poc_control.fbs create mode 100644 score/tests/ipc_poc/poc_grpc.cpp create mode 100644 score/tests/ipc_poc/poc_low_level.cpp diff --git a/MODULE.bazel b/MODULE.bazel index 8cdb18024..ef5b15fe2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -339,6 +339,9 @@ single_version_override( # Build settings utilities (bool_flag, string_flag, etc.) bazel_dep(name = "bazel_skylib", version = "1.9.2") +# s-core communication +bazel_dep(name = "score_communication", version = "0.3.0") + # Testing bazel_dep(name = "googletest", version = "1.18.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 3163c40bf..7906f379c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -896,6 +896,7 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_bazel_platforms/1.0.0/MODULE.bazel": "e5e386e6fc0a447a2f4a47e872e3747822e9d58994b7b55e248c78e5a3e382c0", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_bazel_platforms/1.1.0/MODULE.bazel": "e5c656abcad121a89cbabad3550bb20167a6865de219ef6e1c68ca30567e1c32", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_bazel_platforms/1.1.0/source.json": "9797d56e9dd68e9d3e8b674402df2d6d5e2174f84184ca4cbee78fc07dc117d9", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_communication/0.3.0/MODULE.bazel": "7674d445cb51ea31e7ea69bc5c98f9dd6b81beeced915cb9c4e21b99debb61e0", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_communication/0.4.0/MODULE.bazel": "19e98105c4b1ca674027dfa4c5010eebbdf74d7cbd9524cb84a8fad63a8bcebe", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_communication/0.4.0/source.json": "a3378155debe445305828cfa94464ad5f53d5b7e27488b3c053b02e49801aaeb", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_cr_checker/0.2.2/MODULE.bazel": "dc36d9c35543db918c3fb5b93a8e684431f56c7c784cf2a1b90f35802a373c98", diff --git a/score/crypto/docs/index.rst b/score/crypto/docs/index.rst index b120b447c..e94824cdb 100644 --- a/score/crypto/docs/index.rst +++ b/score/crypto/docs/index.rst @@ -165,6 +165,7 @@ Additional documentation for relevant Crypto subcomponents can be found here: :maxdepth: 1 ../src/daemon/data_manager/docs/index + ../src/ipc/docs/index Component Detail Information ============================ diff --git a/score/crypto/src/ipc/docs/index.rst b/score/crypto/src/ipc/docs/index.rst new file mode 100644 index 000000000..b5fc4e1e4 --- /dev/null +++ b/score/crypto/src/ipc/docs/index.rst @@ -0,0 +1,330 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +IPC Design Decisions +==================== + +Selection of LoLa Message Passing as IPC +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. dec_rec:: Selection of LoLa Message Passing as IPC + :id: dec_rec__crypto__lola_message_passing_ipc + :version: 1 + :status: draft + :context: doc__crypto_architecture + :decision: Use LoLa Message Passing, implemented with the score::message_passing API, as the crypto daemon control-plane IPC transport. + + .. :affects: comp__crypto + +LoLa Message Passing, implemented with the ``score::message_passing`` API, is +selected as the target control-plane transport for communication between client +processes and the daemon. Production integration and implementation validation +are follow-up work. + +Context +******* + +The IPC transport must be suitable for use in an ASIL-B environment. This +includes deterministic and bounded execution and resource behavior, controlled +heap/allocation use on ASIL-relevant paths, suitable isolation for the +deployment, and appropriate quality artifacts such as requirements and design +traceability, analysis, verification, and compliance or qualification evidence. +It must also provide bounded client-side waiting, allow concurrent calls from +multiple client processes and threads, expose the identity of the peer, avoid +imposing the daemon's worker-thread model, and keep the application protocol +independent of operating-system transport details. + +Decision +******** + +Use ``score::message_passing`` with FlatBuffers as the serialized control-plane +payload format. The application uses the ``ClientFactory`` and +``ServerFactory`` abstractions and does not depend directly on Unix-domain +socket or QNX message-passing APIs. + +The deployment configuration shall establish fixed system-level upper bounds: + +* ``N``: maximum number of client processes and server connections. +* ``T``: maximum number of concurrent in-flight requests per client process. +* ``M``: maximum serialized payload size in bytes. + +The client and server must use consistent protocol and queue-size settings. +The relevant capacities shall be derived from ``N``, ``T``, and ``M``, and the +expected server connections shall be pre-allocated. These values are +integration parameters, not runtime-adaptive values, and must be reviewed when +the deployment topology or concurrency assumptions change. + +Consequences +************ + +**Positive:** + +* Kernel-mediated transport provides process isolation without a shared + writable memory region between client and daemon processes. +* The server can retrieve kernel-provided peer credentials from an active + connection, avoiding reliance on a client-supplied identity. +* The application-facing API is OS-agnostic; the framework selects the native + backend for the target operating system. +* The daemon retains control of its execution model and can dispatch work to a + fixed worker pool with appropriate implementation. +* The communication module documents safety-oriented quality tooling and ASIL-B + qualification, while the LoLa Message Passing design supports fixed + resource bounds, preallocation, and pool/monotonic allocation. +* Multiple client processes and multiple concurrent client threads are + supported. A shared connection per client process avoids the + client-process-times-thread connection growth of a per-thread design. +* A service identifier is sufficient to locate the endpoint; no per-client + service configuration or service registry is required by the application. + +**Negative:** + +* Safe operation requires explicit worst-case sizing for ``N``, ``T``, and + ``M``. In particular, an undersized QNX notify queue can cause ``ENOBUFS`` + and a lost response, while inconsistent payload sizes can cause send or + receive failures. +* The selected LoLa Message Passing implementation does not provide + inter-VM communication. Inter-VM deployment depends on a future framework + backend. The separate connection implementation suggested by the crypto IPC + abstraction is only a possible migration direction. +* The transport is not shared-memory based, so it may have higher per-message + overhead than LoLa for very high-throughput data transfer. This is not a + control-plane requirement because bulk data is handled separately. + +Alternatives Considered +*********************** + +gRPC over Unix-Domain Socket +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +gRPC provides mature request/response and inter-VM channel options, but the +general-purpose library uses framework-managed threads and dynamic resources and +does not provide an ASIL-oriented deterministic resource profile or safety +qualification artifacts. The evaluated blocking call also does not establish +the required bounded waiting behaviour. It remains suitable for QM-to-QM +communication and is the strongest option for an immediate network or inter-VM +channel. + +LoLa Full SOA Abstraction +^^^^^^^^^^^^^^^^^^^^^^^^^ + +LoLa is documented as safety-oriented/ASIL-B qualified, with custom +memory-management infrastructure and quality tooling. The synchronous method +lacks an application escape from an indefinitely blocked call. The asynchronous +method layer built on top during POCs, improves server threading and response +timeout handling, but its internal method call still blocks without a timeout +and its broadcast response model requires additional per-client skeleton +instances and response routing. + +LoLa Message Passing +^^^^^^^^^^^^^^^^^^^^ + +The message-passing abstraction exposes platform-independent client and server +factories while keeping the operating-system transport behind the IPC layer. +Using ``SendWithCallback`` with ``Reply`` and ``Notify`` separates request +acceptance from operation completion. A shared connection per client process +supports concurrent client threads through explicit request identifiers and a +server-side worker pool, without scaling connections as ``N * T``. The design +requires explicit bounds for client processes, in-flight requests, payload size, +and transport queues, but preserves process isolation and the daemon's freedom +to choose its worker model. + +Comparison Summary +****************** + +The alternatives were assessed against ASIL-B suitability, bounded waiting, +connection-loss handling, authentic peer identification, configuration effort, +server-threading flexibility, concurrency, one-to-one communication, latency, +and inter-VM support. + +* **gRPC over Unix-domain sockets** provides mature request/reply semantics and + the strongest network and inter-VM path. Its general-purpose runtime does not + provide the deterministic resource profile or safety evidence required for + the intended ASIL-B use. The current adapter also has no configured deadline, + and peer authentication would require an additional security mechanism. +* **LoLa Full SOA synchronous Method** provides a safety-oriented service model + and one-to-one responses, but its blocking call offers no application escape + when the server stalls. Its service configuration is also more extensive than + the selected solution requires. +* **LoLa Full SOA asynchronous Method plus Event** allows independent server + worker execution and bounded waiting for the final event. The initial method + call remains blocking without a framework change, and broadcast event + delivery requires additional per-client instances and response routing. +* **LoLa Message Passing** provides platform-independent client and server + factories, process-isolating transport, kernel-provided peer credentials, + and a server-controlled worker model can be added easily. ``SendWithCallback`` + followed by ``Reply`` and ``Notify`` supports concurrent calls over one + connection per client process, with application-level request identifiers + for response routing. It requires explicit resource bounds and a typed + timeout and connection-loss contract. It does not provide inter-VM + communication. + +LoLa Message Passing is selected because it best satisfies the ASIL-B, +bounded-client-waiting, peer-identity, concurrency, and server-threading goals +while keeping operating-system transport details below the application IPC +interface. The remaining backend, error-contract, resource-sizing, and inter-VM +limitations are explicit implementation and verification work. + +Justification for the Decision +****************************** + +LoLa Message Passing is the only evaluated option that appears capable of +meeting the broader ASIL-B use goals and the bounded client call-path goal based +on its library design and documented quality properties. Meeting the timeout +and connection-loss requirements still depends on the production IPC wrapper, +and a typed error contract. The lack of inter-VM communication is accepted, +the control-plane IPC abstraction can potentially be used to offer a separate +inter-VM transport in the future. + +Two-Phase Request/Reply Acknowledgement and Notify Completion +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. dec_rec:: Two-Phase Request/Reply Acknowledgement and Notify Completion + :id: dec_rec__crypto__two_phase_request_reply_notify + :version: 1 + :status: draft + :context: doc__crypto_architecture + :decision: Use SendWithCallback for a non-blocking request acknowledgement and Notify for the independently produced operation response. + + .. :affects: comp__crypto + +The selected IPC protocol uses two phases: an immediate ``Reply`` acknowledgement +for the accepted request, followed by a ``Notify`` message containing the +operation result. + +Context +******* + +Using LoLa Message Passing, the REQUEST/REPLY protocol serializes requests on one +connection: the server does not process the next request on that connection until +the current request has received a ``Reply``. A client process may nevertheless +have many threads issuing independent operations, and the server must be able to +process those operations concurrently. + +Using a single long-running request callback would keep the shared connection +occupied for the duration of the cryptographic operation. Using +``SendWaitReply`` would additionally block the client thread inside the IPC +library and does not provide the application with a timeout escape if the +server stalls. The selected communication model therefore separates request +acceptance from operation completion. + +Decision +******** + +The protocol is defined as follows: + +1. The client assigns a unique non-zero ``request_id``, inserts a pending-call + record, and calls ``SendWithCallback`` with the FlatBuffer ``ControlRequest``. +2. The server validates the request and copies the complete request into the + application work queue. +3. The server calls ``Reply`` with a minimal acknowledgement carrying the + ``request_id``. After the acknowledgement attempt, it wakes a worker. A + successful acknowledgement means the request was admitted to the work queue. +4. A server worker performs the operation and calls ``Notify`` on the same + ``IServerConnection`` with a ``ControlResponse`` carrying the original + ``request_id`` and the result payload. +5. The client ``NotifyCallback`` routes the response by ``request_id`` and + signals the waiting application thread. The application waits with a + bounded timeout and retires the pending-call record after completion or + timeout. + +The acknowledgement is an acceptance signal, not the operation result. The +transport does not automatically retry requests. An accepted request is +attempted once; if the operation result is not observed because of a timeout or +delivery failure, the outcome is unknown to the caller. The application may +explicitly retry when the operation semantics allow it. ``ReplyCallback`` is +used to observe send or acknowledgement failure; the client does not complete +the operation merely because the acknowledgement arrived. + +Consequences +************ + +**Positive:** + +* The server releases the per-connection REQUEST/REPLY serialization window + before doing the potentially slow operation. Subsequent requests can be + accepted while earlier requests execute in the worker pool. +* ``SendWithCallback`` returns without holding the caller inside the IPC + operation. A production wrapper can use a mandatory application-level + ``wait_for`` to provide a bounded wait for the final response and report a + typed timeout. +* One connection can be shared by all threads in a client process. The + application-level ``request_id`` protocol provides deterministic response + demultiplexing without requiring one connection per thread. +* ``Notify`` is point-to-point, so a response is delivered only to the client + connection that issued the request. This avoids the cross-client response + leakage risk of a broadcast event model. +* The server can use a fixed worker pool and must protect connection lifetime + while workers complete delayed notifications. + +**Negative:** + +* The protocol and implementation are more complex than a single synchronous + call. Pending-call state, request identifiers, response parsing, timeout + cleanup, and late-notification handling are required. +* The acknowledgement does not prove that the operation completed; callers + must handle both acknowledgement failure and final-notification timeout. +* Queue capacities must cover the configured concurrency. Client async-reply + and send queues and the server notify queue must be sized consistently. + An undersized queue can reject a send or drop a notification. +* A worker may finish after the client has timed out. The server must detect a + disconnected connection or safely skip the notification, and the client + must discard late notifications for retired request identifiers. +* ``Notify`` can still be transport-dependent in its blocking behaviour. + Bounded notification semantics require support from the transport or an + additional connection lease abstraction. + +Alternatives Considered +*********************** + +Single SendWaitReply Call +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The client sends a request and waits for the server to return the final result +through the REQUEST/REPLY exchange. This is simpler and naturally matches one +request to one response, but it blocks the caller inside the IPC library for +the whole operation and provides no application-level timeout. With a shared +connection, the server callback also serializes all requests until work is +complete. + +Long-Running SendWithCallback Reply +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The server could defer ``Reply`` until the worker has finished and return the +final response through ``ReplyCallback``. This preserves a single response +channel, but retains the per-connection serialization window for the entire +operation. It prevents the shared-connection design from accepting concurrent +requests at the intended rate and makes the server callback lifetime depend on +operation duration. + +LoLa Method plus Broadcast Event +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The LoLa asynchronous method plus event uses a short method call followed by an +event, which has similar separation of acceptance and completion. Its event is +broadcast, however, so the design needs one skeleton instance per client and +additional routing to prevent clients from observing one another's responses. +The ``Notify`` primitive provides the same asynchronous completion model as a +point-to-point message on the existing client connection. + +Justification for the Decision +****************************** + +Immediate acknowledgement is the smallest operation that satisfies the +low-level protocol's serialization rule while allowing the server to dispatch +work independently. The separate ``Notify`` completion keeps the client +thread out of the IPC library's blocking path, enables an explicit application +timeout, and preserves one connection per client process. The request ID and +pending-call lifecycle are deliberate complexity: they are required to obtain +bounded, concurrent operation completion from a shared connection without +using broadcast communication or shared writable memory. diff --git a/score/crypto/src/ipc/docs/ipc_comparison.md b/score/crypto/src/ipc/docs/ipc_comparison.md new file mode 100644 index 000000000..2f8b45106 --- /dev/null +++ b/score/crypto/src/ipc/docs/ipc_comparison.md @@ -0,0 +1,135 @@ + + +# DAR — IPC Mechanism for score-crypto Daemon + +| | | +|---|---| +| **Status** | Draft | +| **Date** | 2026-08-13 | +| **Author** | ETAS | +| **Context** | score-crypto daemon IPC transport selection | + +--- + +## 1. Problem Statement + +The score-crypto daemon requires an IPC transport to serve cryptographic operations to client processes. The current implementation uses gRPC. The target deployment environment is automotive ECUs with strict functional-safety requirements (ISO 26262), potentially running mixed ASIL-level workloads, possibly across VM boundaries. A replacement or continuation decision is needed. + +--- + +## 2. Requirements + +| ID | Requirement | Rationale | +|---|---|---| +| R1 | Suitability for ASIL-B use | The library must be designed and documented to support deterministic and bounded execution and resource behavior, controlled heap/allocation use on ASIL-relevant paths, suitable isolation for the deployment, and appropriate quality artifacts such as requirements/design traceability, analysis, verification, and compliance or qualification evidence. | +| R2 | Bounded timeout on every call | WCET must be provable. A stalled QM server must never indefinitely block an ASIL-B client thread. | +| R3 | Resilience to server crash / connection loss | If the server process dies or the connection is lost, the client must receive a typed error. The client must not block indefinitely or silently operate on a dead connection. | +| R4 | Authentic peer identification | Server must be able to identify the calling process via an OS-enforced mechanism, not a client-supplied value. | +| R5 | Minimal or zero configuration | The IPC usage should require minimal configuration and be fully configurable programmatically. | +| R6 | Flexible server-side threading model | The IPC transport must not impose a threading model. The server must be free to choose single-thread, fixed pool, or per-request threads. | +| R7 | Concurrent calls from one process | Multiple threads in one client process must be able to issue independent calls simultaneously. | +| R8 | Multiple concurrent client processes | Multiple independent processes must each be able to connect and call the server simultaneously. | +| R9 | One-to-one communication | Each request maps to exactly one response. No broadcast, no fan-out. | +| R10 | Low latency | IPC overhead should be low. Data throughput is not a priority, since crypto provides a separate data plane for transfer of bigger data elements. | +| R11 | Inter-VM communication | The mechanism must either natively support inter-VM communication on a safety hypervisor, or have a minimal and safe migration path to do so. Crypto has an IPC / connection abstraction which allows the usage of different IPC / connection mechanisms. | + +--- + +## 3. Options Considered + +### Option A — gRPC over Unix Domain Socket (current) + +gRPC using FlatBuffers serialisation, connected via `unix://` channel. In production use in the repo today via `GrpcControlClient` / `GrpcControlServer`. + +### Option B — LoLa Full SOA Abstraction (synchronous Method) + +S-CORE `mw::com` LoLa binding, synchronous Method call. Single `Execute(IpcBuffer) → IpcBuffer` method over shared memory. Evaluated in the synchronous LoLa prototype. + +### Option C — LoLa Full SOA Abstraction (asynchronous Method + Event) + +S-CORE `mw::com` LoLa binding, two-phase protocol: Phase 1 — short blocking Method call returns a ticket; Phase 2 — result delivered via a broadcast Event. Evaluated in the asynchronous LoLa prototype. + +### Option D — LoLa Message Passing Abstraction (SendWithCallback + Notify) + +The platform-independent message-passing abstraction uses the S-CORE +`score::message_passing` API with `SendWithCallback` / `Reply` / `Notify` +primitives. The transport backend is OS-specific and provided by the framework: +Unix domain socket on Linux, QNX message passing on QNX. The application code is +OS-agnostic. Evaluated in the low-level message-passing prototype. + +--- + +## 4. Decision Matrix + +Options C and D use a two-phase protocol (enqueue + async response); R2 applies to both phases independently. For Options A and B, which are single-phase, R2 describes a single blocking call. + +The matrix describes target library and architecture capabilities, not completed +production guarantees. ``✅`` means the option appears capable of meeting the +requirement architecturally; ``⚠️`` means the result depends on a wrapper, +backend, configuration, or follow-up implementation; ``❌`` means an identified +architectural mismatch; and ``❓`` means it was not evaluated. The POCs provide +feasibility evidence for selected points, while the production implementation +must validate the remaining requirements. + +| Requirement | Option A — gRPC | Option B — LoLa Full SOA (sync) | Option C — LoLa Full SOA (async) | Option D — LoLa Message Passing Abstraction | +|---|---|---|---|---| +| **R1** ASIL-B suitability | ❌ general-purpose gRPC uses framework-managed threads and dynamic runtime resources; the library does not provide an ASIL-oriented deterministic resource profile or safety qualification artifacts | ✅ the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use and ASIL-B deployment | ✅ the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use and ASIL-B deployment, including its method/event model | ✅ the message-passing library design supports fixed resource bounds, preallocation, and pool/monotonic allocation; the communication module documents safety-oriented quality tooling and ASIL-B qualification | +| **R2** Bounded timeout | ⚠️ current adapter uses a blocking call without a configured deadline; deadline-based or async gRPC was not analyzed | ❌ single-phase blocking call — blocks indefinitely if server stalls | ⚠️ Phase 2 (response event): `wait_for()` indicated in POC; Phase 1 (enqueue): ❌ `DoCall()` holds caller's thread inside LoLa framework with no timeout API — requires framework change | ⚠️ `SendWithCallback()` is non-blocking by design and can support an application-level timeout; mandatory timeout API and typed error handling remain production work, and server-side `Notify()` bounds are backend-dependent | +| **R3** Server crash / connection loss | ❓ not analysed in POC | ❓ not analysed in POC | ❓ not analysed in POC | ⚠️ transport disconnect was observed in the POC; typed propagation to all pending calls and cleanup policy remain production work | +| **R4** Authentic peer identification | ❌ no authentication mechanism | ⚠️ config specifies which UIDs may use a service instance — non-listed UIDs are rejected at runtime by the framework; no API to actively query the connected UID | ⚠️ config specifies which UIDs may use a service instance — non-listed UIDs are rejected at runtime by the framework; no API to actively query the connected UID | ✅ `score::message_passing` exposes an API to retrieve kernel-provided peer credentials from an active connection | +| **R5** Minimal config | ✅ socket path only | ❌ LoLa requires service-oriented configuration for the service, instance, method/event identifiers, application identity, and safety-relevant properties; configuration must be provided consistently to the participating applications and composed with any other `mw::com` services used in the same process | ❌ LoLa requires service-oriented configuration for the service, instance, method/event identifiers, application identity, and safety-relevant properties; configuration must be provided consistently to the participating applications and composed with any other `mw::com` services used in the same process | ⚠️ service identifier only, but worst-case buffer sizes must be configured explicitly: max client processes (N), max concurrent threads per client (T), and max payload size — all must be set to system-wide upper bounds at compile/startup time | +| **R6** Flexible server threading | ⚠️ gRPC owns an internal thread pool; handler is called on gRPC threads. Flexible configuration of that pool may not be sufficient for the required threading model, while a complete replacement of it may be feasible but was not evaluated | ⚠️ skeleton thread processes one call at a time and blocks until the handler returns; handler may dispatch to a pool internally but must block the skeleton on the result — skeleton is always occupied during work | ✅ flexible threading model can be built on top — POC demonstrates a server-side thread pool receiving work via the Phase 1 callback and completing it independently | ✅ `Reply()` callable from any thread at any time — server fully controls threading | +| **R7** Concurrent calls / same process | ✅ channel is thread-safe, no external mechanism needed | ⚠️ concurrent calls depend on the LoLa proxy and method queue configuration; the service abstraction does not make the required parallelism transparent to the application | ⚠️ concurrent calls depend on the LoLa proxy, method queue, and event subscription configuration; the service abstraction does not make the required parallelism transparent to the application | ⚠️ the library supports concurrent use, while request-ID assignment, pending-call tracking, and response multiplexing must be implemented above the library; sufficient resource configuration is also needed | +| **R8** Multiple client processes | ✅ works out of the box | ⚠️ supported through the service-oriented deployment model, but requires consistent application/service configuration across participating processes — see R5 | ⚠️ supported through the service-oriented deployment model, but requires consistent application/service configuration across participating processes — see R5 | ✅ the library supports one server communicating with multiple client processes through independent client/server sessions; sufficient connection and queue resource configuration is needed | +| **R9** One-to-one | ✅ each call gets exactly one response | ✅ each call gets exactly one response | ⚠️ response is a broadcast event — requires one skeleton instance per client to prevent cross-client response leakage | ✅ each call gets exactly one response | +| **R10** Latency | ❌ indicative POC measurement ~430 µs (full request + response) | ❓ not measured | ⚠️ indicative POC measurement ~130 µs (lowest measured value, but still significant IPC overhead) | ⚠️ indicative POC measurement ~160 µs (second-lowest measured value, but still significant IPC overhead) | +| **R11** Inter-VM | ⚠️ gRPC supports network channels, but the current adapter hardcodes Unix-domain endpoints; an endpoint/configuration change and validation of the inter-VM transport and peer-authentication model are required | ⚠️ current LoLa binding is SHM-only (single-kernel); the service-oriented architecture could in principle support a network binding without changing the service API, but no such binding exists today | ⚠️ current LoLa binding is SHM-only (single-kernel); the service-oriented architecture could in principle support a network binding without changing the service API, but no such binding exists today | ❌ current `score::message_passing` backends are local Unix-domain socket and QNX message passing; the library provides no inter-VM transport. A new framework backend would be required. | + +**Note:** R1 and R2 are hard safety blockers — any ❌ on these disqualifies an option for safety use regardless of performance on other requirements. Options A, B, and C all carry at least one ❌ on R1 or R2. + +**POC note:** The LoLa POCs generated per-client configuration and used unique +application identifiers to exercise multiple clients. These are prototype +workarounds and should not be interpreted as the library's fundamental +configuration model. + +--- + +## 5. Analysis + +**Option B (LoLa Full SOA, synchronous)** meets the R1 library-capability assessment: the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use. It fails R2 (timeout): there is no two-phase workaround available — the single blocking call holds the caller's thread until the handler returns with no escape path. The synchronous prototype was only validated in-process (skeleton and proxy on separate threads within a single test binary); cross-process concurrency was not tested. The config burden is high and would compound if LoLa is used elsewhere in the same process. + +**Option C (LoLa Full SOA, asynchronous)** meets the R1 library-capability assessment: the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use, including its method/event model. Its indicative POC latency of approximately 130 microseconds is the lowest measured value, but still represents significant IPC overhead and should remain a warning rather than an unqualified pass. It fails partially on R2: Phase 2 timeout is implemented and flexible server threading is achievable, but Phase 1 timeout requires a LoLa framework change — it cannot be fixed in application code. On server crash, Phase 2 never fires and the client hangs until the application-level timeout expires; the crash itself is not detected independently. The broadcast-event response model requires one skeleton instance per client, adding complexity and config overhead. Notably, LoLa's service-oriented architecture is transport-agnostic by design; a future network binding could enable inter-VM communication without application-level changes — a meaningful long-term advantage that does not resolve the current safety gaps. + +**Option A (gRPC)** fails R1 for the stated ASIL-B use because the general-purpose library uses framework-managed threads and dynamic resources and does not provide an ASIL-oriented deterministic resource profile or safety qualification artifacts. The current adapter also does not configure a deadline; deadline-based and asynchronous gRPC were not analyzed. It handles server crash via gRPC status errors but cannot distinguish a crashed server from a hung one without a configured deadline. Peer authentication requires a PKI (no `SO_PEERCRED` equivalent). It remains the strongest option for inter-VM, because gRPC provides network channel support, but the current adapter hardcodes Unix-domain endpoints and requires endpoint/configuration changes before that path is available. It is appropriate for QM-to-QM communication where safety certification is not required. + +**Option D (LoLa Message Passing Abstraction)** meets the R1 library-capability assessment: its design supports fixed resource bounds, preallocation, and pool/monotonic allocation, and the communication module documents safety-oriented quality tooling and ASIL-B qualification. Its indicative POC latency of approximately 160 microseconds is the second-lowest measured value, but still represents significant IPC overhead and should remain a warning rather than an unqualified pass. The application code is OS-agnostic; the framework provides the OS-specific transport backend (Unix domain socket on Linux, QNX message passing on QNX). The low-level message-passing prototype provides feasibility evidence for the intended skeleton: non-blocking `SendWithCallback` / `Reply` decoupling, application-level `request_id` multiplexing for concurrent threads, application-level bounded waiting, and detection of server death via socket EOF. Typed error propagation, mandatory timeout APIs, and backend-specific bounded-notification behavior remain implementation work. The selected library does not support inter-VM communication: its current backends are local, and a new framework backend would be required. The `IConnection` abstraction may provide a migration direction, but this has not been demonstrated and is not evidence of current library support. + +--- + +## 6. Decision + +**Option D — LoLa Message Passing Abstraction** is selected as the IPC transport for the score-crypto daemon, using `score::message_passing` as its current implementation basis. + +--- + +## 7. Consequences + +**Accepted trade-offs:** +- Indicative POC measurements show ~160 µs for message_passing, ~130 µs for LoLa Full SOA (async), and ~430 µs for gRPC. These figures are not a controlled benchmark and must be validated with the proper implementation using a common platform, payload, concurrency, warm-up, and measurement method. The observed gap between LoLa Full SOA and message_passing is partly attributed to configuration: the LoLa Full SOA asynchronous prototype was measured with QM-only settings, whereas the message_passing prototype was configured for mixed usage (`truly_async=true`). Configuring message_passing for QM-only reduced its observed round-trip to ~140 µs; the remaining difference was not analyzed. Throughput would be higher with LoLa Full SOA due to shared memory, but this is not a control-plane requirement: larger data transfers are expected to use a dedicated data plane. +- The selected library does not currently support inter-VM communication. The existing `IConnection` abstraction may guide a second, dedicated IPC mechanism, but that migration path has not been demonstrated and remains separate follow-up work. + +**Production implementation follow-up:** + +The formal implementation of ``IConnection`` and ``IControlServer`` interfaces +replacing the gRPC adapter is still pending will be taken up. See the [message-passing POC and QNX usage overview](../../../../tests/ipc_poc/message_passing_overview.md) for more detailed follow up tasks. diff --git a/score/tests/ipc_poc/.gitignore b/score/tests/ipc_poc/.gitignore new file mode 100644 index 000000000..cadb8fd56 --- /dev/null +++ b/score/tests/ipc_poc/.gitignore @@ -0,0 +1,2 @@ +mw_com_config_async_consumer_*.json +mw_com_config_async_producer.json diff --git a/score/tests/ipc_poc/BUILD b/score/tests/ipc_poc/BUILD new file mode 100644 index 000000000..41af2c3c3 --- /dev/null +++ b/score/tests/ipc_poc/BUILD @@ -0,0 +1,78 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") + +# Generate C++ header from the FlatBuffer schema. +# No --grpc flag: we only need plain C++ data tables, not gRPC stubs. +genrule( + name = "generated_poc_control", + srcs = ["poc_control.fbs"], + outs = ["poc_control_generated.h"], + cmd = "$(location @flatbuffers//:flatc) --cpp --size-prefixed -o $(@D) $(SRCS)", + tools = ["@flatbuffers//:flatc"], +) + +cc_library( + name = "async_control_plane_service", + hdrs = [ + "async_control_interface.h", + "ipc_buffer.h", + ":generated_poc_control", + ], + include_prefix = "tests/score_com_poc", + deps = [ + "@flatbuffers", + "@score_communication//score/mw/com", + ], +) + +cc_test( + name = "poc_async", + srcs = ["poc_async.cpp"], + deps = [ + ":async_control_plane_service", + ], +) + +cc_library( + name = "low_level_poc_deps", + hdrs = [ + "ipc_buffer.h", + ":generated_poc_control", + ], + include_prefix = "tests/score_com_poc", + deps = [ + "@flatbuffers", + "@score_communication//score/message_passing", + ], +) + +cc_test( + name = "poc_low_level", + srcs = ["poc_low_level.cpp"], + deps = [ + ":low_level_poc_deps", + ], +) + +cc_test( + name = "poc_grpc", + srcs = ["poc_grpc.cpp"], + dynamic_deps = ["//third_party/grpc:libgrpc_shared"], + deps = [ + "//score/crypto/src/daemon/control_plane", + "//score/crypto/src/ipc/grpc_adapter:grpc_control_client", + "//score/crypto/src/ipc/grpc_adapter:grpc_control_server", + ], +) diff --git a/score/tests/ipc_poc/async_control_interface.h b/score/tests/ipc_poc/async_control_interface.h new file mode 100644 index 000000000..558671e22 --- /dev/null +++ b/score/tests/ipc_poc/async_control_interface.h @@ -0,0 +1,57 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#pragma once + +#include + +#include "score/mw/com/types.h" +#include "score/tests/ipc_poc/ipc_buffer.h" + +namespace score::crypto::ipc::control +{ + +/// S-CORE com service interface for the async crypto daemon control plane POC. +/// +/// The new communication model uses a two-phase interaction: +/// +/// Phase 1 — Short-lived Method call: +/// The client sends a ControlRequest (IpcBuffer) via the "Request" method. +/// The skeleton handler enqueues the work and immediately returns the +/// request_id (== ticket number) to the caller. The round-trip is very +/// short; no processing happens on the calling thread. +/// +/// Phase 2 — Event-based response: +/// A server-side background worker dequeues the request, does the actual +/// work, then calls response.Send() to broadcast a ControlResponse +/// (IpcBuffer) to all subscribers. The IpcBuffer carries the original +/// request_id so every client can match the event to its outstanding ticket. +template +class AsyncControlInterface : public Trait::Base +{ + public: + using Trait::Base::Base; + + /// Phase 1: client sends a ControlRequest, server returns the ticket number. + typename Trait::template Method request{*this, "Request"}; + + /// Phase 2: server broadcasts a ControlResponse when work is complete. + /// The IpcBuffer payload is a size-prefixed ControlResponse FlatBuffer + /// whose request_id field equals the ticket returned by Phase 1. + typename Trait::template Event response{*this, "Response"}; +}; + +using AsyncControlProxy = score::mw::com::AsProxy; +using AsyncControlSkeleton = score::mw::com::AsSkeleton; + +} // namespace score::crypto::ipc::control diff --git a/score/tests/ipc_poc/ipc_buffer.h b/score/tests/ipc_poc/ipc_buffer.h new file mode 100644 index 000000000..662d8044d --- /dev/null +++ b/score/tests/ipc_poc/ipc_buffer.h @@ -0,0 +1,92 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +namespace score::crypto::ipc::control +{ + +/// Maximum size (bytes) of a serialized FlatBuffer payload that can be +/// transported via mw::com Method RPC. This is an ABI boundary: changing this +/// value requires rebuilding both sides of the IPC (client and server) together. +constexpr std::size_t kMaxIpcBufferSize = 4096U; + +/// Trivially copyable envelope for transporting a size-prefixed FlatBuffer over +/// S-CORE LoLa shared-memory Method RPC. +/// +/// S-CORE LoLa requires method argument and return types to be trivially +/// copyable (used via reinterpret_cast in shared memory). The payload is stored +/// in size-prefixed FlatBuffer format: the first 4 bytes are a little-endian +/// uint32 encoding the size of the remaining data, so no separate length field +/// is needed. +struct IpcBuffer +{ + std::array payload{}; +}; + +static_assert(std::is_trivially_copyable_v, + "IpcBuffer must be trivially copyable for LoLa shared-memory transport"); +static_assert(sizeof(IpcBuffer) == kMaxIpcBufferSize, "IpcBuffer layout must have no padding"); + +/// Serialize raw size-prefixed FlatBuffer bytes into an IpcBuffer. +/// Returns a zero-initialised buffer (IsValid() == false) if size exceeds kMaxIpcBufferSize. +inline IpcBuffer PackFlatBuffer(const std::uint8_t* data, const std::size_t size) noexcept +{ + IpcBuffer buf{}; + if (size <= kMaxIpcBufferSize) + { + std::memcpy(buf.payload.data(), data, size); + } + return buf; +} + +/// Write size-prefixed FlatBuffer bytes directly into an existing IpcBuffer (e.g. a +/// MethodInArgPtr-backed SHM slot) to avoid an intermediate copy. +/// On success returns true; if size exceeds kMaxIpcBufferSize the buffer is +/// zero-initialised and false is returned. +inline bool PackFlatBufferInto(IpcBuffer& buf, const std::uint8_t* data, const std::size_t size) noexcept +{ + if (size > kMaxIpcBufferSize) + { + buf = IpcBuffer{}; + return false; + } + std::memcpy(buf.payload.data(), data, size); + return true; +} + +/// Read the size prefix embedded in a size-prefixed FlatBuffer. +/// Returns the total number of valid bytes (4-byte prefix + data). +inline std::uint32_t GetPayloadSize(const IpcBuffer& buf) noexcept +{ + std::uint32_t prefix{}; + std::memcpy(&prefix, buf.payload.data(), sizeof(prefix)); + return prefix + static_cast(sizeof(prefix)); +} + +/// Returns true if the buffer contains a non-empty, in-bounds payload. +inline bool IsValid(const IpcBuffer& buf) noexcept +{ + std::uint32_t prefix{}; + std::memcpy(&prefix, buf.payload.data(), sizeof(prefix)); + const auto total = static_cast(prefix) + sizeof(prefix); + return prefix > 0U && total <= kMaxIpcBufferSize; +} + +} // namespace score::crypto::ipc::control diff --git a/score/tests/ipc_poc/message_passing_overview.md b/score/tests/ipc_poc/message_passing_overview.md new file mode 100644 index 000000000..b5fe8d7bd --- /dev/null +++ b/score/tests/ipc_poc/message_passing_overview.md @@ -0,0 +1,234 @@ + + +# score::message_passing POC and QNX Usage Overview + +## Scope + +This document describes the low-level POC in [poc_low_level.cpp](poc_low_level.cpp) and the `score::message_passing` implementation selected by the platform build. The dependency is `score_communication` version `0.3.0`, as declared in [MODULE.bazel](../../../MODULE.bazel). + +The public API is platform-independent. `client_factory.h` selects `QnxDispatchClientFactory` and `QnxDispatchEngine` when `__QNX__` is defined; otherwise it selects `UnixDomainClientFactory` and `UnixDomainEngine`. + +## Executive Summary + +| Concern | Linux | QNX | +|---|---|---| +| Transport | Unix-domain `SOCK_STREAM` socket | QNX resource-manager path and message-passing events | +| Client engine | One `UnixDomainEngine` background thread per engine | One `QnxDispatchEngine` dispatch thread per engine | +| Server engine | One engine thread polls the listening socket and client sockets | One dispatch thread runs `dispatch_block()` / `dispatch_handler()` | +| Application workers | POC-owned server worker pool performs the request | Same POC-owned worker pool performs the request | +| Client send path | `SendWithCallback()` copies into a queued send slot, then engine calls `sendmsg()` | `SendWithCallback()` copies into a queued send slot, then engine calls `writev()` | +| Server receive path | Engine receives into a shared receive buffer, then invokes the handler span | Resource-manager `io_write` supplies a span, then the handler is invoked | +| `Reply()` | `sendmsg(REPLY)` from the calling thread; it can block in the socket layer | Copies into `reply_message_`, queues it, and signals the client with `MsgDeliverEvent`; the payload is returned later through `io_read` / `MsgReplyv` | +| `Notify()` | `sendmsg(NOTIFY)` from the worker thread; it can block in the socket layer | Copies into a preallocated notify slot and signals the client with `MsgDeliverEvent`; the payload is returned later through `io_read` / `MsgReplyv`; returns `ENOBUFS` when the per-connection pool is full | +| Peer disconnect | EOF or `EPIPE` transitions the client to `kStopping` | QNX connection-death pulse or read failure transitions the client to `kStopping` | + +The POC deliberately uses two phases because the library serializes `REQUEST` messages on one connection until the server calls `Reply()`: + +1. `SendWithCallback()` sends a request and receives a small acknowledgement through `Reply()`. +2. The server performs the work independently and sends the full result with `Notify()`. + +The acknowledgement is a protocol release, not the operation result. + +## Diagram 1: POC QNX Deployment and Call Flow + +The process layout is created by `fork()`: the parent is the server and each child is a client. IPC objects are created after the fork. Each client process has one shared connection for all of its client threads. + +```mermaid +sequenceDiagram + autonumber + participant CT as Client thread + participant CM as Client pending map / CV + participant CE as QNX client engine thread + participant QNX as QNX kernel + participant SE as QNX server dispatch thread + participant SH as sent_with_reply_cb + participant WQ as Server work queue + participant SW as Server worker thread + + CM->>CE: Start and open QNX resource manager path + CE->>QNX: open QNX resource manager path + QNX->>SE: Invoke io_open / ProcessConnect + CE->>QNX: MsgRegisterEvent and sticky select io_msg + QNX->>SE: Store rcvid, select_event, and ping_event + CT->>CT: Build ControlRequest and assign request id + CT->>CM: Insert PendingCall with request id before send + CT->>CE: SendWithCallback request span and ReplyCallback + CT->>CE: C1 copy span into queued send slot + CE->>QNX: writev REQUEST and queued bytes + QNX->>QNX: C2 copy message across process boundary + QNX->>SE: Invoke resource manager io_write with REQUEST + SE->>SH: Invoke sent_with_reply_cb with request span + SH->>SH: Verify FlatBuffer and extract request_id + SH->>WQ: Copy request bytes into WorkItem and enqueue + SH->>SE: Reply ack bytes + SE->>SE: C3 copy ack into reply queue on successful Reply + SH->>WQ: Wake worker after Reply returns (success or failure) + SE->>QNX: MsgDeliverEvent with select event + QNX-->>CE: Deliver select pulse + CE->>QNX: read fd after pulse + QNX->>SE: Invoke resource manager io_read + SE->>SE: ProcessReadRequest builds iov from queued REPLY + SE->>QNX: MsgReplyv returns REPLY data + QNX-->>CE: read returns REPLY bytes + CE->>CT: ReplyCallback with ack or error + CE->>CT: Success ack is ignored and error marks call failed + WQ->>SW: Dequeue WorkItem + SW->>SW: ProcessRequestBytes and build full response + SW->>SW: C4 create separate response byte vector + SW->>SE: Notify response span + SE->>SE: C5 copy response into notify slot and enqueue NOTIFY + SE->>QNX: MsgDeliverEvent with select event + QNX-->>CE: Deliver select pulse + CE->>QNX: read fd after pulse + QNX->>SE: Invoke resource manager io_read + SE->>SE: ProcessReadRequest builds iov from queued NOTIFY + SE->>QNX: MsgReplyv returns NOTIFY data + QNX-->>CE: C6 read returns NOTIFY bytes + CE->>CM: NotifyCallback parses request_id and updates PendingCall + CM-->>CT: Notify CV + CT->>CT: Validate result, erase request_id, continue +``` + +### QNX copy and lifetime rules + +- The span passed to `SendWithCallback()`, `Reply()`, or `Notify()` is a borrowed view. The caller owns the source bytes. +- `ClientConnection::TryQueueMessage()` copies a client request into a library-owned `SendCommand.message`. With `truly_async=true`, this copy is required before the caller can return safely. +- `ServerConnection::Reply()` copies the acknowledgement into the library-owned `reply_message_` before it is placed on the server send queue. +- `ServerConnection::Notify()` copies the response into one of the preallocated `notify_storage_` slots. The slot is returned to `notify_pool_` only after the queued message has been delivered. +- The QNX resource manager and kernel perform the process-boundary transport. The callback receives a span valid for the callback invocation; it must not retain that span. +- The POC performs another application copy when it stores the received request in `WorkItem::request_bytes`. This is necessary because the worker runs after `sent_with_reply_cb` returns. +- The POC's worker creates a separate response vector. It is then copied again by `Notify()` into the QNX notify pool. +- `max_queued_notifies >= T` is required for the POC's maximum number of concurrent calls per client connection. If the pool is exhausted, `Notify()` returns `ENOBUFS`; the client eventually reaches its application timeout. + +### QNX threading view + +```mermaid +flowchart TB + subgraph SERVER[Parent server process] + SM[Main thread
exists after fork
setup, wait for clients, shutdown] + SW[Server worker pool
created before StartListening
process requests and call Notify] + SD[QNX server dispatch thread
created with QnxDispatchEngine
run resource-manager callbacks] + SM -->|create| SW + SM -->|construct factory / engine| SD + SM -->|StartListening attaches service path| SD + end + + subgraph CLIENT[Each child client process] + CM[Main thread
exists after fork
setup, join workers, shutdown] + CE[QNX client engine thread
created with QnxDispatchEngine
run dispatch loop and client callbacks] + CW[Client worker threads
created after kReady
issue calls and wait for results] + CM -->|construct factory / engine| CE + CM -->|Start opens connection and registers endpoint| CE + CE -->|kReady| CM + CM -->|create| CW + end +``` + +### QNX delivery and failure semantics + +- Before any response can be delivered, the client engine registers `select_event_` and `ping_event_` with `MsgRegisterEvent()` and sends a sticky-select `io_msg`. The server stores the resulting `rcvid_` and event descriptions on the `ServerConnection`. +- `Reply()` and `Notify()` are queue operations on the server connection. `MsgDeliverEvent()` delivers a pulse that wakes the client engine; it does not carry the response payload. +- After the pulse, the client calls `read(fd)`. The QNX resource manager invokes `io_read`, which calls `ProcessReadRequest`; that method uses `MsgReplyv()` to return the queued protocol code and bytes to the client read. +- `Notify()` can fail with `EMSGSIZE` for an oversized response or `ENOBUFS` when the per-connection notify pool is exhausted. A successful return means the response was accepted into the server queue, not that the client callback has completed. +- The implementation ignores the return values of `MsgDeliverEvent()` and `MsgReplyv()`. Pulse delivery, client read, or queued-read completion can therefore fail after `Reply()` or `Notify()` has returned successfully; the client may then stop or time out. +- A valid `REQUEST` must receive `Reply()` promptly. Until the reply callback runs, the client does not release the per-connection request serialization and cannot send the next queued request on that connection. + +## Request and Connection State Models + +### POC request state + +```mermaid +stateDiagram + [*] --> Built: request_id + FlatBuffer built + Built --> Pending: insert PendingCall into map + Pending --> SendQueued: SendWithCallback accepted + Pending --> Failed: SendWithCallback returns error + SendQueued --> Acked: ReplyCallback receives ack + SendQueued --> Failed: ReplyCallback receives error + Acked --> WaitingNotify: server work is independent + WaitingNotify --> Completed: NotifyCallback sets ready + result + WaitingNotify --> TimedOut: application CV wait_for expires + Completed --> Retired: erase request_id from pending map + TimedOut --> Retired: erase request_id, late Notify discarded + Failed --> Retired: erase request_id + Retired --> [*] + +``` + +`Acked` is a logical POC state. The POC does not store an explicit acknowledgement flag because a successful ack is not the result; it keeps waiting for `Notify()`. A failed acknowledgement sets `PendingCall::ready` and wakes the caller as a failure. + +### Library connection state + +```mermaid +stateDiagram + [*] --> Stopped: constructed + Stopped --> Starting: Start or Restart + Starting --> Ready: transport connected + Starting --> Stopping: non-retryable connect failure or Stop + Ready --> Stopping: peer disconnect, I/O error, or Stop + Stopping --> Stopped: callbacks drained and engine cleanup complete + Stopped --> Starting: Restart + +``` + +### Server-side request lifecycle + +```mermaid +stateDiagram + [*] --> Received: io_write receives REQUEST + Received --> Validated: verify FlatBuffer + request_id + Validated --> WorkQueued: copy request into WorkItem + WorkQueued --> AckQueued: Reply ack + AckQueued --> Processing: wake worker + WorkQueued --> AckFailed: Reply fails; wake worker + AckFailed --> Processing: worker may still execute admitted work + Processing --> NotifyQueued: Notify response + NotifyQueued --> Delivered: pulse, read, and MsgReplyv complete + NotifyQueued --> Dropped: EMSGSIZE, ENOBUFS, disconnect, or delivery failure + Validated --> Rejected: invalid request + Delivered --> [*] + Dropped --> [*] + Rejected --> [*] + +``` + +## Linux versus QNX: What the POC Can Assume + +The application-level flow and state model are the same on both platforms. The important portability boundaries are: + +1. **The asynchronous send guarantee is configuration-dependent.** The POC sets `truly_async=true` and sizes `max_queued_sends` and `max_async_replies` to `T`, so client call threads only enqueue into library-owned storage. +2. **The server worker lifetime guard is required on both platforms.** A worker stores a raw `IServerConnection*`; the POC pairs it with a per-connection `shared_ptr` and holds the live-connection mutex across the check and `Notify()` call. +3. **The QNX resource limits are real queue limits.** `max_queued_notifies` is used by QNX. The corresponding `ServerConfig` fields are ignored by the Linux Unix-domain implementation, which relies on socket buffers and the kernel's stream transport instead. +4. **`Notify()` is not equally bounded.** The QNX implementation can fail quickly with `ENOBUFS`, while the Linux implementation calls `sendmsg()` from the worker and may block in the socket layer. The POC comments explicitly call out this difference; the application timeout bounds the client wait, not necessarily the duration of a server-side `Notify()` call. +5. **The public connection state is richer than the POC's ready/stopped handling.** The library exposes `kStarting`, `kReady`, `kStopping`, and `kStopped`; the POC waits for `kReady` and observes `kStopped`, while the intermediate transitions remain library-internal to the POC. + +## To be considered for proper impl: + +- POC focuses mainly on the good path. Basic errors checks are implemented, but they are not yet sufficient or thorough enough. +- Define the config parameter values (number of threads, ... , size of transferred elements) + - Enforce configuration consistently: number of clients (`N`), concurrent requests per client (`T`), maximum payload (`M`), worker count, transport queue depths, application work-queue depth +- Make timeout mandatory and passed by called -> Interface change? +- Make SerDe lib exchangable +- Think about connection policy e.g. one connection per UID or N per UID. Ideally switchable + - Or bring out the admission call to allow custom behaviour +- Use proper state machines for connection state, pending request state and work item state +- Use base-libs thread pools +- Job cancellation is not implemented. + - Define cancellation and shutdown behavior for queued and in-flight work, client disconnects, server termination, and worker teardown. + - Define typed timeout, connection-loss, cancellation, and delivery errors, including cleanup of pending request state and handling of late notifications. + - We propably need a stop / cancel token in the workitems, however if we can actually cancel a running request depends on actual processors of the requests + - Need to also think how to properly give preference to "cancel requests" and if they need to be handled differently than normal ones +- Bound the application work queue and define admission behavior when it is full. A request must not be acknowledged unless admission and the associated resource reservation have succeeded. +- Define authenticated peer identity, endpoint permissions, authorization, and behavior for rejected or reconnecting clients. +- If acknowledgement delivery fails after queue admission, either cancel the queued work or expose the resulting "may have executed" outcome and define retry rules, especially for non-idempotent operations. diff --git a/score/tests/ipc_poc/mw_com_message_passing.drawio.svg b/score/tests/ipc_poc/mw_com_message_passing.drawio.svg new file mode 100644 index 000000000..41941cc5f --- /dev/null +++ b/score/tests/ipc_poc/mw_com_message_passing.drawio.svg @@ -0,0 +1,2114 @@ + + + + + + + + + + +
+
+
+ Server / Worker Thread +
+
+
+ + + Server / Worker Thread + + + + + + + + + + + +
+
+
+ Main Server Thread is idle after setup and till stop +
+
+
+ + + Main Server Thread is idle after... + + + + + + + + + + + +
+
+
+ Engine Thread +
+
+
+ + + Engine Thread + + + + + + + + + + + +
+
+
+ Client Thread +
+
+
+ + + Client Thread + + + + + + + + + + + +
+
+
+ Engine Thread +
+
+
+ + + Engine Thread + + + + + + + + + + + +
+
+
+ Client +
+
+
+ + + Client + + + + + + + + + + + + + + + + + +
+
+
+ IClientConnection +
+
+
+ + + IClientConnection + + + + + + + + + + + +
+
+
+ Start(state_callback, notify_callback) +
+ SendWithCallback(msg, callback) +
+
+
+ + + Start(state_callback, notify_callback... + + + + + + + + + + + + + +
+
+
+ QnxDispatchEngine +
+
+
+ + + QnxDispatchEngine + + + + + + + + + + + + + +
+
+
+ open() +
+ writev() +
+ read() +
+
+
+ + + open()... + + + + + + + + + + + +
+
+
+
+ Engine +
+
+ Thread +
+
+ (Unix|QNX) +
+
+
+
+ + + Engine... + + + + + + + + + + + + + + + + + +
+
+
+ + ClientFactory + +
+
+
+ + + ClientFactory + + + + + + + + + + + +
+
+
+ Create(protocol_config, client_config) +
+
+
+ + + Create(protocol_config, client_confi... + + + + + + + + + + + + +
+
+
+ creates +
+
+
+ + + creates + + + + + + + + + + + + + + + + + +
+
+
+ ClientConfig +
+
+
+ + + ClientConfig + + + + + + + + + + + +
+
+
+ max_async_replies +
+
+
+ + + max_async_replies + + + + + + + + + + + +
+
+
+ max_queued_sends +
+
+
+ + + max_queued_sends + + + + + + + + + + + +
+
+
+ fully_ordered +
+
+
+ + + fully_ordered + + + + + + + + + + + +
+
+
+ truly_async +
+
+
+ + + truly_async + + + + + + + + + + + +
+
+
+ sync_first_connect +
+
+
+ + + sync_first_connect + + + + + + + + + + + + + + + + + +
+
+
+ (Send&Reply) Queue +
+
+
+ + + (Send&... + + + + + + + + + + + + + + + + +
+
+
+ State +
+ Callback +
+
+
+ + + State... + + + + + + + + + + + + +
+
+
+ Notify +
+ Callback +
+
+
+ + + Notify... + + + + + + + + + + + + +
+
+
+ SendWithCallback +
+ Callback +
+
+
+ + + SendWithCallback... + + + + + + + + + + + +
+
+
+ Server +
+
+
+ + + Server + + + + + + + + + + + + + + + + + +
+
+
+ QnxDispatchServer +
+
+
+ + + QnxDispatchServer + + + + + + + + + + + +
+
+
+ StartListening(connect_callback, disconnect_callback, send_and_reply_callback) +
+ StopListening() +
+
+
+ + + StartListening(connect_callback, disconnect_callback, send_a... + + + + + + + + + + + + + +
+
+
+ QnxDispatchEngine +
+
+
+ + + QnxDispatchEngine + + + + + + + + + + + +
+
+
+
+ Engine +
+
+ Thread +
+
+ (Unix|QNX) +
+
+
+
+ + + Engine... + + + + + + + + + + + + + +
+
+
+ + ServerFactory + +
+
+
+ + + ServerFactory + + + + + + + + + + + +
+
+
+ Create(protocol_config, client_config) +
+
+
+ + + Create(protocol_config, client_confi... + + + + + + + + + + + + +
+
+
+ creates +
+
+
+ + + creates + + + + + + + + + + + + + + + + + +
+
+
+ ServerConfig +
+
+
+ + + ServerConfig + + + + + + + + + + + +
+
+
+ max_queued_sends +
+
+
+ + + max_queued_sends + + + + + + + + + + + +
+
+
+ pre_alloc_connections +
+
+
+ + + pre_alloc_connections + + + + + + + + + + + +
+
+
+ max_queued_notifies +
+
+
+ + + max_queued_notifies + + + + + + + + + + + + + +
+
+
+ create +
+
+
+ + + create + + + + + + + + + + + + +
+
+
+ Connect +
+ Callback +
+
+
+ + + Connect... + + + + + + + + + + + + +
+
+
+ Disconnect +
+ Callback +
+
+
+ + + Disconnect... + + + + + + + + + + + + +
+
+
+ SendAndReply +
+ Callback +
+
+
+ + + SendAndReply... + + + + + + + + + + + + + +
+
+
+ ServiceProtocolConfig +
+
+
+ + + ServiceProtocolConfig + + + + + + + + + + + +
+
+
+ identifier +
+
+
+ + + identifier + + + + + + + + + + + +
+
+
+ max_send_size +
+
+
+ + + max_send_size + + + + + + + + + + + +
+
+
+ max_reply_size +
+
+
+ + + max_reply_size + + + + + + + + + + + +
+
+
+ max_notify_size +
+
+
+ + + max_notify_size + + + + + + + + + + + + + +
+
+
+ Setup Dispatch & +
+ resmgr_attach() +
+
+
+ + + Setup Dispatch &... + + + + + + + + + + + + + + +
+
+
+ ResourceManager +
+ Endpoint +
+
+
+ + + ResourceMa... + + + + + + + + + + + + + + + + + + + + +
+
+
+ OCB per Client +
+
+
+ + + OCB per Cl... + + + + + + + + + + + + + +
+
+
+ ServerConnection +
+
+
+ + + ServerConnection + + + + + + + + + + + +
+
+
+ Reply(msg) +
+ Notify(msg) +
+ GetClientIdenty() +
+
+
+
+
+
+ + + Reply(msg)... + + + + + + + + + + + + +
+
+
+ create +
+
+
+ + + create + + + + + + + + + + + + + + + + +
+
+
+ (Notify) Queue +
+
+
+ + + (Notif... + + + + + + + + + + + + + + + +
+
+
+ max_queued_send & pre_alloc_connection does not seem to be used +
+
+
+ + + max_queued_send & pre_alloc_c... + + + + + + + + + + + +
+
+
+ Self owning object. Destroyed at ProcessDisconnect() +
+
+ We need to ensure it's still alive when attempting to Notify() from the worker thread. +
+
+ Use Connect and Disconnet callbacks +
+
+
+ + + Self owning object. Destroyed at ProcessDisconne... + + + + + + + + + + + + + + + +
+
+
+ SendWithCallback() +
+ (Queue Message and Arm Send Queue) +
+
+
+ + + SendWithCallback()... + + + + + + + + + + + + + + + + + + + +
+
+
+ Enqueue Pending +
+
+
+ + + Enqueue Pending + + + + + + + + + + + + + + + +
+
+
+ Generate RequestId +
+
+
+ + + Generate RequestId + + + + + + + + + + + +
+
+
+
+ + Return Result or Error + +
+
+
+
+ + + Return Result or Error + + + + + + + + + + + + +
+
+
+ no +
+
+
+ + + no + + + + + + + + + + + + +
+
+
+ yes +
+
+
+ + + yes + + + + + + + + + + + +
+
+
+ Timeout? +
+
+
+ + + Timeout? + + + + + + + + + + + +
+
+
+
+ + Return Timeout Error + +
+
+
+
+ + + Return Timeout Error + + + + + + + + + + + + + + + +
+
+
+ sent_with_reply_callback +
+
+
+ + + sent_with_reply_callback + + + + + + + + + + + + + + + + + + + +
+
+
+ Enqueue Request +
+
+
+ + + Enqueue Request + + + + + + + + + + + + +
+
+
+ pulse(select) +
+
+
+ + + pulse(select) + + + + + + + + + + + +
+
+
+ Reply() with Ack +
+
+
+ + + Reply() with Ack + + + + + + + + + + + + + + + +
+
+
+ Wait on Queue +
+
+
+ + + Wait on Queue + + + + + + + + + + + + + + + +
+
+
+ Process Request +
+
+
+ + + Process Request + + + + + + + + + + + + + + + + +
+
+
+ pulse(select) +
+
+
+ + + pulse(select) + + + + + + + + + + + +
+
+
+ work_item.conn.Notify() with Result +
+
+
+ + + work_item.conn.Notify() with R... + + + + + + + + + + + + + + + +
+
+
+ dispatch_handler +
+
+
+ + + dispatch_handler + + + + + + + + + + + +
+
+
+ io_write_callback +
+
+
+ + + io_write_callback + + + + + + + + + + + +
+
+
+ writev +
+
+
+ + + writev + + + + + + + + + + + + +
+
+
+ io_write +
+
+
+ + + io_write + + + + + + + + + + + +
+
+
+ io_read_callback +
+
+
+ + + io_read_callback + + + + + + + + + + + + + + + +
+
+
+ read +
+
+
+ + + read + + + + + + + + + + + + +
+
+
+ io_read +
+
+
+ + + io_read + + + + + + + + + + + + + + + +
+
+
+ Set Pending.ConditionVar +
+
+
+ + + Set Pending.ConditionVar + + + + + + + + + + + + + + + +
+
+
+ notify_callback +
+
+
+ + + notify_callback + + + + + + + + + + + + + + + +
+
+
+ select_pulse_callback +
+
+
+ + + select_pulse_callback + + + + + + + + + + + + + + + +
+
+
+ select_ + + pulse_callback + +
+
+
+ + + select_pulse_callback + + + + + + + + + + + + + + + + +
+
+
+ io_read +
+
+
+ + + io_read + + + + + + + + + + + +
+
+
+ read +
+
+
+ + + read + + + + + + + + + + + +
+
+
+ + io_read_callback + +
+
+
+ + + io_read_callback + + + + + + + + + + + +
+
+
+ dispatch_handler +
+
+
+ + + dispatch_handler + + + + + + + + + + + + + + + + + + + +
+
+
+ Mark enqueue / waiting for notification +
+
+
+ + + Mark enqueue / waiting for not... + + + + + + + + + + + + + + + +
+
+
+
+ + Wait for Pending.ConditionVar with Timeout + +
+
+
+
+ + + Wait for Pending.ConditionVar... + + + + + + + + + + + + + + + +
+
+
+ reply_callback +
+
+
+ + + reply_callback + + + + + + + + + + Text is not SVG - cannot display + + + + diff --git a/score/tests/ipc_poc/poc_async.cpp b/score/tests/ipc_poc/poc_async.cpp new file mode 100644 index 000000000..55577eb86 --- /dev/null +++ b/score/tests/ipc_poc/poc_async.cpp @@ -0,0 +1,1321 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This file exists only to test and understand IPC mechanisms and to guide a +// proper implementation. It is kept temporarily as reference and will be +// removed once confidence has been gained in the proper implementation. +// ============================================================================= + +/// POC: Short-Lived Async Method Call + Event Response + Single-Binary Fork Model +/// +/// Communication model (two phases): +/// +/// Phase 1 — Short-lived Method call +/// The client builds a ControlRequest FlatBuffer and embeds a self-chosen +/// request_id (the "ticket number"). It calls AsyncControlProxy::request(), +/// which blocks only until the server enqueues the work and returns the +/// ticket. No real processing happens on the method-handler thread. +/// +/// Phase 2 — Event-based response +/// A server background worker dequeues each request, does the actual work, +/// and calls AsyncControlSkeleton::response.Send(). The ControlResponse +/// FlatBuffer in the event payload carries the original request_id so every +/// subscribing client can match the event to its outstanding ticket. +/// +/// Process model: +/// A single binary is forked before InitializeRuntime is called so every +/// child process starts with a clean mw::com runtime state. +/// +/// Parent process → server (skeleton + thread pool of worker threads) +/// Child process → client (proxy + event subscriber) +/// +/// The parent waits for all children via waitpid before stopping the service. +/// +/// Usage: +/// bazel run //tests/score_com_poc:poc_async +/// bazel run //tests/score_com_poc:poc_async -- --client_count=3 --call_count=5 +/// bazel run //tests/score_com_poc:poc_async -- --client_count=3 --call_count=5 +/// bazel run //tests/score_com_poc:poc_async -- --client_count=3 --call_count=5 --client_threads=4 +/// bazel run //tests/score_com_poc:poc_async -- --client_count=3 --call_count=5 --server_threads=2 +/// bazel run //tests/score_com_poc:poc_async -- --client_count=3 --call_count=5 --client_threads=4 --server_threads=2 +/// bazel run //tests/score_com_poc:poc_async -- --sleep_milliseconds=500 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flatbuffers/flatbuffers.h" +#include "score/mw/com/impl/proxy_base.h" +#include "score/mw/com/runtime.h" +#include "score/mw/com/runtime_configuration.h" +#include "score/mw/com/types.h" +#include "score/tests/ipc_poc/async_control_interface.h" +#include "score/tests/ipc_poc/ipc_buffer.h" +#include "score/tests/ipc_poc/poc_control_generated.h" + +// --------------------------------------------------------------------------- +// Global parameters (set before fork; never mutated after) +// --------------------------------------------------------------------------- + +static int g_client_count = 2; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_call_count = 2; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_threads = 2; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_server_threads = 8; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +// --------------------------------------------------------------------------- +// Config generation +// --------------------------------------------------------------------------- + +namespace score::crypto::ipc::control +{ + +/// Returns the shared "serviceTypes" JSON block (without trailing comma). +static std::string ServiceTypesJson() +{ + return + R"( "serviceTypes": [ + { + "serviceTypeName": "/score/crypto/poc/AsyncControlService", + "version": { "major": 1, "minor": 0 }, + "bindings": [ + { + "binding": "SHM", + "serviceId": 0, + "methods": [ + { "methodName": "Request", "methodId": 1 } + ], + "events": [ + { "eventName": "Response", "eventId": 2 } + ] + } + ] + } + ])"; +} + +/// Returns a single "serviceInstances" entry JSON block for the given instance_id. +static std::string ServiceInstanceJson(const int instance_id) +{ + // It seems the queueSize of the method shall match the numberOfSampleSlots of the event + // considering we have only one subscriber. + + std::ostringstream ss; + ss << " {\n" + << " \"instanceSpecifier\": \"poc/AsyncControlPort_" << instance_id << "\",\n" + << " \"serviceTypeName\": \"/score/crypto/poc/AsyncControlService\",\n" + << " \"version\": { \"major\": 1, \"minor\": 0 },\n" + << " \"instances\": [\n" + << " {\n" + << " \"instanceId\": " << instance_id << ",\n" + << " \"asil-level\": \"QM\",\n" + << " \"binding\": \"SHM\",\n" + << " \"methods\": [\n" + << " { \"methodName\": \"Request\", \"queueSize\": " + << std::max(8, 2 * g_call_count * g_client_threads) << " }\n" + << " ],\n" + << " \"events\": [\n" + << " {\n" + << " \"eventName\": \"Response\",\n" + << " \"numberOfSampleSlots\": " << std::max(8, 2 * 100 * g_client_threads) + << ",\n" + // This even accounts for multiple threads per client + // Thus if we want 2 threads waiting "method results", we need to allow at least 2 subsribers + << " \"maxSubscribers\": " << g_client_threads << "\n" + << " }\n" + << " ]\n" + << " }\n" + << " ]\n" + << " }"; + return ss.str(); +} + +/// Generates and writes all mw::com JSON config files needed for client_count consumers. +/// +/// Producer config (score/tests/ipc_poc/mw_com_config_async_producer.json): +/// Contains all client_count service instances. applicationID = 1000. +/// +/// Consumer config i (score/tests/ipc_poc/mw_com_config_async_consumer_.json): +/// Contains only service instance i. applicationID = 1001 + i. +static void SetupConfigs(const int client_count) +{ + // --- Producer config (all instances) --- + { + std::ostringstream instances; + for (int i = 0; i < client_count; ++i) + { + if (i > 0) + { + instances << ",\n"; + } + instances << ServiceInstanceJson(i); + } + + std::ostringstream json; + json << "{\n" + << ServiceTypesJson() << ",\n" + << " \"serviceInstances\": [\n" + << instances.str() << "\n" + << " ],\n" + << " \"global\": {\n" + << " \"asil-level\": \"QM\",\n" + << " \"applicationID\": 1000\n" + << " }\n" + << "}\n"; + + std::ofstream f("score/tests/ipc_poc/mw_com_config_async_producer.json"); + if (!f.is_open()) + { + std::cerr << "[SetupConfigs] Failed to open producer config for writing\n"; + return; + } + f << json.str(); + } + + // --- Per-consumer configs (one service instance each) --- + for (int i = 0; i < client_count; ++i) + { + std::ostringstream json; + json << "{\n" + << ServiceTypesJson() << ",\n" + << " \"serviceInstances\": [\n" + << ServiceInstanceJson(i) << "\n" + << " ],\n" + << " \"global\": {\n" + << " \"asil-level\": \"QM\",\n" + << " \"applicationID\": " << (1001 + i) << "\n" + << " }\n" + << "}\n"; + + const std::string path = "score/tests/ipc_poc/mw_com_config_async_consumer_" + std::to_string(i) + ".json"; + std::ofstream f(path); + if (!f.is_open()) + { + std::cerr << "[SetupConfigs] Failed to open consumer config " << i << " for writing\n"; + return; + } + f << json.str(); + } + + std::printf("[SetupConfigs] wrote producer + %d consumer config(s)\n", client_count); +} + +} // namespace score::crypto::ipc::control + +// --------------------------------------------------------------------------- +// Shared logging helper (mutex protects stdout across threads) +// --------------------------------------------------------------------------- + +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +static void Log(const std::string& line) +{ + std::lock_guard lk(g_log_mutex); + std::cout << line << "\n"; +} + +// --------------------------------------------------------------------------- +// FlatBuffer helpers (shared between server and client) +// --------------------------------------------------------------------------- + +namespace score::crypto::ipc::control +{ + +/// Serialise a ControlResponse carrying a single String result. +static IpcBuffer BuildControlResponse(const std::uint64_t request_id, const std::string& combined) +{ + flatbuffers::FlatBufferBuilder fbb(512); + + auto str_val = fbb.CreateString(combined); + auto str_tbl = CreateString(fbb, str_val); + + std::vector resp_param_types{OperationParameter_String}; + std::vector> resp_param_values{str_tbl.Union()}; + + auto resp_op = CreateSingleOperationResponse(fbb, + CreateOperationIdentifier(fbb, 0U, 0U), + CreateOperationResult(fbb, 0U), + fbb.CreateVector(resp_param_types), + fbb.CreateVector(resp_param_values)); + + auto resp_batch = CreateOperationResponseBatch(fbb, fbb.CreateVector({resp_op})); + fbb.FinishSizePrefixed(CreateControlResponse(fbb, request_id, resp_batch)); + return PackFlatBuffer(fbb.GetBufferPointer(), fbb.GetSize()); +} + +/// Deserialise a ControlRequest and produce a ControlResponse: result = "_". +static IpcBuffer ProcessRequest(const IpcBuffer& request_buf) +{ + flatbuffers::Verifier verifier{reinterpret_cast(request_buf.payload.data()), + GetPayloadSize(request_buf)}; + + if (!VerifySizePrefixedControlRequestBuffer(verifier)) + { + std::cerr << "[server/worker] FlatBuffer verification failed\n"; + return IpcBuffer{}; + } + + const auto* req = flatbuffers::GetSizePrefixedRoot(request_buf.payload.data()); + if (req == nullptr || req->operation_batch() == nullptr || req->operation_batch()->operations() == nullptr || + req->operation_batch()->operations()->size() == 0U) + { + return IpcBuffer{}; + } + + const auto* op = req->operation_batch()->operations()->Get(0U); + if (op == nullptr || op->parameter() == nullptr) + { + return IpcBuffer{}; + } + + std::string str_value; + std::uint64_t uint64_value = 0U; + + for (flatbuffers::uoffset_t i = 0U; i < op->parameter()->size(); ++i) + { + const auto ptype = static_cast(op->parameter_type()->Get(i)); + if (ptype == OperationParameter_String) + { + const auto* s = reinterpret_cast(op->parameter()->Get(i)); + if (s != nullptr && s->val() != nullptr) + { + str_value = s->val()->str(); + } + } + else if (ptype == OperationParameter_ValueUint64) + { + const auto* v = reinterpret_cast(op->parameter()->Get(i)); + if (v != nullptr) + { + uint64_value = v->val(); + } + } + } + + const std::string combined = str_value + "_" + std::to_string(uint64_value); + + { + std::stringstream ss; + ss << "[server/worker] ticket=" << req->request_id() << " -> combined=\"" << combined << "\""; + Log(ss.str()); + } + + return BuildControlResponse(req->request_id(), combined); +} + +// --------------------------------------------------------------------------- +// Server +// --------------------------------------------------------------------------- + +/// Work item pushed to the background worker queue. +struct WorkItem +{ + std::uint64_t ticket; + IpcBuffer request; + std::size_t skeleton_index; ///< Which service instance this request came from +}; + +/// Runs the server: skeleton + background worker. +/// Blocks until all client processes have exited (waitpid). +static int RunServer(const std::vector& child_pids) +{ + using score::mw::com::InstanceSpecifier; + + const int client_count = static_cast(child_pids.size()); + + score::mw::com::runtime::InitializeRuntime(score::mw::com::runtime::RuntimeConfiguration{ + score::filesystem::Path{"score/tests/ipc_poc/mw_com_config_async_producer.json"}}); + + // Create skeletons for both service instances (0 and 1). + std::vector skeletons; + for (int instance_id = 0; instance_id < client_count; ++instance_id) + { + const auto instance_spec = + InstanceSpecifier::Create(std::string{"poc/AsyncControlPort_" + std::to_string(instance_id)}).value(); + + auto skel_result = AsyncControlSkeleton::Create(instance_spec); + if (!skel_result.has_value()) + { + std::stringstream ss; + ss << "[server] AsyncControlSkeleton::Create failed for instance " << instance_id; + std::cerr << ss.str() << "\n"; + return 1; + } + skeletons.push_back(std::move(skel_result.value())); + + std::stringstream ss; + ss << "[server] created skeleton for poc/AsyncControlPort_" << instance_id; + Log(ss.str()); + } + + // ------------------------------------------------------------------ + // Thread pool: g_server_threads workers dequeue requests, perform work, + // and fire response events. + // ------------------------------------------------------------------ + std::mutex queue_mutex; + std::condition_variable queue_cv; + std::queue work_queue; + std::atomic stop_worker{false}; + + // One mutex per skeleton: response.Send() is not thread-safe when called + // concurrently on the same skeleton instance from different worker threads. + // Workers for different skeletons can still proceed in parallel. + std::vector> send_mutexes; + send_mutexes.reserve(skeletons.size()); + for (std::size_t idx = 0; idx < skeletons.size(); ++idx) + { + send_mutexes.push_back(std::make_unique()); + } + + const int worker_count = g_server_threads; + std::vector workers; + workers.reserve(static_cast(worker_count)); + for (int widx = 0; widx < worker_count; ++widx) + { + workers.emplace_back([&, widx]() { + while (true) + { + std::unique_lock lk(queue_mutex); + queue_cv.wait(lk, [&] { + return !work_queue.empty() || stop_worker.load(); + }); + + if (stop_worker.load() && work_queue.empty()) + { + break; + } + + WorkItem item = std::move(work_queue.front()); + work_queue.pop(); + lk.unlock(); + + IpcBuffer resp_buf = ProcessRequest(item.request); + if (!IsValid(resp_buf)) + { + std::cerr << "[server/worker " << widx << "] ProcessRequest returned invalid buffer " + << "for ticket=" << item.ticket << "\n"; + continue; + } + + // Simulate processing time (configurable via --sleep_milliseconds). + if (g_sleep_milliseconds > 0) + { + Log("[server/worker " + std::to_string(widx) + "] simulating work by sleeping for " + + std::to_string(g_sleep_milliseconds) + " ms"); + std::this_thread::sleep_for(std::chrono::milliseconds(g_sleep_milliseconds)); + } + + // Send the response to subscriber via the event of the correct instance. + // Serialise concurrent Sends on the same skeleton (response.Send is not + // thread-safe when called from multiple workers for the same instance). + auto send_result = [&]() { + std::lock_guard send_lk(*send_mutexes[item.skeleton_index]); + return skeletons[item.skeleton_index].response.Send(resp_buf); + }(); + if (!send_result.has_value()) + { + std::stringstream ss; + ss << "[server/worker " << widx << "] response.Send() failed on skeleton " << item.skeleton_index + << " for ticket=" << item.ticket; + Log(ss.str()); + } + else + { + std::stringstream ss; + ss << "[server/worker " << widx << "] response event fired on skeleton " << item.skeleton_index + << " for ticket=" << item.ticket; + Log(ss.str()); + } + } + }); + } + Log("[server] started " + std::to_string(worker_count) + " worker thread(s)"); + + // ------------------------------------------------------------------ + // Register the Method handler on all skeletons: enqueue work, return ticket immediately. + // ------------------------------------------------------------------ + for (std::size_t skel_idx = 0; skel_idx < skeletons.size(); ++skel_idx) + { + // Capture skel_idx by value so each handler knows its instance index. + auto reg_result = skeletons[skel_idx].request.RegisterHandler([&, skel_idx](std::uint64_t& result, + const IpcBuffer& req_buf) { + const auto* req = flatbuffers::GetSizePrefixedRoot(req_buf.payload.data()); + + std::stringstream ss1; + ss1 << "[server/handler] instance=" << skel_idx << " req ptr=" << req; + Log(ss1.str()); + + const std::uint64_t ticket = (req != nullptr) ? req->request_id() : 0U; + + if (req == nullptr || ticket == 0U) + { + std::stringstream ss_err; + ss_err << "[server/handler] instance=" << skel_idx << " WARNING: req=" << req << " ticket=" << ticket + << " — FlatBuffer parse failed or zero request_id;" + " response will never match a pending client call"; + Log(ss_err.str()); + } + + { + std::lock_guard lk(queue_mutex); + work_queue.push({ticket, req_buf, skel_idx}); + } + queue_cv.notify_one(); + + std::stringstream ss; + ss << "[server/handler] instance=" << skel_idx << " queued ticket=" << ticket << " (returning immediately)"; + Log(ss.str()); + + result = ticket; // Phase 1 return: the ticket number. + }); + + if (!reg_result.has_value()) + { + std::stringstream ss; + ss << "[server] RegisterHandler failed for skeleton " << skel_idx; + std::cerr << ss.str() << "\n"; + stop_worker.store(true); + queue_cv.notify_all(); + for (auto& worker_thread : workers) + { + worker_thread.join(); + } + return 1; + } + } + + // Offer service on all skeletons + for (std::size_t i = 0; i < skeletons.size(); ++i) + { + if (!skeletons[i].OfferService().has_value()) + { + std::stringstream ss; + ss << "[server] OfferService failed for skeleton " << i; + std::cerr << ss.str() << "\n"; + stop_worker.store(true); + queue_cv.notify_all(); + for (auto& worker_thread : workers) + { + worker_thread.join(); + } + return 1; + } + } + + Log("[server] service offered — waiting for all clients to finish..."); + + // ------------------------------------------------------------------ + // Wait for every child process to terminate. + // ------------------------------------------------------------------ + int overall_status = 0; + for (std::size_t i = 0U; i < child_pids.size(); ++i) + { + int wstatus = 0; + pid_t pid = waitpid(-1, &wstatus, 0); + if (pid > 0) + { + const bool ok = WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0; + std::stringstream ss; + ss << "[server] child pid=" << pid << (ok ? " exited OK" : " FAILED"); + Log(ss.str()); + if (!ok) + { + overall_status = 1; + } + } + } + + // ------------------------------------------------------------------ + // Graceful shutdown. + // ------------------------------------------------------------------ + { + std::lock_guard lk(queue_mutex); + stop_worker.store(true); + } + queue_cv.notify_all(); + for (auto& worker_thread : workers) + { + worker_thread.join(); + } + + for (std::size_t i = 0; i < skeletons.size(); ++i) + { + skeletons[i].StopOfferService(); + } + Log("[server] shutdown complete."); + return overall_status; +} + +// --------------------------------------------------------------------------- +// Client +// --------------------------------------------------------------------------- + +/// Per-call wait state shared between the issuing thread and the receive handler. +struct PendingCall +{ + std::uint64_t ticket{0}; ///< Ticket we are currently waiting for. + bool ready{false}; ///< Set to true when the matching event arrives. + bool ok{false}; ///< Whether the response was valid. + std::string result_value; ///< Extracted combined string from the response. + std::mutex mutex; + std::condition_variable cv; +}; + +static bool RunClient(const int client_index, const int call_count, const int thread_count) +{ + using score::mw::com::InstanceSpecifier; + using score::mw::com::SamplePtr; + using score::mw::com::impl::ProxyBase; + + // Small delay so the server has time to offer the service. + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // Per client config to provide applicationIds + std::string config_path = + "score/tests/ipc_poc/mw_com_config_async_consumer_" + std::to_string(client_index) + ".json"; + score::mw::com::runtime::InitializeRuntime( + score::mw::com::runtime::RuntimeConfiguration{score::filesystem::Path{config_path}}); + + const auto instance_spec = + InstanceSpecifier::Create(std::string{"poc/AsyncControlPort_" + std::to_string(client_index)}).value(); + + // Wait until the server has offered the service. + score::Result> handles; + while (true) + { + handles = ProxyBase::FindService(instance_spec); + if (handles.has_value() && !handles->empty()) + { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::stringstream ss_found; + ss_found << "[client " << client_index << "] service found"; + Log(ss_found.str()); + + // ------------------------------------------------------------------ + // Spawn thread_count worker threads. + // + // IMPORTANT: Each thread owns its own AsyncControlProxy instance. + // The proxy-side call-queue is hardcoded to size 1 in the mw::com + // implementation (ProxyMethodBase::kCallQueueSize = 1, see + // score/mw/com/impl/methods/proxy_method_base.h). Concurrent calls + // from multiple threads on a single proxy would race on that slot and + // one thread's Allocate() would always fail with kCallQueueFull. + // Creating one proxy per thread avoids this limitation completely. + // + // Ticket scheme: (client+1)*100_000 + (thread+1)*1_000 + (call+1) + // Supports up to 99 clients, 99 threads/client, 999 calls/thread. + // ------------------------------------------------------------------ + const std::size_t kMaxSamples = static_cast(call_count * 2); + + std::atomic total_failures{0}; + + // [LoLa framework limitation — see TODO item 9] + // LoLa does not support truly concurrent in-flight method calls from + // different proxy instances to the same skeleton instance. + // + // Root cause: ClientConnection::SendWaitReply serialises outgoing messages + // on a single per-process POSIX socket. When two proxy threads call + // DoCall() simultaneously, the second message is queued; when the first + // reply arrives, ProcessInputEvent() sends the queued message to the server + // AND fires the first reply callback in that order. The server can begin + // writing the return value to the second proxy's SHM buffer before the + // first proxy has read from its buffer, leaving a zero-initialised slot and + // yielding ticket=0 back to the caller. + // + // Workaround: serialise Phase 1 (the blocking RPC) across all threads of + // this client with phase1_mutex. Phase 2 (event wait) runs concurrently. + std::mutex phase1_mutex; + + std::vector threads; + threads.reserve(static_cast(thread_count)); + + // [LoLa framework limitation — see TODO item 8] + // AsyncControlProxy::Create() must complete for ALL threads before any + // thread may issue a method call. + // + // Root cause: Create() calls SetupMethods() → SubscribeServiceMethod() + // → ClientConnection::SendWaitReply(). This subscribe IPC message + // (sizeof = 12 bytes) travels over the exact same POSIX socket as method + // calls (sizeof = 24 bytes). If a concurrent thread is mid-call when + // Create() runs, the two messages interleave in the send queue; the server + // then receives the wrong payload size, logs: + // "Wrong payload size, got 12, expected 24" + // and tears down the connection, killing the whole client process. + // + // Workaround: create and subscribe all proxies here, sequentially, before + // any worker thread is spawned. + std::vector proxies; + proxies.reserve(static_cast(thread_count)); + for (int t = 0; t < thread_count; ++t) + { + auto proxy_result = AsyncControlProxy::Create(handles.value()[0]); + if (!proxy_result.has_value()) + { + Log("[client " + std::to_string(client_index) + "/thread " + std::to_string(t) + + "] AsyncControlProxy::Create failed (pre-create)"); + total_failures.fetch_add((thread_count - t) * call_count, std::memory_order_relaxed); + break; + } + if (!proxy_result.value().response.Subscribe(kMaxSamples).has_value()) + { + Log("[client " + std::to_string(client_index) + "/thread " + std::to_string(t) + + "] Subscribe failed (pre-create)"); + total_failures.fetch_add((thread_count - t) * call_count, std::memory_order_relaxed); + break; + } + proxies.push_back(std::move(proxy_result.value())); + } + + for (int t = 0; t < static_cast(proxies.size()); ++t) + { + threads.emplace_back([&, t]() { + auto& proxy = proxies[static_cast(t)]; + + // Within a single thread calls are sequential, so at most one + // PendingCall is in-flight at a time — no map needed. + PendingCall pending; + + // Bundle state for the receive handler lambda (must fit ≤32 bytes). + struct HandlerState + { + AsyncControlProxy& proxy; + PendingCall& pending; + std::size_t max_samples; + }; + HandlerState handler_state{proxy, pending, kMaxSamples}; + + auto set_handler_result = proxy.response.SetReceiveHandler([hs = &handler_state]() { + auto get_result = hs->proxy.response.GetNewSamples( + [hs](SamplePtr sample) noexcept { + if (!IsValid(*sample)) + { + // Silent failure: zero-filled or oversized slot — pending.ready + // would never be set, causing a 30 s timeout with no prior log. + std::cerr << "[receive handler] discarding invalid (empty/oversized) sample" + " — pending call will time out\n"; + return; + } + + const auto* resp = flatbuffers::GetSizePrefixedRoot(sample->payload.data()); + if (resp == nullptr) + { + std::cerr << "[receive handler] GetSizePrefixedRoot returned nullptr" + " — pending call will time out\n"; + return; + } + + const std::uint64_t ticket = resp->request_id(); + + // Extract the result BEFORE taking the lock so the critical + // section stays short. + std::string result_value; + bool ok = false; + if (resp->operation_batch() != nullptr && resp->operation_batch()->operations() != nullptr && + resp->operation_batch()->operations()->size() > 0U) + { + const auto* op = resp->operation_batch()->operations()->Get(0U); + if (op != nullptr && op->parameter() != nullptr && op->parameter()->size() > 0U && + op->parameter_type() != nullptr && + static_cast(op->parameter_type()->Get(0U)) == + OperationParameter_String) + { + const auto* str = reinterpret_cast(op->parameter()->Get(0U)); + if (str != nullptr && str->val() != nullptr) + { + result_value = str->val()->str(); + ok = true; + } + } + } + + if (!ok) + { + std::cerr << "[receive handler] failed to parse String result from" + " response ticket=" + << ticket << "\n"; + } + + // Single lock for both the stale-check and the ready-set to + // eliminate the race where pending.ticket could advance between + // two separate lock acquisitions. + PendingCall& pending = hs->pending; + std::uint64_t current_pending_ticket = 0U; + bool stale = false; + { + std::unique_lock lk(pending.mutex); + if (ticket != pending.ticket) + { + current_pending_ticket = pending.ticket; + stale = true; + } + else + { + pending.result_value = std::move(result_value); + pending.ok = ok; + pending.ready = true; + pending.cv.notify_one(); + } + } + if (stale) + { + // Log outside the lock to keep g_log_mutex and pending.mutex + // acquisition order consistent. + std::ostringstream oss; + oss << "[receive handler] stale event discarded:" + << " received ticket=" << ticket << " pending.ticket=" << current_pending_ticket; + Log(oss.str()); + } + }, + hs->max_samples); + + if (!get_result.has_value()) + { + std::cerr << "[receive handler] GetNewSamples failed\n"; + } + else if (*get_result == hs->max_samples) + { + // Buffer saturated: further samples may be queued and will trigger + // another receive-handler callback, but if the subscription slot + // count equals max_samples the oldest events could have been dropped. + std::ostringstream oss; + oss << "[receive handler] WARNING: GetNewSamples returned max (" << *get_result + << ") samples — subscription buffer may be saturated"; + Log(oss.str()); + } + }); + + if (!set_handler_result.has_value()) + { + Log("[client " + std::to_string(client_index) + "/thread " + std::to_string(t) + + "] SetReceiveHandler failed"); + proxy.response.Unsubscribe(); + total_failures.fetch_add(call_count, std::memory_order_relaxed); + return; + } + + int failures = 0; + + // Wait till setup is done on server side + // std::this_thread::sleep_for(std::chrono::seconds(3)); + + auto times = std::vector(call_count); + + for (int c = 0; c < call_count; ++c) + { + auto start = std::chrono::system_clock::now(); + + const std::string str_param = "client" + std::to_string(client_index + 1); + const std::uint64_t uint64_param = static_cast(c + 1); + // Unique ticket encodes client, thread and call indices. + const std::uint64_t request_id = static_cast(client_index + 1) * 100'000ULL + + static_cast(t + 1) * 1'000ULL + + static_cast(c + 1); + + const std::string expected = str_param + "_" + std::to_string(uint64_param); + + // Set up the pending state BEFORE sending so the receive handler + // can never miss the response even on a very fast server. + { + std::lock_guard lk(pending.mutex); + pending.ticket = request_id; + pending.ready = false; + pending.ok = false; + pending.result_value.clear(); + } + + // --- Phase 1 (serialised across threads — see phase1_mutex comment above) --- + bool phase1_failed = false; + std::uint64_t ticket = 0U; + { + std::lock_guard phase1_lk(phase1_mutex); + do // single-exit wrapper so 'break' acts like a labelled continue + { + auto alloc_result = proxy.request.Allocate(); + if (!alloc_result.has_value()) + { + Log("[client " + std::to_string(client_index) + "/thread " + std::to_string(t) + + "] request.Allocate() failed"); + ++failures; + phase1_failed = true; + break; + } + auto& arg = std::get<0>(alloc_result.value()); + + // Build the FlatBuffer and write directly into the SHM-backed slot. + { + flatbuffers::FlatBufferBuilder fbb(512); + + auto str_val = fbb.CreateString(str_param); + auto str_tbl = CreateString(fbb, str_val); + auto u64_tbl = CreateValueUint64(fbb, uint64_param); + + std::vector param_types{OperationParameter_String, OperationParameter_ValueUint64}; + std::vector> param_values{str_tbl.Union(), u64_tbl.Union()}; + + auto op_id = CreateOperationIdentifier(fbb, /*actor=*/1U, /*action=*/1U); + auto single_op = CreateSingleOperationRequest( + fbb, op_id, fbb.CreateVector(param_types), fbb.CreateVector(param_values)); + auto batch = CreateOperationRequestBatch(fbb, fbb.CreateVector({single_op})); + fbb.FinishSizePrefixed(CreateControlRequest(fbb, + request_id, + /*client_id=*/0U, + /*data_node_id=*/0U, + batch)); + + if (!PackFlatBufferInto(*arg, fbb.GetBufferPointer(), fbb.GetSize())) + { + Log("[client " + std::to_string(client_index) + "/thread " + std::to_string(t) + + "] PackFlatBufferInto failed"); + ++failures; + phase1_failed = true; + break; + } + } + + { + std::stringstream ss; + ss << "[client " << client_index << "/thread " << t << "] -> request_id=" << request_id + << " str=\"" << str_param << "\" uint64=" << uint64_param + << " (Phase 1: short-lived call)"; + Log(ss.str()); + } + + auto call_result = proxy.request(std::move(arg)); + if (!call_result.has_value()) + { + std::ostringstream oss; + oss << "[client " << client_index << "/thread " << t + << "] request() call failed for ticket=" << request_id; + Log(oss.str()); + ++failures; + phase1_failed = true; + break; + } + ticket = *call_result.value(); + + if (ticket != request_id) + { + std::ostringstream oss; + oss << "[client " << client_index << "/thread " << t + << "] Phase 1 ticket mismatch: sent request_id=" << request_id + << " but server returned ticket=" << ticket; + Log(oss.str()); + ++failures; + phase1_failed = true; + break; + } + + { + std::stringstream ss; + ss << "[client " << client_index << "/thread " << t << "] <- ticket=" << ticket + << " (Phase 1 complete \u2014 now waiting for Phase 2 event)"; + Log(ss.str()); + } + } while (false); + } // phase1_mutex released — Phase 2 runs concurrently + + if (phase1_failed) + { + continue; + } + + // --- Phase 2: wait for the matching response event --- + constexpr auto kTimeout = std::chrono::seconds(30); + bool timed_out = false; + { + std::unique_lock lk(pending.mutex); + timed_out = !pending.cv.wait_for(lk, kTimeout, [&] { + return pending.ready; + }); + } + + if (timed_out) + { + std::ostringstream oss; + oss << "[client " << client_index << "/thread " << t + << "] TIMEOUT waiting for event ticket=" << ticket << " (pending.ticket=" << pending.ticket + << " pending.ready=" << pending.ready << ")"; + Log(oss.str()); + ++failures; + continue; + } + + if (!pending.ok || pending.result_value != expected) + { + std::ostringstream oss; + oss << "[client " << client_index << "/thread " << t << "] MISMATCH ticket=" << ticket + << ": expected=\"" << expected << "\" got=\"" << pending.result_value << "\""; + Log(oss.str()); + ++failures; + continue; + } + + { + std::stringstream ss; + ss << "[client " << client_index << "/thread " << t << "] OK ticket=" << ticket << " result=\"" + << pending.result_value << "\" (Phase 2 complete)"; + Log(ss.str()); + } + + auto end = std::chrono::system_clock::now(); + auto elapsed = end - start; + times[c] = elapsed; + + // std::this_thread::sleep_for(std::chrono::milliseconds(5000)); + } + + auto sum = std::accumulate(times.begin(), times.end(), std::chrono::duration::zero()); + + for (int c = 0; c < call_count; ++c) + { + std::ostringstream oss; + oss << "[client " << client_index << "/thread " << t << "] call " << (c + 1) << "/" << call_count + << ": " << std::chrono::duration_cast(times[c]).count() << " us"; + Log(oss.str()); + } + + if (failures == 0) + { + std::ostringstream oss; + oss << "- [client " + std::to_string(client_index) + "/thread " + std::to_string(t) + "] completed " + + std::to_string(call_count) + " calls with " + + std::to_string(std::chrono::duration_cast(sum).count()) + + " us elapsed, average " + + std::to_string(std::chrono::duration_cast(sum).count() / + call_count) + + " us per call"; + Log(oss.str()); + } + + if (failures == 0) + { + auto c = call_count - 1; + auto s = sum - times[0]; + + std::ostringstream oss; + oss << "SKIP FIRST [client " + std::to_string(client_index) + "/thread " + std::to_string(t) + + "] completed " + std::to_string(c) + " calls with " + + std::to_string(std::chrono::duration_cast(s).count()) + + " us elapsed, average " + + std::to_string(std::chrono::duration_cast(s).count() / c) + + " us per call"; + Log(oss.str()); + } + + proxy.response.UnsetReceiveHandler(); + proxy.response.Unsubscribe(); + + total_failures.fetch_add(failures, std::memory_order_relaxed); + }); + } + + for (auto& th : threads) + { + th.join(); + } + + const int total = call_count * thread_count; + const int failures = total_failures.load(); + const int success = total - failures; + std::stringstream ss_summary; + ss_summary << "[client " << client_index << "] Results: " << success << "/" << total << " calls succeeded " + << failures << "/" << total << " calls failed (" << thread_count << " thread(s) x " << call_count + << " call(s))"; + Log(ss_summary.str()); + + return failures == 0; +} + +} // namespace score::crypto::ipc::control + +// --------------------------------------------------------------------------- +// main — fork before InitializeRuntime for a clean per-process runtime state +// --------------------------------------------------------------------------- + +int main(int argc, char** argv) +{ + // Parse optional flags. + const std::string kClientPrefix{"--client_count="}; + const std::string kCallPrefix{"--call_count="}; + const std::string kClientThreadsPrefix{"--client_threads="}; + const std::string kServerThreadsPrefix{"--server_threads="}; + const std::string kSleepPrefix{"--sleep_milliseconds="}; + for (int i = 1; i < argc; ++i) + { + const std::string arg{argv[i]}; + try + { + if (arg.rfind(kClientPrefix, 0) == 0) + { + g_client_count = std::stoi(arg.substr(kClientPrefix.size())); + } + else if (arg.rfind(kCallPrefix, 0) == 0) + { + g_call_count = std::stoi(arg.substr(kCallPrefix.size())); + } + else if (arg.rfind(kClientThreadsPrefix, 0) == 0) + { + g_client_threads = std::stoi(arg.substr(kClientThreadsPrefix.size())); + } + else if (arg.rfind(kServerThreadsPrefix, 0) == 0) + { + g_server_threads = std::stoi(arg.substr(kServerThreadsPrefix.size())); + } + else if (arg.rfind(kSleepPrefix, 0) == 0) + { + g_sleep_milliseconds = std::stoi(arg.substr(kSleepPrefix.size())); + } + } + catch (const std::exception& ex) + { + std::fprintf(stderr, "[main] invalid argument '%s': %s\n", argv[i], ex.what()); + return 1; + } + } + if (g_client_count < 1) + { + std::fprintf(stderr, "[main] --client_count must be >= 1\n"); + return 1; + } + if (g_call_count < 1) + { + std::fprintf(stderr, "[main] --call_count must be >= 1\n"); + return 1; + } + if (g_client_threads < 1) + { + std::fprintf(stderr, "[main] --client_threads must be >= 1\n"); + return 1; + } + if (g_server_threads < 1) + { + std::fprintf(stderr, "[main] --server_threads must be >= 1\n"); + return 1; + } + + std::printf("[main] client_count=%d call_count=%d client_threads=%d server_threads=%d sleep_milliseconds=%d\n", + g_client_count, + g_call_count, + g_client_threads, + g_server_threads, + g_sleep_milliseconds); + + score::crypto::ipc::control::SetupConfigs(g_client_count); + + // Flush before forking so buffered output is not duplicated. + std::fflush(nullptr); + + // ------------------------------------------------------------------ + // Fork all client processes BEFORE InitializeRuntime so each child + // gets a completely clean mw::com state. + // ------------------------------------------------------------------ + std::vector child_pids; + int my_client_index = -1; // -1 means "I am the server" + + for (int i = 0; i < g_client_count; ++i) + { + pid_t pid = ::fork(); + if (pid < 0) + { + std::perror("[main] fork"); + // Kill already-spawned children before exiting. + for (pid_t cpid : child_pids) + { + ::kill(cpid, SIGTERM); + } + return 1; + } + if (pid == 0) + { +#if false + // With configured uniuqe applicationIds + // we do not necessarily need unique UIDS + // If applicationId is not set in config, it falls back to UID + + // Child: drop to a per-client UID before touching anything else. + // Client 0 → UID 1000, client 1 → UID 1001, etc. + const auto target_uid = static_cast(1000 + i); + if (::setresuid(target_uid, target_uid, target_uid) != 0) + { + std::perror("[child] setresuid"); + return 1; + } +#endif + + my_client_index = i; + break; + } + // Parent: record the child PID and continue forking. + child_pids.push_back(pid); + } + + if (my_client_index == -1) + { + // ---- PARENT = SERVER ---- + return score::crypto::ipc::control::RunServer(child_pids); + } + else + { + // ---- CHILD = CLIENT ---- + const bool ok = score::crypto::ipc::control::RunClient(my_client_index, g_call_count, g_client_threads); + return ok ? 0 : 1; + } +} + +// clang-format off + +// TODO: Issues +// 1. There is some UID / Application ID issue +// If both clients run unter the same UID, (any no Application ID is set) they seem to conflict with each other +// We get flatbuffer validation errors, indicating that both may write to the same SHM locations +// -> Not tested yet. +// -> But we could assign ApplicationIds in the json. We would however need multiple json files with different +// ApplicationIds +// -> See +// https://github.com/eclipse-score/communication/blob/main/score/mw/com/example/com-api-example/etc/mw_com_config.json#L28 +// -> With dedicated configs and applicationIds, it seems to also work without unique UIDs +// -> In reality however, we would very likely want unique UIDs +// -> It is however not very nice, that we just get a very obscure error in this case +// -> It really seems that the second client does overwrite the request of the first one (ticker 1001 is overwriten to 2001) +// 2. When running without --sleep_milliseconds > 0 +// We get issues while waiting for the response event, leading to timeouts and such +// -> This actually seems to be caused by a maxSample count of 1 in the Subscribe +// -> With higher counts we seems to not run into this issue +// 3. With the current config, we publish to each subscriber, I guess that means each Subscriber +// could read the event and therefore also the operation response of others +// How to do this, espacailly also the routing to an event from a method call? +// We could potentially use more service instances, then it would be clear, not sure however how this would look in code and how it would scale +// 4. Occasionally i see the following +// 2026/05/13 13:59:01.741300 258889237 000 ECU1 NONE shm log error verbose 4 LockFile::Create failed to open file: /tmp/lola-ctl-0000000000000101-00003_lock | Error: An OS error has occurred with error code: File exists +// This however does not seem to cause any issue +// 5. Occasionally, i see the following: +// 2026/05/13 14:19:46.1986422 271340454 000 ECU1 NONE lola log warn verbose 5 MessagePassingService: Received NotifyEventUpdateMessage for event: ElementFqId{S:101, E:2, I:3, T:1} from node 187843 although we don't have currently any registered handlers. Might be an acceptable race, if it happens seldom! +// This however also does not seem to cause any issue +// 6. Multiple threads within one client issuing calls in parallel +// Some issue with the Allocation, looks like on client side, the queueSize is not used, but hardcoded to 1: +// See: https://github.com/eclipse-score/communication/blob/7e8cb889e17ba79b1004d6015c6216ceebd38056/score/mw/com/impl/methods/proxy_method_base.h#L71-L73 +// We may be able to get around this by having separate proxy instances +// 7. Also need to test with multiple worker threads on the server +// Server Worker Thread pool is in use +// +// 8. [LoLa limitation — WORKED AROUND] Proxy creation races with method calls on shared ClientConnection +// AsyncControlProxy::Create() calls SetupMethods() → SubscribeServiceMethod() → +// ClientConnection::SendWaitReply(). This subscribe message (12 bytes, SubscribeServiceMethodUnserializedPayload) +// uses the same POSIX message-passing socket as method calls (24 bytes, MethodCallUnserializedPayload). +// If a concurrent thread has a method call in-flight, the two messages are serialised by the +// ClientConnection queue; the server receives the 12-byte payload when it expects 24 bytes, +// logs "Wrong payload size" and disconnects the client. +// Workaround: pre-create all proxies sequentially in the main thread before spawning worker threads. +// See: ClientConnection::TryQueueMessage / ProcessSendQueueUnderLock in score_communication. +// +// 9. [LoLa limitation — WORKED AROUND] Concurrent DoCall from distinct proxy instances to the same skeleton +// All proxy instances in a process share a single ClientConnection to the skeleton. +// ClientConnection::ProcessInputEvent fires the queued next message BEFORE returning the current +// reply to its caller (see the REPLY case in ProcessInputEvent). This means the server can start +// processing call N+1 and write to its SHM return buffer before the caller of call N has read its +// own return value, leaving a zero-initialised slot and yielding ticket=0. +// Workaround: serialise Phase 1 (the DoCall / SendWaitReply) across all threads with phase1_mutex. +// See: ClientConnection::ProcessInputEvent → ProcessSendQueueUnderLock in score_communication. +// +// 10. Looks like the current Method call in Lola may not have a timeout, or it is not set. +// Isn't that a problem considering our use-case for FFI Apps communicating with a QM daemon +// In case the QM callback "gets stuck", we are also stuck on the client side +// 11. Mit event subsribers > 5 (?), we get +// 2026/06/03 06:42:40.8960242 5665993 000 ECU1 NONE lola log error verbose 4 MessagePassingServiceInstance: NotifyEventLocally failed to call ALL registered event receive handlers for event_id ElementFqId{S:0, E:2, I:1, T:1} , because number is exceeding 5 +// Not sure if this is relevant, both the remote and local notification paths seems to be called +// (I guess both are needed, since we have 1->n pattern and some receivers could be remote and some local) +// However, I could not yet increase the number of receptients +// I gues I need to redesign the client side to have a "engine thread", which solely pull out the events + + +// Current Problems: +// - Config needed. (We can request some programmatic API for this) +// Ideally, we should be config-less. +// - Keeping config in sync between Server and Clients. Even if we can inject it programmatically, +// currently both sides need both ISpec as well as the underlying Ids in their config. This +// must then also be kept in sync. +// What's the scope of this? +// ServiceIds are system-wide? Do we compete with all Services on the system for these? +// - How will this be done if a Client wants to make use of Crypto and "a regular mw com service"? +// Do we need to sync, merge, resolve conflicts in configs? Do we need to enhance configs, and e.g. +// delay com runtime init? +// - Even if we could inject the config programmatically, we still need to scale it to the number +// of users, since we need to separate ServiceInstances to separate return data via events +// (This we may change with async methods) +// * UID retrieval currently not there (I would assume this can also change in future) +// * Methods do not have a timeout, we can implement a timeout on top, but I do not see that +// being sufficient for the mixed QM / ASIL / FFI use-case, if our "short lived" enqueue call +// could already block the FFI Apps (Again, this may change with update of methods) +// - Workarounds needed for desired threading model for method calls (we can work around this) +// - Interference if multiple clients run under the same UID and no ApplicationId +// (May be changed, to base ApplicationId on PID instead of UID, but interference is surprising) +// - Naming conventions for ISpecs to detect CryptoISpecs +// - Global elements in config but Application specific parts + +// Hint // + +// Method queue_size does seem to block when full. +// E.g. with queue_size=1, we get the second request only after the send back the ticket of the first one +// The second request is blocked, till the first one is dequeued (and handled ? (just ticketing)) +// With queue_size>2, we see both clients sending their request, then first one get its ticket, then the second one. + +// OLD // + +// TODO: When running this multiple times in parallels, we get broken messages +// Quick look with strace also seems to suggests, that both open the same (two) SHM files +// Two threads in one process seems to be fine (seem's there is some serialization), +// but currently we only have one server thread (???) so it is run in series + +// Queue size, on a first glance dos not seem to make a difference + +// Multiple instances within one serviceInstance (same ISpec) does not seem to be working +// Got an error "Multi-Binding support right now not supported"; +// https://github.com/eclipse-score/communication/blob/main/score/mw/com/impl/configuration/config_parser.cpp#L774 +// Even tough, the same binding was used for both + +// With multiple ServiceInstances, we should have multiple ISpecs +// Looks like we actually need create another Skeleton Instance for the second instance +// ISpec is passed at Skeleton creation + +// Looks like we get into trouble when I register multiple handlers on the same thread +// When using serviceInstanceA and the serviceInstanceB (or vice versa) + +// Seems rather related to the UID +// (or the ApplicationId according to Copilot, seems UID is the fallback for ApplicationId) +// https://github.com/eclipse-score/communication/blob/fc50b6ae9801b118810484d0688abf5084096eb5/score/mw/com/impl/bindings/lola/runtime.cpp#L39 + +// Same as above, but run both client with different UIDs, then both work +// All is done serially, because we only have one server thread tough +// This also does work, if both clients use the same serviceInstance + +// clang-format on diff --git a/score/tests/ipc_poc/poc_control.fbs b/score/tests/ipc_poc/poc_control.fbs new file mode 100644 index 000000000..c64ccdaa4 --- /dev/null +++ b/score/tests/ipc_poc/poc_control.fbs @@ -0,0 +1,131 @@ +// ============================================================================= +// C O P Y R I G H T +// ----------------------------------------------------------------------------- +// Copyright (c) 2026 by ETAS GmbH. All rights reserved. +// +// The reproduction, distribution and utilization of this file as +// well as the communication of its contents to others without express +// authorization is prohibited. Offenders will be held liable for the +// payment of damages. All rights reserved in the event of the grant +// of a patent, utility model or design. +// ============================================================================= + +namespace score.crypto.ipc.control; + +// Basic Operation Structures + +union OperationParameter { + NoParam, + + ValueBool, + ValueUint8, + ValueUint16, + ValueUint32, + ValueUint64, + + DataBufferInBand, + DataBufferDataPlane, + + String, +} + +table SingleOperationRequest { + operation_id: OperationIdentifier; + parameter: [OperationParameter]; +} + +table SingleOperationResponse { + operation_id: OperationIdentifier; + result: OperationResult; + parameter: [OperationParameter]; +} + +table OperationIdentifier { + operation_actor: uint16; + operation_action: uint16; +} + +table OperationResult { + val: uint32; +} + +// Operation Parameter + +table NoParam { +} + +table ValueBool { + val: bool; +} + +table ValueUint8 { + val: uint8; +} + +table ValueUint16 { + val: uint16; +} + +table ValueUint32 { + val: uint32; +} + +table ValueUint64 { + val: uint64; +} + +table String { + val: string; +} + +table DataBufferInBand { + val: [uint8]; +} + +table DataBufferDataPlane { + // TODO: Whatever is needed here +} + +// Operation batching to allow transfer of multiple operations in a single IPC interaction + +table OperationRequestBatch { + operations: [SingleOperationRequest]; +} + +table OperationResponseBatch { + operations: [SingleOperationResponse]; +} + +// IPC messages +table ControlRequest { + // Identifier to match a Response to a Request + // Obsolete in sync GRPC + request_id: uint64; + + // Identifier of the client + // Goal is an authentic identifier of the caller + // Currently implemented as (PID | UID) + // with INSECURE retrieval and transfer in the GRPC based implementation + client_id: uint64; + + // Identifier of state-full DataNodes in the DataNodeManager + // Binds API side elements to DataNodes + // The concrete type of DataNode depends on the context of use + // e.g. session or hashContext + data_node_id: uint64; + + // OperationRequest + operation_batch: OperationRequestBatch; +} + +table ControlResponse { + // Identifier to match a Response to a Request + // Obsolete in sync GRPC + request_id: uint64; + + operation_batch: OperationResponseBatch; +} + +// ControlRequest is the natural entry point for the serialised payload. +// Use flatbuffers::GetSizePrefixedRoot for the response. +root_type ControlRequest; diff --git a/score/tests/ipc_poc/poc_grpc.cpp b/score/tests/ipc_poc/poc_grpc.cpp new file mode 100644 index 000000000..7ebdf89f4 --- /dev/null +++ b/score/tests/ipc_poc/poc_grpc.cpp @@ -0,0 +1,515 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This file exists only to test and understand IPC mechanisms and to guide a +// proper implementation. It is kept temporarily as reference and will be +// removed once confidence has been gained in the proper implementation. +// ============================================================================= + +/// POC: gRPC round-trip benchmark — standalone, no daemon or crypto specifics +/// +/// Mirrors poc_low_level.cpp in structure: +/// - fork model: parent = server, children = clients +/// - same CLI flags: --client_count, --call_count, --client_threads, +/// --server_threads (accepted but unused — gRPC controls +/// its own thread pool), --sleep_milliseconds +/// - same per-thread timings and "SKIP FIRST" summary output +/// +/// Communication model: +/// Each client thread calls GrpcControlClient::SendRequest() synchronously. +/// The call is blocking: one request in → one response out. No ticket map +/// or correlation logic is needed because ordering is guaranteed by gRPC. +/// +/// The server's EchoRequestHandler concatenates the two input parameters +/// ("_") and returns the result as an OwnedString in the +/// response, mirroring what poc_low_level does with its FlatBuffer echo. +/// +/// Handler chain: +/// EchoHandlerFactory → EchoRequestHandler +/// No daemon, no data_manager, no crypto, no config. +/// +/// Note on --server_threads: +/// GrpcControlServer currently hard-codes MIN/MAX_POLLERS = 1. The flag +/// is accepted for CLI parity with the other POCs but has no effect on the +/// actual gRPC thread count. +/// +/// Usage: +/// bazel run //tests/score_com_poc:poc_grpc +/// bazel run //tests/score_com_poc:poc_grpc -- --client_count=3 --call_count=5 +/// bazel run //tests/score_com_poc:poc_grpc -- --client_count=3 --call_count=5 --client_threads=4 +/// bazel run //tests/score_com_poc:poc_grpc -- --sleep_milliseconds=50 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "score/crypto/src/daemon/control_plane/control_protocol.h" +#include "score/crypto/src/daemon/control_plane/i_handler_chain_factory.hpp" +#include "score/crypto/src/daemon/control_plane/i_request_handler.hpp" +#include "score/crypto/src/ipc/grpc_adapter/grpc_control_client.h" +#include "score/crypto/src/ipc/grpc_adapter/grpc_control_server.h" + +// --------------------------------------------------------------------------- +// Global parameters (set before fork; never mutated after) +// --------------------------------------------------------------------------- + +static int g_client_count = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_call_count = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_threads = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_server_threads = 8; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +// --------------------------------------------------------------------------- +// Shared logging helper +// --------------------------------------------------------------------------- + +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +static void Log(const std::string& line) +{ + std::lock_guard lk(g_log_mutex); + std::cout << line << "\n"; +} + +// --------------------------------------------------------------------------- +// Echo handler — concatenates "_" and returns it +// No daemon, no data_manager, no config +// --------------------------------------------------------------------------- + +namespace score::crypto::poc::grpc +{ + +namespace proto = daemon::control_plane::protocol; + +class EchoRequestHandler : public daemon::control_plane::IRequestHandler +{ + public: + daemon::control_plane::ControlResponse processRequest(daemon::control_plane::ControlRequest& request) override + { + proto::ControlResponse response; + response.request_id = request.request_id; + + if (request.operation.operations.empty()) + { + return response; + } + + const auto& op = request.operation.operations[0]; + + // Extract string (arrives as string_view into the FlatBuffer — safe here, we copy it) + std::string str_value; + if (!op.parameters.empty() && std::holds_alternative(op.parameters[0])) + { + str_value = std::string(std::get(op.parameters[0])); + } + + // Extract uint64 + std::uint64_t uint64_value = 0U; + if (op.parameters.size() >= 2 && std::holds_alternative(op.parameters[1])) + { + uint64_value = std::get(op.parameters[1]); + } + + if (g_sleep_milliseconds > 0) + { + std::this_thread::sleep_for(std::chrono::milliseconds(g_sleep_milliseconds)); + } + + const std::string combined = str_value + "_" + std::to_string(uint64_value); + + { + std::ostringstream ss; + ss << "[server/handler] request_id=" << request.request_id << " -> combined=\"" << combined << "\""; + Log(ss.str()); + } + + // Return combined string as OwnedString and the uint64 echo. + // OperationResponseBuilder has no return_value_string(), so we push the + // OwnedString directly into the last operation's parameters after build(). + proto::OperationResponseBuilder builder; + builder.operation(op.operationId).return_success().return_value_uint64(uint64_value); + + auto built = builder.build(); + if (built.has_value()) + { + if (!built.value().operations.empty()) + { + built.value().operations.back().parameters.push_back(proto::OwnedString{combined}); + } + response.operation = std::move(built.value()); + } + + return response; + } +}; + +class EchoHandlerFactory : public daemon::control_plane::IHandlerChainFactory +{ + public: + std::unique_ptr CreateRequestHandler() override + { + return std::make_unique(); + } +}; + +// --------------------------------------------------------------------------- +// Server +// --------------------------------------------------------------------------- + +static int RunServer(const std::string& socket_path, const std::vector& child_pids) +{ + auto factory = std::make_unique(); + ipc::GrpcControlServer server(std::move(factory)); + + std::thread server_thread([&server, &socket_path]() { + server.Start(socket_path); + server.WaitForTermination(); + }); + + Log("[server] started on " + socket_path + " — waiting for all clients to finish..."); + + int overall_status = 0; + for (std::size_t i = 0U; i < child_pids.size(); ++i) + { + int wstatus = 0; + const pid_t pid = ::waitpid(-1, &wstatus, 0); + if (pid > 0) + { + const bool ok = WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0; + std::ostringstream ss; + ss << "[server] child pid=" << pid << (ok ? " exited OK" : " FAILED"); + Log(ss.str()); + if (!ok) + { + overall_status = 1; + } + } + } + + server.Stop(); + if (server_thread.joinable()) + { + server_thread.join(); + } + + Log("[server] shutdown complete."); + return overall_status; +} + +// --------------------------------------------------------------------------- +// Client +// --------------------------------------------------------------------------- + +static bool RunClient(const std::string& socket_path, + const int client_index, + const int call_count, + const int thread_count) +{ + // Give the server time to bind before the first connection attempt. + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // One gRPC channel shared by all threads — gRPC channels are thread-safe. + // SendRequest() blocks the calling thread until the response arrives; + // multiple threads can issue concurrent calls without extra synchronization. + ipc::GrpcControlClient client(socket_path); + + std::atomic total_failures{0}; + std::vector threads; + threads.reserve(static_cast(thread_count)); + + for (int t = 0; t < thread_count; ++t) + { + threads.emplace_back([&, t]() { + int failures = 0; + auto times = std::vector(call_count); + + for (int c = 0; c < call_count; ++c) + { + auto start_time = std::chrono::system_clock::now(); + + const std::string str_param = "client" + std::to_string(client_index + 1); + const std::uint64_t uint64_param = static_cast(c + 1); + const std::string expected_combined = str_param + "_" + std::to_string(uint64_param); + + // operationActor encodes the client index so the server log is readable. + const daemon::common::OperationIdentifier opId{ + /*operationActor=*/static_cast(client_index + 1), + /*operationAction=*/1U, + }; + + auto requestResult = proto::ControlRequestBuilder() + .forDataNodeId(0) + .operation(opId) + .with_in_string(str_param) + .with_in_val_uint64(uint64_param) + .build(); + + if (!requestResult.has_value()) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] build() failed for call " << c; + Log(ss.str()); + ++failures; + continue; + } + + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] -> SendRequest() str=\"" << str_param + << "\" uint64=" << uint64_param; + Log(ss.str()); + } + + // Synchronous blocking call — no ticket map needed. + // GrpcControlClient overwrites request_id internally; the response + // request_id matches the auto-generated one used on the wire. + auto responseResult = client.SendRequest(requestResult.value()); + + if (!responseResult.has_value()) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] SendRequest() failed for call " << c; + Log(ss.str()); + ++failures; + continue; + } + + const auto& resp = responseResult.value(); + + // Validate: one operation, success result, uint64 echo matches, combined string matches. + bool ok = !resp.operation.operations.empty() && + resp.operation.operations[0].result == proto::OPERATION_RESULT_SUCCESS; + + std::string got_combined; + if (ok) + { + const auto& params = resp.operation.operations[0].parameters; + // param[0] = uint64 echo + auto u64 = resp.operation.operations[0].getParameter(0); + ok = u64.has_value() && (u64.value() == uint64_param); + + // param[1] = combined OwnedString + if (ok && params.size() >= 2) + { + auto str = resp.operation.operations[0].getParameter(1); + if (str.has_value()) + { + got_combined = str.value(); + ok = (got_combined == expected_combined); + } + else + { + ok = false; + } + } + } + + if (!ok) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] MISMATCH or error for call " << c + << ": expected combined=\"" << expected_combined << "\" got=\"" << got_combined << "\""; + Log(ss.str()); + ++failures; + continue; + } + + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] <- OK combined=\"" << got_combined + << "\""; + Log(ss.str()); + } + + auto end_time = std::chrono::system_clock::now(); + times[c] = end_time - start_time; + } + + // Per-call timings + for (int c = 0; c < call_count; ++c) + { + std::ostringstream oss; + oss << "[client " << client_index << "/thread " << t << "] call " << (c + 1) << "/" << call_count + << ": " << std::chrono::duration_cast(times[c]).count() << " us"; + Log(oss.str()); + } + + // Summary + if (failures == 0) + { + auto sum = std::chrono::duration::zero(); + for (int c = 0; c < call_count; ++c) + { + sum += times[c]; + } + + std::ostringstream oss; + oss << "- [client " << client_index << "/thread " << t << "] completed " << call_count << " calls with " + << std::chrono::duration_cast(sum).count() << " us elapsed, average " + << std::chrono::duration_cast(sum).count() / call_count + << " us per call"; + Log(oss.str()); + + if (call_count > 1) + { + auto s = sum - times[0]; + std::ostringstream oss2; + oss2 << "SKIP FIRST [client " << client_index << "/thread " << t << "] completed " + << (call_count - 1) << " calls with " + << std::chrono::duration_cast(s).count() << " us elapsed, average " + << std::chrono::duration_cast(s).count() / (call_count - 1) + << " us per call"; + Log(oss2.str()); + } + } + + total_failures.fetch_add(failures, std::memory_order_relaxed); + }); + } + + for (auto& th : threads) + { + th.join(); + } + + const int total = call_count * thread_count; + const int failures = total_failures.load(); + const int success = total - failures; + std::ostringstream ss; + ss << "[client " << client_index << "] Results: " << success << "/" << total << " calls succeeded, " << failures + << "/" << total << " calls failed (" << thread_count << " thread(s) x " << call_count << " call(s))"; + Log(ss.str()); + + return failures == 0; +} + +} // namespace score::crypto::poc::grpc + +// --------------------------------------------------------------------------- +// main — fork before any gRPC setup for a clean per-process state +// --------------------------------------------------------------------------- + +int main(int argc, char** argv) +{ + const std::string kClientPrefix{"--client_count="}; + const std::string kCallPrefix{"--call_count="}; + const std::string kClientThreadsPrefix{"--client_threads="}; + const std::string kServerThreadsPrefix{"--server_threads="}; + const std::string kSleepPrefix{"--sleep_milliseconds="}; + + for (int i = 1; i < argc; ++i) + { + const std::string arg{argv[i]}; + try + { + if (arg.rfind(kClientPrefix, 0) == 0) + g_client_count = std::stoi(arg.substr(kClientPrefix.size())); + else if (arg.rfind(kCallPrefix, 0) == 0) + g_call_count = std::stoi(arg.substr(kCallPrefix.size())); + else if (arg.rfind(kClientThreadsPrefix, 0) == 0) + g_client_threads = std::stoi(arg.substr(kClientThreadsPrefix.size())); + else if (arg.rfind(kServerThreadsPrefix, 0) == 0) + g_server_threads = std::stoi(arg.substr(kServerThreadsPrefix.size())); + else if (arg.rfind(kSleepPrefix, 0) == 0) + g_sleep_milliseconds = std::stoi(arg.substr(kSleepPrefix.size())); + } + catch (const std::exception& ex) + { + std::fprintf(stderr, "[main] invalid argument '%s': %s\n", argv[i], ex.what()); + return 1; + } + } + + if (g_client_count < 1) + { + std::fprintf(stderr, "[main] --client_count must be >= 1\n"); + return 1; + } + if (g_call_count < 1) + { + std::fprintf(stderr, "[main] --call_count must be >= 1\n"); + return 1; + } + if (g_client_threads < 1) + { + std::fprintf(stderr, "[main] --client_threads must be >= 1\n"); + return 1; + } + if (g_server_threads < 1) + { + std::fprintf(stderr, "[main] --server_threads must be >= 1\n"); + return 1; + } + + std::printf( + "[main] client_count=%d call_count=%d client_threads=%d" + " server_threads=%d (note: gRPC manages its own pool) sleep_milliseconds=%d\n", + g_client_count, + g_call_count, + g_client_threads, + g_server_threads, + g_sleep_milliseconds); + + // Unique socket per run to avoid collisions between concurrent bazel invocations. + const std::string socket_path = "/tmp/score_poc_grpc_" + std::to_string(::getpid()) + ".sock"; + + std::fflush(nullptr); + + std::vector child_pids; + int my_client_index = -1; + + for (int i = 0; i < g_client_count; ++i) + { + const pid_t pid = ::fork(); + if (pid < 0) + { + std::perror("[main] fork"); + for (const pid_t cpid : child_pids) + { + ::kill(cpid, SIGTERM); + } + return 1; + } + if (pid == 0) + { + my_client_index = i; + break; + } + child_pids.push_back(pid); + } + + if (my_client_index == -1) + { + return score::crypto::poc::grpc::RunServer(socket_path, child_pids); + } + else + { + const bool ok = + score::crypto::poc::grpc::RunClient(socket_path, my_client_index, g_call_count, g_client_threads); + return ok ? 0 : 1; + } +} diff --git a/score/tests/ipc_poc/poc_low_level.cpp b/score/tests/ipc_poc/poc_low_level.cpp new file mode 100644 index 000000000..baf2a3295 --- /dev/null +++ b/score/tests/ipc_poc/poc_low_level.cpp @@ -0,0 +1,1393 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This file exists only to test and understand IPC mechanisms and to guide a +// proper implementation. It is kept temporarily as reference and will be +// removed once confidence has been gained in the proper implementation. +// ============================================================================= + +/// POC: SendWithCallback-ack + Notify async model — one shared connection per client process +/// +/// This replicates the functionality of poc_async.cpp using only the low-level +/// score::message_passing API, without any score::mw elements. +/// +/// Communication model (two phases, mirroring poc_async's method+event approach): +/// +/// Phase 1 — SendWithCallback for non-blocking acknowledgment: +/// The client thread calls SendWithCallback() with a ControlRequest and a +/// ReplyCallback. The call enqueues the message and returns immediately +/// (non-blocking, ASIL-B safe). The server's sent_with_reply_callback +/// enqueues the work, calls Reply() with the client-assigned request_id as an ack, +/// and then returns. The engine thread fires ReplyCallback +/// on the client side when the ack arrives. The calling thread waits on an +/// application-level CV with a timeout — if the server stalls, the thread is +/// unblocked by the timeout rather than hanging forever inside the library. +/// +/// Phase 2 — Server Notify (analogous to the Response event): +/// A server pool worker dequeues the request, does the work, and calls +/// Notify() on the IServerConnection. The ControlResponse payload carries +/// the original request_id. The client's NotifyCallback routes by +/// request_id and wakes the waiting thread. +/// +/// Why the two-phase approach is needed: +/// The REQUEST/REPLY protocol serializes per connection: the server does not +/// process the next REQUEST until Reply() has been called on the current one. +/// With multiple threads sharing one connection, their SendWithCallback() calls +/// are serialized at the protocol level. By calling Reply() immediately in +/// the callback (before the actual work), the serialization window is minimal, +/// By enqueueing the work and then calling Reply() immediately in the callback +/// (before the actual work), the serialization window is minimal, +/// so request throughput is still high while the pool workers run concurrently. +/// NOTIFY messages go from server to client independently, so they do not +/// block incoming requests. +/// +/// Why SendWithCallback instead of SendWaitReply: +/// SendWaitReply() blocks the calling thread inside the library with no timeout. +/// A misbehaving QM server (slow callback, scheduling starvation without crash) +/// would hold an ASIL-B thread blocked indefinitely. SendWithCallback() is +/// non-blocking; the library guarantees the calling thread is never held inside +/// the IPC layer. The application-level wait_for() provides the safety timeout. +/// +/// Connection model: +/// One IClientConnection per client process (shared by all threads). +/// The request_id in the FlatBuffer payload is used to route each Notify +/// back to the thread that issued the corresponding SendWithCallback. +/// Because Notify() is point-to-point (reaches only this connection's client), +/// no cross-client leakage occurs — unlike poc_async's broadcast events which +/// required one skeleton instance per client. +/// +/// Process model: same as poc_async — fork before any IPC setup, parent=server, +/// children=clients. +/// +/// Usage: +/// bazel run //tests/score_com_poc:poc_low_level +/// bazel run //tests/score_com_poc:poc_low_level -- --client_count=3 --call_count=5 +/// bazel run //tests/score_com_poc:poc_low_level -- --client_count=3 --call_count=5 --client_threads=4 +/// bazel run //tests/score_com_poc:poc_low_level -- --client_count=3 --call_count=5 --server_threads=2 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flatbuffers/flatbuffers.h" +#include "score/message_passing/client_factory.h" +#include "score/message_passing/i_client_connection.h" +#include "score/message_passing/i_server_connection.h" +#include "score/message_passing/i_server_factory.h" +#include "score/message_passing/server_factory.h" +#include "score/message_passing/service_protocol_config.h" +#include "score/tests/ipc_poc/ipc_buffer.h" +#include "score/tests/ipc_poc/poc_control_generated.h" + +// --------------------------------------------------------------------------- +// Global parameters (set before fork; never mutated after) +// --------------------------------------------------------------------------- + +static int g_client_count = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_call_count = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_threads = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_server_threads = 8; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +// --------------------------------------------------------------------------- +// Shared logging helper +// --------------------------------------------------------------------------- + +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +static void Log(const std::string& line) +{ + std::lock_guard lk(g_log_mutex); + std::cout << line << "\n"; +} + +static void LogErr(const std::string& line) +{ + std::stringstream ss; + ss << "[ERROR] " << line << "\n"; + std::lock_guard lk(g_log_mutex); + std::cerr << ss.str(); +} + +// --------------------------------------------------------------------------- +// Protocol config — identifier resolves to an abstract Unix domain socket +// --------------------------------------------------------------------------- + +namespace score::crypto::ipc::control +{ + +static constexpr std::string_view kServiceIdentifier{"score_crypto_poc_ll"}; + +// --------------------------------------------------------------------------- +// Configuration +// +// All queue sizes derive from two logical inputs: +// +// N — number of distinct client processes (== distinct UIDs the server +// accepts). Controls how many connections the server manages and +// sets the capacity of the global server-side receive queue. +// +// T — maximum number of threads per client process that may have +// concurrent in-flight requests at any one time. Controls the +// per-connection queue depths. +// +// Payload size M is fixed by the IPC protocol (sizeof(IpcBuffer)). +// --------------------------------------------------------------------------- + +/// Parameters shared by both server and client sides. +/// Both sides must be constructed from the same ServiceParams values; +/// a mismatch in any size field causes EMSGSIZE on send or silent +/// truncation on receive. +struct ServiceParams +{ + /// Logical identifier of the service. Maps to an abstract Unix domain + /// socket name on Linux and to a QNX resource-manager path on QNX. + std::string_view identifier; + + /// Maximum byte size of a client→server message (ControlRequest). + /// Must be >= sizeof(the largest FlatBuffer payload sent by any client). + std::uint32_t max_payload_bytes; +}; + +/// Parameters that only the server side needs. +struct ServerParams +{ + /// Number of distinct client processes expected to connect. + /// Used to: + /// - size the global server-side receive queue (N * T slots total) + /// - limit accepted connections to at most N (one per UID) + std::uint32_t max_client_processes; // N + + /// Maximum number of threads per client process that may have concurrent + /// in-flight requests. Used to size per-connection notify queues. + /// Under-sizing this causes Notify() to return ENOBUFS on QNX (response + /// lost, client times out) or blocks the engine thread on Linux. + std::uint32_t max_threads_per_client; // T + + /// Number of server-side pool threads processing requests. + std::uint32_t worker_threads; +}; + +/// Parameters that only the client side needs. +struct ClientParams +{ + /// Maximum number of threads in this process that may have concurrent + /// in-flight requests. Sizes both max_async_replies and max_queued_sends + /// in the client config so that T concurrent SendWithCallback() calls can + /// be in-flight simultaneously without getting ENOBUFS. + std::uint32_t max_concurrent_threads; // T +}; + +// --------------------------------------------------------------------------- +// Config factory functions +// --------------------------------------------------------------------------- + +static score::message_passing::ServiceProtocolConfig MakeProtocolConfig(const ServiceParams& p) +{ + return score::message_passing::ServiceProtocolConfig{ + p.identifier, + // max_send_size: upper bound for a client→server ControlRequest FlatBuffer. + /*max_send_size=*/p.max_payload_bytes, + // max_reply_size: upper bound for the ack ControlResponse sent by Reply(). + // The ack only carries request_id and an empty operation batch, so it is much + // smaller than max_payload_bytes in practice. Using the same value keeps both + // sides in sync without a second size constant; the slight over-allocation in the + // client receive buffer is acceptable. + /*max_reply_size=*/p.max_payload_bytes, + // max_notify_size: upper bound for the full ControlResponse sent by Notify(). + // Must be at least as large as the largest response payload the server produces. + /*max_notify_size=*/p.max_payload_bytes, + }; +} + +static score::message_passing::IServerFactory::ServerConfig MakeServerConfig(const ServerParams& p) +{ + const std::uint32_t n = p.max_client_processes; + const std::uint32_t t = p.max_threads_per_client; + + // NOTE: ServerConfig is read only by the QNX implementation; the Linux/Unix-domain + // implementation ignores all three fields and relies on kernel socket buffers instead. + // The values are set correctly here so that the same code works on QNX without changes. + return score::message_passing::IServerFactory::ServerConfig{ + // Server-side ring buffer for incoming SEND and REQUEST messages (QNX). + // The REQUEST/REPLY protocol serializes one REQUEST per connection: the server + // does not accept the next REQUEST on a connection until Reply() has been called. + // With one connection per client process we therefore have at most N simultaneous + // in-flight REQUESTs — one per client — regardless of how many threads each client + // has. N slots are sufficient; N*T would be an over-allocation. + /*max_queued_sends=*/n, + + // Number of ServerConnection objects pre-allocated at startup (QNX). + // Avoids runtime heap allocation when clients connect, which is required for + // monotonic/bounded memory in safety contexts. Set to N (one per expected client). + /*pre_alloc_connections=*/n, + + // Per-connection NOTIFY queue depth on the server side (QNX). + // Each in-flight SendWithCallback() on the client side will eventually receive one + // Notify() from the server. With T threads sharing one connection, up to T + // Notify() calls may be queued before the client drains them. + // If this queue overflows, Notify() returns ENOBUFS on QNX — the response is + // silently dropped and the client hangs until the 30-second timeout. + // This is the most critical parameter to size correctly: must be >= T. + /*max_queued_notifies=*/t, + }; +} + +static score::message_passing::IClientFactory::ClientConfig MakeClientConfig(const ClientParams& p) +{ + const std::uint32_t t = p.max_concurrent_threads; + + return score::message_passing::IClientFactory::ClientConfig{ + // One async-reply slot per concurrent thread: each in-flight SendWithCallback() + // holds one slot until its ReplyCallback fires. Must be >= T. + /*max_async_replies=*/t, + + // SendWithCallback() with truly_async=true always queues into the send queue + // before the engine thread picks it up. One slot per concurrent thread. + // Must be >= T; shared pool with max_async_replies. + /*max_queued_sends=*/t, + + // Serialize SEND (fire-and-forget) delivery relative to REQUEST/REPLY messages. + // This POC never calls Send(), so ordering across message types is irrelevant. + /*fully_ordered=*/false, + + // Route SendWithCallback() through the engine's background thread so the + // calling thread is never held inside the IPC layer (non-blocking guarantee). + // Required when max_queued_sends > 0. Mandatory for safety clients sending + // to QM servers where the server callback duration is not bounded. + /*truly_async=*/true, + + // Do not block the calling thread on the first connection attempt. + // Start() is called before the server socket exists (child processes start + // 300 ms after the fork); the background engine thread retries until the + // server is ready and fires the kReady state callback. + /*sync_first_connect=*/false, + }; +} + +// --------------------------------------------------------------------------- +// FlatBuffer helpers +// --------------------------------------------------------------------------- + +/// Builds a minimal ControlResponse carrying only request_id and no operation +/// payload. Used as the immediate ack sent by Reply() in the +/// sent_with_reply_callback so the client's ReplyCallback fires quickly. +static std::vector BuildAcknowledgeReply(const std::uint64_t request_id) +{ + flatbuffers::FlatBufferBuilder fbb(128); + auto empty_batch = CreateOperationResponseBatch( + fbb, fbb.CreateVector(std::vector>{})); + fbb.FinishSizePrefixed(CreateControlResponse(fbb, request_id, empty_batch)); + return std::vector(fbb.GetBufferPointer(), fbb.GetBufferPointer() + fbb.GetSize()); +} + +static std::vector BuildResponseBytes(const std::uint64_t request_id, const std::string& combined) +{ + flatbuffers::FlatBufferBuilder fbb(512); + + auto str_val = fbb.CreateString(combined); + auto str_tbl = CreateString(fbb, str_val); + + std::vector resp_param_types{OperationParameter_String}; + std::vector> resp_param_values{str_tbl.Union()}; + + auto resp_op = CreateSingleOperationResponse(fbb, + CreateOperationIdentifier(fbb, 0U, 0U), + CreateOperationResult(fbb, 0U), + fbb.CreateVector(resp_param_types), + fbb.CreateVector(resp_param_values)); + + auto resp_batch = CreateOperationResponseBatch(fbb, fbb.CreateVector({resp_op})); + fbb.FinishSizePrefixed(CreateControlResponse(fbb, request_id, resp_batch)); + return std::vector(fbb.GetBufferPointer(), fbb.GetBufferPointer() + fbb.GetSize()); +} + +static std::vector ProcessRequestBytes(const std::uint64_t request_id, + score::cpp::span message) +{ + flatbuffers::Verifier verifier{message.data(), static_cast(message.size())}; + if (!VerifySizePrefixedControlRequestBuffer(verifier)) + { + LogErr("[server/worker] FlatBuffer verification failed"); + return {}; + } + + const auto* req = flatbuffers::GetSizePrefixedRoot(message.data()); + if (req == nullptr || req->operation_batch() == nullptr || req->operation_batch()->operations() == nullptr || + req->operation_batch()->operations()->size() == 0U) + { + return {}; + } + + const auto* op = req->operation_batch()->operations()->Get(0U); + if (op == nullptr || op->parameter() == nullptr) + { + return {}; + } + + std::string str_value; + std::uint64_t uint64_value = 0U; + + for (flatbuffers::uoffset_t i = 0U; i < op->parameter()->size(); ++i) + { + const auto ptype = static_cast(op->parameter_type()->Get(i)); + if (ptype == OperationParameter_String) + { + const auto* s = reinterpret_cast(op->parameter()->Get(i)); + if (s != nullptr && s->val() != nullptr) + { + str_value = s->val()->str(); + } + } + else if (ptype == OperationParameter_ValueUint64) + { + const auto* v = reinterpret_cast(op->parameter()->Get(i)); + if (v != nullptr) + { + uint64_value = v->val(); + } + } + } + + const std::string combined = str_value + "_" + std::to_string(uint64_value); + + { + std::ostringstream ss; + ss << "[server/worker] request_id=" << request_id << " -> combined=\"" << combined << "\""; + Log(ss.str()); + } + + return BuildResponseBytes(request_id, combined); +} + +// --------------------------------------------------------------------------- +// Server +// --------------------------------------------------------------------------- + +struct WorkItem +{ + score::message_passing::IServerConnection* conn; + std::shared_ptr alive; // per-connection lifetime token; set to false by disconnect_cb + std::uint64_t request_id; + std::vector request_bytes; +}; + +static int RunServer(const std::vector& child_pids) +{ + const ServiceParams service_params{ + kServiceIdentifier, + /*max_payload_bytes=*/static_cast(sizeof(IpcBuffer)), + }; + const ServerParams server_params{ + /*max_client_processes=*/static_cast(g_client_count), + /*max_threads_per_client=*/static_cast(g_client_threads), + /*worker_threads=*/static_cast(g_server_threads), + }; + + score::message_passing::ServerFactory server_factory; + const auto protocol_config = MakeProtocolConfig(service_params); + const auto server_config = MakeServerConfig(server_params); + + auto server = server_factory.Create(protocol_config, server_config); + if (!server) + { + LogErr("[server] failed to create server"); + return 1; + } + + // ------------------------------------------------------------------ + // Live-connection guard via per-connection lifetime token. + // + // Worker threads hold a shared_ptr (alive token) inside WorkItem, + // captured at enqueue time. The library destroys a ServerConnection as + // soon as the client disconnects. Without coordination, a worker that + // dequeued a WorkItem before the disconnect fires could call Notify() on + // a destroyed object — or on a new connection that reused the same address. + // + // Fix: each connection gets a shared_ptr initialised to true. + // disconnect_cb sets it to false under live_conn->mutex before the library + // destroys the object. The worker checks the flag under the same mutex + // before calling Notify(), so a false flag always wins the race. + // + // Aliasing is impossible: the WorkItem holds its own shared_ptr copy whose + // control block is unique to that connection's lifetime; a new connection + // that reuses the same address gets a brand-new shared_ptr(true). + // + // Notify() is called while holding live_conn->mutex because the transport + // does not expose a connection lifetime lease. This ordering is required: + // disconnect_cb cannot destroy the connection until a worker has finished + // using its raw pointer. It does not make Notify() bounded. The Unix + // backend uses a blocking sendmsg(), while the QNX backend takes its own + // send mutex and can return ENOBUFS when its notify pool is exhausted. + // Consequently, a blocked Notify() can delay disconnect_cb and therefore + // client admission. Production code must provide a bounded/non-blocking + // Notify() operation or a library-owned connection lease before removing + // this lock or claiming a bounded disconnect path. + // + // Lock order: live_conn->mutex must NOT be taken while holding any + // score::message_passing internal lock. Workers take it only around + // the alive check + Notify(); they release it before touching the work + // queue again. + // + // connect_cb and disconnect_cb use [&] capture and access live_conn directly + // by reference — no heap allocation needed. sent_with_reply_cb accesses it + // via SentWithReplyCtx (a shared_ptr-boxed struct required for the 32-byte + // callback limit), which holds a LiveConnections& into the same frame. + // ------------------------------------------------------------------ + struct LiveConnections + { + std::mutex mutex; + // Was facing pointer-reuse issues, when just using the Connection address for alive checks + // Thus the additional shared_ptr per connection, used in work_items + std::unordered_map> alive_map; + }; + LiveConnections live_conn; + + // ------------------------------------------------------------------ + // Thread pool: workers dequeue requests, do the work, and call + // Notify() on the stored IServerConnection*. + // Notify() is thread-safe and safe to call from a pool thread. + // Multiple concurrent Notify() calls on the same connection are + // serialized by the library; the NotifyCallback on the client side + // routes by request_id so ordering does not matter. + // ------------------------------------------------------------------ + std::mutex queue_mutex; + std::condition_variable queue_cv; + std::queue work_queue; + std::atomic stop_workers{false}; + + std::vector workers; + workers.reserve(server_params.worker_threads); + for (std::uint32_t w = 0U; w < server_params.worker_threads; ++w) + { + workers.emplace_back([&, w]() { + while (true) + { + std::unique_lock lk(queue_mutex); + + auto start_time = std::chrono::system_clock::now(); + + queue_cv.wait(lk, [&] { + return !work_queue.empty() || stop_workers.load(); + }); + + auto end_time = std::chrono::system_clock::now(); + auto diff = end_time - start_time; + Log("[server/worker " + std::to_string(w) + "] Took: " + + std::to_string(std::chrono::duration_cast(diff).count()) + " us\n"); + + if (stop_workers.load() && work_queue.empty()) + { + break; + } + + WorkItem item = std::move(work_queue.front()); + work_queue.pop(); + lk.unlock(); + + if (g_sleep_milliseconds > 0) + { + Log("[server/worker " + std::to_string(w) + "] simulating work, sleeping " + + std::to_string(g_sleep_milliseconds) + " ms"); + std::this_thread::sleep_for(std::chrono::milliseconds(g_sleep_milliseconds)); + } + + auto response_bytes = ProcessRequestBytes( + item.request_id, + score::cpp::span{item.request_bytes.data(), item.request_bytes.size()}); + + // Guard against use-after-free: check the alive token under live_conn->mutex, + // which disconnect_cb also holds when it flips the flag to false. + // Notify() is called under the same lock so no window exists between the check + // and the call. This protects the raw connection pointer, but the transport + // call itself is not guaranteed to return within a bounded time; see the + // LiveConnections note above. + std::lock_guard live_lk(live_conn.mutex); + if (!*item.alive) + { + LogErr("[server/worker " + std::to_string(w) + + "] Notify() skipped — connection already disconnected " + "(request_id=" + + std::to_string(item.request_id) + ")"); + continue; + } + + std::ostringstream notify_log; + notify_log << "[server/worker " << w << "] Calling Notify() for request_id=" << item.request_id + << " with " << response_bytes.size() << " bytes"; + Log(notify_log.str()); + + auto notify_result = item.conn->Notify( + score::cpp::span{response_bytes.data(), response_bytes.size()}); + + if (!notify_result.has_value()) + { + // ENOBUFS: max_queued_notifies was too small — response dropped, + // client will hang until its timeout expires. + LogErr("[server/worker " + std::to_string(w) + "] Notify() failed for request_id=" + + std::to_string(item.request_id) + " — check max_queued_notifies >= max_threads_per_client"); + } + else + { + Log("[server/worker " + std::to_string(w) + + "] Notify() succeeded for request_id=" + std::to_string(item.request_id)); + } + } + }); + } + Log("[server] started " + std::to_string(server_params.worker_threads) + " worker thread(s)"); + + // ------------------------------------------------------------------ + // UID admission control. + // + // The server enforces at most one active connection per UID. This: + // - prevents a single client from starving others by opening N*T + // connections and consuming the entire server receive queue; + // - binds resource consumption (queue slots, connection objects) to + // the number of authenticated client processes, not to thread count. + // + // All server callbacks for the same IServer instance are called + // sequentially on the library's internal thread (doc §Server callbacks), + // so connected_uids needs no external mutex. + // + // Rejection policy: + // EAGAIN — the UID is already connected; the library will tell the + // client to retry. Used instead of EACCES so that a client + // which reconnects after a crash is not permanently locked out + // while the previous disconnect callback has not yet fired. + // + // Hint: The idea here is not access control as we did it earlier + // but to prevent resource starvation, since we pre-allocate x buffer + // we have a limit on how many simultaneous connections we can handle + // enforcing one connection per UID is a simple way to prevent a + // single client from consuming all resources and starving others. + // ------------------------------------------------------------------ + std::unordered_set connected_uids; + + auto connect_cb = [&](score::message_passing::IServerConnection& conn) + -> score::cpp::expected { + const uid_t uid = conn.GetClientIdentity().uid; +#if ENFORCE_SINGLE_CONNECTION_PER_UID + if (connected_uids.count(uid) != 0U) + { + std::ostringstream ss; + ss << "[server] rejected connection from uid=" << uid << " (already connected) — client will retry"; + Log(ss.str()); + // EAGAIN: instructs the client library to retry the connection + // rather than transitioning to kStopped with kPermission reason. + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EAGAIN)); + } +#endif + connected_uids.insert(uid); + { + std::lock_guard live_lk(live_conn.mutex); + live_conn.alive_map[&conn] = std::make_shared(true); + } + std::ostringstream ss; + ss << "[server] accepted connection from uid=" << uid << " (" << connected_uids.size() << "/" + << server_params.max_client_processes << " slots used)"; + Log(ss.str()); + return score::message_passing::UserData{static_cast(nullptr)}; + }; + + auto disconnect_cb = [&](score::message_passing::IServerConnection& conn) { + const uid_t uid = conn.GetClientIdentity().uid; + connected_uids.erase(uid); + { + // Flip the alive token to false before the library destroys the + // ServerConnection object. Workers hold a shared_ptr copy of the + // same token and check it under live_conn->mutex before Notify(), + // so a false flag always wins the race against pointer reuse. + std::lock_guard live_lk(live_conn.mutex); + auto it = live_conn.alive_map.find(&conn); + if (it != live_conn.alive_map.end()) + { + *it->second = false; + live_conn.alive_map.erase(it); + } + } + std::ostringstream ss; + ss << "[server] client disconnected uid=" << uid << " (" << connected_uids.size() << "/" + << server_params.max_client_processes << " slots used)"; + Log(ss.str()); + }; + + // Box the captured references into a heap struct so the lambda fits + // in the 32-byte inline capacity of score::cpp::callback<>. + struct SentWithReplyCtx + { + std::mutex& queue_mutex; + std::condition_variable& queue_cv; + std::queue& work_queue; + LiveConnections& live_conn; + }; + auto swr_ctx = std::make_shared(SentWithReplyCtx{queue_mutex, queue_cv, work_queue, live_conn}); + + auto sent_with_reply_cb = + [swr_ctx](score::message_passing::IServerConnection& conn, + score::cpp::span message) -> score::cpp::expected_blank { + // Read the client-assigned request_id from the FlatBuffer. + // The client guarantees it is non-zero and unique within its process. + flatbuffers::Verifier verifier{message.data(), static_cast(message.size())}; + if (!VerifySizePrefixedControlRequestBuffer(verifier)) + { + LogErr("[server/handler] FlatBuffer verification failed — dropping request"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + const auto* req = flatbuffers::GetSizePrefixedRoot(message.data()); + const std::uint64_t request_id = (req != nullptr) ? req->request_id() : 0U; + if (request_id == 0U) + { + LogErr("[server/handler] received request with zero request_id — dropping"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + + // Fetch the alive token for this connection. The connection is guaranteed + // live at this point (connect_cb has fired, disconnect_cb has not), so the + // entry must exist in alive_map. + std::shared_ptr alive; + { + std::lock_guard live_lk(swr_ctx->live_conn.mutex); + auto it = swr_ctx->live_conn.alive_map.find(&conn); + if (it != swr_ctx->live_conn.alive_map.end()) + { + alive = it->second; + } + } + if (!alive) + { + LogErr("[server/handler] alive token missing for request_id=" + std::to_string(request_id) + + " — connection not found in alive_map (unexpected)"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(ENOENT)); + } + + // Enqueue the full work for the pool worker before acknowledging the + // request. A successful acknowledgement therefore means the work is + // admitted to the application queue. + std::vector bytes(message.begin(), message.end()); + { + std::lock_guard lk(swr_ctx->queue_mutex); + swr_ctx->work_queue.push({&conn, alive, request_id, std::move(bytes)}); + } + + // Reply immediately as ack to end the REQUEST/REPLY window. The worker + // is woken only after this call so completion cannot normally race ahead + // of a successful acknowledgement. + // This allows the next REQUEST from any thread sharing this connection + // to be processed without waiting for the actual work to complete. + // The ack payload is minimal — the client already knows its request_id. + auto ack_bytes = BuildAcknowledgeReply(request_id); + auto reply_result = conn.Reply(score::cpp::span{ack_bytes.data(), ack_bytes.size()}); + if (!reply_result.has_value()) + { + // We should still wake up the worker + // TODO: But maybe we need to invalidate the added workqueue element, + // such that is shall not be processed + swr_ctx->queue_cv.notify_one(); + + LogErr("[server/handler] Reply() (ack) failed for request_id=" + std::to_string(request_id)); + return score::cpp::make_unexpected(reply_result.error()); + } + + swr_ctx->queue_cv.notify_one(); + + Log("[server/handler] REQUEST received, ack request_id=" + std::to_string(request_id) + + " sent, request queued"); + return {}; + }; + + // Pass sent_with_reply_cb as the REQUEST/REPLY callback (4th arg). + // The 3rd (fire-and-forget) arg is left empty — we no longer use Send(). + auto start_result = server->StartListening(connect_cb, disconnect_cb, /*sent_cb=*/{}, sent_with_reply_cb); + if (!start_result.has_value()) + { + LogErr("[server] StartListening failed"); + stop_workers.store(true); + queue_cv.notify_all(); + for (auto& t : workers) + { + t.join(); + } + return 1; + } + + Log("[server] listening — waiting for all clients to finish..."); + + int overall_status = 0; + for (std::size_t i = 0U; i < child_pids.size(); ++i) + { + int wstatus = 0; + pid_t pid = waitpid(-1, &wstatus, 0); + if (pid > 0) + { + const bool ok = WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0; + std::ostringstream ss; + ss << "[server] child pid=" << pid << (ok ? " exited OK" : " FAILED"); + (ok ? Log : LogErr)(ss.str()); + if (!ok) + { + overall_status = 1; + } + } + } + + stop_workers.store(true); + queue_cv.notify_all(); + for (auto& t : workers) + { + t.join(); + } + + server->StopListening(); + Log("[server] shutdown complete."); + return overall_status; +} + +// --------------------------------------------------------------------------- +// Client — per-call pending state for notify-based demultiplexing +// --------------------------------------------------------------------------- + +struct PendingCall +{ + bool ready{false}; + bool ok{false}; + std::string result_value; + std::mutex mutex; + std::condition_variable cv; +}; + +static bool RunClient(const int client_index, const int call_count, const int thread_count) +{ + const ServiceParams service_params{ + kServiceIdentifier, + /*max_payload_bytes=*/static_cast(sizeof(IpcBuffer)), + }; + const ClientParams client_params{ + /*max_concurrent_threads=*/static_cast(thread_count), + }; + + const auto protocol_config = MakeProtocolConfig(service_params); + const auto client_config = MakeClientConfig(client_params); + + score::message_passing::ClientFactory client_factory; + + // ------------------------------------------------------------------ + // Client-assigned request IDs: pid in the upper 32 bits, per-process + // counter in the lower 32 bits. Unique within this client process and + // distinguishable across processes (different pids), so the server can + // echo them back without any server-side ID assignment. + // + // Pending-call map: each in-flight request inserts its shared PendingCall + // BEFORE calling SendWithCallback(), keyed by its pre-assigned id. + // NotifyCallback looks up by id — the entry is always present because + // the insert happens before the send, eliminating the gap that previously + // required a generation counter and two-phase wait. + // + // Lock order: always pending_map_mutex before PendingCall::mutex. + // ------------------------------------------------------------------ + const std::uint64_t pid_upper = static_cast(::getpid()) << 32U; + std::atomic call_counter{1U}; + std::mutex pending_map_mutex; + std::unordered_map> pending_map; + + // ------------------------------------------------------------------ + // Create ONE shared connection for the whole process. + // All threads share it — SendWithCallback() is safe to call concurrently + // because truly_async=true routes all sends through the library's + // background thread without blocking the caller. + // ------------------------------------------------------------------ + auto client = client_factory.Create(protocol_config, client_config); + if (!client) + { + LogErr("[client " + std::to_string(client_index) + "] Create failed"); + return false; + } + + // The state callback is stored in a score::cpp::callback<> with a fixed + // 32-byte inline capacity. Capturing four objects (two promises + two + // atomics) exceeds that limit, so we box them onto the heap and capture + // a single pointer — sizeof(void*) == 8 bytes. + struct ConnectionState + { + std::promise ready_promise; + std::promise stopped_promise; + std::atomic ready_set{false}; + std::atomic stopped_set{false}; + }; + auto conn_state = std::make_shared(); + auto ready_future = conn_state->ready_promise.get_future(); + auto stopped_future = conn_state->stopped_promise.get_future(); + + // Box the NotifyCallback captures into a heap struct so the lambda fits + // in the 32-byte inline capacity of score::cpp::callback<>. + struct NotifyCtx + { + std::mutex& pending_map_mutex; + std::unordered_map>& pending_map; + int client_index; + }; + // notify_ctx is shared across all calls; the per-call generation guard is + // stored inside each PendingCall and checked under pending_map_mutex. + auto notify_ctx = std::make_shared(NotifyCtx{pending_map_mutex, pending_map, client_index}); + + client->Start( + [conn_state, client_index](score::message_passing::IClientConnection::State state) { + if (state == score::message_passing::IClientConnection::State::kReady) + { + if (!conn_state->ready_set.exchange(true)) + { + Log("[client " + std::to_string(client_index) + "] connection ready"); + conn_state->ready_promise.set_value(); + } + } + else if (state == score::message_passing::IClientConnection::State::kStopped) + { + if (!conn_state->stopped_set.exchange(true)) + { + conn_state->stopped_promise.set_value(); + } + } + }, + // NotifyCallback — runs on the library's engine thread. + // Runs sequentially with ReplyCallback, so the PendingCall is always + // already in pending_map when a Notify arrives (no early-notify map needed). + // MUST NOT call any blocking message_passing operation (doc §Client + // Connection callbacks). Only parse the payload, look up the pending + // call by request_id, and signal the condition variable. + [notify_ctx](score::cpp::span message) { + auto start_time = std::chrono::system_clock::now(); + + // ControlResponse is not root_type in the schema — parse directly. + const auto* resp = flatbuffers::GetSizePrefixedRoot(message.data()); + if (resp == nullptr) + { + LogErr("[client " + std::to_string(notify_ctx->client_index) + "] NotifyCallback: null response root"); + return; + } + + const std::uint64_t request_id = resp->request_id(); + std::string result_value; + bool parse_ok = false; + + if (resp->operation_batch() != nullptr && resp->operation_batch()->operations() != nullptr && + resp->operation_batch()->operations()->size() > 0U) + { + const auto* op = resp->operation_batch()->operations()->Get(0U); + if (op != nullptr && op->parameter() != nullptr && op->parameter()->size() > 0U && + op->parameter_type() != nullptr && + static_cast(op->parameter_type()->Get(0U)) == OperationParameter_String) + { + const auto* str = reinterpret_cast(op->parameter()->Get(0U)); + if (str != nullptr && str->val() != nullptr) + { + result_value = str->val()->str(); + parse_ok = true; + } + } + } + + // Look up and signal the waiting thread. + // The entry was inserted before SendWithCallback() so it is always + // present when Notify arrives. A missing entry means the call already + // timed out and was erased by the calling thread — discard silently. + std::lock_guard map_lk(notify_ctx->pending_map_mutex); + auto it = notify_ctx->pending_map.find(request_id); + if (it != notify_ctx->pending_map.end()) + { + const auto pending = it->second; + { + std::lock_guard call_lk(pending->mutex); + pending->result_value = std::move(result_value); + pending->ok = parse_ok; + pending->ready = true; + } + pending->cv.notify_one(); + } + else + { + LogErr("[client " + std::to_string(notify_ctx->client_index) + + "] NotifyCallback: late notify for timed-out request_id=" + std::to_string(request_id) + + " — discarded"); + } + + auto end_time = std::chrono::system_clock::now(); + auto diff = end_time - start_time; + Log("[client " + std::to_string(notify_ctx->client_index) + + "] NotifyCallback request_id=" + std::to_string(request_id) + " took " + + std::to_string(std::chrono::duration_cast(diff).count()) + " us"); + }); + + if (ready_future.wait_for(std::chrono::seconds(120)) != std::future_status::ready) + { + LogErr("[client " + std::to_string(client_index) + "] timed out waiting for connection"); + client->Stop(); + return false; + } + + // ------------------------------------------------------------------ + // Spawn all client threads. They all share the single connection. + // ------------------------------------------------------------------ + std::atomic total_failures{0}; + std::vector threads; + threads.reserve(static_cast(thread_count)); + + for (int t = 0; t < thread_count; ++t) + { + threads.emplace_back([&, t]() { + int failures = 0; + auto times = std::vector(call_count); + + for (int c = 0; c < call_count; ++c) + { + auto start_time = std::chrono::system_clock::now(); + + // Build request + const std::string str_param = "client" + std::to_string(client_index + 1); + const std::uint64_t uint64_param = static_cast(c + 1); + const std::string expected = str_param + "_" + std::to_string(uint64_param); + + // Assign the request_id before building the FlatBuffer so it can + // be embedded in the payload and inserted into the pending_map — + // all before the send. Upper 32 bits = pid (process-unique prefix), + // lower 32 bits = per-process monotonic counter (thread-unique within + // this process). Zero is never produced (counter starts at 1). + const std::uint64_t request_id = pid_upper | call_counter.fetch_add(1U, std::memory_order_relaxed); + + // Build request FlatBuffer with the pre-assigned request_id. + // Persisted in a shared_ptr so it outlives the async send queue. + flatbuffers::FlatBufferBuilder fbb(512); + auto str_val = fbb.CreateString(str_param); + auto str_tbl = CreateString(fbb, str_val); + auto u64_tbl = CreateValueUint64(fbb, uint64_param); + std::vector param_types{OperationParameter_String, OperationParameter_ValueUint64}; + std::vector> param_values{str_tbl.Union(), u64_tbl.Union()}; + auto op_id = CreateOperationIdentifier(fbb, /*actor=*/1U, /*action=*/1U); + auto single_op = CreateSingleOperationRequest( + fbb, op_id, fbb.CreateVector(param_types), fbb.CreateVector(param_values)); + auto batch = CreateOperationRequestBatch(fbb, fbb.CreateVector({single_op})); + fbb.FinishSizePrefixed( + CreateControlRequest(fbb, request_id, /*client_id=*/0U, /*data_node_id=*/0U, batch)); + + auto request_buffer = std::make_shared>( + fbb.GetBufferPointer(), fbb.GetBufferPointer() + fbb.GetSize()); + score::cpp::span request_span{request_buffer->data(), request_buffer->size()}; + + // Insert into the pending map BEFORE the send so the entry is + // guaranteed to exist when NotifyCallback fires. No generation + // counter needed: request_ids are never reused within a process. + auto pending = std::make_shared(); + { + std::lock_guard map_lk(pending_map_mutex); + pending_map[request_id] = pending; + } + + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] -> SendWithCallback() str=\"" << str_param + << "\" uint64=" << uint64_param << " request_id=" << request_id; + Log(ss.str()); + } + + // ReplyCallback is kept minimal: it only signals the waiting thread + // on send failure. On success the ack is ignored — the client + // already knows its request_id and waits only for the Notify. + // Box captures into a heap struct to stay within the 32-byte inline + // capacity of score::cpp::callback<>. + struct ReplyCtx + { + std::shared_ptr pending; + std::shared_ptr> request_buffer; // keep buffer alive + int client_index; + int thread_index; + }; + auto reply_ctx = std::make_shared(ReplyCtx{pending, request_buffer, client_index, t}); + + auto reply_callback = + [reply_ctx]( + score::cpp::expected, score::os::Error> ack_expected) { + if (!ack_expected.has_value()) + { + std::ostringstream ss; + ss << "[client " << reply_ctx->client_index << "/thread " << reply_ctx->thread_index + << "] ReplyCallback: ack failed: " << ack_expected.error(); + LogErr(ss.str()); + std::lock_guard call_lk(reply_ctx->pending->mutex); + reply_ctx->pending->ok = false; + reply_ctx->pending->ready = true; + reply_ctx->pending->cv.notify_one(); + } + }; + + auto send_result = client->SendWithCallback(request_span, std::move(reply_callback)); + if (!send_result.has_value()) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t + << "] SendWithCallback() failed: " << send_result.error(); + LogErr(ss.str()); + std::lock_guard map_lk(pending_map_mutex); + pending_map.erase(request_id); + ++failures; + continue; + } + + // Wait for NotifyCallback to signal ready. + constexpr auto kNotifyTimeout = std::chrono::seconds(300); + bool timed_out = false; + { + std::unique_lock call_lk(pending->mutex); + timed_out = !pending->cv.wait_for(call_lk, kNotifyTimeout, [&] { + return pending->ready; + }); + } + + // Retire: erase from map. The shared state remains alive until all + // deferred ReplyCallback and NotifyCallback instances release it. + { + std::lock_guard map_lk(pending_map_mutex); + pending_map.erase(request_id); + } + + if (timed_out) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t + << "] TIMEOUT waiting for Notify request_id=" << request_id; + LogErr(ss.str()); + ++failures; + continue; + } + + if (!pending->ok || pending->result_value != expected) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] MISMATCH request_id=" << request_id + << ": expected=\"" << expected << "\" got=\"" + << (pending->ok ? pending->result_value : "") << "\""; + LogErr(ss.str()); + ++failures; + continue; + } + + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] <- OK request_id=" << request_id + << " result=\"" << pending->result_value << "\" (Phase 2: Notify received, round-trip complete)"; + Log(ss.str()); + } + + auto end_time = std::chrono::system_clock::now(); + times[c] = end_time - start_time; + + // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + auto sum = std::chrono::duration::zero(); + for (int c = 0; c < call_count; ++c) + { + sum += times[c]; + std::ostringstream oss; + oss << "[client " << client_index << "/thread " << t << "] call " << (c + 1) << "/" << call_count + << ": " << std::chrono::duration_cast(times[c]).count() << " us"; + Log(oss.str()); + } + + if (failures == 0) + { + std::ostringstream oss; + oss << "- [client " << client_index << "/thread " << t << "] completed " << call_count << " calls with " + << std::chrono::duration_cast(sum).count() << " us elapsed, average " + << std::chrono::duration_cast(sum).count() / call_count + << " us per call"; + Log(oss.str()); + + if (call_count > 1) + { + auto s = sum - times[0]; + std::ostringstream oss2; + oss2 << "SKIP FIRST [client " << client_index << "/thread " << t << "] completed " + << (call_count - 1) << " calls with " + << std::chrono::duration_cast(s).count() << " us elapsed, average " + << std::chrono::duration_cast(s).count() / (call_count - 1) + << " us per call"; + Log(oss2.str()); + } + } + + total_failures.fetch_add(failures, std::memory_order_relaxed); + }); + } + + for (auto& th : threads) + { + th.join(); + } + + client->Stop(); + if (stopped_future.wait_for(std::chrono::seconds(30)) != std::future_status::ready) + { + LogErr("[client " + std::to_string(client_index) + "] timed out waiting for connection to stop"); + } + + const int total = call_count * thread_count; + const int failures = total_failures.load(); + const int success = total - failures; + std::ostringstream ss; + ss << "[client " << client_index << "] Results: " << success << "/" << total << " calls succeeded, " << failures + << "/" << total << " calls failed (" << thread_count << " thread(s) x " << call_count << " call(s))"; + Log(ss.str()); + + return failures == 0; +} + +} // namespace score::crypto::ipc::control + +// --------------------------------------------------------------------------- +// main — fork before any IPC setup for a clean per-process state +// --------------------------------------------------------------------------- + +int main(int argc, char** argv) +{ + const std::string kClientPrefix{"--client_count="}; + const std::string kCallPrefix{"--call_count="}; + const std::string kClientThreadsPrefix{"--client_threads="}; + const std::string kServerThreadsPrefix{"--server_threads="}; + const std::string kSleepPrefix{"--sleep_milliseconds="}; + for (int i = 1; i < argc; ++i) + { + const std::string arg{argv[i]}; + try + { + if (arg.rfind(kClientPrefix, 0) == 0) + { + g_client_count = std::stoi(arg.substr(kClientPrefix.size())); + } + else if (arg.rfind(kCallPrefix, 0) == 0) + { + g_call_count = std::stoi(arg.substr(kCallPrefix.size())); + } + else if (arg.rfind(kClientThreadsPrefix, 0) == 0) + { + g_client_threads = std::stoi(arg.substr(kClientThreadsPrefix.size())); + } + else if (arg.rfind(kServerThreadsPrefix, 0) == 0) + { + g_server_threads = std::stoi(arg.substr(kServerThreadsPrefix.size())); + } + else if (arg.rfind(kSleepPrefix, 0) == 0) + { + g_sleep_milliseconds = std::stoi(arg.substr(kSleepPrefix.size())); + } + } + catch (const std::exception& ex) + { + LogErr("[main] invalid argument '" + arg + "': " + ex.what()); + return 1; + } + } + if (g_client_count < 1) + { + LogErr("[main] --client_count must be >= 1"); + return 1; + } + if (g_call_count < 1) + { + LogErr("[main] --call_count must be >= 1"); + return 1; + } + if (g_client_threads < 1) + { + LogErr("[main] --client_threads must be >= 1"); + return 1; + } + if (g_server_threads < 1) + { + LogErr("[main] --server_threads must be >= 1"); + return 1; + } + + Log("[main] client_count=" + std::to_string(g_client_count) + " call_count=" + std::to_string(g_call_count) + + " client_threads=" + std::to_string(g_client_threads) + " server_threads=" + + std::to_string(g_server_threads) + " sleep_milliseconds=" + std::to_string(g_sleep_milliseconds)); + + std::vector child_pids; + int my_client_index = -1; + + for (int i = 0; i < g_client_count; ++i) + { + pid_t pid = ::fork(); + if (pid < 0) + { + std::perror("[main] fork"); + for (pid_t cpid : child_pids) + { + ::kill(cpid, SIGTERM); + } + return 1; + } + if (pid == 0) + { + my_client_index = i; + break; + } + child_pids.push_back(pid); + } + + if (my_client_index == -1) + { + return score::crypto::ipc::control::RunServer(child_pids); + } + else + { + const bool ok = score::crypto::ipc::control::RunClient(my_client_index, g_call_count, g_client_threads); + return ok ? 0 : 1; + } +} + +// clang-format off + +// ============================================================================= +// Design comparison: per-thread connection (initial) vs. shared connection +// with Send+Notify (this version) +// ============================================================================= +// +// APPROACH A — One IClientConnection per thread, SendWaitReply +// ------------------------------------------------------------- +// Client threads each own a dedicated connection to the server. +// SendWaitReply() blocks the calling thread until the server calls Reply(). +// The server uses sent_with_reply_callback + pool threads that call Reply(). +// +// Max parallel requests from one client process: +// Exactly thread_count. Each connection carries at most one in-flight +// REQUEST at a time (per-connection REQUEST/REPLY serialization). +// Adding a thread automatically adds a connection and one more parallel +// slot — no explicit queue configuration needed. +// +// Memory / configuration (N client processes, T threads each): +// Server allocates N*T connection objects, each with its own receive +// buffer, reply slot, and notify queue. No application-level queue +// sizing is required beyond the connection count itself — the parallelism +// limit is implicit in the number of connections. +// Server max_queued_sends must cover all N*T concurrent sends. +// +// Pros: +// - Simple: no ticket tracking, no response routing, no shared state on the +// client side. The transport itself matches requests to replies by +// connection ordering. +// - Configuration is trivially correct: parallelism = thread count = connection +// count, with no additional parameters to keep in sync. +// - reply_buffer can be stack-allocated per thread. +// +// Cons: +// - One OS connection (socket fd pair) per thread. For N client processes +// each with T threads, the server holds N*T open connections. +// - The REQUEST/REPLY protocol serializes the server's sent_with_reply +// callback per connection (doc §Server callbacks): while Reply() has not +// been called, no further REQUEST from the same connection is processed. +// This is fine with one connection per thread (each thread sends one +// request at a time), but it means the connection count cannot be reduced +// without losing parallelism. +// - SendWaitReply() blocks the calling thread inside the library during the +// entire round-trip; there is no built-in timeout. +// +// APPROACH B — One IClientConnection per process, SendWithCallback + Notify (this file) +// -------------------------------------------------------------------------------------- +// One connection is shared by all threads. Threads call SendWithCallback() with a +// per-call ReplyCallback. The ReplyCallback (engine thread) parses the ack and inserts +// into the pending_map. The NotifyCallback (same engine thread) routes by request_id +// and signals the waiting thread. The caller waits on an application-level CV with a +// timeout — it is never blocked inside the library. +// +// Max parallel requests from one client process: +// Bounded by the minimum of four independently configured sizes: +// client max_async_replies (one slot per in-flight SendWithCallback) +// client max_queued_sends (one slot per concurrent enqueue) +// server max_queued_sends (server-side receive queue, shared across clients) +// server max_queued_notifies (per-connection notify queue) +// All must be set to >= thread_count. If any one is undersized, +// SendWithCallback() returns ENOBUFS or Notify() is dropped — with no +// automatic backpressure to the calling thread. +// +// Memory / configuration (N client processes, T threads each): +// Server allocates N connection objects (not N*T). Total queue capacity: +// +// server max_queued_sends — shared across ALL connections (QNX). +// At most N simultaneous REQUESTs (one per +// connection), so N slots suffice. +// +// server max_queued_notifies — per connection (QNX). +// Must hold T responses for that client's +// threads. Independent of N. +// +// client max_async_replies — per connection, per client process. +// Must hold T concurrent in-flight callbacks. +// +// client max_queued_sends — per connection, per client process. +// Must hold T concurrent enqueues. +// +// All four client sizes must be kept in sync with T; the coupling is +// enforced by convention only, not by the API. +// +// Pros: +// - O(clients) connections instead of O(clients * threads). +// - SendWithCallback() is non-blocking — the calling thread is never held +// inside the IPC layer regardless of server behavior. +// - Application-level wait_for() timeout bounds how long the thread can wait, +// which is a prerequisite for use in safety-relevant contexts. +// - ReplyCallback and NotifyCallback run sequentially on the engine thread, +// so the pending_map needs no early-notify buffer: the entry is always +// present by the time Notify() arrives. +// - NotifyCallback runs on the library's own thread — no dedicated receive +// thread is needed on the client side. +// +// Cons: +// - Requires ticket tracking (request_id map + mutex) on the client side. +// - ReplyCallback and NotifyCallback must not call any blocking message_passing +// operation (doc §Client Connection callbacks); only signal/mutex work is allowed. +// - Four queue sizes must all be kept >= thread_count. Getting any one wrong +// causes ENOBUFS on send or silent response drops rather than a clean error. +// - The response FlatBuffer (ControlResponse) must carry the request_id +// for response routing; a simpler protocol without a correlation id could +// not use this model. +// +// ANALOGY TO poc_async (mw::com): +// poc_async faced the same root constraint at the higher abstraction level: +// mw::com methods also serialize the handler per skeleton instance. The +// workaround there was identical in spirit — use a short-lived method call +// (enqueue only) + a separate event channel (broadcast) for the response. +// The key difference is that mw::com events broadcast to ALL subscribers, +// requiring one skeleton per client to prevent cross-client leakage and +// making the ticket essential for routing. message_passing Notify() is +// point-to-point per connection, so one connection per process suffices and +// the ticket is only needed for intra-process thread response routing. +// +// clang-format on diff --git a/third_party/grpc/grpc_symbol_reference.cpp b/third_party/grpc/grpc_symbol_reference.cpp index 83568514a..bcced18c3 100644 --- a/third_party/grpc/grpc_symbol_reference.cpp +++ b/third_party/grpc/grpc_symbol_reference.cpp @@ -40,7 +40,8 @@ namespace score::crypto::ipc::internal void* ForceServerSymbols() { // These will never be called, but create references the linker needs - static bool initialized = false; + // Use volatile to prevent compiler optimizations from removing the code + static volatile bool initialized = false; if (initialized) { // Create references to server APIs @@ -53,7 +54,7 @@ void* ForceServerSymbols() // Force inclusion of client-side symbols void* ForceClientSymbols() { - static bool initialized = false; + static volatile bool initialized = false; if (initialized) { // Create references to client APIs From 6b55f51ade62227f0065f45919bbe2f3002ec209 Mon Sep 17 00:00:00 2001 From: Lobsinger Sebastian Date: Mon, 17 Aug 2026 12:16:26 +0000 Subject: [PATCH 2/4] Fix doc build issues --- score/crypto/src/ipc/docs/index.rst | 40 +++++++++++---------- score/crypto/src/ipc/docs/ipc_comparison.md | 2 +- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/score/crypto/src/ipc/docs/index.rst b/score/crypto/src/ipc/docs/index.rst index b5fc4e1e4..881092888 100644 --- a/score/crypto/src/ipc/docs/index.rst +++ b/score/crypto/src/ipc/docs/index.rst @@ -21,7 +21,7 @@ Selection of LoLa Message Passing as IPC .. dec_rec:: Selection of LoLa Message Passing as IPC :id: dec_rec__crypto__lola_message_passing_ipc :version: 1 - :status: draft + :status: proposed :context: doc__crypto_architecture :decision: Use LoLa Message Passing, implemented with the score::message_passing API, as the crypto daemon control-plane IPC transport. @@ -190,9 +190,9 @@ Two-Phase Request/Reply Acknowledgement and Notify Completion ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. dec_rec:: Two-Phase Request/Reply Acknowledgement and Notify Completion - :id: dec_rec__crypto__two_phase_request_reply_notify + :id: dec_rec__crypto__two_phase_ipc_interaction :version: 1 - :status: draft + :status: proposed :context: doc__crypto_architecture :decision: Use SendWithCallback for a non-blocking request acknowledgement and Notify for the independently produced operation response. @@ -223,20 +223,18 @@ Decision The protocol is defined as follows: -1. The client assigns a unique non-zero ``request_id``, inserts a pending-call - record, and calls ``SendWithCallback`` with the FlatBuffer ``ControlRequest``. -2. The server validates the request and copies the complete request into the - application work queue. -3. The server calls ``Reply`` with a minimal acknowledgement carrying the - ``request_id``. After the acknowledgement attempt, it wakes a worker. A - successful acknowledgement means the request was admitted to the work queue. -4. A server worker performs the operation and calls ``Notify`` on the same - ``IServerConnection`` with a ``ControlResponse`` carrying the original - ``request_id`` and the result payload. -5. The client ``NotifyCallback`` routes the response by ``request_id`` and - signals the waiting application thread. The application waits with a - bounded timeout and retires the pending-call record after completion or - timeout. +1. The client assigns a unique non-zero ``request_id``, inserts a pending-call record, and calls + ``SendWithCallback`` with the FlatBuffer ``ControlRequest``. +2. The server validates the request and copies the complete request into the application work + queue. +3. The server calls ``Reply`` with a minimal acknowledgement carrying the ``request_id``. After + the acknowledgement attempt, it wakes a worker. A successful acknowledgement means the request + was admitted to the work queue. +4. A server worker performs the operation and calls ``Notify`` on the same ``IServerConnection`` + with a ``ControlResponse`` carrying the original ``request_id`` and the result payload. +5. The client ``NotifyCallback`` routes the response by ``request_id`` and signals the waiting + application thread. The application waits with a bounded timeout and retires the pending-call + record after completion or timeout. The acknowledgement is an acceptance signal, not the operation result. The transport does not automatically retry requests. An accepted request is @@ -298,7 +296,7 @@ connection, the server callback also serializes all requests until work is complete. Long-Running SendWithCallback Reply -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The server could defer ``Reply`` until the worker has finished and return the final response through ``ReplyCallback``. This preserves a single response @@ -328,3 +326,9 @@ timeout, and preserves one connection per client process. The request ID and pending-call lifecycle are deliberate complexity: they are required to obtain bounded, concurrent operation completion from a shared connection without using broadcast communication or shared writable memory. + +Reference +~~~~~~~~~ + +.. include:: ipc_comparison.md + :parser: myst_parser.sphinx_ diff --git a/score/crypto/src/ipc/docs/ipc_comparison.md b/score/crypto/src/ipc/docs/ipc_comparison.md index 2f8b45106..bb37ff175 100644 --- a/score/crypto/src/ipc/docs/ipc_comparison.md +++ b/score/crypto/src/ipc/docs/ipc_comparison.md @@ -132,4 +132,4 @@ configuration model. **Production implementation follow-up:** The formal implementation of ``IConnection`` and ``IControlServer`` interfaces -replacing the gRPC adapter is still pending will be taken up. See the [message-passing POC and QNX usage overview](../../../../tests/ipc_poc/message_passing_overview.md) for more detailed follow up tasks. +replacing the gRPC adapter is still pending will be taken up. From a34a4d1bf37d25d92e87dce8e8b8e8f439116919 Mon Sep 17 00:00:00 2001 From: Lobsinger Sebastian Date: Thu, 20 Aug 2026 11:35:54 +0200 Subject: [PATCH 3/4] Add new POCs and update decision --- MODULE.bazel.lock | 1 + score/crypto/src/ipc/docs/index.rst | 156 ++- score/crypto/src/ipc/docs/ipc_comparison.md | 112 +- score/tests/ipc_poc/BUILD | 156 ++- score/tests/ipc_poc/measure_client_scaling.sh | 314 +++++ .../tests/ipc_poc/message_passing_overview.md | 11 +- score/tests/ipc_poc/mw_com_methods.drawio.svg | 4 + score/tests/ipc_poc/poc_async.cpp | 357 ++--- score/tests/ipc_poc/poc_control.fbs | 4 + score/tests/ipc_poc/poc_engine.cpp | 680 +++++++++ score/tests/ipc_poc/poc_engine_sync.cpp | 606 ++++++++ score/tests/ipc_poc/poc_grpc.cpp | 530 +++---- score/tests/ipc_poc/poc_helper.hpp | 466 +++++++ score/tests/ipc_poc/poc_low_level.cpp | 351 ++--- .../tests/ipc_poc/poc_low_level_no_reply.cpp | 1236 +++++++++++++++++ ...oc_low_level_no_reply_notify_in_thread.cpp | 1209 ++++++++++++++++ .../tests/ipc_poc/poc_qnx_message_passing.cpp | 434 ++++++ score/tests/ipc_poc/poc_thread_pool_queue.cpp | 213 +++ score/tests/ipc_poc/poc_unix_socket.cpp | 459 ++++++ score/tests/utility/BUILD | 13 + .../utility/process_resource_measurement.cpp | 311 +++++ .../utility/process_resource_measurement.hpp | 49 + score/tests/utility/runtime_measurement.hpp | 281 ++++ 23 files changed, 7003 insertions(+), 950 deletions(-) create mode 100755 score/tests/ipc_poc/measure_client_scaling.sh create mode 100644 score/tests/ipc_poc/mw_com_methods.drawio.svg create mode 100644 score/tests/ipc_poc/poc_engine.cpp create mode 100644 score/tests/ipc_poc/poc_engine_sync.cpp create mode 100644 score/tests/ipc_poc/poc_helper.hpp create mode 100644 score/tests/ipc_poc/poc_low_level_no_reply.cpp create mode 100644 score/tests/ipc_poc/poc_low_level_no_reply_notify_in_thread.cpp create mode 100644 score/tests/ipc_poc/poc_qnx_message_passing.cpp create mode 100644 score/tests/ipc_poc/poc_thread_pool_queue.cpp create mode 100644 score/tests/ipc_poc/poc_unix_socket.cpp create mode 100644 score/tests/utility/process_resource_measurement.cpp create mode 100644 score/tests/utility/process_resource_measurement.hpp create mode 100644 score/tests/utility/runtime_measurement.hpp diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7906f379c..c6a28ca73 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -969,6 +969,7 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_toolchains_rust/0.10.0/source.json": "8bda773be264da16d2a82a03ebb737421dd4a35855f1e9a5d03d9722d84c1df5", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/1.0.2/MODULE.bazel": "e70f396375b9d612b4f41ebceff7f18f68ab423b14625c138a354cc01bc62a10", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/1.1.2/MODULE.bazel": "56d08309931cfad67c2b6691207bb5f761a3946830d620c630d2436630e6b499", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/1.2.0/MODULE.bazel": "982db97a4f8440356ea935f103204f644fb2b84d8188df63533c7312c82afc37", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/1.3.1/MODULE.bazel": "4ce18de0fdfb447ba750b98febb24e12471862cbbd5d08f5f683f27886767367", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/2.0.2/MODULE.bazel": "edf7730454fc1c7951d903895524d57efbe059edbf2572cf9bdc0e7a20d6cc55", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/2.2.0/MODULE.bazel": "178ba4862246b6ba2bbcd96b7e9e728299b19fb94bcf23d315dc2299aabf7178", diff --git a/score/crypto/src/ipc/docs/index.rst b/score/crypto/src/ipc/docs/index.rst index 881092888..ca35fd18c 100644 --- a/score/crypto/src/ipc/docs/index.rst +++ b/score/crypto/src/ipc/docs/index.rst @@ -40,6 +40,9 @@ includes deterministic and bounded execution and resource behavior, controlled heap/allocation use on ASIL-relevant paths, suitable isolation for the deployment, and appropriate quality artifacts such as requirements and design traceability, analysis, verification, and compliance or qualification evidence. +The safety-relevant deployment target for this decision is QNX. Linux support +is retained for development, testing, and non-safety or QM deployments; Linux +backend limitations are not acceptance blockers for the QNX safety target. It must also provide bounded client-side waiting, allow concurrent calls from multiple client processes and threads, expose the identity of the peer, avoid imposing the daemon's worker-thread model, and keep the application protocol @@ -131,13 +134,15 @@ LoLa Message Passing The message-passing abstraction exposes platform-independent client and server factories while keeping the operating-system transport behind the IPC layer. -Using ``SendWithCallback`` with ``Reply`` and ``Notify`` separates request -acceptance from operation completion. A shared connection per client process -supports concurrent client threads through explicit request identifiers and a -server-side worker pool, without scaling connections as ``N * T``. The design -requires explicit bounds for client processes, in-flight requests, payload size, -and transport queues, but preserves process isolation and the daemon's freedom -to choose its worker model. +Using ``Send`` with ``Notify`` separates request submission from operation +completion without the per-connection REQUEST/REPLY serialization window. A +shared connection per client process supports concurrent client threads through +explicit request identifiers and a server-side worker pool, without scaling +connections as ``N * T``. Server-side validation or work-queue submission +failure is reported through a negative ``Notify`` result. The design requires +explicit bounds for client processes, in-flight requests, payload size, and +transport queues, but preserves process isolation and the daemon's freedom to +choose its worker model. Comparison Summary ****************** @@ -162,12 +167,12 @@ and inter-VM support. delivery requires additional per-client instances and response routing. * **LoLa Message Passing** provides platform-independent client and server factories, process-isolating transport, kernel-provided peer credentials, - and a server-controlled worker model can be added easily. ``SendWithCallback`` - followed by ``Reply`` and ``Notify`` supports concurrent calls over one - connection per client process, with application-level request identifiers - for response routing. It requires explicit resource bounds and a typed - timeout and connection-loss contract. It does not provide inter-VM - communication. + and a server-controlled worker model. ``Send`` followed by ``Notify`` supports + concurrent calls over one connection per client process, with application- + level request identifiers for response routing. A negative ``Notify`` result + reports server-side validation or work-queue submission failure. It requires + explicit resource bounds and a typed timeout and connection-loss contract. It + does not provide inter-VM communication. LoLa Message Passing is selected because it best satisfies the ASIL-B, bounded-client-waiting, peer-identity, concurrency, and server-threading goals @@ -186,76 +191,76 @@ and a typed error contract. The lack of inter-VM communication is accepted, the control-plane IPC abstraction can potentially be used to offer a separate inter-VM transport in the future. -Two-Phase Request/Reply Acknowledgement and Notify Completion -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Send and Notify Completion +~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. dec_rec:: Two-Phase Request/Reply Acknowledgement and Notify Completion - :id: dec_rec__crypto__two_phase_ipc_interaction +.. dec_rec:: Send and Notify Completion + :id: dec_rec__crypto__send_notify_ipc_interaction :version: 1 :status: proposed :context: doc__crypto_architecture - :decision: Use SendWithCallback for a non-blocking request acknowledgement and Notify for the independently produced operation response. + :decision: Use Send for non-blocking request submission and Notify for the operation result or a negative server-side submission result. .. :affects: comp__crypto -The selected IPC protocol uses two phases: an immediate ``Reply`` acknowledgement -for the accepted request, followed by a ``Notify`` message containing the -operation result. +The selected IPC protocol uses a non-blocking ``Send`` request followed by a +point-to-point ``Notify`` message containing either the operation result or a +negative result when server-side validation or work-queue submission fails. Context ******* -Using LoLa Message Passing, the REQUEST/REPLY protocol serializes requests on one -connection: the server does not process the next request on that connection until -the current request has received a ``Reply``. A client process may nevertheless -have many threads issuing independent operations, and the server must be able to -process those operations concurrently. +Using LoLa Message Passing, the selected ``Send`` path avoids the per-connection +REQUEST/REPLY serialization window. A client process may have many threads +issuing independent operations, and the server must be able to process those +operations concurrently. Using a single long-running request callback would keep the shared connection -occupied for the duration of the cryptographic operation. Using -``SendWaitReply`` would additionally block the client thread inside the IPC -library and does not provide the application with a timeout escape if the -server stalls. The selected communication model therefore separates request -acceptance from operation completion. +occupied for the duration of the cryptographic operation. The selected +communication model returns from the ``Send`` callback after validation and +work-queue submission, then performs the operation in a worker and sends the +terminal result with ``Notify``. The client uses an application-level timeout; +it does not wait indefinitely inside the IPC library. Decision ******** The protocol is defined as follows: -1. The client assigns a unique non-zero ``request_id``, inserts a pending-call record, and calls - ``SendWithCallback`` with the FlatBuffer ``ControlRequest``. -2. The server validates the request and copies the complete request into the application work - queue. -3. The server calls ``Reply`` with a minimal acknowledgement carrying the ``request_id``. After - the acknowledgement attempt, it wakes a worker. A successful acknowledgement means the request - was admitted to the work queue. -4. A server worker performs the operation and calls ``Notify`` on the same ``IServerConnection`` - with a ``ControlResponse`` carrying the original ``request_id`` and the result payload. -5. The client ``NotifyCallback`` routes the response by ``request_id`` and signals the waiting - application thread. The application waits with a bounded timeout and retires the pending-call - record after completion or timeout. - -The acknowledgement is an acceptance signal, not the operation result. The -transport does not automatically retry requests. An accepted request is -attempted once; if the operation result is not observed because of a timeout or -delivery failure, the outcome is unknown to the caller. The application may -explicitly retry when the operation semantics allow it. ``ReplyCallback`` is -used to observe send or acknowledgement failure; the client does not complete -the operation merely because the acknowledgement arrived. +1. The client assigns a unique non-zero ``request_id``, inserts a pending-call + record, and calls ``Send`` with the FlatBuffer ``ControlRequest``. +2. The server validates the request and copies the complete request into the + application work queue. If validation or queue submission fails, the server + sends a negative ``Notify`` result carrying the request ID when it can be + recovered from the request. +3. A server worker performs the operation and calls ``Notify`` on the same + ``IServerConnection`` with a ``ControlResponse`` carrying the original + ``request_id`` and the result payload. +4. The client ``NotifyCallback`` routes the response by ``request_id`` and + signals the waiting application thread. The application waits with a + bounded timeout and retires the pending-call record after completion, error, + or timeout. + +The ``Send`` call confirms only that the request was accepted by the client-side +send path; it is not an immediate admission acknowledgement. The transport +does not automatically retry requests. If a result or negative ``Notify`` is +not observed because of a timeout or delivery failure, the outcome is unknown +to the caller. The application may explicitly retry when the operation +semantics allow it. A malformed request for which the server cannot recover a +request ID cannot be correlated to a pending call and must be handled through +the connection-error or timeout path. Consequences ************ **Positive:** -* The server releases the per-connection REQUEST/REPLY serialization window - before doing the potentially slow operation. Subsequent requests can be - accepted while earlier requests execute in the worker pool. -* ``SendWithCallback`` returns without holding the caller inside the IPC - operation. A production wrapper can use a mandatory application-level - ``wait_for`` to provide a bounded wait for the final response and report a - typed timeout. +* ``Send`` avoids the per-connection REQUEST/REPLY serialization window before + the potentially slow operation. Subsequent requests can be accepted while + earlier requests execute in the worker pool. +* ``Send`` returns without holding the caller inside the IPC operation. A + production wrapper can use a mandatory application-level ``wait_for`` to + provide a bounded wait for the final response and report a typed timeout. * One connection can be shared by all threads in a client process. The application-level ``request_id`` protocol provides deterministic response demultiplexing without requiring one connection per thread. @@ -270,17 +275,21 @@ Consequences * The protocol and implementation are more complex than a single synchronous call. Pending-call state, request identifiers, response parsing, timeout cleanup, and late-notification handling are required. -* The acknowledgement does not prove that the operation completed; callers - must handle both acknowledgement failure and final-notification timeout. -* Queue capacities must cover the configured concurrency. Client async-reply - and send queues and the server notify queue must be sized consistently. - An undersized queue can reject a send or drop a notification. +* ``Send`` does not provide an immediate admission acknowledgement; callers + must handle negative ``Notify`` results and final-notification timeouts. +* Queue capacities must cover the configured concurrency. The client send queue and server + notify queue must still be sized consistently. An undersized queue can reject + a send or drop a notification. * A worker may finish after the client has timed out. The server must detect a disconnected connection or safely skip the notification, and the client must discard late notifications for retired request identifiers. -* ``Notify`` can still be transport-dependent in its blocking behaviour. - Bounded notification semantics require support from the transport or an - additional connection lease abstraction. +* ``Notify`` has backend-dependent blocking behaviour. The QNX backend used by + the safety target provides bounded preallocated notification capacity and + reports exhaustion as ``ENOBUFS``; queue sizing and the resulting failure + handling remain part of the QNX production configuration. The Linux backend + may block in the socket layer, which is an accepted limitation for + development, testing, and non-safety or QM deployments and must not be used + as evidence for a Linux safety deployment. Alternatives Considered *********************** @@ -318,14 +327,13 @@ point-to-point message on the existing client connection. Justification for the Decision ****************************** -Immediate acknowledgement is the smallest operation that satisfies the -low-level protocol's serialization rule while allowing the server to dispatch -work independently. The separate ``Notify`` completion keeps the client -thread out of the IPC library's blocking path, enables an explicit application -timeout, and preserves one connection per client process. The request ID and -pending-call lifecycle are deliberate complexity: they are required to obtain -bounded, concurrent operation completion from a shared connection without -using broadcast communication or shared writable memory. +``Send`` plus ``Notify`` is selected because it avoids the request/reply +serialization window and the client-side acknowledgement queue while allowing +the server to dispatch work independently. The ``Notify`` completion keeps the +client thread out of the IPC library's blocking path, enables an explicit +application timeout, and preserves one connection per client process. A +negative ``Notify`` communicates server-side submission failure when the +request ID is available. Reference ~~~~~~~~~ diff --git a/score/crypto/src/ipc/docs/ipc_comparison.md b/score/crypto/src/ipc/docs/ipc_comparison.md index bb37ff175..ed395234a 100644 --- a/score/crypto/src/ipc/docs/ipc_comparison.md +++ b/score/crypto/src/ipc/docs/ipc_comparison.md @@ -35,7 +35,7 @@ The score-crypto daemon requires an IPC transport to serve cryptographic operati | R1 | Suitability for ASIL-B use | The library must be designed and documented to support deterministic and bounded execution and resource behavior, controlled heap/allocation use on ASIL-relevant paths, suitable isolation for the deployment, and appropriate quality artifacts such as requirements/design traceability, analysis, verification, and compliance or qualification evidence. | | R2 | Bounded timeout on every call | WCET must be provable. A stalled QM server must never indefinitely block an ASIL-B client thread. | | R3 | Resilience to server crash / connection loss | If the server process dies or the connection is lost, the client must receive a typed error. The client must not block indefinitely or silently operate on a dead connection. | -| R4 | Authentic peer identification | Server must be able to identify the calling process via an OS-enforced mechanism, not a client-supplied value. | +| R4 | Authentic peer identification | Server must be able to identify the calling identity via an OS-enforced mechanism, not a client-supplied value. | | R5 | Minimal or zero configuration | The IPC usage should require minimal configuration and be fully configurable programmatically. | | R6 | Flexible server-side threading model | The IPC transport must not impose a threading model. The server must be free to choose single-thread, fixed pool, or per-request threads. | | R7 | Concurrent calls from one process | Multiple threads in one client process must be able to issue independent calls simultaneously. | @@ -60,7 +60,7 @@ S-CORE `mw::com` LoLa binding, synchronous Method call. Single `Execute(IpcBuffe S-CORE `mw::com` LoLa binding, two-phase protocol: Phase 1 — short blocking Method call returns a ticket; Phase 2 — result delivered via a broadcast Event. Evaluated in the asynchronous LoLa prototype. -### Option D — LoLa Message Passing Abstraction (SendWithCallback + Notify) +### Option D — LoLa Message Passing Abstraction (SendWithCallback + Reply + Notify) The platform-independent message-passing abstraction uses the S-CORE `score::message_passing` API with `SendWithCallback` / `Reply` / `Notify` @@ -68,11 +68,32 @@ primitives. The transport backend is OS-specific and provided by the framework: Unix domain socket on Linux, QNX message passing on QNX. The application code is OS-agnostic. Evaluated in the low-level message-passing prototype. +### Option E — LoLa Message Passing Abstraction (Send + Notify, no Reply) + +The same platform-independent message-passing abstraction, using `Send` for +non-blocking request submission and `Notify` for the operation result. The +server does not send an immediate `Reply`; it copies each request into its +application work queue and a worker sends the terminal response. The client +uses the request ID in the payload to route the point-to-point notification. +Evaluated in `poc_low_level_no_reply.cpp`. + +### Option F — Message Passing Engine (bootstrap + per-thread Reply) + +The `score::message_passing` Engine model uses a bootstrap endpoint to create +one on-demand service endpoint and server Engine session per client thread. +Each client thread then owns one connection and uses `SendWithCallback`; the +server Engine performs the complete operation in its request callback and +returns the result with `Reply()`. There is no application-owned work queue or +worker pool. Evaluated in `poc_engine.cpp`. + --- ## 4. Decision Matrix -Options C and D use a two-phase protocol (enqueue + async response); R2 applies to both phases independently. For Options A and B, which are single-phase, R2 describes a single blocking call. +Options C, D, and E use an enqueue + asynchronous response protocol. Option F +uses a bootstrap handshake followed by a blocking server callback; R2 applies +to the client submission/wait and the server callback separately. For Options +A and B, which are single-phase, R2 describes a single blocking call. The matrix describes target library and architecture capabilities, not completed production guarantees. ``✅`` means the option appears capable of meeting the @@ -82,21 +103,21 @@ architectural mismatch; and ``❓`` means it was not evaluated. The POCs provide feasibility evidence for selected points, while the production implementation must validate the remaining requirements. -| Requirement | Option A — gRPC | Option B — LoLa Full SOA (sync) | Option C — LoLa Full SOA (async) | Option D — LoLa Message Passing Abstraction | -|---|---|---|---|---| -| **R1** ASIL-B suitability | ❌ general-purpose gRPC uses framework-managed threads and dynamic runtime resources; the library does not provide an ASIL-oriented deterministic resource profile or safety qualification artifacts | ✅ the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use and ASIL-B deployment | ✅ the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use and ASIL-B deployment, including its method/event model | ✅ the message-passing library design supports fixed resource bounds, preallocation, and pool/monotonic allocation; the communication module documents safety-oriented quality tooling and ASIL-B qualification | -| **R2** Bounded timeout | ⚠️ current adapter uses a blocking call without a configured deadline; deadline-based or async gRPC was not analyzed | ❌ single-phase blocking call — blocks indefinitely if server stalls | ⚠️ Phase 2 (response event): `wait_for()` indicated in POC; Phase 1 (enqueue): ❌ `DoCall()` holds caller's thread inside LoLa framework with no timeout API — requires framework change | ⚠️ `SendWithCallback()` is non-blocking by design and can support an application-level timeout; mandatory timeout API and typed error handling remain production work, and server-side `Notify()` bounds are backend-dependent | -| **R3** Server crash / connection loss | ❓ not analysed in POC | ❓ not analysed in POC | ❓ not analysed in POC | ⚠️ transport disconnect was observed in the POC; typed propagation to all pending calls and cleanup policy remain production work | -| **R4** Authentic peer identification | ❌ no authentication mechanism | ⚠️ config specifies which UIDs may use a service instance — non-listed UIDs are rejected at runtime by the framework; no API to actively query the connected UID | ⚠️ config specifies which UIDs may use a service instance — non-listed UIDs are rejected at runtime by the framework; no API to actively query the connected UID | ✅ `score::message_passing` exposes an API to retrieve kernel-provided peer credentials from an active connection | -| **R5** Minimal config | ✅ socket path only | ❌ LoLa requires service-oriented configuration for the service, instance, method/event identifiers, application identity, and safety-relevant properties; configuration must be provided consistently to the participating applications and composed with any other `mw::com` services used in the same process | ❌ LoLa requires service-oriented configuration for the service, instance, method/event identifiers, application identity, and safety-relevant properties; configuration must be provided consistently to the participating applications and composed with any other `mw::com` services used in the same process | ⚠️ service identifier only, but worst-case buffer sizes must be configured explicitly: max client processes (N), max concurrent threads per client (T), and max payload size — all must be set to system-wide upper bounds at compile/startup time | -| **R6** Flexible server threading | ⚠️ gRPC owns an internal thread pool; handler is called on gRPC threads. Flexible configuration of that pool may not be sufficient for the required threading model, while a complete replacement of it may be feasible but was not evaluated | ⚠️ skeleton thread processes one call at a time and blocks until the handler returns; handler may dispatch to a pool internally but must block the skeleton on the result — skeleton is always occupied during work | ✅ flexible threading model can be built on top — POC demonstrates a server-side thread pool receiving work via the Phase 1 callback and completing it independently | ✅ `Reply()` callable from any thread at any time — server fully controls threading | -| **R7** Concurrent calls / same process | ✅ channel is thread-safe, no external mechanism needed | ⚠️ concurrent calls depend on the LoLa proxy and method queue configuration; the service abstraction does not make the required parallelism transparent to the application | ⚠️ concurrent calls depend on the LoLa proxy, method queue, and event subscription configuration; the service abstraction does not make the required parallelism transparent to the application | ⚠️ the library supports concurrent use, while request-ID assignment, pending-call tracking, and response multiplexing must be implemented above the library; sufficient resource configuration is also needed | -| **R8** Multiple client processes | ✅ works out of the box | ⚠️ supported through the service-oriented deployment model, but requires consistent application/service configuration across participating processes — see R5 | ⚠️ supported through the service-oriented deployment model, but requires consistent application/service configuration across participating processes — see R5 | ✅ the library supports one server communicating with multiple client processes through independent client/server sessions; sufficient connection and queue resource configuration is needed | -| **R9** One-to-one | ✅ each call gets exactly one response | ✅ each call gets exactly one response | ⚠️ response is a broadcast event — requires one skeleton instance per client to prevent cross-client response leakage | ✅ each call gets exactly one response | -| **R10** Latency | ❌ indicative POC measurement ~430 µs (full request + response) | ❓ not measured | ⚠️ indicative POC measurement ~130 µs (lowest measured value, but still significant IPC overhead) | ⚠️ indicative POC measurement ~160 µs (second-lowest measured value, but still significant IPC overhead) | -| **R11** Inter-VM | ⚠️ gRPC supports network channels, but the current adapter hardcodes Unix-domain endpoints; an endpoint/configuration change and validation of the inter-VM transport and peer-authentication model are required | ⚠️ current LoLa binding is SHM-only (single-kernel); the service-oriented architecture could in principle support a network binding without changing the service API, but no such binding exists today | ⚠️ current LoLa binding is SHM-only (single-kernel); the service-oriented architecture could in principle support a network binding without changing the service API, but no such binding exists today | ❌ current `score::message_passing` backends are local Unix-domain socket and QNX message passing; the library provides no inter-VM transport. A new framework backend would be required. | - -**Note:** R1 and R2 are hard safety blockers — any ❌ on these disqualifies an option for safety use regardless of performance on other requirements. Options A, B, and C all carry at least one ❌ on R1 or R2. +| Requirement | Option A — gRPC | Option B — LoLa Full SOA (sync) | Option C — LoLa Full SOA (async) | Option D — LoLa Message Passing (ack + notify) | Option E — LoLa Message Passing (send + notify) | Option F — Message Passing Engine (per-thread Reply) | +|---|---|---|---|---|---|---| +| **R1** ASIL-B suitability | ❌ general-purpose gRPC uses framework-managed threads and dynamic runtime resources; the library does not provide an ASIL-oriented deterministic resource profile or safety qualification artifacts | ✅ the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use and ASIL-B deployment | ✅ the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use and ASIL-B deployment, including its method/event model | ✅ the message-passing library design supports fixed resource bounds, preallocation, and pool/monotonic allocation; the communication module documents safety-oriented quality tooling and ASIL-B qualification | ✅ the same message-passing library capabilities apply; the POC uses bounded queues and an application-owned worker pool | ⚠️ the same library provides safety-oriented capabilities, but the POC dynamically creates one endpoint and Engine session per client thread; bounded startup/resource evidence for production use is still required | +| **R2** Bounded timeout | ⚠️ current adapter uses a blocking call without a configured deadline; deadline-based or async gRPC was not analyzed | ❌ single-phase blocking call — blocks indefinitely if server stalls | Phase 2 (response event) has a `wait_for()` in the POC, but Phase 1 (enqueue) `DoCall()` holds the caller's thread inside LoLa framework with no timeout API — because R2 applies to every phase, the option fails overall and requires a framework change | ⚠️ `SendWithCallback()` is non-blocking by design and can support an application-level timeout; mandatory timeout API and typed error handling remain production work. For QNX, configured Notify capacity is bounded; Linux socket blocking is outside the safety scope | ⚠️ `Send()` is non-blocking when the send queue and `truly_async` mode are configured; the client can bound its wait for `Notify()`. For QNX, configured Notify capacity is bounded; Linux socket blocking is outside the safety scope | ⚠️ client waits use an application timeout, but the server Engine callback performs the operation synchronously and has no operation deadline or cancellation path; a stalled callback can occupy the session indefinitely | +| **R3** Server crash / connection loss | ❓ not analysed in POC | ❓ not analysed in POC | ❓ not analysed in POC | ⚠️ transport disconnect was observed in the POC; typed propagation to all pending calls and cleanup policy remain production work | ⚠️ the POC wakes pending calls when the client connection reaches stopped, but typed error propagation, late notifications, and all delivery-failure paths remain production work | ⚠️ connection state is observed, but the current response wait is not directly completed by a stopped callback; crash/error propagation and bootstrap/session cleanup remain production work | +| **R4** Authentic peer identification | ❌ no authentication mechanism | ⚠️ config specifies which UIDs may use a service instance — non-listed UIDs are rejected at runtime by the framework; no API to actively query the connected UID | ⚠️ config specifies which UIDs may use a service instance — non-listed UIDs are rejected at runtime by the framework; no API to actively query the connected UID | ✅ `score::message_passing` exposes an API to retrieve kernel-provided peer credentials from an active connection | ✅ same as Option D; the transport exposes kernel-provided peer credentials from an active connection | ✅ the server reads kernel-provided UID credentials in bootstrap and session connect callbacks | +| **R5** Minimal config | ✅ socket path only | ❌ LoLa requires service-oriented configuration for the service, instance, method/event identifiers, application identity, and safety-relevant properties; configuration must be provided consistently to the participating applications and composed with any other `mw::com` services used in the same process | ❌ LoLa requires service-oriented configuration for the service, instance, method/event identifiers, application identity, and safety-relevant properties; configuration must be provided consistently to the participating applications and composed with any other `mw::com` services used in the same process | ⚠️ service identifier only, but worst-case buffer sizes must be configured explicitly: max client processes (N), max concurrent threads per client (T), and max payload size — all must be set to system-wide upper bounds at compile/startup time | ⚠️ service identifier only, but N, T, M, send-queue, and notify-queue bounds still must be configured explicitly; the no-`Reply` model removes the async-reply queue from the client-side sizing | ❌ requires a bootstrap endpoint, dynamically generated per-thread service identifiers, one session endpoint per thread, and separate connection/Engine lifecycle management | +| **R6** Flexible server threading | ⚠️ gRPC owns an internal thread pool; handler is called on gRPC threads. Flexible configuration of that pool may not be sufficient for the required threading model, while a complete replacement of it may be feasible but was not evaluated | ⚠️ skeleton thread processes one call at a time and blocks until the handler returns; handler may dispatch to a pool internally but must block the skeleton on the result — skeleton is always occupied during work | ✅ flexible threading model can be built on top — POC demonstrates a server-side thread pool receiving work via the Phase 1 callback and completing it independently | ✅ `Reply()` callable from any thread at any time — server fully controls threading | ✅ `Notify()` callable from worker threads; the server queues work before returning from the `Send` callback, so it retains full control of the worker model | ❌ operation work runs on the message-passing Engine callback thread; the model does not provide an application-controlled worker-pool or per-request dispatch boundary | +| **R7** Concurrent calls / same process | ✅ channel is thread-safe, no external mechanism needed | ⚠️ concurrent calls depend on the LoLa proxy and method queue configuration; the service abstraction does not make the required parallelism transparent to the application | ⚠️ concurrent calls depend on the LoLa proxy, method queue, and event subscription configuration; the service abstraction does not make the required parallelism transparent to the application | ⚠️ the library supports concurrent use, while request-ID assignment, pending-call tracking, and response multiplexing must be implemented above the library; sufficient resource configuration is also needed | ⚠️ the library supports concurrent `Send()` calls, while request-ID assignment, pending-call tracking, and response multiplexing remain application responsibilities; fewer queue types need sizing than Option D | ⚠️ each client thread has its own endpoint and connection, so no application-level response multiplexing is required; however, concurrency scales open channels and Engine resources with thread count, and QNX imposes platform/deployment limits on the number of open channels | +| **R8** Multiple client processes | ✅ works out of the box | ⚠️ supported through the service-oriented deployment model, but requires consistent application/service configuration across participating processes — see R5 | ⚠️ supported through the service-oriented deployment model, but requires consistent application/service configuration across participating processes — see R5 | ✅ the library supports one server communicating with multiple client processes through independent client/server sessions; sufficient connection and queue resource configuration is needed | ✅ same as Option D; one server can communicate with multiple client processes through independent sessions, subject to connection and queue bounds | ⚠️ the POC exercises one client process; supporting multiple client processes requires bootstrap/session ownership and endpoint namespace rules beyond the demonstrated setup, and the per-thread channel model is constrained by QNX platform/deployment limits on the number of open channels | +| **R9** One-to-one | ✅ each call gets exactly one response | ✅ each call gets exactly one response | ⚠️ response is a broadcast event — requires one skeleton instance per client to prevent cross-client response leakage | ✅ each call gets exactly one response | ✅ each `Send()` gets one point-to-point `Notify()` response | ✅ each per-thread request receives one `Reply()` on its dedicated connection | +| **R10** Latency (WSL on Performance Laptop / QNX on RPi4) | ⚠️ indicative POC measurement ~420 µs / 1244 µs | ❓ not measured | ⚠️ indicative POC measurement ~130 µs / - | ⚠️ indicative POC measurement ~155 µs / 342 µs | ⚠️ indicative POC measurement ~141 µs / 219 µs | ⚠️ indicative POC measurement ~99 µs / 185 µs | +| **R11** Inter-VM | ⚠️ gRPC supports network channels, but the current adapter hardcodes Unix-domain endpoints; an endpoint/configuration change and validation of the inter-VM transport and peer-authentication model are required | ⚠️ current LoLa binding is SHM-only (single-kernel); the service-oriented architecture could in principle support a network binding without changing the service API, but no such binding exists today | ⚠️ current LoLa binding is SHM-only (single-kernel); the service-oriented architecture could in principle support a network binding without changing the service API, but no such binding exists today | ❌ current `score::message_passing` backends are local Unix-domain socket and QNX message passing; the library provides no inter-VM transport. A new framework backend would be required. | ❌ same as Option D; the no-`Reply` protocol does not change the available local-only backends and a new framework backend would be required. | ❌ uses the same local Unix-domain socket and QNX message-passing backends; the bootstrap/session arrangement does not add inter-VM transport | + +**Note:** R1 and R2 are hard safety blockers — any ❌ on these disqualifies an option for safety use regardless of performance on other requirements. Options A, B, and C each fail at least one of these requirements: A on R1, B on R2, and C on R2 because its Phase 1 enqueue has no timeout, even though its Phase 2 response wait is bounded. **POC note:** The LoLa POCs generated per-client configuration and used unique application identifiers to exercise multiple clients. These are prototype @@ -109,27 +130,70 @@ configuration model. **Option B (LoLa Full SOA, synchronous)** meets the R1 library-capability assessment: the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use. It fails R2 (timeout): there is no two-phase workaround available — the single blocking call holds the caller's thread until the handler returns with no escape path. The synchronous prototype was only validated in-process (skeleton and proxy on separate threads within a single test binary); cross-process concurrency was not tested. The config burden is high and would compound if LoLa is used elsewhere in the same process. -**Option C (LoLa Full SOA, asynchronous)** meets the R1 library-capability assessment: the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use, including its method/event model. Its indicative POC latency of approximately 130 microseconds is the lowest measured value, but still represents significant IPC overhead and should remain a warning rather than an unqualified pass. It fails partially on R2: Phase 2 timeout is implemented and flexible server threading is achievable, but Phase 1 timeout requires a LoLa framework change — it cannot be fixed in application code. On server crash, Phase 2 never fires and the client hangs until the application-level timeout expires; the crash itself is not detected independently. The broadcast-event response model requires one skeleton instance per client, adding complexity and config overhead. Notably, LoLa's service-oriented architecture is transport-agnostic by design; a future network binding could enable inter-VM communication without application-level changes — a meaningful long-term advantage that does not resolve the current safety gaps. +**Option C (LoLa Full SOA, asynchronous)** meets the R1 library-capability assessment: the LoLa library is documented as safety-oriented/ASIL-B qualified and provides custom memory-management infrastructure suitable for bounded resource use, including its method/event model. Its indicative POC latency is approximately 130 microseconds on WSL. It fails R2 overall: Phase 2 timeout is implemented and flexible server threading is achievable, but Phase 1 timeout requires a LoLa framework change — it cannot be fixed in application code. Because R2 applies to every phase, the bounded Phase 2 wait does not compensate for the unbounded Phase 1 call. On server crash, Phase 2 never fires and the client hangs until the application-level timeout expires; the crash itself is not detected independently. The broadcast-event response model requires one skeleton instance per client, adding complexity and config overhead. Notably, LoLa's service-oriented architecture is transport-agnostic by design; a future network binding could enable inter-VM communication without application-level changes — a meaningful long-term advantage that does not resolve the current safety gaps. **Option A (gRPC)** fails R1 for the stated ASIL-B use because the general-purpose library uses framework-managed threads and dynamic resources and does not provide an ASIL-oriented deterministic resource profile or safety qualification artifacts. The current adapter also does not configure a deadline; deadline-based and asynchronous gRPC were not analyzed. It handles server crash via gRPC status errors but cannot distinguish a crashed server from a hung one without a configured deadline. Peer authentication requires a PKI (no `SO_PEERCRED` equivalent). It remains the strongest option for inter-VM, because gRPC provides network channel support, but the current adapter hardcodes Unix-domain endpoints and requires endpoint/configuration changes before that path is available. It is appropriate for QM-to-QM communication where safety certification is not required. -**Option D (LoLa Message Passing Abstraction)** meets the R1 library-capability assessment: its design supports fixed resource bounds, preallocation, and pool/monotonic allocation, and the communication module documents safety-oriented quality tooling and ASIL-B qualification. Its indicative POC latency of approximately 160 microseconds is the second-lowest measured value, but still represents significant IPC overhead and should remain a warning rather than an unqualified pass. The application code is OS-agnostic; the framework provides the OS-specific transport backend (Unix domain socket on Linux, QNX message passing on QNX). The low-level message-passing prototype provides feasibility evidence for the intended skeleton: non-blocking `SendWithCallback` / `Reply` decoupling, application-level `request_id` multiplexing for concurrent threads, application-level bounded waiting, and detection of server death via socket EOF. Typed error propagation, mandatory timeout APIs, and backend-specific bounded-notification behavior remain implementation work. The selected library does not support inter-VM communication: its current backends are local, and a new framework backend would be required. The `IConnection` abstraction may provide a migration direction, but this has not been demonstrated and is not evidence of current library support. +**Option D (LoLa Message Passing Abstraction)** meets the R1 library-capability assessment: its design supports fixed resource bounds, preallocation, and pool/monotonic allocation, and the communication module documents safety-oriented quality tooling and ASIL-B qualification. Its indicative POC latency is approximately 155 microseconds on WSL and 342 microseconds on QNX in the collected runs. The application code is OS-agnostic; the framework provides the OS-specific transport backend (Unix domain socket on Linux, QNX message passing on QNX). The low-level message-passing prototype provides feasibility evidence for the intended skeleton: non-blocking `SendWithCallback` / `Reply` decoupling, application-level `request_id` multiplexing for concurrent threads, application-level bounded waiting, and detection of server death via socket EOF. Typed error propagation, mandatory timeout APIs, and backend-specific bounded-notification behavior remain implementation work. The selected library does not support inter-VM communication: its current backends are local, and a new framework backend would be required. The `IConnection` abstraction may provide a migration direction, but this has not been demonstrated and is not evidence of current library support. + +**Option E (LoLa Message Passing Abstraction, send plus notify)** uses the same +OS-agnostic library and shared connection model, but removes the immediate +`Reply` acknowledgement. `Send()` is configured as fire-and-forget and the +server's callback only validates and queues the request; a worker later sends +the result with `Notify()`. This avoids the per-connection REQUEST/REPLY +serialization window and the client-side async-reply queue, making the +protocol simpler and potentially reducing submission overhead. It does not +provide an immediate admission acknowledgement: a successful `Send()` only +means the request was accepted by the client-side send path. If server-side +validation or work-queue submission fails, the server can return a negative +`Notify()` result; otherwise, `Notify()` carries the operation result. The POC +still requires application-level request IDs, pending-call tracking, bounded +waiting, queue sizing, and connection-lifetime protection for worker +notifications. Its indicative POC latency is approximately 141 microseconds on +WSL and 219 microseconds on QNX in the collected runs. Like Option D, it does +not provide inter-VM communication. + +**Option F (Message Passing Engine)** demonstrates a bootstrap handshake that +creates a dedicated service endpoint, server Engine session, and client +connection for each client thread. The server Engine performs the complete +operation in its request callback and returns the response with `Reply()`, so +the POC needs no application work queue, worker pool, request-ID map, or +`Notify()` path. This makes one-to-one response matching straightforward and +keeps the server implementation small, but it couples the server's execution +capacity to the number of Engine sessions and client threads. The callback has +no operation deadline or cancellation path, so a stalled operation occupies +its Engine session and the client's application wait can expire without +interrupting the server work. The bootstrap and per-thread endpoint lifecycle +also add substantial configuration and resource-management complexity; the POC +does not establish the multiple-client-process or bounded-resource behavior +needed for production. As with Options D and E, the current backends do not +provide inter-VM communication. On QNX, the maximum number of open channels is +configurable, but the per-thread design consumes one channel and Engine session +per client thread against that limit, leading to many channels open at the +server side. QNX recommends sharing a single channel among multiple threads +instead, so this POC's scaling model is not the platform-preferred approach +and would need a deployment-specific channel-count assessment or redesign. Its +indicative POC latency is approximately 99 microseconds on WSL and 185 +microseconds on QNX in the collected runs, but this performance does not offset +its R2, R6, and resource-scaling limitations. --- ## 6. Decision -**Option D — LoLa Message Passing Abstraction** is selected as the IPC transport for the score-crypto daemon, using `score::message_passing` as its current implementation basis. +**Option E — LoLa Message Passing Abstraction (Send + Notify)** is selected as the IPC transport for the score-crypto daemon, using `score::message_passing` as its current implementation basis. The no-`Reply` protocol avoids the per-connection request/reply serialization window and still allows server-side submission failures to be reported through a negative `Notify()` result. --- ## 7. Consequences **Accepted trade-offs:** -- Indicative POC measurements show ~160 µs for message_passing, ~130 µs for LoLa Full SOA (async), and ~430 µs for gRPC. These figures are not a controlled benchmark and must be validated with the proper implementation using a common platform, payload, concurrency, warm-up, and measurement method. The observed gap between LoLa Full SOA and message_passing is partly attributed to configuration: the LoLa Full SOA asynchronous prototype was measured with QM-only settings, whereas the message_passing prototype was configured for mixed usage (`truly_async=true`). Configuring message_passing for QM-only reduced its observed round-trip to ~140 µs; the remaining difference was not analyzed. Throughput would be higher with LoLa Full SOA due to shared memory, but this is not a control-plane requirement: larger data transfers are expected to use a dedicated data plane. +- Option E is not the fastest measured option: the indicative results show lower round-trip latency for Option F and, in some measurements, for the asynchronous LoLa POC. Option E is selected because it provides a better overall balance across the requirements, including bounded client-side waiting, concurrent client and server operation, point-to-point responses, peer identification, configuration, and server-threading flexibility. Further preliminary measurements indicated that the IPC overhead was below 20% of the overall operation time (of the target daemon architecture), reducing the practical impact of the latency difference. +- Option E does not provide an immediate admission acknowledgement. A successful `Send()` confirms only client-side enqueueing; server-side validation or work-queue submission failure must be returned through a negative `Notify()`, and the client must handle final-result timeouts and late notifications. In rare cases (Server cannot retrieve request_id), the notify cannot be matched to a specific client request. This however is considered as a rare case and could be partially mitigated e.g. by relying on the timeout mechanism. - The selected library does not currently support inter-VM communication. The existing `IConnection` abstraction may guide a second, dedicated IPC mechanism, but that migration path has not been demonstrated and remains separate follow-up work. **Production implementation follow-up:** -The formal implementation of ``IConnection`` and ``IControlServer`` interfaces -replacing the gRPC adapter is still pending will be taken up. +The formal implementation of the ``IConnection`` and ``IControlServer`` +interfaces to replace the gRPC adapter remains pending and will be addressed +separately. diff --git a/score/tests/ipc_poc/BUILD b/score/tests/ipc_poc/BUILD index 41af2c3c3..dd507c81f 100644 --- a/score/tests/ipc_poc/BUILD +++ b/score/tests/ipc_poc/BUILD @@ -13,13 +13,16 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") -# Generate C++ header from the FlatBuffer schema. -# No --grpc flag: we only need plain C++ data tables, not gRPC stubs. +# Generate C++ and gRPC bindings from the shared POC FlatBuffer schema. genrule( name = "generated_poc_control", srcs = ["poc_control.fbs"], - outs = ["poc_control_generated.h"], - cmd = "$(location @flatbuffers//:flatc) --cpp --size-prefixed -o $(@D) $(SRCS)", + outs = [ + "poc_control_generated.h", + "poc_control.grpc.fb.h", + "poc_control.grpc.fb.cc", + ], + cmd = "$(location @flatbuffers//:flatc) --cpp --grpc --size-prefixed -o $(@D) $(SRCS)", tools = ["@flatbuffers//:flatc"], ) @@ -42,7 +45,11 @@ cc_test( srcs = ["poc_async.cpp"], deps = [ ":async_control_plane_service", + ":poc_helper", + "//score/tests/utility:runtime_measurement", + "//score/tests/utility:process_resource_measurement", ], + tags = ["manual"], ) cc_library( @@ -58,21 +65,154 @@ cc_library( ], ) +cc_library( + name = "direct_socket_poc_deps", + hdrs = [ + "ipc_buffer.h", + ":generated_poc_control", + ], + include_prefix = "tests/score_com_poc", + deps = [ + "@flatbuffers", + ], +) + +cc_library( + name = "poc_helper", + hdrs = [ + "ipc_buffer.h", + "poc_helper.hpp", + ], + deps = [ + ":generated_poc_control", + "@flatbuffers", + ], +) + cc_test( name = "poc_low_level", srcs = ["poc_low_level.cpp"], deps = [ ":low_level_poc_deps", + ":poc_helper", + "//score/tests/utility:runtime_measurement", + "//score/tests/utility:process_resource_measurement", + ], + tags = ["manual"], +) + +cc_test( + name = "poc_low_level_no_reply", + srcs = ["poc_low_level_no_reply.cpp"], + deps = [ + ":low_level_poc_deps", + ":poc_helper", + "//score/tests/utility:runtime_measurement", + "//score/tests/utility:process_resource_measurement", + ], + tags = ["manual"], +) + +cc_test( + name = "poc_low_level_no_reply_notify_in_thread", + srcs = ["poc_low_level_no_reply_notify_in_thread.cpp"], + deps = [ + ":low_level_poc_deps", + ":poc_helper", + "//score/tests/utility:runtime_measurement", + "//score/tests/utility:process_resource_measurement", + ], + tags = ["manual"], +) + +cc_test( + name = "poc_thread_pool_queue", + srcs = ["poc_thread_pool_queue.cpp"], + deps = [ + "//score/tests/utility:runtime_measurement", + ], + tags = ["manual"], +) + +cc_test( + name = "poc_engine", + srcs = ["poc_engine.cpp"], + deps = [ + ":low_level_poc_deps", + ":poc_helper", + "//score/tests/utility:runtime_measurement", + "//score/tests/utility:process_resource_measurement", + ], + tags = ["manual"], +) + +cc_test( + name = "poc_engine_sync", + srcs = ["poc_engine_sync.cpp"], + deps = [ + ":low_level_poc_deps", + ":poc_helper", + "//score/tests/utility:runtime_measurement", + "//score/tests/utility:process_resource_measurement", + ], + tags = ["manual"], +) + +cc_test( + name = "poc_unix_socket", + srcs = ["poc_unix_socket.cpp"], + linkopts = select({ + "//platforms:is_qnx_aarch64": ["-lsocket"], + "//platforms:is_qnx_x86_64": ["-lsocket"], + "//conditions:default": [], + }), + deps = [ + ":direct_socket_poc_deps", + ":poc_helper", + "//score/tests/utility:runtime_measurement", + "//score/tests/utility:process_resource_measurement", + ], + tags = ["manual"], +) + +cc_test( + name = "poc_qnx_message_passing", + srcs = ["poc_qnx_message_passing.cpp"], + target_compatible_with = select({ + "//platforms:is_qnx_aarch64": [], + "//platforms:is_qnx_x86_64": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + deps = [ + ":direct_socket_poc_deps", + ":poc_helper", + "//score/tests/utility:runtime_measurement", + "//score/tests/utility:process_resource_measurement", + ], + tags = ["manual"], +) + +cc_library( + name = "poc_grpc_protocol", + srcs = [":generated_poc_control"], + hdrs = [ + ":generated_poc_control", + ], + deps = [ + "@flatbuffers", + "@grpc//:grpc++_unsecure", ], ) cc_test( name = "poc_grpc", srcs = ["poc_grpc.cpp"], - dynamic_deps = ["//third_party/grpc:libgrpc_shared"], deps = [ - "//score/crypto/src/daemon/control_plane", - "//score/crypto/src/ipc/grpc_adapter:grpc_control_client", - "//score/crypto/src/ipc/grpc_adapter:grpc_control_server", + "//score/tests/utility:runtime_measurement", + ":poc_helper", + ":poc_grpc_protocol", + "//score/tests/utility:process_resource_measurement", + "@grpc//:grpc++_unsecure", ], + tags = ["manual"], ) diff --git a/score/tests/ipc_poc/measure_client_scaling.sh b/score/tests/ipc_poc/measure_client_scaling.sh new file mode 100755 index 000000000..601a84d7a --- /dev/null +++ b/score/tests/ipc_poc/measure_client_scaling.sh @@ -0,0 +1,314 @@ +#!/bin/sh + +# Run the IPC POCs for burst and staggered client-scaling measurements. +# +set -u + +readonly SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +readonly WORKSPACE_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) +readonly DEFAULT_OUTPUT_ROOT="$WORKSPACE_ROOT/measurement-results" +readonly DEFAULT_POC_BIN_DIR="$WORKSPACE_ROOT/bazel-bin/score/tests/ipc_poc" + +OUTPUT_ROOT="$DEFAULT_OUTPUT_ROOT" +POC_BIN_DIR="$DEFAULT_POC_BIN_DIR" +CLIENT_COUNT=1 +CLIENT_THREADS="1 2 4 8 16 32 64 128" +ITERATIONS=10 +CAPTURE_LOGS=true +CALL_COUNT=100 +SERVER_THREADS=8 +SLEEP_MILLISECONDS=0 +INCLUDE_QNX_POC=auto +POC_NAMES="poc_low_level poc_low_level_no_reply poc_engine poc_engine_sync poc_unix_socket poc_grpc poc_qnx_message_passing" + +case "$CAPTURE_LOGS" in + true|false) ;; + *) + echo "CAPTURE_LOGS must be true or false" >&2 + exit 2 + ;; +esac + +case "$INCLUDE_QNX_POC" in + true|false) ;; + auto) + case "$(uname -s 2>/dev/null || printf unknown)" in + QNX|QNX*) INCLUDE_QNX_POC=true ;; + *) INCLUDE_QNX_POC=false ;; + esac + ;; + *) + echo "INCLUDE_QNX_POC must be true, false, or auto" >&2 + exit 2 + ;; +esac + +case " $POC_NAMES " in + *" poc_qnx_message_passing "*) + if [ "$INCLUDE_QNX_POC" != true ]; then + POC_NAMES=$(printf '%s' "$POC_NAMES" | sed 's/[[:space:]][[:space:]]*poc_qnx_message_passing\([[:space:]]*\|$\)//') + fi + ;; +esac + +mkdir -p "$OUTPUT_ROOT" +MANIFEST="$OUTPUT_ROOT/results.csv" +ITERATION_MANIFEST="$OUTPUT_ROOT/iteration_results.csv" +: > "$ITERATION_MANIFEST" +printf '%s\n' 'iteration,poc,mode,client_count,client_threads,server_threads,call_count,sleep_milliseconds,random_wait,round_trip_mean_ns,server_rss_delta_bytes,server_cpu_total_time_delta_ns,client_cpu_total_time_delta_ns,status,log' > "$ITERATION_MANIFEST" + +safe_value() +{ + awk -v value="$1" 'BEGIN { + gsub(/[^A-Za-z0-9._-]/, "_", value) + printf "%s", value + }' +} + +read_round_trip_mean_ns() +{ + awk '/\[Timing\] average .*RoundTrip:/{ print $4; exit }' "$1" +} + +read_server_rss_delta_bytes() +{ + awk ' + /\[Resources\].*_server_thread_creation:/ { + for (field = 1; field <= NF; field++) { + if ($field ~ /^resident_delta_bytes=/) { + split($field, value, "=") + total += value[2] + found = 1 + } + } + } + END { + if (found) { + print total + } + } + ' "$1" +} + +read_server_cpu_total_time_delta_ns() +{ + awk ' + /\[Resources\].*_server_workload:/ { + for (field = 1; field <= NF; field++) { + if ($field ~ /^cpu_total_time_delta_ns=/) { + split($field, value, "=") + total += value[2] + found = 1 + } + } + } + END { + if (found) { + print total + } + } + ' "$1" +} + +read_client_cpu_total_time_delta_ns() +{ + awk ' + /\[Resources\].*_client_workload:/ { + for (field = 1; field <= NF; field++) { + if ($field ~ /^cpu_total_time_delta_ns=/) { + split($field, value, "=") + total += value[2] + found = 1 + } + } + } + END { + if (found) { + print total + } + } + ' "$1" +} + +run_poc_binary() +{ + target=$1 + shift + case "$(uname -s 2>/dev/null || printf unknown)" in + QNX|QNX*) + env LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:$target.runfiles/_main/_solib_aarch64le/" "$target" "$@" + ;; + *) + "$target" "$@" + ;; + esac +} + +run_poc() +{ + iteration=$1 + mode=$2 + poc=$3 + client_count=$4 + client_threads=$5 + random_wait=$6 + poc_binary="$POC_BIN_DIR/$poc" + if [ "$CAPTURE_LOGS" = true ]; then + log_file="$OUTPUT_ROOT/$(safe_value "$poc")_$(safe_value "$mode")_iteration$(printf '%02d' "$iteration")_clients$(safe_value "$client_count")_threads$(safe_value "$client_threads").log" + log_name=$(basename "$log_file") + else + log_file=$(mktemp "${TMPDIR:-/tmp}/ipc_poc_measurement.XXXXXX") + log_name= + fi + + printf '\n[%s] %s clients=%s client_threads=%s server_threads=%s calls=%s wait=%s\n' \ + "$mode" "$poc" "$client_count" "$client_threads" "$SERVER_THREADS" "$CALL_COUNT" "$random_wait" + { + printf '[measurement] iteration=%s mode=%s poc=%s client_count=%s client_threads=%s server_threads=%s call_count=%s sleep_milliseconds=%s random_wait=%s\n' \ + "$iteration" "$mode" "$poc" "$client_count" "$client_threads" "$SERVER_THREADS" "$CALL_COUNT" "$SLEEP_MILLISECONDS" "$random_wait" + if [ ! -x "$poc_binary" ]; then + printf '[measurement] missing executable: %s\n' "$poc_binary" + exit 127 + else + run_poc_binary "$poc_binary" \ + "--client_count=$client_count" \ + "--client_threads=$client_threads" \ + "--server_threads=$SERVER_THREADS" \ + "--call_count=$CALL_COUNT" \ + "--sleep_milliseconds=$SLEEP_MILLISECONDS" \ + "--random_wait=$random_wait" + fi + } > "$log_file" 2>&1 + status=$? + round_trip_mean_ns=$(read_round_trip_mean_ns "$log_file") + server_rss_delta_bytes=$(read_server_rss_delta_bytes "$log_file") + server_cpu_total_time_delta_ns=$(read_server_cpu_total_time_delta_ns "$log_file") + client_cpu_total_time_delta_ns=$(read_client_cpu_total_time_delta_ns "$log_file") + [ -n "$round_trip_mean_ns" ] || round_trip_mean_ns=NA + [ -n "$server_rss_delta_bytes" ] || server_rss_delta_bytes=NA + [ -n "$server_cpu_total_time_delta_ns" ] || server_cpu_total_time_delta_ns=NA + [ -n "$client_cpu_total_time_delta_ns" ] || client_cpu_total_time_delta_ns=NA + + printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \ + "$iteration" "$poc" "$mode" "$client_count" "$client_threads" "$SERVER_THREADS" "$CALL_COUNT" \ + "$SLEEP_MILLISECONDS" "$random_wait" "$round_trip_mean_ns" "$server_rss_delta_bytes" \ + "$server_cpu_total_time_delta_ns" "$client_cpu_total_time_delta_ns" "$status" \ + "$log_name" >> "$ITERATION_MANIFEST" + + if [ "$CAPTURE_LOGS" = false ]; then + rm -f "$log_file" + fi + + if [ "$status" -eq 0 ]; then + if [ "$CAPTURE_LOGS" = true ]; then + printf '[done] %s\n' "$log_file" + else + printf '[done] %s (log capture disabled)\n' "$poc" + fi + else + if [ "$CAPTURE_LOGS" = true ]; then + printf '[failed status=%s] %s\n' "$status" "$log_file" >&2 + else + printf '[failed status=%s] %s (log capture disabled)\n' "$status" "$poc" >&2 + fi + fi +} + +run_skipped_qnx_row() +{ + iteration=$1 + mode=$2 + client_count=$3 + client_threads=$4 + random_wait=$5 + printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \ + "$iteration" "poc_qnx_message_passing" "$mode" "$client_count" "$client_threads" "$SERVER_THREADS" "$CALL_COUNT" \ + "$SLEEP_MILLISECONDS" "$random_wait" "NA" "NA" "NA" "NA" "SKIPPED_HOST" "" >> "$ITERATION_MANIFEST" +} + +iteration=1 +while [ "$iteration" -le "$ITERATIONS" ]; do + printf '\n=== iteration %s/%s ===\n' "$iteration" "$ITERATIONS" + for poc in $POC_NAMES; do + for mode in burst staggered; do + if [ "$mode" = burst ]; then + random_wait=false + else + random_wait=true + fi + + for client_threads in $CLIENT_THREADS; do + if [ "$poc" = poc_qnx_message_passing ] && [ "$INCLUDE_QNX_POC" != true ]; then + run_skipped_qnx_row "$iteration" "$mode" "$CLIENT_COUNT" "$client_threads" "$random_wait" + else + run_poc "$iteration" "$mode" "$poc" "$CLIENT_COUNT" "$client_threads" "$random_wait" + fi + done + done + done + iteration=$((iteration + 1)) +done + +awk -F, ' + NR == 1 { + next + } + { + key = $2 FS $3 FS $4 FS $5 FS $6 FS $7 FS $8 FS $9 + if (!(key in first)) { + first[key] = $0 + order[++count] = key + } + iterations[key]++ + if ($14 == 0) { + successful[key]++ + if ($10 != "NA") { + round_trip_total[key] += $10 + round_trip_samples[key]++ + } + if ($11 != "NA") { + rss_total[key] += $11 + rss_samples[key]++ + } + if ($12 != "NA") { + server_cpu_total[key] += $12 + server_cpu_samples[key]++ + } + if ($13 != "NA") { + client_cpu_total[key] += $13 + client_cpu_samples[key]++ + } + } else if ($14 == "SKIPPED_HOST") { + skipped[key]++ + } + } + END { + print "poc,mode,client_count,client_threads,server_threads,call_count,sleep_milliseconds,random_wait,round_trip_mean_ns,server_rss_delta_bytes,server_cpu_total_time_delta_ns,client_cpu_total_time_delta_ns,iterations,successful_iterations,status,log" + for (row_number = 1; row_number <= count; row_number++) { + key = order[row_number] + split(first[key], fields, FS) + round_trip_mean = (round_trip_samples[key] ? sprintf("%.0f", round_trip_total[key] / round_trip_samples[key]) : "NA") + server_rss_delta = (rss_samples[key] ? sprintf("%.0f", rss_total[key] / rss_samples[key]) : "NA") + server_cpu_mean = (server_cpu_samples[key] ? sprintf("%.0f", server_cpu_total[key] / server_cpu_samples[key]) : "NA") + client_cpu_mean = (client_cpu_samples[key] ? sprintf("%.0f", client_cpu_total[key] / client_cpu_samples[key]) : "NA") + if (successful[key] == iterations[key]) { + status = "OK" + } else if (successful[key] > 0) { + status = "PARTIAL" + } else if (skipped[key] == iterations[key]) { + status = "SKIPPED_HOST" + } else { + status = "FAILED" + } + printf "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n", fields[2], fields[3], fields[4], fields[5], fields[6], fields[7], fields[8], fields[9], round_trip_mean, server_rss_delta, server_cpu_mean, client_cpu_mean, iterations[key], successful[key] + 0, status, "iteration_results.csv" + } + } +' "$ITERATION_MANIFEST" > "$MANIFEST" + +printf '\nResults manifest: %s\n' "$MANIFEST" +printf 'Logs directory: %s\n' "$OUTPUT_ROOT" +printf 'Per-iteration manifest: %s\n' "$ITERATION_MANIFEST" +if awk -F, 'NR > 1 && $15 != "OK" && $15 != "SKIPPED_HOST" { failed = 1 } END { exit failed }' "$MANIFEST"; then + exit 0 +fi +exit 1 diff --git a/score/tests/ipc_poc/message_passing_overview.md b/score/tests/ipc_poc/message_passing_overview.md index b5fe8d7bd..397162f30 100644 --- a/score/tests/ipc_poc/message_passing_overview.md +++ b/score/tests/ipc_poc/message_passing_overview.md @@ -224,11 +224,12 @@ The application-level flow and state model are the same on both platforms. The i - Or bring out the admission call to allow custom behaviour - Use proper state machines for connection state, pending request state and work item state - Use base-libs thread pools -- Job cancellation is not implemented. - - Define cancellation and shutdown behavior for queued and in-flight work, client disconnects, server termination, and worker teardown. - - Define typed timeout, connection-loss, cancellation, and delivery errors, including cleanup of pending request state and handling of late notifications. - - We propably need a stop / cancel token in the workitems, however if we can actually cancel a running request depends on actual processors of the requests - - Need to also think how to properly give preference to "cancel requests" and if they need to be handled differently than normal ones +- Job cancellation is implemented as cooperative cancellation. + - The client sends a cancellation request but keeps the original pending request until a final completion notification arrives. + - Queued work can produce a final cancelled response; running work receives a stop token and its handler result remains authoritative. + - If cancellation cannot be resolved before the deadline or connection loss, the client returns the typed `kOperationCancelledMayExecute` outcome. + - Cancellation and shutdown behavior for queued and in-flight work, client disconnects, server termination, and worker teardown is implemented, but provider-level interruption remains handler-dependent. + - Need to also think how to properly give preference to "cancel requests" and if they need to be handled differently than normal ones - Bound the application work queue and define admission behavior when it is full. A request must not be acknowledged unless admission and the associated resource reservation have succeeded. - Define authenticated peer identity, endpoint permissions, authorization, and behavior for rejected or reconnecting clients. - If acknowledgement delivery fails after queue admission, either cancel the queued work or expose the resulting "may have executed" outcome and define retry rules, especially for non-idempotent operations. diff --git a/score/tests/ipc_poc/mw_com_methods.drawio.svg b/score/tests/ipc_poc/mw_com_methods.drawio.svg new file mode 100644 index 000000000..d452c4a54 --- /dev/null +++ b/score/tests/ipc_poc/mw_com_methods.drawio.svg @@ -0,0 +1,4 @@ + + + +
e.g. RAM filesystem
e.g. RAM filesystem
Persistent storage
Persistent storage
Client 3 (UID: 15, ASIL-B)
Client 3 (UID: 15, ASIL-B)
Client 1 (UID: 10, QM)
Client 1 (UID: 10, QM)
Server (e.g. Crypto Daemon) QM
Server (e.g. Crypto Daemon) QM
reads
config
reads...
AsSkeleton<MyInterface>
AsSkeleton<MyInterface>
Create(instance_spec)
Create(instance_spec)
Proxy<MyInterface>
Proxy<MyInterface>
execute.Allocate()
execute.Allocate()
execute()
execute()
MySkeleton
MySkeleton
OfferService()
OfferService()
execute.RegisterHandler()
execute.RegisterHandler()
StopOfferService()
StopOfferService()
Engine
Thread
(Unix|QNX)
Engine...
Message passing
unix domain socket
Message passing...
(File)
Service
Discovery
(File)Serv...
creates
service
discovery
file
creates...
checks for
service
discovery
file
checks for...
- serviceTypes
  - serviceTypeName: "/My/Service/Name"
  - version: { 1, 0 }
  - bindings
    - binding: SHM
    - serviceId: 100
- serviceInstance
  - instanceSpecifier: "my/Identifier"
  - serviceTypeName: "/My/Service/Name"
  - version
  - instances (Server Instance)
    {
- instanceId: 1 
    - asil-level: QM
    - uid: [10, 11, 12, ..., 15]
    - methods:
        - methodName: Execute, queueSize: 1
    },
- serviceTypes...
reads
config
reads...
ProxyBase
ProxyBase
FindService(instance_spec)
FindService(instance_spec)
creates
SHM
creates...
AsProxy<MyInterface>
AsProxy<MyInterface>
uses returned 
handle
uses returned...
Create(   )
Create(   )
synchronous call
synchronous call
Required change of POC
- method -> Send Request
  - Define request Id (ticket number)
  - Handler dispatch to threadpool (request id)
  - return only request id
- client waits for request id via event
  - timeout support possible
Required change of POC...
mw_com_config.json
mw_com_config.json
service id + instance id (dependent)
service id + instance id (dependent)
service id + instance id (dependent)
service id + instance id (dependent)
Shared Memory
Shared Memory
MyIn
MyIn
MyOut
MyOut
myHandler
myHandler
 set callback
 set callback
return MyOut
lola handles
(buffer?)
return MyOut...
Different
Shared
memory
for out?
Different...
Client 2 (UID: 11, QM)
Client 2 (UID: 11, QM)
SHM
SHM
SHM
SHM
get slot
from buffer
get slot...
create
threads
create...
mw::com::runtime
mw::com::runtime
InitializeRuntime()
InitializeRuntime()
Message
Receiver
1
Message...
Message
Receiver
2
Message...
Service
Discovery
(notify)
Service...
Engine
Thread
Engine...
Only required by events / fields?
Only required...
We only offer a service
We only offer...
Threadpool
Threadpool
Engine
Thread
(Unix|QNX)
Engine...
mw::com::runtime
mw::com::runtime
InitializeRuntime()
InitializeRuntime()
Message
Receiver
1
Message...
Message
Receiver
2
Message...
Service
Discovery
(notify)
Service...
Only required by events / fields?
Only required...
create
threads
create...
calling
Thread
calling...
used for
method
subscritpion
used for...
cannot dispatch to threadpool
as the engine thread has to wait for the result
to handle the method return
cannot dispatch to threadpool...
Send Request
Send Request
Reply
request id (ticket)
Reply...
Dispatch to threadpool
Dispatch to threadpool
Threadpool
Threadpool
Send Result
request id (via event)
Send Result...
Send
Send
Get request id
Get request id
Return timeout error
Return timeout error
yes
yes
no
no
timeout?
timeout?
React on event
React on event
Return result
Return result
How ot acquire Client UID?
How ot acquire Clie...Text is not SVG - cannot display \ No newline at end of file diff --git a/score/tests/ipc_poc/poc_async.cpp b/score/tests/ipc_poc/poc_async.cpp index 55577eb86..489014990 100644 --- a/score/tests/ipc_poc/poc_async.cpp +++ b/score/tests/ipc_poc/poc_async.cpp @@ -70,14 +70,15 @@ #include #include -#include "flatbuffers/flatbuffers.h" #include "score/mw/com/impl/proxy_base.h" #include "score/mw/com/runtime.h" #include "score/mw/com/runtime_configuration.h" #include "score/mw/com/types.h" +#include "score/tests/utility/runtime_measurement.hpp" #include "score/tests/ipc_poc/async_control_interface.h" #include "score/tests/ipc_poc/ipc_buffer.h" -#include "score/tests/ipc_poc/poc_control_generated.h" +#include "score/tests/ipc_poc/poc_helper.hpp" +#include "score/tests/utility/process_resource_measurement.hpp" // --------------------------------------------------------------------------- // Global parameters (set before fork; never mutated after) @@ -88,6 +89,9 @@ static int g_call_count = 2; // NOLINT(cppcoreguidelines-avoid-non-cons static int g_client_threads = 2; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) static int g_server_threads = 8; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static bool g_random_wait = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +static void Log(const std::string& line); // --------------------------------------------------------------------------- // Config generation @@ -96,6 +100,8 @@ static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-cons namespace score::crypto::ipc::control { +namespace helper = score::crypto::ipc::poc_helper; + /// Returns the shared "serviceTypes" JSON block (without trailing comma). static std::string ServiceTypesJson() { @@ -223,7 +229,7 @@ static void SetupConfigs(const int client_count) f << json.str(); } - std::printf("[SetupConfigs] wrote producer + %d consumer config(s)\n", client_count); + Log("[SetupConfigs] wrote producer + " + std::to_string(client_count) + " consumer config(s)"); } } // namespace score::crypto::ipc::control @@ -233,9 +239,15 @@ static void SetupConfigs(const int client_count) // --------------------------------------------------------------------------- static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static constexpr bool kEnableVerboseOutput{false}; +static constexpr bool kEnableLatencyVerbose{false}; static void Log(const std::string& line) { + if (!kEnableVerboseOutput) + { + return; + } std::lock_guard lk(g_log_mutex); std::cout << line << "\n"; } @@ -247,86 +259,24 @@ static void Log(const std::string& line) namespace score::crypto::ipc::control { -/// Serialise a ControlResponse carrying a single String result. -static IpcBuffer BuildControlResponse(const std::uint64_t request_id, const std::string& combined) -{ - flatbuffers::FlatBufferBuilder fbb(512); - - auto str_val = fbb.CreateString(combined); - auto str_tbl = CreateString(fbb, str_val); - - std::vector resp_param_types{OperationParameter_String}; - std::vector> resp_param_values{str_tbl.Union()}; - - auto resp_op = CreateSingleOperationResponse(fbb, - CreateOperationIdentifier(fbb, 0U, 0U), - CreateOperationResult(fbb, 0U), - fbb.CreateVector(resp_param_types), - fbb.CreateVector(resp_param_values)); - - auto resp_batch = CreateOperationResponseBatch(fbb, fbb.CreateVector({resp_op})); - fbb.FinishSizePrefixed(CreateControlResponse(fbb, request_id, resp_batch)); - return PackFlatBuffer(fbb.GetBufferPointer(), fbb.GetSize()); -} - /// Deserialise a ControlRequest and produce a ControlResponse: result = "_". static IpcBuffer ProcessRequest(const IpcBuffer& request_buf) { - flatbuffers::Verifier verifier{reinterpret_cast(request_buf.payload.data()), - GetPayloadSize(request_buf)}; - - if (!VerifySizePrefixedControlRequestBuffer(verifier)) + helper::Response workload_response; + if (!helper::ProcessRequestBuffer(request_buf, workload_response)) { std::cerr << "[server/worker] FlatBuffer verification failed\n"; return IpcBuffer{}; } - const auto* req = flatbuffers::GetSizePrefixedRoot(request_buf.payload.data()); - if (req == nullptr || req->operation_batch() == nullptr || req->operation_batch()->operations() == nullptr || - req->operation_batch()->operations()->size() == 0U) - { - return IpcBuffer{}; - } - - const auto* op = req->operation_batch()->operations()->Get(0U); - if (op == nullptr || op->parameter() == nullptr) - { - return IpcBuffer{}; - } - - std::string str_value; - std::uint64_t uint64_value = 0U; - - for (flatbuffers::uoffset_t i = 0U; i < op->parameter()->size(); ++i) - { - const auto ptype = static_cast(op->parameter_type()->Get(i)); - if (ptype == OperationParameter_String) - { - const auto* s = reinterpret_cast(op->parameter()->Get(i)); - if (s != nullptr && s->val() != nullptr) - { - str_value = s->val()->str(); - } - } - else if (ptype == OperationParameter_ValueUint64) - { - const auto* v = reinterpret_cast(op->parameter()->Get(i)); - if (v != nullptr) - { - uint64_value = v->val(); - } - } - } - - const std::string combined = str_value + "_" + std::to_string(uint64_value); - { std::stringstream ss; - ss << "[server/worker] ticket=" << req->request_id() << " -> combined=\"" << combined << "\""; + ss << "[server/worker] ticket=" << workload_response.request_id << " -> combined=\"" + << workload_response.string_value << "\""; Log(ss.str()); } - return BuildControlResponse(req->request_id(), combined); + return helper::BuildResponseBuffer(workload_response); } // --------------------------------------------------------------------------- @@ -345,6 +295,7 @@ struct WorkItem /// Blocks until all client processes have exited (waitpid). static int RunServer(const std::vector& child_pids) { + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; using score::mw::com::InstanceSpecifier; const int client_count = static_cast(child_pids.size()); @@ -394,11 +345,16 @@ static int RunServer(const std::vector& child_pids) } const int worker_count = g_server_threads; + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); std::vector workers; workers.reserve(static_cast(worker_count)); + helper::ThreadStartBarrier worker_start_barrier{static_cast(worker_count)}; for (int widx = 0; widx < worker_count; ++widx) { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::Async::ServerThreadCreation"}; workers.emplace_back([&, widx]() { + worker_start_barrier.ArriveAndWait(); while (true) { std::unique_lock lk(queue_mutex); @@ -455,6 +411,11 @@ static int RunServer(const std::vector& child_pids) } }); } + worker_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_async_server_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + worker_start_barrier.Release(); Log("[server] started " + std::to_string(worker_count) + " worker thread(s)"); // ------------------------------------------------------------------ @@ -465,18 +426,18 @@ static int RunServer(const std::vector& child_pids) // Capture skel_idx by value so each handler knows its instance index. auto reg_result = skeletons[skel_idx].request.RegisterHandler([&, skel_idx](std::uint64_t& result, const IpcBuffer& req_buf) { - const auto* req = flatbuffers::GetSizePrefixedRoot(req_buf.payload.data()); + helper::Request request; + const bool parse_ok = helper::ParseRequestBuffer(req_buf, request); + const std::uint64_t ticket = parse_ok ? request.request_id : 0U; std::stringstream ss1; - ss1 << "[server/handler] instance=" << skel_idx << " req ptr=" << req; + ss1 << "[server/handler] instance=" << skel_idx << " ticket=" << ticket; Log(ss1.str()); - const std::uint64_t ticket = (req != nullptr) ? req->request_id() : 0U; - - if (req == nullptr || ticket == 0U) + if (!parse_ok || ticket == 0U) { std::stringstream ss_err; - ss_err << "[server/handler] instance=" << skel_idx << " WARNING: req=" << req << " ticket=" << ticket + ss_err << "[server/handler] instance=" << skel_idx << " WARNING: ticket=" << ticket << " — FlatBuffer parse failed or zero request_id;" " response will never match a pending client call"; Log(ss_err.str()); @@ -589,6 +550,7 @@ struct PendingCall static bool RunClient(const int client_index, const int call_count, const int thread_count) { + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; using score::mw::com::InstanceSpecifier; using score::mw::com::SamplePtr; using score::mw::com::impl::ProxyBase; @@ -678,6 +640,8 @@ static bool RunClient(const int client_index, const int call_count, const int th proxies.reserve(static_cast(thread_count)); for (int t = 0; t < thread_count; ++t) { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement connection_scope{ + latency_measurement, "POC::Async::ClientConnectionSetup"}; auto proxy_result = AsyncControlProxy::Create(handles.value()[0]); if (!proxy_result.has_value()) { @@ -696,9 +660,14 @@ static bool RunClient(const int client_index, const int call_count, const int th proxies.push_back(std::move(proxy_result.value())); } + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + helper::ThreadStartBarrier thread_start_barrier{static_cast(proxies.size())}; for (int t = 0; t < static_cast(proxies.size()); ++t) { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::Async::ClientThreadCreation"}; threads.emplace_back([&, t]() { + thread_start_barrier.ArriveAndWait(); auto& proxy = proxies[static_cast(t)]; // Within a single thread calls are sequential, so at most one @@ -726,37 +695,10 @@ static bool RunClient(const int client_index, const int call_count, const int th return; } - const auto* resp = flatbuffers::GetSizePrefixedRoot(sample->payload.data()); - if (resp == nullptr) - { - std::cerr << "[receive handler] GetSizePrefixedRoot returned nullptr" - " — pending call will time out\n"; - return; - } - - const std::uint64_t ticket = resp->request_id(); - - // Extract the result BEFORE taking the lock so the critical - // section stays short. - std::string result_value; - bool ok = false; - if (resp->operation_batch() != nullptr && resp->operation_batch()->operations() != nullptr && - resp->operation_batch()->operations()->size() > 0U) - { - const auto* op = resp->operation_batch()->operations()->Get(0U); - if (op != nullptr && op->parameter() != nullptr && op->parameter()->size() > 0U && - op->parameter_type() != nullptr && - static_cast(op->parameter_type()->Get(0U)) == - OperationParameter_String) - { - const auto* str = reinterpret_cast(op->parameter()->Get(0U)); - if (str != nullptr && str->val() != nullptr) - { - result_value = str->val()->str(); - ok = true; - } - } - } + helper::Response parsed_response; + const bool ok = helper::ParseResponseBuffer(*sample, parsed_response); + const std::uint64_t ticket = parsed_response.request_id; + const std::string result_value = parsed_response.string_value; if (!ok) { @@ -828,20 +770,14 @@ static bool RunClient(const int client_index, const int call_count, const int th // Wait till setup is done on server side // std::this_thread::sleep_for(std::chrono::seconds(3)); - auto times = std::vector(call_count); - for (int c = 0; c < call_count; ++c) { - auto start = std::chrono::system_clock::now(); - - const std::string str_param = "client" + std::to_string(client_index + 1); - const std::uint64_t uint64_param = static_cast(c + 1); - // Unique ticket encodes client, thread and call indices. - const std::uint64_t request_id = static_cast(client_index + 1) * 100'000ULL + - static_cast(t + 1) * 1'000ULL + - static_cast(c + 1); + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement latency_scope{ + latency_measurement, "POC::Async::RoundTrip"}; - const std::string expected = str_param + "_" + std::to_string(uint64_param); + const auto workload_request = helper::CreateRequest(client_index, t, c); + const auto request_id = workload_request.request_id; // Set up the pending state BEFORE sending so the receive handler // can never miss the response even on a very fast server. @@ -871,41 +807,20 @@ static bool RunClient(const int client_index, const int call_count, const int th } auto& arg = std::get<0>(alloc_result.value()); - // Build the FlatBuffer and write directly into the SHM-backed slot. + if (!helper::BuildRequestInto(*arg, workload_request)) { - flatbuffers::FlatBufferBuilder fbb(512); - - auto str_val = fbb.CreateString(str_param); - auto str_tbl = CreateString(fbb, str_val); - auto u64_tbl = CreateValueUint64(fbb, uint64_param); - - std::vector param_types{OperationParameter_String, OperationParameter_ValueUint64}; - std::vector> param_values{str_tbl.Union(), u64_tbl.Union()}; - - auto op_id = CreateOperationIdentifier(fbb, /*actor=*/1U, /*action=*/1U); - auto single_op = CreateSingleOperationRequest( - fbb, op_id, fbb.CreateVector(param_types), fbb.CreateVector(param_values)); - auto batch = CreateOperationRequestBatch(fbb, fbb.CreateVector({single_op})); - fbb.FinishSizePrefixed(CreateControlRequest(fbb, - request_id, - /*client_id=*/0U, - /*data_node_id=*/0U, - batch)); - - if (!PackFlatBufferInto(*arg, fbb.GetBufferPointer(), fbb.GetSize())) - { - Log("[client " + std::to_string(client_index) + "/thread " + std::to_string(t) + - "] PackFlatBufferInto failed"); - ++failures; - phase1_failed = true; - break; - } + Log("[client " + std::to_string(client_index) + "/thread " + std::to_string(t) + + "] PackFlatBufferInto failed"); + ++failures; + phase1_failed = true; + break; } { std::stringstream ss; ss << "[client " << client_index << "/thread " << t << "] -> request_id=" << request_id - << " str=\"" << str_param << "\" uint64=" << uint64_param + << " str=\"" << workload_request.string_value << "\" uint64=" + << workload_request.uint64_value << " (Phase 1: short-lived call)"; Log(ss.str()); } @@ -970,11 +885,13 @@ static bool RunClient(const int client_index, const int call_count, const int th continue; } - if (!pending.ok || pending.result_value != expected) + if (!pending.ok || + !helper::Matches(workload_request, helper::Response{ticket, pending.result_value})) { std::ostringstream oss; oss << "[client " << client_index << "/thread " << t << "] MISMATCH ticket=" << ticket - << ": expected=\"" << expected << "\" got=\"" << pending.result_value << "\""; + << ": expected=\"" << helper::ProcessRequest(workload_request).string_value + << "\" got=\"" << pending.result_value << "\""; Log(oss.str()); ++failures; continue; @@ -987,49 +904,9 @@ static bool RunClient(const int client_index, const int call_count, const int th Log(ss.str()); } - auto end = std::chrono::system_clock::now(); - auto elapsed = end - start; - times[c] = elapsed; - // std::this_thread::sleep_for(std::chrono::milliseconds(5000)); - } - - auto sum = std::accumulate(times.begin(), times.end(), std::chrono::duration::zero()); - - for (int c = 0; c < call_count; ++c) - { - std::ostringstream oss; - oss << "[client " << client_index << "/thread " << t << "] call " << (c + 1) << "/" << call_count - << ": " << std::chrono::duration_cast(times[c]).count() << " us"; - Log(oss.str()); - } - - if (failures == 0) - { - std::ostringstream oss; - oss << "- [client " + std::to_string(client_index) + "/thread " + std::to_string(t) + "] completed " + - std::to_string(call_count) + " calls with " + - std::to_string(std::chrono::duration_cast(sum).count()) + - " us elapsed, average " + - std::to_string(std::chrono::duration_cast(sum).count() / - call_count) + - " us per call"; - Log(oss.str()); - } - - if (failures == 0) - { - auto c = call_count - 1; - auto s = sum - times[0]; - - std::ostringstream oss; - oss << "SKIP FIRST [client " + std::to_string(client_index) + "/thread " + std::to_string(t) + - "] completed " + std::to_string(c) + " calls with " + - std::to_string(std::chrono::duration_cast(s).count()) + - " us elapsed, average " + - std::to_string(std::chrono::duration_cast(s).count() / c) + - " us per call"; - Log(oss.str()); + } + helper::WaitAfterCall(g_random_wait); } proxy.response.UnsetReceiveHandler(); @@ -1039,6 +916,12 @@ static bool RunClient(const int client_index, const int call_count, const int th }); } + thread_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_async_client_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + thread_start_barrier.Release(); + for (auto& th : threads) { th.join(); @@ -1064,71 +947,39 @@ static bool RunClient(const int client_index, const int call_count, const int th int main(int argc, char** argv) { - // Parse optional flags. - const std::string kClientPrefix{"--client_count="}; - const std::string kCallPrefix{"--call_count="}; - const std::string kClientThreadsPrefix{"--client_threads="}; - const std::string kServerThreadsPrefix{"--server_threads="}; - const std::string kSleepPrefix{"--sleep_milliseconds="}; - for (int i = 1; i < argc; ++i) + score::crypto::ipc::poc_helper::PocArguments defaults; + defaults.client_count = g_client_count; + defaults.call_count = g_call_count; + defaults.client_threads = g_client_threads; + defaults.server_threads = g_server_threads; + defaults.sleep_milliseconds = g_sleep_milliseconds; + std::string parse_error; + const auto parsed_arguments = score::crypto::ipc::poc_helper::ParseArguments(argc, argv, parse_error, defaults); + if (!parsed_arguments.has_value()) { - const std::string arg{argv[i]}; - try - { - if (arg.rfind(kClientPrefix, 0) == 0) - { - g_client_count = std::stoi(arg.substr(kClientPrefix.size())); - } - else if (arg.rfind(kCallPrefix, 0) == 0) - { - g_call_count = std::stoi(arg.substr(kCallPrefix.size())); - } - else if (arg.rfind(kClientThreadsPrefix, 0) == 0) - { - g_client_threads = std::stoi(arg.substr(kClientThreadsPrefix.size())); - } - else if (arg.rfind(kServerThreadsPrefix, 0) == 0) - { - g_server_threads = std::stoi(arg.substr(kServerThreadsPrefix.size())); - } - else if (arg.rfind(kSleepPrefix, 0) == 0) - { - g_sleep_milliseconds = std::stoi(arg.substr(kSleepPrefix.size())); - } - } - catch (const std::exception& ex) - { - std::fprintf(stderr, "[main] invalid argument '%s': %s\n", argv[i], ex.what()); - return 1; - } - } - if (g_client_count < 1) - { - std::fprintf(stderr, "[main] --client_count must be >= 1\n"); - return 1; - } - if (g_call_count < 1) - { - std::fprintf(stderr, "[main] --call_count must be >= 1\n"); + std::fprintf(stderr, "[main] %s\n", parse_error.c_str()); return 1; } - if (g_client_threads < 1) - { - std::fprintf(stderr, "[main] --client_threads must be >= 1\n"); - return 1; - } - if (g_server_threads < 1) - { - std::fprintf(stderr, "[main] --server_threads must be >= 1\n"); - return 1; - } - - std::printf("[main] client_count=%d call_count=%d client_threads=%d server_threads=%d sleep_milliseconds=%d\n", - g_client_count, - g_call_count, - g_client_threads, - g_server_threads, - g_sleep_milliseconds); + g_client_count = parsed_arguments->client_count; + g_call_count = parsed_arguments->call_count; + g_client_threads = parsed_arguments->client_threads; + g_server_threads = parsed_arguments->server_threads; + g_sleep_milliseconds = parsed_arguments->sleep_milliseconds; + g_random_wait = parsed_arguments->random_wait; + + std::cout << score::crypto::ipc::poc_helper::SettingsSummary( + "poc_async", + g_client_count, + g_call_count, + g_client_threads, + g_server_threads, + g_random_wait) + << '\n' + << std::flush; + Log("[main] client_count=" + std::to_string(g_client_count) + " call_count=" + + std::to_string(g_call_count) + " client_threads=" + std::to_string(g_client_threads) + + " server_threads=" + std::to_string(g_server_threads) + + " sleep_milliseconds=" + std::to_string(g_sleep_milliseconds)); score::crypto::ipc::control::SetupConfigs(g_client_count); diff --git a/score/tests/ipc_poc/poc_control.fbs b/score/tests/ipc_poc/poc_control.fbs index c64ccdaa4..ec55e7003 100644 --- a/score/tests/ipc_poc/poc_control.fbs +++ b/score/tests/ipc_poc/poc_control.fbs @@ -126,6 +126,10 @@ table ControlResponse { operation_batch: OperationResponseBatch; } +rpc_service PocControlService { + Execute(ControlRequest): ControlResponse; +} + // ControlRequest is the natural entry point for the serialised payload. // Use flatbuffers::GetSizePrefixedRoot for the response. root_type ControlRequest; diff --git a/score/tests/ipc_poc/poc_engine.cpp b/score/tests/ipc_poc/poc_engine.cpp new file mode 100644 index 000000000..4c2b47814 --- /dev/null +++ b/score/tests/ipc_poc/poc_engine.cpp @@ -0,0 +1,680 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This POC demonstrates a bootstrap connection that creates one on-demand +// message_passing endpoint/Engine session per client thread. Each server Engine +// performs the operation in its callback and returns the complete FlatBuffer +// response with Reply(). No application worker thread or Notify() path is used. +// ============================================================================= + +/// Usage: +/// bazel run //tests/score_com_poc:poc_engine +/// bazel run //tests/score_com_poc:poc_engine -- --call_count=5 +/// bazel run //tests/score_com_poc:poc_engine -- --client_threads=3 --call_count=5 +/// bazel run //tests/score_com_poc:poc_engine -- --call_count=3 --sleep_milliseconds=10 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "score/message_passing/client_factory.h" +#include "score/message_passing/i_client_connection.h" +#include "score/message_passing/i_server_connection.h" +#include "score/message_passing/server_factory.h" +#include "score/message_passing/service_protocol_config.h" +#include "score/tests/utility/runtime_measurement.hpp" +#include "score/tests/ipc_poc/ipc_buffer.h" +#include "score/tests/ipc_poc/poc_helper.hpp" +#include "score/tests/utility/process_resource_measurement.hpp" + +namespace score::crypto::ipc::control +{ + +namespace helper = score::crypto::ipc::poc_helper; + +static constexpr std::string_view kBootstrapServiceIdentifier{"score_crypto_poc_engine_bootstrap"}; + +static helper::PocArguments g_arguments{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static std::atomic g_session_counter{0U}; +static constexpr bool kEnableVerboseOutput{false}; +static constexpr bool kEnableLatencyVerbose{false}; + +static void Log(const std::string& line) +{ + if (!kEnableVerboseOutput) + { + return; + } + std::lock_guard lock{g_log_mutex}; + std::cout << line << "\n"; +} + +static void LogErr(const std::string& line) +{ + std::lock_guard lock{g_log_mutex}; + std::cerr << "[ERROR] " << line << "\n"; +} + +static std::string MakeSessionServiceIdentifier() +{ + const auto session_number = g_session_counter.fetch_add(1U, std::memory_order_relaxed); + return std::string{kBootstrapServiceIdentifier} + "_session_" + std::to_string(session_number); +} + +static score::message_passing::ServiceProtocolConfig MakeProtocolConfig(const std::string_view service_identifier) +{ + return score::message_passing::ServiceProtocolConfig{ + service_identifier, + /*max_send_size=*/static_cast(sizeof(IpcBuffer)), + /*max_reply_size=*/static_cast(sizeof(IpcBuffer)), + /*max_notify_size=*/static_cast(sizeof(IpcBuffer)), + }; +} + +static score::message_passing::IServerFactory::ServerConfig MakeBootstrapServerConfig(const int client_threads) +{ + return score::message_passing::IServerFactory::ServerConfig{ + /*max_queued_sends=*/static_cast(client_threads), + /*pre_alloc_connections=*/0U, + /*max_queued_notifies=*/0U, + }; +} + +static score::message_passing::IServerFactory::ServerConfig MakeSessionServerConfig() +{ + return score::message_passing::IServerFactory::ServerConfig{ + /*max_queued_sends=*/1U, + /*pre_alloc_connections=*/0U, + /*max_queued_notifies=*/0U, + }; +} + +static score::message_passing::IClientFactory::ClientConfig MakeClientConfig() +{ + return score::message_passing::IClientFactory::ClientConfig{ + /*max_async_replies=*/1U, + /*max_queued_sends=*/1U, + /*fully_ordered=*/true, + /*truly_async=*/true, + /*sync_first_connect=*/false, + }; +} + +struct EngineSession +{ + explicit EngineSession(std::string identifier) : endpoint_identifier{std::move(identifier)} {} + + std::string endpoint_identifier; + std::unique_ptr server_factory; + score::cpp::pmr::unique_ptr server; +}; + +struct BootstrapState +{ + explicit BootstrapState(const int maximum_sessions) : maximum_sessions{maximum_sessions} {} + + const int maximum_sessions; + std::mutex sessions_mutex; + std::condition_variable sessions_condition; + std::size_t ready_sessions{0U}; + std::vector> sessions; +}; + +static bool StartEngineSession(const std::shared_ptr& session) +{ + const auto connect_callback = [](score::message_passing::IServerConnection& connection) + -> score::cpp::expected { + std::ostringstream log; + log << "[server] accepted uid=" << connection.GetClientIdentity().uid; + Log(log.str()); + return score::message_passing::UserData{static_cast(nullptr)}; + }; + + const auto disconnect_callback = [](score::message_passing::IServerConnection& connection) { + Log("[server] client disconnected uid=" + std::to_string(connection.GetClientIdentity().uid)); + }; + + // This callback is executed by the server's message_passing Engine thread. + // Deliberately doing the operation here demonstrates the requested model: + // there is no POC-owned work queue and no application worker thread. + const auto request_callback = [](score::message_passing::IServerConnection& connection, + score::cpp::span message) + -> score::cpp::expected_blank { + helper::Response response; + if (!helper::ProcessRequestBytes(message.data(), message.size(), response)) + { + LogErr("[server/engine] request validation or processing failed"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + + const auto response_bytes = helper::BuildResponseBytes(response); + + if (g_arguments.sleep_milliseconds > 0) + { + Log("[server/engine] simulating work for " + std::to_string(g_arguments.sleep_milliseconds) + " ms"); + std::this_thread::sleep_for(std::chrono::milliseconds(g_arguments.sleep_milliseconds)); + } + + Log("[server/engine] processed request_id=" + std::to_string(response.request_id) + + ", replying with FlatBuffer"); + return connection.Reply(score::cpp::span{response_bytes.data(), response_bytes.size()}); + }; + + session->server_factory = std::make_unique(); + session->server = session->server_factory->Create( + MakeProtocolConfig(session->endpoint_identifier), MakeSessionServerConfig()); + if (!session->server) + { + LogErr("[server] failed to create endpoint " + session->endpoint_identifier); + return false; + } + + const auto start_result = session->server->StartListening( + connect_callback, disconnect_callback, /*sent_callback=*/{}, request_callback); + if (!start_result.has_value()) + { + LogErr("[server] failed to start endpoint " + session->endpoint_identifier); + return false; + } + + Log("[server] started endpoint " + session->endpoint_identifier); + return true; +} + +static bool RunServer(const pid_t client_pid, const int thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement setup_measurement{kEnableLatencyVerbose}; + auto bootstrap_state = std::make_shared(thread_count); + bootstrap_state->sessions.reserve(static_cast(thread_count)); + + const auto connect_callback = [](score::message_passing::IServerConnection& connection) + -> score::cpp::expected { + Log("[bootstrap] accepted uid=" + std::to_string(connection.GetClientIdentity().uid)); + return score::message_passing::UserData{static_cast(nullptr)}; + }; + + const auto disconnect_callback = [](score::message_passing::IServerConnection& connection) { + Log("[bootstrap] client disconnected uid=" + std::to_string(connection.GetClientIdentity().uid)); + }; + + const auto request_callback = [bootstrap_state]( + score::message_passing::IServerConnection& connection, + score::cpp::span message) + -> score::cpp::expected_blank { + helper::Request request; + if (!helper::ParseRequestBytes(message.data(), message.size(), request)) + { + LogErr("[bootstrap] invalid session request"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + + auto session = std::make_shared(MakeSessionServiceIdentifier()); + { + std::lock_guard lock{bootstrap_state->sessions_mutex}; + if (bootstrap_state->sessions.size() >= static_cast(bootstrap_state->maximum_sessions)) + { + LogErr("[bootstrap] session limit reached"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EBUSY)); + } + bootstrap_state->sessions.push_back(session); + } + + if (!StartEngineSession(session)) + { + LogErr("[bootstrap] Engine session failed to start"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EIO)); + } + + { + std::lock_guard lock{bootstrap_state->sessions_mutex}; + ++bootstrap_state->ready_sessions; + } + bootstrap_state->sessions_condition.notify_one(); + + const helper::Response response{request.request_id, session->endpoint_identifier}; + const auto response_bytes = helper::BuildResponseBytes(response); + return connection.Reply(score::cpp::span{response_bytes.data(), response_bytes.size()}); + }; + + auto bootstrap_factory = std::make_unique(); + auto bootstrap_server = bootstrap_factory->Create( + MakeProtocolConfig(kBootstrapServiceIdentifier), MakeBootstrapServerConfig(thread_count)); + if (!bootstrap_server) + { + LogErr("[bootstrap] failed to create endpoint"); + return false; + } + + const auto start_result = bootstrap_server->StartListening( + connect_callback, disconnect_callback, /*sent_callback=*/{}, request_callback); + if (!start_result.has_value()) + { + LogErr("[bootstrap] failed to start endpoint"); + return false; + } + Log("[bootstrap] started endpoint " + std::string{kBootstrapServiceIdentifier}); + + { + std::unique_lock lock{bootstrap_state->sessions_mutex}; + if (!bootstrap_state->sessions_condition.wait_for(lock, std::chrono::seconds(30), [&] { + return bootstrap_state->ready_sessions == static_cast(thread_count); + })) + { + LogErr("[server] timed out waiting for all Engine sessions to start"); + } + } + + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_engine_server_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + + int wait_status = 0; + const auto waited_pid = waitpid(client_pid, &wait_status, 0); + const bool client_ok = waited_pid == client_pid && WIFEXITED(wait_status) && WEXITSTATUS(wait_status) == 0; + if (client_ok) + { + Log("[server] client exited OK"); + } + else + { + LogErr("[server] client failed"); + } + + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_engine_server_workload", resources_after_thread_creation, resources_after_workload); + bootstrap_server->StopListening(); + for (const auto& session : bootstrap_state->sessions) + { + if (session->server) + { + session->server->StopListening(); + } + } + Log("[server] shutdown complete"); + return client_ok; +} + +struct ResponseState +{ + std::mutex mutex; + std::condition_variable condition; + bool ready{false}; + bool ok{false}; + std::string value; +}; + +static void SetResponse(ResponseState& state, const bool ok, std::string value) +{ + { + std::lock_guard lock{state.mutex}; + state.ok = ok; + state.value = std::move(value); + state.ready = true; + } + state.condition.notify_one(); +} + +static bool RunBootstrapHandshake(const int thread_index, std::string& endpoint_identifier) +{ + score::message_passing::ClientFactory client_factory; + auto client = client_factory.Create(MakeProtocolConfig(kBootstrapServiceIdentifier), MakeClientConfig()); + if (!client) + { + LogErr("[client " + std::to_string(thread_index) + "] failed to create bootstrap connection"); + return false; + } + + struct ConnectionState + { + std::mutex mutex; + std::condition_variable condition; + bool ready{false}; + bool stopped{false}; + } connection_state; + + client->Start( + [&connection_state](score::message_passing::IClientConnection::State state) { + std::lock_guard lock{connection_state.mutex}; + if (state == score::message_passing::IClientConnection::State::kReady) + { + connection_state.ready = true; + } + else if (state == score::message_passing::IClientConnection::State::kStopped) + { + connection_state.stopped = true; + } + connection_state.condition.notify_one(); + }, + /*notify_callback=*/{}); + + { + std::unique_lock lock{connection_state.mutex}; + if (!connection_state.condition.wait_for(lock, std::chrono::seconds(30), [&] { + return connection_state.ready; + })) + { + LogErr("[client " + std::to_string(thread_index) + "] bootstrap connection timed out"); + client->Stop(); + return false; + } + } + + auto response_state = std::make_shared(); + const auto request = helper::CreateRequest(0, thread_index, 0); + const auto request_bytes = helper::BuildRequestBytes(request); + { + const auto send_result = client->SendWithCallback( + score::cpp::span{request_bytes.data(), request_bytes.size()}, + [response_state, request_id = request.request_id]( + score::cpp::expected, score::os::Error> response) { + if (!response.has_value()) + { + SetResponse(*response_state, false, {}); + return; + } + helper::Response parsed_response; + if (!helper::ParseResponseBytes(response->data(), response->size(), parsed_response) || + parsed_response.request_id != request_id) + { + SetResponse(*response_state, false, {}); + return; + } + SetResponse(*response_state, true, std::move(parsed_response.string_value)); + }); + if (!send_result.has_value()) + { + LogErr("[client " + std::to_string(thread_index) + "] bootstrap request failed"); + client->Stop(); + return false; + } + } + + { + std::unique_lock lock{response_state->mutex}; + if (!response_state->condition.wait_for(lock, std::chrono::seconds(30), [&] { + return response_state->ready; + }) || !response_state->ok) + { + LogErr("[client " + std::to_string(thread_index) + "] bootstrap response timed out"); + client->Stop(); + return false; + } + endpoint_identifier = response_state->value; + } + + client->Stop(); + { + std::unique_lock lock{connection_state.mutex}; + if (!connection_state.condition.wait_for(lock, std::chrono::seconds(30), [&] { + return connection_state.stopped; + })) + { + LogErr("[client " + std::to_string(thread_index) + "] bootstrap shutdown timed out"); + return false; + } + } + return true; +} + +static bool RunClientThread(const int thread_index, + const int call_count, + score::crypto::daemon::common::RuntimeMeasurement& latency_measurement, + helper::ThreadStartBarrier& bootstrap_ready_barrier, + helper::ThreadStartBarrier& connection_ready_barrier) +{ + std::string endpoint_identifier; + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement setup_scope{ + latency_measurement, "POC::Engine::ClientThreadCreationAndBootstrapHandshake"}; + if (!RunBootstrapHandshake(thread_index, endpoint_identifier)) + { + bootstrap_ready_barrier.ArriveAndWait(); + connection_ready_barrier.ArriveAndWait(); + return false; + } + bootstrap_ready_barrier.ArriveAndWait(); + } + + score::message_passing::ClientFactory client_factory; + decltype(client_factory.Create(MakeProtocolConfig(endpoint_identifier), MakeClientConfig())) client; + + struct ConnectionState + { + std::mutex mutex; + std::condition_variable condition; + bool ready{false}; + bool stopped{false}; + } connection_state; + + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement setup_scope{ + latency_measurement, "POC::Engine::ClientConnectionSetup"}; + client = client_factory.Create(MakeProtocolConfig(endpoint_identifier), MakeClientConfig()); + if (!client) + { + LogErr("[client " + std::to_string(thread_index) + "] failed to create connection"); + connection_ready_barrier.ArriveAndWait(); + return false; + } + + client->Start( + [&connection_state, thread_index](score::message_passing::IClientConnection::State state) { + std::lock_guard lock{connection_state.mutex}; + if (state == score::message_passing::IClientConnection::State::kReady) + { + connection_state.ready = true; + connection_state.condition.notify_one(); + Log("[client " + std::to_string(thread_index) + "] connection ready"); + } + else if (state == score::message_passing::IClientConnection::State::kStopped) + { + connection_state.stopped = true; + connection_state.condition.notify_one(); + } + }, + // The response callback also runs on the client Engine thread. + [](score::cpp::span) {}); + + std::unique_lock lock{connection_state.mutex}; + if (!connection_state.condition.wait_for(lock, std::chrono::seconds(30), [&] { + return connection_state.ready; + })) + { + LogErr("[client " + std::to_string(thread_index) + "] timed out waiting for connection"); + client->Stop(); + connection_ready_barrier.ArriveAndWait(); + return false; + } + } + + connection_ready_barrier.ArriveAndWait(); + + bool success = true; + for (int call = 0; call < call_count; ++call) + { + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement latency_scope{ + latency_measurement, "POC::Engine::RoundTrip"}; + + const auto workload_request = helper::CreateRequest(0, thread_index, call); + const auto request_id = workload_request.request_id; + const auto request_bytes = helper::BuildRequestBytes(workload_request); + auto response_state = std::make_shared(); + + Log("[client] SendWithCallback request_id=" + std::to_string(request_id)); + const auto send_result = client->SendWithCallback( + score::cpp::span{request_bytes.data(), request_bytes.size()}, + [response_state, request_id]( + score::cpp::expected, score::os::Error> response) { + if (!response.has_value()) + { + SetResponse(*response_state, false, {}); + LogErr("[client/engine] ReplyCallback failed for request_id=" + std::to_string(request_id)); + return; + } + + helper::Response parsed_response; + if (!helper::ParseResponseBytes(response->data(), response->size(), parsed_response) || + parsed_response.request_id != request_id) + { + SetResponse(*response_state, false, {}); + LogErr("[client/engine] response string missing for request_id=" + + std::to_string(request_id)); + return; + } + SetResponse(*response_state, true, std::move(parsed_response.string_value)); + }); + + if (!send_result.has_value()) + { + LogErr("[client " + std::to_string(thread_index) + "] SendWithCallback failed for request_id=" + + std::to_string(request_id)); + success = false; + break; + } + + std::unique_lock lock{response_state->mutex}; + const bool received = response_state->condition.wait_for(lock, std::chrono::seconds(30), [&] { + return response_state->ready; + }); + if (!received || !response_state->ok || + !helper::Matches(workload_request, helper::Response{request_id, response_state->value})) + { + LogErr("[client " + std::to_string(thread_index) + "] response mismatch or timeout for request_id=" + + std::to_string(request_id)); + success = false; + break; + } + } + + helper::WaitAfterCall(g_arguments.random_wait); + } + + client->Stop(); + { + std::unique_lock lock{connection_state.mutex}; + if (!connection_state.condition.wait_for(lock, std::chrono::seconds(30), [&] { + return connection_state.stopped; + })) + { + LogErr("[client] timed out waiting for connection shutdown"); + success = false; + } + } + return success; +} + +static bool RunClient(const int thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; + std::atomic failures{0}; + std::vector threads; + threads.reserve(static_cast(thread_count)); + helper::ThreadStartBarrier thread_start_barrier{static_cast(thread_count)}; + helper::ThreadStartBarrier bootstrap_ready_barrier{static_cast(thread_count)}; + helper::ThreadStartBarrier connection_ready_barrier{static_cast(thread_count)}; + + for (int thread_index = 0; thread_index < thread_count; ++thread_index) + { + threads.emplace_back([&, thread_index] { + thread_start_barrier.ArriveAndWait(); + if (!RunClientThread(thread_index, + g_arguments.call_count, + latency_measurement, + bootstrap_ready_barrier, + connection_ready_barrier)) + { + failures.fetch_add(1, std::memory_order_relaxed); + } + }); + } + + thread_start_barrier.WaitForAll(); + thread_start_barrier.Release(); + bootstrap_ready_barrier.WaitForAll(); + bootstrap_ready_barrier.Release(); + + connection_ready_barrier.WaitForAll(); + const auto resources_after_connection_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta("poc_engine_client_connection_thread_creation", + resources_before_thread_creation, + resources_after_connection_thread_creation); + connection_ready_barrier.Release(); + + for (auto& thread : threads) + { + thread.join(); + } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_engine_client_workload", resources_after_connection_thread_creation, resources_after_workload); + return failures.load(std::memory_order_relaxed) == 0; +} + +} // namespace score::crypto::ipc::control + +int main(int argc, char** argv) +{ + std::string parse_error; + const auto parsed_arguments = score::crypto::ipc::poc_helper::ParseArguments(argc, argv, parse_error); + if (!parsed_arguments.has_value()) + { + score::crypto::ipc::control::LogErr("[main] " + parse_error); + return 1; + } + score::crypto::ipc::control::g_arguments = *parsed_arguments; + + std::cout << score::crypto::ipc::poc_helper::SettingsSummary( + "poc_engine", + 1, + score::crypto::ipc::control::g_arguments.call_count, + score::crypto::ipc::control::g_arguments.client_threads, + score::crypto::ipc::control::g_arguments.client_threads, + score::crypto::ipc::control::g_arguments.random_wait) + << '\n' + << std::flush; + const auto client_pid = ::fork(); + if (client_pid < 0) + { + std::perror("[main] fork"); + return 1; + } + if (client_pid == 0) + { + return score::crypto::ipc::control::RunClient(score::crypto::ipc::control::g_arguments.client_threads) ? 0 : 1; + } + return score::crypto::ipc::control::RunServer(client_pid, score::crypto::ipc::control::g_arguments.client_threads) ? 0 : 1; +} diff --git a/score/tests/ipc_poc/poc_engine_sync.cpp b/score/tests/ipc_poc/poc_engine_sync.cpp new file mode 100644 index 000000000..e1b542409 --- /dev/null +++ b/score/tests/ipc_poc/poc_engine_sync.cpp @@ -0,0 +1,606 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This POC mirrors poc_engine, but uses a bootstrap connection to create one +// on-demand message_passing endpoint/Engine session per client thread. Each +// child connection uses blocking IClientConnection::SendWaitReply(). +// ============================================================================= + +/// Usage: +/// bazel run //tests/score_com_poc:poc_engine_sync +/// bazel run //tests/score_com_poc:poc_engine_sync -- --call_count=5 +/// bazel run //tests/score_com_poc:poc_engine_sync -- --client_threads=3 --call_count=5 +/// bazel run //tests/score_com_poc:poc_engine_sync -- --call_count=3 --sleep_milliseconds=10 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "score/message_passing/client_factory.h" +#include "score/message_passing/i_client_connection.h" +#include "score/message_passing/i_server_connection.h" +#include "score/message_passing/server_factory.h" +#include "score/message_passing/service_protocol_config.h" +#include "score/tests/utility/runtime_measurement.hpp" +#include "score/tests/ipc_poc/ipc_buffer.h" +#include "score/tests/ipc_poc/poc_helper.hpp" +#include "score/tests/utility/process_resource_measurement.hpp" + +namespace score::crypto::ipc::control +{ + +namespace helper = score::crypto::ipc::poc_helper; + +static constexpr std::string_view kBootstrapServiceIdentifier{"score_crypto_poc_engine_sync_bootstrap"}; + +static helper::PocArguments g_arguments{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static std::atomic g_session_counter{0U}; +static constexpr bool kEnableVerboseOutput{false}; +static constexpr bool kEnableLatencyVerbose{false}; + +static void Log(const std::string& line) +{ + if (!kEnableVerboseOutput) + { + return; + } + std::lock_guard lock{g_log_mutex}; + std::cout << line << "\n"; +} + +static void LogErr(const std::string& line) +{ + std::lock_guard lock{g_log_mutex}; + std::cerr << "[ERROR] " << line << "\n"; +} + +static std::string MakeSessionServiceIdentifier() +{ + const auto session_number = g_session_counter.fetch_add(1U, std::memory_order_relaxed); + return std::string{kBootstrapServiceIdentifier} + "_session_" + std::to_string(session_number); +} + +static score::message_passing::ServiceProtocolConfig MakeProtocolConfig(const std::string_view service_identifier) +{ + return score::message_passing::ServiceProtocolConfig{ + service_identifier, + /*max_send_size=*/static_cast(sizeof(IpcBuffer)), + /*max_reply_size=*/static_cast(sizeof(IpcBuffer)), + /*max_notify_size=*/static_cast(sizeof(IpcBuffer)), + }; +} + +static score::message_passing::IServerFactory::ServerConfig MakeBootstrapServerConfig(const int client_threads) +{ + return score::message_passing::IServerFactory::ServerConfig{ + /*max_queued_sends=*/static_cast(client_threads), + /*pre_alloc_connections=*/0U, + /*max_queued_notifies=*/0U, + }; +} + +static score::message_passing::IServerFactory::ServerConfig MakeSessionServerConfig() +{ + return score::message_passing::IServerFactory::ServerConfig{ + /*max_queued_sends=*/1U, + /*pre_alloc_connections=*/0U, + /*max_queued_notifies=*/0U, + }; +} + +static score::message_passing::IClientFactory::ClientConfig MakeClientConfig() +{ + return score::message_passing::IClientFactory::ClientConfig{ + /*max_async_replies=*/1U, + /*max_queued_sends=*/1U, + /*fully_ordered=*/true, + /*truly_async=*/false, + /*sync_first_connect=*/false, + }; +} + +struct EngineSession +{ + explicit EngineSession(std::string identifier) : endpoint_identifier{std::move(identifier)} {} + + std::string endpoint_identifier; + std::unique_ptr server_factory; + score::cpp::pmr::unique_ptr server; +}; + +struct BootstrapState +{ + explicit BootstrapState(const int maximum_sessions) : maximum_sessions{maximum_sessions} {} + + const int maximum_sessions; + std::mutex sessions_mutex; + std::condition_variable sessions_condition; + std::size_t ready_sessions{0U}; + std::vector> sessions; +}; + +static bool StartEngineSession(const std::shared_ptr& session) +{ + const auto connect_callback = [](score::message_passing::IServerConnection& connection) + -> score::cpp::expected { + std::ostringstream log; + log << "[server] accepted uid=" << connection.GetClientIdentity().uid; + Log(log.str()); + return score::message_passing::UserData{static_cast(nullptr)}; + }; + + const auto disconnect_callback = [](score::message_passing::IServerConnection& connection) { + Log("[server] client disconnected uid=" + std::to_string(connection.GetClientIdentity().uid)); + }; + + const auto request_callback = [](score::message_passing::IServerConnection& connection, + score::cpp::span message) + -> score::cpp::expected_blank { + helper::Response workload_response; + if (!helper::ProcessRequestBytes(message.data(), message.size(), workload_response)) + { + LogErr("[server/engine] request validation or processing failed"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + + const auto response_bytes = helper::BuildResponseBytes(workload_response); + if (g_arguments.sleep_milliseconds > 0) + { + Log("[server/engine] simulating work for " + std::to_string(g_arguments.sleep_milliseconds) + " ms"); + std::this_thread::sleep_for(std::chrono::milliseconds(g_arguments.sleep_milliseconds)); + } + + Log("[server/engine] processed request_id=" + std::to_string(workload_response.request_id) + + ", replying with FlatBuffer"); + return connection.Reply(score::cpp::span{response_bytes.data(), response_bytes.size()}); + }; + + session->server_factory = std::make_unique(); + session->server = session->server_factory->Create( + MakeProtocolConfig(session->endpoint_identifier), MakeSessionServerConfig()); + if (!session->server) + { + LogErr("[server] failed to create endpoint " + session->endpoint_identifier); + return false; + } + + const auto start_result = session->server->StartListening( + connect_callback, disconnect_callback, /*sent_callback=*/{}, request_callback); + if (!start_result.has_value()) + { + LogErr("[server] failed to start endpoint " + session->endpoint_identifier); + return false; + } + + Log("[server] started endpoint " + session->endpoint_identifier); + return true; +} + +static bool RunServer(const pid_t client_pid, const int thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement setup_measurement{kEnableLatencyVerbose}; + auto bootstrap_state = std::make_shared(thread_count); + bootstrap_state->sessions.reserve(static_cast(thread_count)); + + const auto connect_callback = [](score::message_passing::IServerConnection& connection) + -> score::cpp::expected { + Log("[bootstrap] accepted uid=" + std::to_string(connection.GetClientIdentity().uid)); + return score::message_passing::UserData{static_cast(nullptr)}; + }; + + const auto disconnect_callback = [](score::message_passing::IServerConnection& connection) { + Log("[bootstrap] client disconnected uid=" + std::to_string(connection.GetClientIdentity().uid)); + }; + + const auto request_callback = [bootstrap_state]( + score::message_passing::IServerConnection& connection, + score::cpp::span message) + -> score::cpp::expected_blank { + helper::Request request; + if (!helper::ParseRequestBytes(message.data(), message.size(), request)) + { + LogErr("[bootstrap] invalid session request"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + + auto session = std::make_shared(MakeSessionServiceIdentifier()); + { + std::lock_guard lock{bootstrap_state->sessions_mutex}; + if (bootstrap_state->sessions.size() >= static_cast(bootstrap_state->maximum_sessions)) + { + LogErr("[bootstrap] session limit reached"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EBUSY)); + } + bootstrap_state->sessions.push_back(session); + } + + if (!StartEngineSession(session)) + { + LogErr("[bootstrap] Engine session failed to start"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EIO)); + } + + { + std::lock_guard lock{bootstrap_state->sessions_mutex}; + ++bootstrap_state->ready_sessions; + } + bootstrap_state->sessions_condition.notify_one(); + + const helper::Response response{request.request_id, session->endpoint_identifier}; + const auto response_bytes = helper::BuildResponseBytes(response); + return connection.Reply(score::cpp::span{response_bytes.data(), response_bytes.size()}); + }; + + auto bootstrap_factory = std::make_unique(); + auto bootstrap_server = bootstrap_factory->Create( + MakeProtocolConfig(kBootstrapServiceIdentifier), MakeBootstrapServerConfig(thread_count)); + if (!bootstrap_server) + { + LogErr("[bootstrap] failed to create endpoint"); + return false; + } + + const auto start_result = bootstrap_server->StartListening( + connect_callback, disconnect_callback, /*sent_callback=*/{}, request_callback); + if (!start_result.has_value()) + { + LogErr("[bootstrap] failed to start endpoint"); + return false; + } + Log("[bootstrap] started endpoint " + std::string{kBootstrapServiceIdentifier}); + + { + std::unique_lock lock{bootstrap_state->sessions_mutex}; + if (!bootstrap_state->sessions_condition.wait_for(lock, std::chrono::seconds(30), [&] { + return bootstrap_state->ready_sessions == static_cast(thread_count); + })) + { + LogErr("[server] timed out waiting for all Engine sessions to start"); + } + } + + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta("poc_engine_sync_server_thread_creation", + resources_before_thread_creation, + resources_after_thread_creation); + + int wait_status = 0; + const auto waited_pid = waitpid(client_pid, &wait_status, 0); + const bool client_ok = waited_pid == client_pid && WIFEXITED(wait_status) && WEXITSTATUS(wait_status) == 0; + if (client_ok) + { + Log("[server] client exited OK"); + } + else + { + LogErr("[server] client failed"); + } + + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_engine_sync_server_workload", resources_after_thread_creation, resources_after_workload); + bootstrap_server->StopListening(); + for (const auto& session : bootstrap_state->sessions) + { + if (session->server) + { + session->server->StopListening(); + } + } + Log("[server] shutdown complete"); + return client_ok; +} + +struct ConnectionState +{ + std::mutex mutex; + std::condition_variable condition; + bool ready{false}; + bool stopped{false}; +}; + +static bool RunBootstrapHandshake(const int thread_index, std::string& endpoint_identifier) +{ + score::message_passing::ClientFactory client_factory; + auto client = client_factory.Create(MakeProtocolConfig(kBootstrapServiceIdentifier), MakeClientConfig()); + if (!client) + { + LogErr("[client " + std::to_string(thread_index) + "] failed to create bootstrap connection"); + return false; + } + + ConnectionState connection_state; + client->Start( + [&connection_state](score::message_passing::IClientConnection::State state) { + std::lock_guard lock{connection_state.mutex}; + if (state == score::message_passing::IClientConnection::State::kReady) + { + connection_state.ready = true; + } + else if (state == score::message_passing::IClientConnection::State::kStopped) + { + connection_state.stopped = true; + } + connection_state.condition.notify_one(); + }, + /*notify_callback=*/{}); + + { + std::unique_lock lock{connection_state.mutex}; + if (!connection_state.condition.wait_for(lock, std::chrono::seconds(30), [&] { + return connection_state.ready; + })) + { + LogErr("[client " + std::to_string(thread_index) + "] bootstrap connection timed out"); + client->Stop(); + return false; + } + } + + const auto request = helper::CreateRequest(0, thread_index, 0); + const auto request_bytes = helper::BuildRequestBytes(request); + std::vector response_buffer(sizeof(IpcBuffer)); + const auto response = client->SendWaitReply( + score::cpp::span{request_bytes.data(), request_bytes.size()}, + score::cpp::span{response_buffer.data(), response_buffer.size()}); + if (!response.has_value()) + { + LogErr("[client " + std::to_string(thread_index) + "] bootstrap request failed"); + client->Stop(); + return false; + } + + helper::Response parsed_response; + if (!helper::ParseResponseBytes(response->data(), response->size(), parsed_response) || + parsed_response.request_id != request.request_id || parsed_response.string_value.empty()) + { + LogErr("[client " + std::to_string(thread_index) + "] invalid bootstrap response"); + client->Stop(); + return false; + } + endpoint_identifier = std::move(parsed_response.string_value); + + client->Stop(); + { + std::unique_lock lock{connection_state.mutex}; + if (!connection_state.condition.wait_for(lock, std::chrono::seconds(30), [&] { + return connection_state.stopped; + })) + { + LogErr("[client " + std::to_string(thread_index) + "] bootstrap shutdown timed out"); + return false; + } + } + return true; +} + +static bool RunClientThread(const int thread_index, + const int call_count, + score::crypto::daemon::common::RuntimeMeasurement& latency_measurement, + helper::ThreadStartBarrier& bootstrap_ready_barrier, + helper::ThreadStartBarrier& connection_ready_barrier) +{ + std::string endpoint_identifier; + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement setup_scope{ + latency_measurement, "POC::EngineSync::ClientThreadCreationAndBootstrapHandshake"}; + if (!RunBootstrapHandshake(thread_index, endpoint_identifier)) + { + bootstrap_ready_barrier.ArriveAndWait(); + connection_ready_barrier.ArriveAndWait(); + return false; + } + bootstrap_ready_barrier.ArriveAndWait(); + } + + score::message_passing::ClientFactory client_factory; + decltype(client_factory.Create(MakeProtocolConfig(endpoint_identifier), MakeClientConfig())) client; + + ConnectionState connection_state; + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement setup_scope{ + latency_measurement, "POC::EngineSync::ClientConnectionSetup"}; + client = client_factory.Create(MakeProtocolConfig(endpoint_identifier), MakeClientConfig()); + if (!client) + { + LogErr("[client " + std::to_string(thread_index) + "] failed to create connection"); + connection_ready_barrier.ArriveAndWait(); + return false; + } + + client->Start( + [&connection_state, thread_index](score::message_passing::IClientConnection::State state) { + std::lock_guard lock{connection_state.mutex}; + if (state == score::message_passing::IClientConnection::State::kReady) + { + connection_state.ready = true; + connection_state.condition.notify_one(); + Log("[client " + std::to_string(thread_index) + "] connection ready"); + } + else if (state == score::message_passing::IClientConnection::State::kStopped) + { + connection_state.stopped = true; + connection_state.condition.notify_one(); + } + }, + /*notify_callback=*/{}); + + std::unique_lock lock{connection_state.mutex}; + if (!connection_state.condition.wait_for(lock, std::chrono::seconds(30), [&] { + return connection_state.ready; + })) + { + LogErr("[client " + std::to_string(thread_index) + "] timed out waiting for connection"); + client->Stop(); + connection_ready_barrier.ArriveAndWait(); + return false; + } + } + + connection_ready_barrier.ArriveAndWait(); + + bool success = true; + std::vector reply_buffer(sizeof(IpcBuffer)); + + for (int call = 0; call < call_count; ++call) + { + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement latency_scope{ + latency_measurement, "POC::EngineSync::RoundTrip"}; + const auto workload_request = helper::CreateRequest(0, thread_index, call); + const auto request_id = workload_request.request_id; + const auto request_bytes = helper::BuildRequestBytes(workload_request); + + Log("[client] SendWaitReply request_id=" + std::to_string(request_id)); + const auto response = client->SendWaitReply( + score::cpp::span{request_bytes.data(), request_bytes.size()}, + score::cpp::span{reply_buffer.data(), reply_buffer.size()}); + if (!response.has_value()) + { + LogErr("[client " + std::to_string(thread_index) + "] SendWaitReply failed for request_id=" + + std::to_string(request_id)); + success = false; + break; + } + + helper::Response parsed_response; + if (!helper::ParseResponseBytes(response->data(), response->size(), parsed_response) || + !helper::Matches(workload_request, parsed_response)) + { + LogErr("[client " + std::to_string(thread_index) + "] invalid or unexpected response for request_id=" + + std::to_string(request_id)); + success = false; + break; + } + + } + + helper::WaitAfterCall(g_arguments.random_wait); + } + + client->Stop(); + { + std::unique_lock lock{connection_state.mutex}; + if (!connection_state.condition.wait_for(lock, std::chrono::seconds(30), [&] { + return connection_state.stopped; + })) + { + LogErr("[client] timed out waiting for connection shutdown"); + success = false; + } + } + return success; +} + +static bool RunClient(const int thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; + std::atomic failures{0}; + std::vector threads; + threads.reserve(static_cast(thread_count)); + helper::ThreadStartBarrier thread_start_barrier{static_cast(thread_count)}; + helper::ThreadStartBarrier bootstrap_ready_barrier{static_cast(thread_count)}; + helper::ThreadStartBarrier connection_ready_barrier{static_cast(thread_count)}; + + for (int thread_index = 0; thread_index < thread_count; ++thread_index) + { + threads.emplace_back([&, thread_index] { + thread_start_barrier.ArriveAndWait(); + if (!RunClientThread(thread_index, + g_arguments.call_count, + latency_measurement, + bootstrap_ready_barrier, + connection_ready_barrier)) + { + failures.fetch_add(1, std::memory_order_relaxed); + } + }); + } + + thread_start_barrier.WaitForAll(); + thread_start_barrier.Release(); + bootstrap_ready_barrier.WaitForAll(); + bootstrap_ready_barrier.Release(); + + connection_ready_barrier.WaitForAll(); + const auto resources_after_connection_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta("poc_engine_sync_client_connection_thread_creation", + resources_before_thread_creation, + resources_after_connection_thread_creation); + connection_ready_barrier.Release(); + + for (auto& thread : threads) + { + thread.join(); + } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_engine_sync_client_workload", resources_after_connection_thread_creation, resources_after_workload); + return failures.load(std::memory_order_relaxed) == 0; +} + +} // namespace score::crypto::ipc::control + +int main(int argc, char** argv) +{ + std::string parse_error; + const auto parsed_arguments = score::crypto::ipc::poc_helper::ParseArguments(argc, argv, parse_error); + if (!parsed_arguments.has_value()) + { + score::crypto::ipc::control::LogErr("[main] " + parse_error); + return 1; + } + score::crypto::ipc::control::g_arguments = *parsed_arguments; + + std::cout << score::crypto::ipc::poc_helper::SettingsSummary( + "poc_engine_sync", + 1, + score::crypto::ipc::control::g_arguments.call_count, + score::crypto::ipc::control::g_arguments.client_threads, + score::crypto::ipc::control::g_arguments.client_threads, + score::crypto::ipc::control::g_arguments.random_wait) + << '\n' + << std::flush; + const auto client_pid = ::fork(); + if (client_pid < 0) + { + std::perror("[main] fork"); + return 1; + } + if (client_pid == 0) + { + return score::crypto::ipc::control::RunClient(score::crypto::ipc::control::g_arguments.client_threads) ? 0 : 1; + } + return score::crypto::ipc::control::RunServer(client_pid, score::crypto::ipc::control::g_arguments.client_threads) ? 0 : 1; +} diff --git a/score/tests/ipc_poc/poc_grpc.cpp b/score/tests/ipc_poc/poc_grpc.cpp index 7ebdf89f4..ade6253e5 100644 --- a/score/tests/ipc_poc/poc_grpc.cpp +++ b/score/tests/ipc_poc/poc_grpc.cpp @@ -19,190 +19,138 @@ // removed once confidence has been gained in the proper implementation. // ============================================================================= -/// POC: gRPC round-trip benchmark — standalone, no daemon or crypto specifics +/// POC: standalone gRPC round-trip benchmark. /// -/// Mirrors poc_low_level.cpp in structure: -/// - fork model: parent = server, children = clients -/// - same CLI flags: --client_count, --call_count, --client_threads, -/// --server_threads (accepted but unused — gRPC controls -/// its own thread pool), --sleep_milliseconds -/// - same per-thread timings and "SKIP FIRST" summary output -/// -/// Communication model: -/// Each client thread calls GrpcControlClient::SendRequest() synchronously. -/// The call is blocking: one request in → one response out. No ticket map -/// or correlation logic is needed because ordering is guaranteed by gRPC. -/// -/// The server's EchoRequestHandler concatenates the two input parameters -/// ("_") and returns the result as an OwnedString in the -/// response, mirroring what poc_low_level does with its FlatBuffer echo. -/// -/// Handler chain: -/// EchoHandlerFactory → EchoRequestHandler -/// No daemon, no data_manager, no crypto, no config. -/// -/// Note on --server_threads: -/// GrpcControlServer currently hard-codes MIN/MAX_POLLERS = 1. The flag -/// is accepted for CLI parity with the other POCs but has no effect on the -/// actual gRPC thread count. -/// -/// Usage: -/// bazel run //tests/score_com_poc:poc_grpc -/// bazel run //tests/score_com_poc:poc_grpc -- --client_count=3 --call_count=5 -/// bazel run //tests/score_com_poc:poc_grpc -- --client_count=3 --call_count=5 --client_threads=4 -/// bazel run //tests/score_com_poc:poc_grpc -- --sleep_milliseconds=50 +/// This POC uses the generated service from poc_control.fbs directly. It does +/// not use the production daemon control-plane adapter or daemon conversions. #include #include #include -#include #include #include #include +#include #include #include #include #include -#include #include -#include "score/crypto/src/daemon/control_plane/control_protocol.h" -#include "score/crypto/src/daemon/control_plane/i_handler_chain_factory.hpp" -#include "score/crypto/src/daemon/control_plane/i_request_handler.hpp" -#include "score/crypto/src/ipc/grpc_adapter/grpc_control_client.h" -#include "score/crypto/src/ipc/grpc_adapter/grpc_control_server.h" +#include "flatbuffers/grpc.h" +#include "grpcpp/grpcpp.h" +#include "score/tests/utility/runtime_measurement.hpp" +#include "score/tests/ipc_poc/poc_control.grpc.fb.h" +#include "score/tests/ipc_poc/poc_control_generated.h" +#include "score/tests/ipc_poc/poc_helper.hpp" +#include "score/tests/utility/process_resource_measurement.hpp" // --------------------------------------------------------------------------- // Global parameters (set before fork; never mutated after) // --------------------------------------------------------------------------- -static int g_client_count = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -static int g_call_count = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -static int g_client_threads = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -static int g_server_threads = 8; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -// --------------------------------------------------------------------------- -// Shared logging helper -// --------------------------------------------------------------------------- +static int g_client_count = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_call_count = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_threads = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_server_threads = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static bool g_random_wait = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static constexpr bool kEnableVerboseOutput{false}; +static constexpr bool kEnableLatencyVerbose{false}; static void Log(const std::string& line) { - std::lock_guard lk(g_log_mutex); + if (!kEnableVerboseOutput) + { + return; + } + std::lock_guard lock{g_log_mutex}; std::cout << line << "\n"; } -// --------------------------------------------------------------------------- -// Echo handler — concatenates "_" and returns it -// No daemon, no data_manager, no config -// --------------------------------------------------------------------------- +static void LogErr(const std::string& line) +{ + std::lock_guard lock{g_log_mutex}; + std::cerr << "[ERROR] " << line << "\n"; +} namespace score::crypto::poc::grpc { -namespace proto = daemon::control_plane::protocol; +namespace control = score::crypto::ipc::control; +namespace helper = score::crypto::ipc::poc_helper; -class EchoRequestHandler : public daemon::control_plane::IRequestHandler +class PocControlService final : public control::PocControlService::Service { public: - daemon::control_plane::ControlResponse processRequest(daemon::control_plane::ControlRequest& request) override + ::grpc::Status Execute(::grpc::ServerContext* /*context*/, + const flatbuffers::grpc::Message* request, + flatbuffers::grpc::Message* response) override { - proto::ControlResponse response; - response.request_id = request.request_id; - - if (request.operation.operations.empty()) + const auto* request_root = request->GetRoot(); + helper::Request workload_request; + if (!helper::ParseRequestRoot(request_root, workload_request)) { - return response; + return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, "Invalid request"}; } - const auto& op = request.operation.operations[0]; - - // Extract string (arrives as string_view into the FlatBuffer — safe here, we copy it) - std::string str_value; - if (!op.parameters.empty() && std::holds_alternative(op.parameters[0])) - { - str_value = std::string(std::get(op.parameters[0])); - } - - // Extract uint64 - std::uint64_t uint64_value = 0U; - if (op.parameters.size() >= 2 && std::holds_alternative(op.parameters[1])) - { - uint64_value = std::get(op.parameters[1]); - } + const auto workload_response = helper::ProcessRequest(workload_request); if (g_sleep_milliseconds > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(g_sleep_milliseconds)); } - const std::string combined = str_value + "_" + std::to_string(uint64_value); + Log("[server] request_id=" + std::to_string(workload_response.request_id) + " -> combined=\"" + + workload_response.string_value + "\""); - { - std::ostringstream ss; - ss << "[server/handler] request_id=" << request.request_id << " -> combined=\"" << combined << "\""; - Log(ss.str()); - } - - // Return combined string as OwnedString and the uint64 echo. - // OperationResponseBuilder has no return_value_string(), so we push the - // OwnedString directly into the last operation's parameters after build(). - proto::OperationResponseBuilder builder; - builder.operation(op.operationId).return_success().return_value_uint64(uint64_value); - - auto built = builder.build(); - if (built.has_value()) - { - if (!built.value().operations.empty()) - { - built.value().operations.back().parameters.push_back(proto::OwnedString{combined}); - } - response.operation = std::move(built.value()); - } - - return response; + flatbuffers::grpc::MessageBuilder builder; + builder.Finish(helper::CreateResponseTable(builder, workload_response)); + *response = builder.GetMessage(); + return ::grpc::Status::OK; } }; -class EchoHandlerFactory : public daemon::control_plane::IHandlerChainFactory +static int RunServer(const std::string& socket_path, const std::vector& child_pids) { - public: - std::unique_ptr CreateRequestHandler() override + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement setup_measurement{kEnableLatencyVerbose}; + ::unlink(socket_path.c_str()); + + PocControlService service; + ::grpc::ServerBuilder builder; + builder.AddListeningPort("unix:" + socket_path, ::grpc::InsecureServerCredentials()); + builder.RegisterService(&service); + builder.SetSyncServerOption(::grpc::ServerBuilder::SyncServerOption::MIN_POLLERS, g_server_threads); + builder.SetSyncServerOption(::grpc::ServerBuilder::SyncServerOption::MAX_POLLERS, g_server_threads); + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + setup_measurement, "POC::Grpc::ServerThreadCreation"}; + auto server = builder.BuildAndStart(); + if (!server) { - return std::make_unique(); + LogErr("Failed to start standalone gRPC server on " + socket_path); + return 1; } -}; - -// --------------------------------------------------------------------------- -// Server -// --------------------------------------------------------------------------- + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_grpc_server_thread_creation", resources_before_thread_creation, resources_after_thread_creation); -static int RunServer(const std::string& socket_path, const std::vector& child_pids) -{ - auto factory = std::make_unique(); - ipc::GrpcControlServer server(std::move(factory)); - - std::thread server_thread([&server, &socket_path]() { - server.Start(socket_path); - server.WaitForTermination(); - }); - - Log("[server] started on " + socket_path + " — waiting for all clients to finish..."); + Log("[server] started on " + socket_path + " - waiting for all clients to finish..."); int overall_status = 0; - for (std::size_t i = 0U; i < child_pids.size(); ++i) + for (std::size_t index = 0U; index < child_pids.size(); ++index) { - int wstatus = 0; - const pid_t pid = ::waitpid(-1, &wstatus, 0); + int wait_status = 0; + const pid_t pid = ::waitpid(-1, &wait_status, 0); if (pid > 0) { - const bool ok = WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0; - std::ostringstream ss; - ss << "[server] child pid=" << pid << (ok ? " exited OK" : " FAILED"); - Log(ss.str()); + const bool ok = WIFEXITED(wait_status) && WEXITSTATUS(wait_status) == 0; + std::ostringstream message; + message << "[server] child pid=" << pid << (ok ? " exited OK" : " FAILED"); + (ok ? Log : LogErr)(message.str()); if (!ok) { overall_status = 1; @@ -210,306 +158,174 @@ static int RunServer(const std::string& socket_path, const std::vector& c } } - server.Stop(); - if (server_thread.joinable()) - { - server_thread.join(); - } - + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_grpc_server_workload", resources_after_thread_creation, resources_after_workload); + server->Shutdown(); + server->Wait(); + ::unlink(socket_path.c_str()); Log("[server] shutdown complete."); return overall_status; } -// --------------------------------------------------------------------------- -// Client -// --------------------------------------------------------------------------- - static bool RunClient(const std::string& socket_path, const int client_index, const int call_count, const int thread_count) { - // Give the server time to bind before the first connection attempt. - std::this_thread::sleep_for(std::chrono::milliseconds(300)); + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); - // One gRPC channel shared by all threads — gRPC channels are thread-safe. - // SendRequest() blocks the calling thread until the response arrives; - // multiple threads can issue concurrent calls without extra synchronization. - ipc::GrpcControlClient client(socket_path); + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + std::unique_ptr stub; + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement connection_scope{ + latency_measurement, "POC::Grpc::ClientConnectionSetup"}; + const auto channel = ::grpc::CreateChannel("unix:" + socket_path, ::grpc::InsecureChannelCredentials()); + stub = control::PocControlService::NewStub(channel); + } std::atomic total_failures{0}; std::vector threads; threads.reserve(static_cast(thread_count)); - for (int t = 0; t < thread_count; ++t) + helper::ThreadStartBarrier thread_start_barrier{static_cast(thread_count)}; + + for (int thread_index = 0; thread_index < thread_count; ++thread_index) { - threads.emplace_back([&, t]() { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::Grpc::ClientThreadCreation"}; + threads.emplace_back([&, thread_index]() { + thread_start_barrier.ArriveAndWait(); int failures = 0; - auto times = std::vector(call_count); - - for (int c = 0; c < call_count; ++c) + for (int call = 0; call < call_count; ++call) { - auto start_time = std::chrono::system_clock::now(); - - const std::string str_param = "client" + std::to_string(client_index + 1); - const std::uint64_t uint64_param = static_cast(c + 1); - const std::string expected_combined = str_param + "_" + std::to_string(uint64_param); - - // operationActor encodes the client index so the server log is readable. - const daemon::common::OperationIdentifier opId{ - /*operationActor=*/static_cast(client_index + 1), - /*operationAction=*/1U, - }; - - auto requestResult = proto::ControlRequestBuilder() - .forDataNodeId(0) - .operation(opId) - .with_in_string(str_param) - .with_in_val_uint64(uint64_param) - .build(); - - if (!requestResult.has_value()) + bool valid = false; { - std::ostringstream ss; - ss << "[client " << client_index << "/thread " << t << "] build() failed for call " << c; - Log(ss.str()); - ++failures; - continue; - } + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement latency_scope{ + latency_measurement, "POC::Grpc::RoundTrip"}; - { - std::ostringstream ss; - ss << "[client " << client_index << "/thread " << t << "] -> SendRequest() str=\"" << str_param - << "\" uint64=" << uint64_param; - Log(ss.str()); - } + const auto workload_request = helper::CreateRequest(client_index, thread_index, call); - // Synchronous blocking call — no ticket map needed. - // GrpcControlClient overwrites request_id internally; the response - // request_id matches the auto-generated one used on the wire. - auto responseResult = client.SendRequest(requestResult.value()); + flatbuffers::grpc::MessageBuilder builder; + builder.Finish(helper::CreateRequestTable(builder, workload_request)); + const auto request = builder.GetMessage(); - if (!responseResult.has_value()) + ::grpc::ClientContext context; + flatbuffers::grpc::Message response; + const auto status = stub->Execute(&context, request, &response); + if (!status.ok()) { - std::ostringstream ss; - ss << "[client " << client_index << "/thread " << t << "] SendRequest() failed for call " << c; - Log(ss.str()); + LogErr("[client " + std::to_string(client_index) + "/thread " + std::to_string(thread_index) + + "] Execute() failed: " + status.error_message()); ++failures; continue; } - const auto& resp = responseResult.value(); - - // Validate: one operation, success result, uint64 echo matches, combined string matches. - bool ok = !resp.operation.operations.empty() && - resp.operation.operations[0].result == proto::OPERATION_RESULT_SUCCESS; - - std::string got_combined; - if (ok) + const auto* response_root = response.GetRoot(); + std::string received; + helper::Response workload_response; + valid = helper::ParseResponseRoot(response_root, workload_response) && + workload_response.request_id == workload_request.request_id && + workload_response.string_value == helper::ProcessRequest(workload_request).string_value; + if (valid) { - const auto& params = resp.operation.operations[0].parameters; - // param[0] = uint64 echo - auto u64 = resp.operation.operations[0].getParameter(0); - ok = u64.has_value() && (u64.value() == uint64_param); - - // param[1] = combined OwnedString - if (ok && params.size() >= 2) - { - auto str = resp.operation.operations[0].getParameter(1); - if (str.has_value()) - { - got_combined = str.value(); - ok = (got_combined == expected_combined); - } - else - { - ok = false; - } - } + received = workload_response.string_value; } - if (!ok) + if (!valid || !helper::Matches(workload_request, workload_response)) { - std::ostringstream ss; - ss << "[client " << client_index << "/thread " << t << "] MISMATCH or error for call " << c - << ": expected combined=\"" << expected_combined << "\" got=\"" << got_combined << "\""; - Log(ss.str()); + LogErr("[client " + std::to_string(client_index) + "/thread " + std::to_string(thread_index) + + "] response mismatch: expected=\"" + helper::ProcessRequest(workload_request).string_value + + "\" got=\"" + received + "\""); ++failures; - continue; } - - { - std::ostringstream ss; - ss << "[client " << client_index << "/thread " << t << "] <- OK combined=\"" << got_combined - << "\""; - Log(ss.str()); } - - auto end_time = std::chrono::system_clock::now(); - times[c] = end_time - start_time; + helper::WaitAfterCall(g_random_wait && valid); } - - // Per-call timings - for (int c = 0; c < call_count; ++c) - { - std::ostringstream oss; - oss << "[client " << client_index << "/thread " << t << "] call " << (c + 1) << "/" << call_count - << ": " << std::chrono::duration_cast(times[c]).count() << " us"; - Log(oss.str()); - } - - // Summary - if (failures == 0) - { - auto sum = std::chrono::duration::zero(); - for (int c = 0; c < call_count; ++c) - { - sum += times[c]; - } - - std::ostringstream oss; - oss << "- [client " << client_index << "/thread " << t << "] completed " << call_count << " calls with " - << std::chrono::duration_cast(sum).count() << " us elapsed, average " - << std::chrono::duration_cast(sum).count() / call_count - << " us per call"; - Log(oss.str()); - - if (call_count > 1) - { - auto s = sum - times[0]; - std::ostringstream oss2; - oss2 << "SKIP FIRST [client " << client_index << "/thread " << t << "] completed " - << (call_count - 1) << " calls with " - << std::chrono::duration_cast(s).count() << " us elapsed, average " - << std::chrono::duration_cast(s).count() / (call_count - 1) - << " us per call"; - Log(oss2.str()); - } - } - total_failures.fetch_add(failures, std::memory_order_relaxed); }); } - for (auto& th : threads) + thread_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_grpc_client_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + thread_start_barrier.Release(); + + for (auto& thread : threads) { - th.join(); + thread.join(); } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_grpc_client_workload", resources_after_thread_creation, resources_after_workload); const int total = call_count * thread_count; const int failures = total_failures.load(); const int success = total - failures; - std::ostringstream ss; - ss << "[client " << client_index << "] Results: " << success << "/" << total << " calls succeeded, " << failures - << "/" << total << " calls failed (" << thread_count << " thread(s) x " << call_count << " call(s))"; - Log(ss.str()); - + Log("[client " + std::to_string(client_index) + "] Results: " + std::to_string(success) + "/" + + std::to_string(total) + " calls succeeded, " + std::to_string(failures) + "/" + std::to_string(total) + + " calls failed"); return failures == 0; } -} // namespace score::crypto::poc::grpc - -// --------------------------------------------------------------------------- -// main — fork before any gRPC setup for a clean per-process state -// --------------------------------------------------------------------------- +} // namespace score::crypto::poc::grpc int main(int argc, char** argv) { - const std::string kClientPrefix{"--client_count="}; - const std::string kCallPrefix{"--call_count="}; - const std::string kClientThreadsPrefix{"--client_threads="}; - const std::string kServerThreadsPrefix{"--server_threads="}; - const std::string kSleepPrefix{"--sleep_milliseconds="}; - - for (int i = 1; i < argc; ++i) - { - const std::string arg{argv[i]}; - try - { - if (arg.rfind(kClientPrefix, 0) == 0) - g_client_count = std::stoi(arg.substr(kClientPrefix.size())); - else if (arg.rfind(kCallPrefix, 0) == 0) - g_call_count = std::stoi(arg.substr(kCallPrefix.size())); - else if (arg.rfind(kClientThreadsPrefix, 0) == 0) - g_client_threads = std::stoi(arg.substr(kClientThreadsPrefix.size())); - else if (arg.rfind(kServerThreadsPrefix, 0) == 0) - g_server_threads = std::stoi(arg.substr(kServerThreadsPrefix.size())); - else if (arg.rfind(kSleepPrefix, 0) == 0) - g_sleep_milliseconds = std::stoi(arg.substr(kSleepPrefix.size())); - } - catch (const std::exception& ex) - { - std::fprintf(stderr, "[main] invalid argument '%s': %s\n", argv[i], ex.what()); - return 1; - } - } - - if (g_client_count < 1) + score::crypto::ipc::poc_helper::PocArguments defaults; + defaults.client_count = g_client_count; + defaults.call_count = g_call_count; + defaults.client_threads = g_client_threads; + defaults.server_threads = g_server_threads; + defaults.sleep_milliseconds = g_sleep_milliseconds; + std::string parse_error; + const auto parsed_arguments = score::crypto::ipc::poc_helper::ParseArguments(argc, argv, parse_error, defaults); + if (!parsed_arguments.has_value()) { - std::fprintf(stderr, "[main] --client_count must be >= 1\n"); + std::fprintf(stderr, "[main] %s\n", parse_error.c_str()); return 1; } - if (g_call_count < 1) - { - std::fprintf(stderr, "[main] --call_count must be >= 1\n"); - return 1; - } - if (g_client_threads < 1) - { - std::fprintf(stderr, "[main] --client_threads must be >= 1\n"); - return 1; - } - if (g_server_threads < 1) - { - std::fprintf(stderr, "[main] --server_threads must be >= 1\n"); - return 1; - } - - std::printf( - "[main] client_count=%d call_count=%d client_threads=%d" - " server_threads=%d (note: gRPC manages its own pool) sleep_milliseconds=%d\n", - g_client_count, - g_call_count, - g_client_threads, - g_server_threads, - g_sleep_milliseconds); - - // Unique socket per run to avoid collisions between concurrent bazel invocations. - const std::string socket_path = "/tmp/score_poc_grpc_" + std::to_string(::getpid()) + ".sock"; - - std::fflush(nullptr); - + g_client_count = parsed_arguments->client_count; + g_call_count = parsed_arguments->call_count; + g_client_threads = parsed_arguments->client_threads; + g_server_threads = parsed_arguments->server_threads; + g_sleep_milliseconds = parsed_arguments->sleep_milliseconds; + g_random_wait = parsed_arguments->random_wait; + + std::cout << score::crypto::ipc::poc_helper::SettingsSummary( + "poc_grpc", + g_client_count, + g_call_count, + g_client_threads, + g_server_threads, + g_random_wait) + << '\n' + << std::flush; + const std::string socket_path = " score_poc_grpc_" + std::to_string(::getpid()) + ".sock"; std::vector child_pids; - int my_client_index = -1; - - for (int i = 0; i < g_client_count; ++i) + int client_index = -1; + for (int index = 0; index < g_client_count; ++index) { const pid_t pid = ::fork(); if (pid < 0) { std::perror("[main] fork"); - for (const pid_t cpid : child_pids) - { - ::kill(cpid, SIGTERM); - } return 1; } if (pid == 0) { - my_client_index = i; + client_index = index; break; } child_pids.push_back(pid); } - if (my_client_index == -1) + if (client_index < 0) { return score::crypto::poc::grpc::RunServer(socket_path, child_pids); } - else - { - const bool ok = - score::crypto::poc::grpc::RunClient(socket_path, my_client_index, g_call_count, g_client_threads); - return ok ? 0 : 1; - } + const bool ok = score::crypto::poc::grpc::RunClient(socket_path, client_index, g_call_count, g_client_threads); + return ok ? 0 : 1; } diff --git a/score/tests/ipc_poc/poc_helper.hpp b/score/tests/ipc_poc/poc_helper.hpp new file mode 100644 index 000000000..492a83e84 --- /dev/null +++ b/score/tests/ipc_poc/poc_helper.hpp @@ -0,0 +1,466 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef SCORE_TESTS_IPC_POC_POC_HELPER_HPP +#define SCORE_TESTS_IPC_POC_POC_HELPER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flatbuffers/flatbuffers.h" +#include "score/tests/ipc_poc/ipc_buffer.h" +#include "score/tests/ipc_poc/poc_control_generated.h" + +namespace score::crypto::ipc::poc_helper +{ + +namespace control = score::crypto::ipc::control; +using Bytes = std::vector; + +class ThreadStartBarrier final +{ + public: + explicit ThreadStartBarrier(const std::size_t participant_count) : participant_count_{participant_count} {} + + void ArriveAndWait() + { + std::unique_lock lock{mutex_}; + ++arrived_count_; + condition_.notify_all(); + condition_.wait(lock, [this] { return released_; }); + } + + void WaitForAll() + { + std::unique_lock lock{mutex_}; + condition_.wait(lock, [this] { return arrived_count_ == participant_count_; }); + } + + void Release() + { + { + std::lock_guard lock{mutex_}; + released_ = true; + } + condition_.notify_all(); + } + + private: + const std::size_t participant_count_; + std::size_t arrived_count_{0U}; + bool released_{false}; + std::mutex mutex_; + std::condition_variable condition_; +}; + +struct PocArguments final +{ + int client_count{1}; + int call_count{1000}; + int client_threads{1}; + int server_threads{1}; + int sleep_milliseconds{0}; + bool random_wait{false}; +}; + +inline std::optional ParseArguments(const int argc, + char** argv, + std::string& error, + const PocArguments defaults = {}) +{ + PocArguments arguments{defaults}; + const std::string client_prefix{"--client_count="}; + const std::string call_prefix{"--call_count="}; + const std::string client_threads_prefix{"--client_threads="}; + const std::string server_threads_prefix{"--server_threads="}; + const std::string sleep_prefix{"--sleep_milliseconds="}; + const std::string random_wait_prefix{"--random_wait="}; + + for (int index = 1; index < argc; ++index) + { + const std::string argument{argv[index]}; + try + { + if (argument.rfind(client_prefix, 0) == 0) + { + arguments.client_count = std::stoi(argument.substr(client_prefix.size())); + } + else if (argument.rfind(call_prefix, 0) == 0) + { + arguments.call_count = std::stoi(argument.substr(call_prefix.size())); + } + else if (argument.rfind(client_threads_prefix, 0) == 0) + { + arguments.client_threads = std::stoi(argument.substr(client_threads_prefix.size())); + } + else if (argument.rfind(server_threads_prefix, 0) == 0) + { + arguments.server_threads = std::stoi(argument.substr(server_threads_prefix.size())); + } + else if (argument.rfind(sleep_prefix, 0) == 0) + { + arguments.sleep_milliseconds = std::stoi(argument.substr(sleep_prefix.size())); + } + else if (argument.rfind(random_wait_prefix, 0) == 0) + { + const auto value = argument.substr(random_wait_prefix.size()); + if (value == "true" || value == "1") + { + arguments.random_wait = true; + } + else if (value == "false" || value == "0") + { + arguments.random_wait = false; + } + else + { + throw std::invalid_argument{"random_wait must be true, false, 1, or 0"}; + } + } + else + { + throw std::invalid_argument{"unknown argument"}; + } + } + catch (const std::exception& exception) + { + error = "invalid argument '" + argument + "': " + exception.what(); + return std::nullopt; + } + } + + if (arguments.client_count < 1 || arguments.call_count < 1 || arguments.client_threads < 1 || + arguments.server_threads < 1 || arguments.sleep_milliseconds < 0) + { + error = "client_count, call_count, client_threads, and server_threads must be >= 1; " + "sleep_milliseconds must be >= 0"; + return std::nullopt; + } + return arguments; +} + +inline void WaitAfterCall(const bool random_wait) +{ + if (!random_wait) + { + return; + } + + thread_local std::mt19937 generator{std::random_device{}()}; + std::uniform_int_distribution distribution{0, 5}; + std::this_thread::sleep_for(std::chrono::milliseconds(distribution(generator))); +} + +struct Request final +{ + std::uint64_t request_id{0U}; + std::string string_value; + std::uint64_t uint64_value{0U}; +}; + +struct Response final +{ + std::uint64_t request_id{0U}; + std::string string_value; +}; + +inline Request CreateRequest(const int client_index, const int thread_index, const int call_index) +{ + return Request{ + static_cast(client_index + 1) * 100'000ULL + + static_cast(thread_index + 1) * 1'000ULL + + static_cast(call_index + 1), + "client" + std::to_string(client_index + 1), + static_cast(call_index + 1), + }; +} + +inline Response ProcessRequest(const Request& request) +{ + return Response{request.request_id, request.string_value + "_" + std::to_string(request.uint64_value)}; +} + +inline bool Matches(const Request& request, const Response& response) +{ + const auto expected = ProcessRequest(request); + return response.request_id == expected.request_id && response.string_value == expected.string_value; +} + +template +auto CreateRequestTable(Builder& builder, const Request& request) +{ + const auto string_table = control::CreateString(builder, builder.CreateString(request.string_value)); + const auto uint64_table = control::CreateValueUint64(builder, request.uint64_value); + const std::vector parameter_types{ + control::OperationParameter_String, + control::OperationParameter_ValueUint64, + }; + const std::vector> parameter_values{ + string_table.Union(), + uint64_table.Union(), + }; + const auto operation = control::CreateSingleOperationRequest( + builder, + control::CreateOperationIdentifier(builder, 1U, 1U), + builder.CreateVector(parameter_types), + builder.CreateVector(parameter_values)); + const auto batch = control::CreateOperationRequestBatch(builder, builder.CreateVector({operation})); + return control::CreateControlRequest(builder, request.request_id, 0U, 0U, batch); +} + +template +auto CreateResponseTable(Builder& builder, + const Response& response, + const std::uint32_t operation_actor = 0U, + const std::uint32_t operation_action = 0U) +{ + const auto result_string = control::CreateString(builder, builder.CreateString(response.string_value)); + const std::vector parameter_types{control::OperationParameter_String}; + const std::vector> parameter_values{result_string.Union()}; + const auto operation = control::CreateSingleOperationResponse( + builder, + control::CreateOperationIdentifier(builder, operation_actor, operation_action), + control::CreateOperationResult(builder, 0U), + builder.CreateVector(parameter_types), + builder.CreateVector(parameter_values)); + const auto batch = control::CreateOperationResponseBatch(builder, builder.CreateVector({operation})); + return control::CreateControlResponse(builder, response.request_id, batch); +} + +inline Bytes BuildRequestBytes(const Request& request) +{ + flatbuffers::FlatBufferBuilder builder(512); + builder.FinishSizePrefixed(CreateRequestTable(builder, request)); + return {builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()}; +} + +inline Bytes BuildResponseBytes(const Response& response) +{ + flatbuffers::FlatBufferBuilder builder(512); + builder.FinishSizePrefixed(CreateResponseTable(builder, response)); + return {builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()}; +} + +inline bool BuildRequestInto(control::IpcBuffer& buffer, const Request& request) +{ + const auto bytes = BuildRequestBytes(request); + return control::PackFlatBufferInto(buffer, bytes.data(), bytes.size()); +} + +inline control::IpcBuffer BuildResponseBuffer(const Response& response) +{ + const auto bytes = BuildResponseBytes(response); + return control::PackFlatBuffer(bytes.data(), bytes.size()); +} + +inline bool ParseRequestRoot(const control::ControlRequest* root, Request& request) +{ + if (root == nullptr || root->operation_batch() == nullptr || root->operation_batch()->operations() == nullptr || + root->operation_batch()->operations()->size() != 1U) + { + return false; + } + const auto* operation = root->operation_batch()->operations()->Get(0U); + if (operation == nullptr || operation->parameter() == nullptr || operation->parameter_type() == nullptr || + operation->parameter()->size() != 2U || operation->parameter()->size() != operation->parameter_type()->size()) + { + return false; + } + + Request parsed{root->request_id(), {}, 0U}; + bool found_string = false; + bool found_uint64 = false; + for (flatbuffers::uoffset_t index = 0U; index < operation->parameter()->size(); ++index) + { + const auto type = static_cast(operation->parameter_type()->Get(index)); + if (type == control::OperationParameter_String) + { + const auto* value = reinterpret_cast(operation->parameter()->Get(index)); + if (value == nullptr || value->val() == nullptr || found_string) + { + return false; + } + parsed.string_value = value->val()->str(); + found_string = true; + } + else if (type == control::OperationParameter_ValueUint64) + { + const auto* value = reinterpret_cast(operation->parameter()->Get(index)); + if (value == nullptr || found_uint64) + { + return false; + } + parsed.uint64_value = value->val(); + found_uint64 = true; + } + else + { + return false; + } + } + if (!found_string || !found_uint64) + { + return false; + } + request = std::move(parsed); + return true; +} + +inline bool ParseRequestBytes(const std::uint8_t* data, const std::size_t size, Request& request) +{ + flatbuffers::Verifier verifier{data, size}; + if (!control::VerifySizePrefixedControlRequestBuffer(verifier)) + { + return false; + } + return ParseRequestRoot(flatbuffers::GetSizePrefixedRoot(data), request); +} + +inline bool ProcessRequestBytes(const std::uint8_t* data, const std::size_t size, Response& response) +{ + Request request; + if (!ParseRequestBytes(data, size, request)) + { + return false; + } + response = ProcessRequest(request); + return true; +} + +inline bool ProcessRequestBuffer(const control::IpcBuffer& buffer, Response& response) +{ + if (!control::IsValid(buffer)) + { + return false; + } + return ProcessRequestBytes(reinterpret_cast(buffer.payload.data()), + control::GetPayloadSize(buffer), + response); +} + +inline bool ParseRequestBuffer(const control::IpcBuffer& buffer, Request& request) +{ + if (!control::IsValid(buffer)) + { + return false; + } + return ParseRequestBytes(reinterpret_cast(buffer.payload.data()), + control::GetPayloadSize(buffer), + request); +} + +inline bool ParseResponseRoot(const control::ControlResponse* root, + Response& response, + std::uint64_t* echoed_uint64 = nullptr) +{ + if (root == nullptr || root->operation_batch() == nullptr || root->operation_batch()->operations() == nullptr || + root->operation_batch()->operations()->size() != 1U) + { + return false; + } + const auto* operation = root->operation_batch()->operations()->Get(0U); + if (operation == nullptr || operation->parameter() == nullptr || operation->parameter_type() == nullptr || + operation->parameter()->size() == 0U || operation->parameter()->size() != operation->parameter_type()->size()) + { + return false; + } + + bool found_string = false; + for (flatbuffers::uoffset_t index = 0U; index < operation->parameter()->size(); ++index) + { + const auto type = static_cast(operation->parameter_type()->Get(index)); + if (type == control::OperationParameter_String) + { + const auto* value = reinterpret_cast(operation->parameter()->Get(index)); + if (value == nullptr || value->val() == nullptr || found_string) + { + return false; + } + response.string_value = value->val()->str(); + found_string = true; + } + else if (type == control::OperationParameter_ValueUint64 && echoed_uint64 != nullptr) + { + const auto* value = reinterpret_cast(operation->parameter()->Get(index)); + if (value == nullptr) + { + return false; + } + *echoed_uint64 = value->val(); + } + else + { + return false; + } + } + if (!found_string) + { + return false; + } + response.request_id = root->request_id(); + return true; +} + +inline bool ParseResponseBytes(const std::uint8_t* data, const std::size_t size, Response& response) +{ + flatbuffers::Verifier verifier{data, size}; + if (!verifier.template VerifySizePrefixedBuffer(nullptr)) + { + return false; + } + return ParseResponseRoot(flatbuffers::GetSizePrefixedRoot(data), response); +} + +inline bool ParseResponseBuffer(const control::IpcBuffer& buffer, Response& response) +{ + if (!control::IsValid(buffer)) + { + return false; + } + return ParseResponseBytes(reinterpret_cast(buffer.payload.data()), + control::GetPayloadSize(buffer), + response); +} + +inline std::string SettingsSummary(const std::string_view poc_name, + const int client_count, + const int call_count, + const int client_threads, + const int worker_threads, + const std::optional random_wait = std::nullopt) +{ + auto summary = "[POC settings] name=" + std::string{poc_name} + " client_count=" + + std::to_string(client_count) + " call_count=" + std::to_string(call_count) + + " client_threads=" + std::to_string(client_threads) + + " worker_threads=" + std::to_string(worker_threads); + if (random_wait.has_value()) + { + summary += " random_wait=" + std::string{*random_wait ? "true" : "false"}; + } + return summary; +} + +} // namespace score::crypto::ipc::poc_helper + +#endif // SCORE_TESTS_IPC_POC_POC_HELPER_HPP diff --git a/score/tests/ipc_poc/poc_low_level.cpp b/score/tests/ipc_poc/poc_low_level.cpp index baf2a3295..b6c637cd1 100644 --- a/score/tests/ipc_poc/poc_low_level.cpp +++ b/score/tests/ipc_poc/poc_low_level.cpp @@ -105,27 +105,37 @@ #include "score/message_passing/i_server_factory.h" #include "score/message_passing/server_factory.h" #include "score/message_passing/service_protocol_config.h" +#include "score/tests/utility/runtime_measurement.hpp" #include "score/tests/ipc_poc/ipc_buffer.h" #include "score/tests/ipc_poc/poc_control_generated.h" +#include "score/tests/ipc_poc/poc_helper.hpp" +#include "score/tests/utility/process_resource_measurement.hpp" // --------------------------------------------------------------------------- // Global parameters (set before fork; never mutated after) // --------------------------------------------------------------------------- -static int g_client_count = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -static int g_call_count = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -static int g_client_threads = 20; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_count = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_call_count = 1000; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_threads = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) static int g_server_threads = 8; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static bool g_random_wait = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) // --------------------------------------------------------------------------- // Shared logging helper // --------------------------------------------------------------------------- static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static constexpr bool kEnableVerboseOutput{false}; +static constexpr bool kEnableLatencyVerbose{false}; static void Log(const std::string& line) { + if (!kEnableVerboseOutput) + { + return; + } std::lock_guard lk(g_log_mutex); std::cout << line << "\n"; } @@ -145,6 +155,8 @@ static void LogErr(const std::string& line) namespace score::crypto::ipc::control { +namespace helper = score::crypto::ipc::poc_helper; + static constexpr std::string_view kServiceIdentifier{"score_crypto_poc_ll"}; // --------------------------------------------------------------------------- @@ -310,83 +322,28 @@ static std::vector BuildAcknowledgeReply(const std::uint64_t reque return std::vector(fbb.GetBufferPointer(), fbb.GetBufferPointer() + fbb.GetSize()); } -static std::vector BuildResponseBytes(const std::uint64_t request_id, const std::string& combined) -{ - flatbuffers::FlatBufferBuilder fbb(512); - - auto str_val = fbb.CreateString(combined); - auto str_tbl = CreateString(fbb, str_val); - - std::vector resp_param_types{OperationParameter_String}; - std::vector> resp_param_values{str_tbl.Union()}; - - auto resp_op = CreateSingleOperationResponse(fbb, - CreateOperationIdentifier(fbb, 0U, 0U), - CreateOperationResult(fbb, 0U), - fbb.CreateVector(resp_param_types), - fbb.CreateVector(resp_param_values)); - - auto resp_batch = CreateOperationResponseBatch(fbb, fbb.CreateVector({resp_op})); - fbb.FinishSizePrefixed(CreateControlResponse(fbb, request_id, resp_batch)); - return std::vector(fbb.GetBufferPointer(), fbb.GetBufferPointer() + fbb.GetSize()); -} - static std::vector ProcessRequestBytes(const std::uint64_t request_id, score::cpp::span message) { - flatbuffers::Verifier verifier{message.data(), static_cast(message.size())}; - if (!VerifySizePrefixedControlRequestBuffer(verifier)) + helper::Response workload_response; + if (!helper::ProcessRequestBytes(message.data(), message.size(), workload_response)) { LogErr("[server/worker] FlatBuffer verification failed"); return {}; } - const auto* req = flatbuffers::GetSizePrefixedRoot(message.data()); - if (req == nullptr || req->operation_batch() == nullptr || req->operation_batch()->operations() == nullptr || - req->operation_batch()->operations()->size() == 0U) { - return {}; + std::ostringstream ss; + ss << "[server/worker] request_id=" << workload_response.request_id << " -> combined=\"" + << workload_response.string_value << "\""; + Log(ss.str()); } - const auto* op = req->operation_batch()->operations()->Get(0U); - if (op == nullptr || op->parameter() == nullptr) + if (workload_response.request_id != request_id) { return {}; } - - std::string str_value; - std::uint64_t uint64_value = 0U; - - for (flatbuffers::uoffset_t i = 0U; i < op->parameter()->size(); ++i) - { - const auto ptype = static_cast(op->parameter_type()->Get(i)); - if (ptype == OperationParameter_String) - { - const auto* s = reinterpret_cast(op->parameter()->Get(i)); - if (s != nullptr && s->val() != nullptr) - { - str_value = s->val()->str(); - } - } - else if (ptype == OperationParameter_ValueUint64) - { - const auto* v = reinterpret_cast(op->parameter()->Get(i)); - if (v != nullptr) - { - uint64_value = v->val(); - } - } - } - - const std::string combined = str_value + "_" + std::to_string(uint64_value); - - { - std::ostringstream ss; - ss << "[server/worker] request_id=" << request_id << " -> combined=\"" << combined << "\""; - Log(ss.str()); - } - - return BuildResponseBytes(request_id, combined); + return helper::BuildResponseBytes(workload_response); } // --------------------------------------------------------------------------- @@ -403,6 +360,9 @@ struct WorkItem static int RunServer(const std::vector& child_pids) { + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; const ServiceParams service_params{ kServiceIdentifier, /*max_payload_bytes=*/static_cast(sizeof(IpcBuffer)), @@ -487,24 +447,23 @@ static int RunServer(const std::vector& child_pids) std::vector workers; workers.reserve(server_params.worker_threads); + helper::ThreadStartBarrier worker_start_barrier{server_params.worker_threads}; for (std::uint32_t w = 0U; w < server_params.worker_threads; ++w) { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::LowLevel::ServerThreadCreation"}; workers.emplace_back([&, w]() { + worker_start_barrier.ArriveAndWait(); while (true) { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement wait_scope{ + latency_measurement, "POC::LowLevel::ServerQueueWait"}; std::unique_lock lk(queue_mutex); - auto start_time = std::chrono::system_clock::now(); - queue_cv.wait(lk, [&] { return !work_queue.empty() || stop_workers.load(); }); - auto end_time = std::chrono::system_clock::now(); - auto diff = end_time - start_time; - Log("[server/worker " + std::to_string(w) + "] Took: " + - std::to_string(std::chrono::duration_cast(diff).count()) + " us\n"); - if (stop_workers.load() && work_queue.empty()) { break; @@ -564,6 +523,11 @@ static int RunServer(const std::vector& child_pids) } }); } + worker_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_server_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + worker_start_barrier.Release(); Log("[server] started " + std::to_string(server_params.worker_threads) + " worker thread(s)"); // ------------------------------------------------------------------ @@ -657,14 +621,13 @@ static int RunServer(const std::vector& child_pids) score::cpp::span message) -> score::cpp::expected_blank { // Read the client-assigned request_id from the FlatBuffer. // The client guarantees it is non-zero and unique within its process. - flatbuffers::Verifier verifier{message.data(), static_cast(message.size())}; - if (!VerifySizePrefixedControlRequestBuffer(verifier)) + helper::Request workload_request; + if (!helper::ParseRequestBytes(message.data(), message.size(), workload_request)) { LogErr("[server/handler] FlatBuffer verification failed — dropping request"); return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); } - const auto* req = flatbuffers::GetSizePrefixedRoot(message.data()); - const std::uint64_t request_id = (req != nullptr) ? req->request_id() : 0U; + const std::uint64_t request_id = workload_request.request_id; if (request_id == 0U) { LogErr("[server/handler] received request with zero request_id — dropping"); @@ -767,6 +730,9 @@ static int RunServer(const std::vector& child_pids) t.join(); } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_server_workload", resources_after_thread_creation, resources_after_workload); server->StopListening(); Log("[server] shutdown complete."); return overall_status; @@ -787,6 +753,8 @@ struct PendingCall static bool RunClient(const int client_index, const int call_count, const int thread_count) { + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; const ServiceParams service_params{ kServiceIdentifier, /*max_payload_bytes=*/static_cast(sizeof(IpcBuffer)), @@ -825,12 +793,7 @@ static bool RunClient(const int client_index, const int call_count, const int th // because truly_async=true routes all sends through the library's // background thread without blocking the caller. // ------------------------------------------------------------------ - auto client = client_factory.Create(protocol_config, client_config); - if (!client) - { - LogErr("[client " + std::to_string(client_index) + "] Create failed"); - return false; - } + decltype(client_factory.Create(protocol_config, client_config)) client; // The state callback is stored in a score::cpp::callback<> with a fixed // 32-byte inline capacity. Capturing four objects (two promises + two @@ -859,7 +822,17 @@ static bool RunClient(const int client_index, const int call_count, const int th // stored inside each PendingCall and checked under pending_map_mutex. auto notify_ctx = std::make_shared(NotifyCtx{pending_map_mutex, pending_map, client_index}); - client->Start( + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement connection_scope{ + latency_measurement, "POC::LowLevel::ClientConnectionSetup"}; + client = client_factory.Create(protocol_config, client_config); + if (!client) + { + LogErr("[client " + std::to_string(client_index) + "] Create failed"); + return false; + } + + client->Start( [conn_state, client_index](score::message_passing::IClientConnection::State state) { if (state == score::message_passing::IClientConnection::State::kReady) { @@ -886,33 +859,13 @@ static bool RunClient(const int client_index, const int call_count, const int th [notify_ctx](score::cpp::span message) { auto start_time = std::chrono::system_clock::now(); - // ControlResponse is not root_type in the schema — parse directly. - const auto* resp = flatbuffers::GetSizePrefixedRoot(message.data()); - if (resp == nullptr) - { - LogErr("[client " + std::to_string(notify_ctx->client_index) + "] NotifyCallback: null response root"); - return; - } - - const std::uint64_t request_id = resp->request_id(); - std::string result_value; - bool parse_ok = false; - - if (resp->operation_batch() != nullptr && resp->operation_batch()->operations() != nullptr && - resp->operation_batch()->operations()->size() > 0U) + helper::Response parsed_response; + const bool parse_ok = helper::ParseResponseBytes(message.data(), message.size(), parsed_response); + const std::uint64_t request_id = parsed_response.request_id; + std::string result_value = std::move(parsed_response.string_value); + if (!parse_ok) { - const auto* op = resp->operation_batch()->operations()->Get(0U); - if (op != nullptr && op->parameter() != nullptr && op->parameter()->size() > 0U && - op->parameter_type() != nullptr && - static_cast(op->parameter_type()->Get(0U)) == OperationParameter_String) - { - const auto* str = reinterpret_cast(op->parameter()->Get(0U)); - if (str != nullptr && str->val() != nullptr) - { - result_value = str->val()->str(); - parse_ok = true; - } - } + LogErr("[client " + std::to_string(notify_ctx->client_index) + "] NotifyCallback: invalid response"); } // Look up and signal the waiting thread. @@ -946,11 +899,12 @@ static bool RunClient(const int client_index, const int call_count, const int th std::to_string(std::chrono::duration_cast(diff).count()) + " us"); }); - if (ready_future.wait_for(std::chrono::seconds(120)) != std::future_status::ready) - { - LogErr("[client " + std::to_string(client_index) + "] timed out waiting for connection"); - client->Stop(); - return false; + if (ready_future.wait_for(std::chrono::seconds(120)) != std::future_status::ready) + { + LogErr("[client " + std::to_string(client_index) + "] timed out waiting for connection"); + client->Stop(); + return false; + } } // ------------------------------------------------------------------ @@ -959,21 +913,24 @@ static bool RunClient(const int client_index, const int call_count, const int th std::atomic total_failures{0}; std::vector threads; threads.reserve(static_cast(thread_count)); + helper::ThreadStartBarrier thread_start_barrier{static_cast(thread_count)}; for (int t = 0; t < thread_count; ++t) { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::LowLevel::ClientThreadCreation"}; threads.emplace_back([&, t]() { + thread_start_barrier.ArriveAndWait(); int failures = 0; - auto times = std::vector(call_count); for (int c = 0; c < call_count; ++c) { - auto start_time = std::chrono::system_clock::now(); + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement latency_scope{ + latency_measurement, "POC::LowLevel::RoundTrip"}; // Build request - const std::string str_param = "client" + std::to_string(client_index + 1); - const std::uint64_t uint64_param = static_cast(c + 1); - const std::string expected = str_param + "_" + std::to_string(uint64_param); + auto workload_request = helper::CreateRequest(client_index, t, c); // Assign the request_id before building the FlatBuffer so it can // be embedded in the payload and inserted into the pending_map — @@ -981,24 +938,11 @@ static bool RunClient(const int client_index, const int call_count, const int th // lower 32 bits = per-process monotonic counter (thread-unique within // this process). Zero is never produced (counter starts at 1). const std::uint64_t request_id = pid_upper | call_counter.fetch_add(1U, std::memory_order_relaxed); + workload_request.request_id = request_id; - // Build request FlatBuffer with the pre-assigned request_id. - // Persisted in a shared_ptr so it outlives the async send queue. - flatbuffers::FlatBufferBuilder fbb(512); - auto str_val = fbb.CreateString(str_param); - auto str_tbl = CreateString(fbb, str_val); - auto u64_tbl = CreateValueUint64(fbb, uint64_param); - std::vector param_types{OperationParameter_String, OperationParameter_ValueUint64}; - std::vector> param_values{str_tbl.Union(), u64_tbl.Union()}; - auto op_id = CreateOperationIdentifier(fbb, /*actor=*/1U, /*action=*/1U); - auto single_op = CreateSingleOperationRequest( - fbb, op_id, fbb.CreateVector(param_types), fbb.CreateVector(param_values)); - auto batch = CreateOperationRequestBatch(fbb, fbb.CreateVector({single_op})); - fbb.FinishSizePrefixed( - CreateControlRequest(fbb, request_id, /*client_id=*/0U, /*data_node_id=*/0U, batch)); - + // Persist the shared-codec request so it outlives the async send queue. auto request_buffer = std::make_shared>( - fbb.GetBufferPointer(), fbb.GetBufferPointer() + fbb.GetSize()); + helper::BuildRequestBytes(workload_request)); score::cpp::span request_span{request_buffer->data(), request_buffer->size()}; // Insert into the pending map BEFORE the send so the entry is @@ -1012,8 +956,9 @@ static bool RunClient(const int client_index, const int call_count, const int th { std::ostringstream ss; - ss << "[client " << client_index << "/thread " << t << "] -> SendWithCallback() str=\"" << str_param - << "\" uint64=" << uint64_param << " request_id=" << request_id; + ss << "[client " << client_index << "/thread " << t + << "] -> SendWithCallback() str=\"" << workload_request.string_value + << "\" uint64=" << workload_request.uint64_value << " request_id=" << request_id; Log(ss.str()); } @@ -1087,11 +1032,12 @@ static bool RunClient(const int client_index, const int call_count, const int th continue; } - if (!pending->ok || pending->result_value != expected) + if (!pending->ok || + !helper::Matches(workload_request, helper::Response{request_id, pending->result_value})) { std::ostringstream ss; ss << "[client " << client_index << "/thread " << t << "] MISMATCH request_id=" << request_id - << ": expected=\"" << expected << "\" got=\"" + << ": expected=\"" << helper::ProcessRequest(workload_request).string_value << "\" got=\"" << (pending->ok ? pending->result_value : "") << "\""; LogErr(ss.str()); ++failures; @@ -1105,52 +1051,28 @@ static bool RunClient(const int client_index, const int call_count, const int th Log(ss.str()); } - auto end_time = std::chrono::system_clock::now(); - times[c] = end_time - start_time; - // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - } - - auto sum = std::chrono::duration::zero(); - for (int c = 0; c < call_count; ++c) - { - sum += times[c]; - std::ostringstream oss; - oss << "[client " << client_index << "/thread " << t << "] call " << (c + 1) << "/" << call_count - << ": " << std::chrono::duration_cast(times[c]).count() << " us"; - Log(oss.str()); - } - - if (failures == 0) - { - std::ostringstream oss; - oss << "- [client " << client_index << "/thread " << t << "] completed " << call_count << " calls with " - << std::chrono::duration_cast(sum).count() << " us elapsed, average " - << std::chrono::duration_cast(sum).count() / call_count - << " us per call"; - Log(oss.str()); - - if (call_count > 1) - { - auto s = sum - times[0]; - std::ostringstream oss2; - oss2 << "SKIP FIRST [client " << client_index << "/thread " << t << "] completed " - << (call_count - 1) << " calls with " - << std::chrono::duration_cast(s).count() << " us elapsed, average " - << std::chrono::duration_cast(s).count() / (call_count - 1) - << " us per call"; - Log(oss2.str()); } + helper::WaitAfterCall(g_random_wait); } total_failures.fetch_add(failures, std::memory_order_relaxed); }); } + thread_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_client_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + thread_start_barrier.Release(); + for (auto& th : threads) { th.join(); } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_client_workload", resources_after_thread_creation, resources_after_workload); client->Stop(); if (stopped_future.wait_for(std::chrono::seconds(30)) != std::future_status::ready) @@ -1177,64 +1099,35 @@ static bool RunClient(const int client_index, const int call_count, const int th int main(int argc, char** argv) { - const std::string kClientPrefix{"--client_count="}; - const std::string kCallPrefix{"--call_count="}; - const std::string kClientThreadsPrefix{"--client_threads="}; - const std::string kServerThreadsPrefix{"--server_threads="}; - const std::string kSleepPrefix{"--sleep_milliseconds="}; - for (int i = 1; i < argc; ++i) + score::crypto::ipc::poc_helper::PocArguments defaults; + defaults.client_count = g_client_count; + defaults.call_count = g_call_count; + defaults.client_threads = g_client_threads; + defaults.server_threads = g_server_threads; + defaults.sleep_milliseconds = g_sleep_milliseconds; + std::string parse_error; + const auto parsed_arguments = score::crypto::ipc::poc_helper::ParseArguments(argc, argv, parse_error, defaults); + if (!parsed_arguments.has_value()) { - const std::string arg{argv[i]}; - try - { - if (arg.rfind(kClientPrefix, 0) == 0) - { - g_client_count = std::stoi(arg.substr(kClientPrefix.size())); - } - else if (arg.rfind(kCallPrefix, 0) == 0) - { - g_call_count = std::stoi(arg.substr(kCallPrefix.size())); - } - else if (arg.rfind(kClientThreadsPrefix, 0) == 0) - { - g_client_threads = std::stoi(arg.substr(kClientThreadsPrefix.size())); - } - else if (arg.rfind(kServerThreadsPrefix, 0) == 0) - { - g_server_threads = std::stoi(arg.substr(kServerThreadsPrefix.size())); - } - else if (arg.rfind(kSleepPrefix, 0) == 0) - { - g_sleep_milliseconds = std::stoi(arg.substr(kSleepPrefix.size())); - } - } - catch (const std::exception& ex) - { - LogErr("[main] invalid argument '" + arg + "': " + ex.what()); - return 1; - } - } - if (g_client_count < 1) - { - LogErr("[main] --client_count must be >= 1"); - return 1; - } - if (g_call_count < 1) - { - LogErr("[main] --call_count must be >= 1"); + LogErr("[main] " + parse_error); return 1; } - if (g_client_threads < 1) - { - LogErr("[main] --client_threads must be >= 1"); - return 1; - } - if (g_server_threads < 1) - { - LogErr("[main] --server_threads must be >= 1"); - return 1; - } - + g_client_count = parsed_arguments->client_count; + g_call_count = parsed_arguments->call_count; + g_client_threads = parsed_arguments->client_threads; + g_server_threads = parsed_arguments->server_threads; + g_sleep_milliseconds = parsed_arguments->sleep_milliseconds; + g_random_wait = parsed_arguments->random_wait; + + std::cout << score::crypto::ipc::poc_helper::SettingsSummary( + "poc_low_level", + g_client_count, + g_call_count, + g_client_threads, + g_server_threads, + g_random_wait) + << '\n' + << std::flush; Log("[main] client_count=" + std::to_string(g_client_count) + " call_count=" + std::to_string(g_call_count) + " client_threads=" + std::to_string(g_client_threads) + " server_threads=" + std::to_string(g_server_threads) + " sleep_milliseconds=" + std::to_string(g_sleep_milliseconds)); diff --git a/score/tests/ipc_poc/poc_low_level_no_reply.cpp b/score/tests/ipc_poc/poc_low_level_no_reply.cpp new file mode 100644 index 000000000..e452a4f37 --- /dev/null +++ b/score/tests/ipc_poc/poc_low_level_no_reply.cpp @@ -0,0 +1,1236 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This file exists only to test and understand IPC mechanisms and to guide a +// proper implementation. It is kept temporarily as reference and will be +// removed once confidence has been gained in the proper implementation. +// ============================================================================= + +/// POC: Send + Notify async model — one shared connection per client process +/// +/// This replicates the functionality of poc_async.cpp using only the low-level +/// score::message_passing API, without any score::mw elements. +/// +/// Communication model: +/// +/// Phase 1 — Send for non-blocking request submission: +/// The client thread calls Send() with a ControlRequest. The call enqueues +/// the message and returns immediately. The server's sent_callback +/// validates and enqueues the work, then returns without sending a reply. +/// +/// Phase 2 — Server Notify (analogous to the Response event): +/// A server pool worker dequeues the request, does the work, and calls +/// Notify() on the IServerConnection. The ControlResponse payload carries +/// the original request_id. The client's NotifyCallback routes by +/// request_id and wakes the waiting thread. +/// +/// Send() avoids the per-connection REQUEST/REPLY serialization window. The +/// server worker pool still executes requests independently, and NOTIFY carries +/// the only terminal result for each request. +/// +/// Why Send instead of SendWaitReply: +/// SendWaitReply() blocks the calling thread inside the library with no timeout. +/// A misbehaving QM server (slow callback, scheduling starvation without crash) +/// would hold an ASIL-B thread blocked indefinitely. Send() is non-blocking +/// when the client send queue is enabled. The application-level wait_for() +/// provides the terminal notification timeout. +/// +/// Connection model: +/// One IClientConnection per client process (shared by all threads). +/// The request_id in the FlatBuffer payload is used to route each Notify +/// back to the thread that issued the corresponding Send. +/// Because Notify() is point-to-point (reaches only this connection's client), +/// no cross-client leakage occurs — unlike poc_async's broadcast events which +/// required one skeleton instance per client. +/// +/// Process model: same as poc_async — fork before any IPC setup, parent=server, +/// children=clients. +/// +/// Usage: +/// bazel run //tests/score_com_poc:poc_low_level +/// bazel run //tests/score_com_poc:poc_low_level -- --client_count=3 --call_count=5 +/// bazel run //tests/score_com_poc:poc_low_level -- --client_count=3 --call_count=5 --client_threads=4 +/// bazel run //tests/score_com_poc:poc_low_level -- --client_count=3 --call_count=5 --server_threads=2 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flatbuffers/flatbuffers.h" +#include "score/message_passing/client_factory.h" +#include "score/message_passing/i_client_connection.h" +#include "score/message_passing/i_server_connection.h" +#include "score/message_passing/i_server_factory.h" +#include "score/message_passing/server_factory.h" +#include "score/message_passing/service_protocol_config.h" +#include "score/tests/utility/runtime_measurement.hpp" +#include "score/tests/ipc_poc/ipc_buffer.h" +#include "score/tests/ipc_poc/poc_control_generated.h" +#include "score/tests/ipc_poc/poc_helper.hpp" +#include "score/tests/utility/process_resource_measurement.hpp" + +// --------------------------------------------------------------------------- +// Global parameters (set before fork; never mutated after) +// --------------------------------------------------------------------------- + +static int g_client_count = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_call_count = 1000; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_threads = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_server_threads = 8; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static bool g_random_wait = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +// --------------------------------------------------------------------------- +// Shared logging helper +// --------------------------------------------------------------------------- + +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static constexpr bool kEnableVerboseOutput{false}; +static constexpr bool kEnableLatencyVerbose{false}; + +static void Log(const std::string& line) +{ + if (!kEnableVerboseOutput) + { + return; + } + std::lock_guard lk(g_log_mutex); + std::cout << line << "\n"; +} + +static void LogErr(const std::string& line) +{ + std::stringstream ss; + ss << "[ERROR] " << line << "\n"; + std::lock_guard lk(g_log_mutex); + std::cerr << ss.str(); +} + +// --------------------------------------------------------------------------- +// Protocol config — identifier resolves to an abstract Unix domain socket +// --------------------------------------------------------------------------- + +namespace score::crypto::ipc::control +{ + +namespace helper = score::crypto::ipc::poc_helper; + +static constexpr std::string_view kServiceIdentifier{"score_crypto_poc_ll_no_reply"}; + +// --------------------------------------------------------------------------- +// Configuration +// +// All queue sizes derive from two logical inputs: +// +// N — number of distinct client processes (== distinct UIDs the server +// accepts). Controls how many connections the server manages and +// sets the capacity of the global server-side receive queue. +// +// T — maximum number of threads per client process that may have +// concurrent in-flight requests at any one time. Controls the +// per-connection queue depths. +// +// Payload size M is fixed by the IPC protocol (sizeof(IpcBuffer)). +// --------------------------------------------------------------------------- + +/// Parameters shared by both server and client sides. +/// Both sides must be constructed from the same ServiceParams values; +/// a mismatch in any size field causes EMSGSIZE on send or silent +/// truncation on receive. +struct ServiceParams +{ + /// Logical identifier of the service. Maps to an abstract Unix domain + /// socket name on Linux and to a QNX resource-manager path on QNX. + std::string_view identifier; + + /// Maximum byte size of a client→server message (ControlRequest). + /// Must be >= sizeof(the largest FlatBuffer payload sent by any client). + std::uint32_t max_payload_bytes; +}; + +/// Parameters that only the server side needs. +struct ServerParams +{ + /// Number of distinct client processes expected to connect. + /// Used to: + /// - size the global server-side receive queue (N * T slots total) + /// - limit accepted connections to at most N (one per UID) + std::uint32_t max_client_processes; // N + + /// Maximum number of threads per client process that may have concurrent + /// in-flight requests. Used to size per-connection notify queues. + /// Under-sizing this causes Notify() to return ENOBUFS on QNX (response + /// lost, client times out) or blocks the engine thread on Linux. + std::uint32_t max_threads_per_client; // T + + /// Number of server-side pool threads processing requests. + std::uint32_t worker_threads; +}; + +/// Parameters that only the client side needs. +struct ClientParams +{ + /// Maximum number of threads in this process that may have concurrent + /// in-flight requests. Sizes max_queued_sends in the client config so that + /// T concurrent Send() calls can + /// be in-flight simultaneously without getting ENOBUFS. + std::uint32_t max_concurrent_threads; // T +}; + +// --------------------------------------------------------------------------- +// Config factory functions +// --------------------------------------------------------------------------- + +static score::message_passing::ServiceProtocolConfig MakeProtocolConfig(const ServiceParams& p) +{ + return score::message_passing::ServiceProtocolConfig{ + p.identifier, + // max_send_size: upper bound for a client→server ControlRequest FlatBuffer. + /*max_send_size=*/p.max_payload_bytes, + // max_reply_size: upper bound for the ack ControlResponse sent by Reply(). + // The ack only carries request_id and an empty operation batch, so it is much + // smaller than max_payload_bytes in practice. Using the same value keeps both + // sides in sync without a second size constant; the slight over-allocation in the + // client receive buffer is acceptable. + /*max_reply_size=*/0U, + // max_notify_size: upper bound for the full ControlResponse sent by Notify(). + // Must be at least as large as the largest response payload the server produces. + /*max_notify_size=*/p.max_payload_bytes, + }; +} + +static score::message_passing::IServerFactory::ServerConfig MakeServerConfig(const ServerParams& p) +{ + const std::uint32_t n = p.max_client_processes; + const std::uint32_t t = p.max_threads_per_client; + + // NOTE: ServerConfig is read only by the QNX implementation; the Linux/Unix-domain + // implementation ignores all three fields and relies on kernel socket buffers instead. + // The values are set correctly here so that the same code works on QNX without changes. + return score::message_passing::IServerFactory::ServerConfig{ + // Server-side ring buffer for incoming SEND and REQUEST messages (QNX). + // The REQUEST/REPLY protocol serializes one REQUEST per connection: the server + // does not accept the next REQUEST on a connection until Reply() has been called. + // With one connection per client process we therefore have at most N simultaneous + // in-flight REQUESTs — one per client — regardless of how many threads each client + // has. N slots are sufficient; N*T would be an over-allocation. + /*max_queued_sends=*/n, + + // Number of ServerConnection objects pre-allocated at startup (QNX). + // Avoids runtime heap allocation when clients connect, which is required for + // monotonic/bounded memory in safety contexts. Set to N (one per expected client). + /*pre_alloc_connections=*/n, + + // Per-connection NOTIFY queue depth on the server side (QNX). + // Each in-flight Send() on the client side will eventually receive one + // Notify() from the server. With T threads sharing one connection, up to T + // Notify() calls may be queued before the client drains them. + // If this queue overflows, Notify() returns ENOBUFS on QNX — the response is + // silently dropped and the client hangs until the 30-second timeout. + // This is the most critical parameter to size correctly: must be >= T. + /*max_queued_notifies=*/t, + }; +} + +static score::message_passing::IClientFactory::ClientConfig MakeClientConfig(const ClientParams& p) +{ + const std::uint32_t t = p.max_concurrent_threads; + + return score::message_passing::IClientFactory::ClientConfig{ + // One async-reply slot per concurrent thread: each in-flight SendWithCallback() + // holds one slot until its ReplyCallback fires. Must be >= T. + /*max_async_replies=*/0U, + + // Send() with truly_async=true always queues into the send queue before + // the engine thread picks it up. One slot per concurrent thread. + // Must be >= T; shared pool with max_async_replies. + /*max_queued_sends=*/t, + + // This POC uses only SEND, so cross-type ordering is irrelevant. + /*fully_ordered=*/false, + + // Route Send() through the engine's background thread so the + // calling thread is never held inside the IPC layer (non-blocking guarantee). + // Required when max_queued_sends > 0. Mandatory for safety clients sending + // to QM servers where the server callback duration is not bounded. + /*truly_async=*/true, + + // Do not block the calling thread on the first connection attempt. + // Start() is called before the server socket exists (child processes start + // 300 ms after the fork); the background engine thread retries until the + // server is ready and fires the kReady state callback. + /*sync_first_connect=*/false, + }; +} + +// --------------------------------------------------------------------------- +// FlatBuffer helpers +// --------------------------------------------------------------------------- + +static std::vector ProcessRequestBytes(const std::uint64_t request_id, + score::cpp::span message) +{ + helper::Response workload_response; + if (!helper::ProcessRequestBytes(message.data(), message.size(), workload_response)) + { + LogErr("[server/worker] FlatBuffer verification failed"); + return {}; + } + + { + std::ostringstream ss; + ss << "[server/worker] request_id=" << workload_response.request_id << " -> combined=\"" + << workload_response.string_value << "\""; + Log(ss.str()); + } + + if (workload_response.request_id != request_id) + { + return {}; + } + return helper::BuildResponseBytes(workload_response); +} + +// --------------------------------------------------------------------------- +// Server +// --------------------------------------------------------------------------- + +struct WorkItem +{ + score::message_passing::IServerConnection* conn; + std::shared_ptr alive; // per-connection lifetime token; set to false by disconnect_cb + std::uint64_t request_id; + std::vector request_bytes; +}; + +static int RunServer(const std::vector& child_pids) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; + const ServiceParams service_params{ + kServiceIdentifier, + /*max_payload_bytes=*/static_cast(sizeof(IpcBuffer)), + }; + const ServerParams server_params{ + /*max_client_processes=*/static_cast(g_client_count), + /*max_threads_per_client=*/static_cast(g_client_threads), + /*worker_threads=*/static_cast(g_server_threads), + }; + + score::message_passing::ServerFactory server_factory; + const auto protocol_config = MakeProtocolConfig(service_params); + const auto server_config = MakeServerConfig(server_params); + + auto server = server_factory.Create(protocol_config, server_config); + if (!server) + { + LogErr("[server] failed to create server"); + return 1; + } + + // ------------------------------------------------------------------ + // Live-connection guard via per-connection lifetime token. + // + // Worker threads hold a shared_ptr (alive token) inside WorkItem, + // captured at enqueue time. The library destroys a ServerConnection as + // soon as the client disconnects. Without coordination, a worker that + // dequeued a WorkItem before the disconnect fires could call Notify() on + // a destroyed object — or on a new connection that reused the same address. + // + // Fix: each connection gets a shared_ptr initialised to true. + // disconnect_cb sets it to false under live_conn->mutex before the library + // destroys the object. The worker checks the flag under the same mutex + // before calling Notify(), so a false flag always wins the race. + // + // Aliasing is impossible: the WorkItem holds its own shared_ptr copy whose + // control block is unique to that connection's lifetime; a new connection + // that reuses the same address gets a brand-new shared_ptr(true). + // + // Notify() is called while holding live_conn->mutex because the transport + // does not expose a connection lifetime lease. This ordering is required: + // disconnect_cb cannot destroy the connection until a worker has finished + // using its raw pointer. It does not make Notify() bounded. The Unix + // backend uses a blocking sendmsg(), while the QNX backend takes its own + // send mutex and can return ENOBUFS when its notify pool is exhausted. + // Consequently, a blocked Notify() can delay disconnect_cb and therefore + // client admission. Production code must provide a bounded/non-blocking + // Notify() operation or a library-owned connection lease before removing + // this lock or claiming a bounded disconnect path. + // + // Lock order: live_conn->mutex must NOT be taken while holding any + // score::message_passing internal lock. Workers take it only around + // the alive check + Notify(); they release it before touching the work + // queue again. + // + // connect_cb and disconnect_cb use [&] capture and access live_conn directly + // by reference — no heap allocation needed. sent_with_reply_cb accesses it + // via SentWithReplyCtx (a shared_ptr-boxed struct required for the 32-byte + // callback limit), which holds a LiveConnections& into the same frame. + // ------------------------------------------------------------------ + struct LiveConnections + { + std::mutex mutex; + // Was facing pointer-reuse issues, when just using the Connection address for alive checks + // Thus the additional shared_ptr per connection, used in work_items + std::unordered_map> alive_map; + }; + LiveConnections live_conn; + + // ------------------------------------------------------------------ + // Thread pool: workers dequeue requests, do the work, and call + // Notify() on the stored IServerConnection*. + // Notify() is thread-safe and safe to call from a pool thread. + // Multiple concurrent Notify() calls on the same connection are + // serialized by the library; the NotifyCallback on the client side + // routes by request_id so ordering does not matter. + // ------------------------------------------------------------------ + std::mutex queue_mutex; + std::condition_variable queue_cv; + std::queue work_queue; + std::atomic stop_workers{false}; + + std::vector workers; + workers.reserve(server_params.worker_threads); + helper::ThreadStartBarrier worker_start_barrier{server_params.worker_threads}; + for (std::uint32_t w = 0U; w < server_params.worker_threads; ++w) + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::LowLevel::ServerThreadCreation"}; + workers.emplace_back([&, w]() { + worker_start_barrier.ArriveAndWait(); + while (true) + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement wait_scope{ + latency_measurement, "POC::LowLevel::ServerQueueWait"}; + std::unique_lock lk(queue_mutex); + + queue_cv.wait(lk, [&] { + return !work_queue.empty() || stop_workers.load(); + }); + + if (stop_workers.load() && work_queue.empty()) + { + break; + } + + WorkItem item = std::move(work_queue.front()); + work_queue.pop(); + lk.unlock(); + + if (g_sleep_milliseconds > 0) + { + Log("[server/worker " + std::to_string(w) + "] simulating work, sleeping " + + std::to_string(g_sleep_milliseconds) + " ms"); + std::this_thread::sleep_for(std::chrono::milliseconds(g_sleep_milliseconds)); + } + + auto response_bytes = ProcessRequestBytes( + item.request_id, + score::cpp::span{item.request_bytes.data(), item.request_bytes.size()}); + + // Guard against use-after-free: check the alive token under live_conn->mutex, + // which disconnect_cb also holds when it flips the flag to false. + // Notify() is called under the same lock so no window exists between the check + // and the call. This protects the raw connection pointer, but the transport + // call itself is not guaranteed to return within a bounded time; see the + // LiveConnections note above. + std::lock_guard live_lk(live_conn.mutex); + if (!*item.alive) + { + LogErr("[server/worker " + std::to_string(w) + + "] Notify() skipped — connection already disconnected " + "(request_id=" + + std::to_string(item.request_id) + ")"); + continue; + } + + std::ostringstream notify_log; + notify_log << "[server/worker " << w << "] Calling Notify() for request_id=" << item.request_id + << " with " << response_bytes.size() << " bytes"; + Log(notify_log.str()); + + auto notify_result = item.conn->Notify( + score::cpp::span{response_bytes.data(), response_bytes.size()}); + + if (!notify_result.has_value()) + { + // ENOBUFS: max_queued_notifies was too small — response dropped, + // client will hang until its timeout expires. + LogErr("[server/worker " + std::to_string(w) + "] Notify() failed for request_id=" + + std::to_string(item.request_id) + " — check max_queued_notifies >= max_threads_per_client"); + } + else + { + Log("[server/worker " + std::to_string(w) + + "] Notify() succeeded for request_id=" + std::to_string(item.request_id)); + } + } + }); + } + worker_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_server_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + worker_start_barrier.Release(); + Log("[server] started " + std::to_string(server_params.worker_threads) + " worker thread(s)"); + + // ------------------------------------------------------------------ + // UID admission control. + // + // The server enforces at most one active connection per UID. This: + // - prevents a single client from starving others by opening N*T + // connections and consuming the entire server receive queue; + // - binds resource consumption (queue slots, connection objects) to + // the number of authenticated client processes, not to thread count. + // + // All server callbacks for the same IServer instance are called + // sequentially on the library's internal thread (doc §Server callbacks), + // so connected_uids needs no external mutex. + // + // Rejection policy: + // EAGAIN — the UID is already connected; the library will tell the + // client to retry. Used instead of EACCES so that a client + // which reconnects after a crash is not permanently locked out + // while the previous disconnect callback has not yet fired. + // + // Hint: The idea here is not access control as we did it earlier + // but to prevent resource starvation, since we pre-allocate x buffer + // we have a limit on how many simultaneous connections we can handle + // enforcing one connection per UID is a simple way to prevent a + // single client from consuming all resources and starving others. + // ------------------------------------------------------------------ + std::unordered_set connected_uids; + + auto connect_cb = [&](score::message_passing::IServerConnection& conn) + -> score::cpp::expected { + const uid_t uid = conn.GetClientIdentity().uid; +#if ENFORCE_SINGLE_CONNECTION_PER_UID + if (connected_uids.count(uid) != 0U) + { + std::ostringstream ss; + ss << "[server] rejected connection from uid=" << uid << " (already connected) — client will retry"; + Log(ss.str()); + // EAGAIN: instructs the client library to retry the connection + // rather than transitioning to kStopped with kPermission reason. + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EAGAIN)); + } +#endif + connected_uids.insert(uid); + { + std::lock_guard live_lk(live_conn.mutex); + live_conn.alive_map[&conn] = std::make_shared(true); + } + std::ostringstream ss; + ss << "[server] accepted connection from uid=" << uid << " (" << connected_uids.size() << "/" + << server_params.max_client_processes << " slots used)"; + Log(ss.str()); + return score::message_passing::UserData{static_cast(nullptr)}; + }; + + auto disconnect_cb = [&](score::message_passing::IServerConnection& conn) { + const uid_t uid = conn.GetClientIdentity().uid; + connected_uids.erase(uid); + { + // Flip the alive token to false before the library destroys the + // ServerConnection object. Workers hold a shared_ptr copy of the + // same token and check it under live_conn->mutex before Notify(), + // so a false flag always wins the race against pointer reuse. + std::lock_guard live_lk(live_conn.mutex); + auto it = live_conn.alive_map.find(&conn); + if (it != live_conn.alive_map.end()) + { + *it->second = false; + live_conn.alive_map.erase(it); + } + } + std::ostringstream ss; + ss << "[server] client disconnected uid=" << uid << " (" << connected_uids.size() << "/" + << server_params.max_client_processes << " slots used)"; + Log(ss.str()); + }; + + // Box the captured references into a heap struct so the lambda fits + // in the 32-byte inline capacity of score::cpp::callback<>. + struct SentCtx + { + std::mutex& queue_mutex; + std::condition_variable& queue_cv; + std::queue& work_queue; + LiveConnections& live_conn; + }; + auto sent_ctx = std::make_shared(SentCtx{queue_mutex, queue_cv, work_queue, live_conn}); + + auto sent_cb = + [sent_ctx](score::message_passing::IServerConnection& conn, + score::cpp::span message) -> score::cpp::expected_blank { + // Read the client-assigned request_id from the FlatBuffer. + // The client guarantees it is non-zero and unique within its process. + helper::Request workload_request; + if (!helper::ParseRequestBytes(message.data(), message.size(), workload_request)) + { + LogErr("[server/handler] FlatBuffer verification failed — dropping request"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + const std::uint64_t request_id = workload_request.request_id; + if (request_id == 0U) + { + LogErr("[server/handler] received request with zero request_id — dropping"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + + // Fetch the alive token for this connection. The connection is guaranteed + // live at this point (connect_cb has fired, disconnect_cb has not), so the + // entry must exist in alive_map. + std::shared_ptr alive; + { + std::lock_guard live_lk(sent_ctx->live_conn.mutex); + auto it = sent_ctx->live_conn.alive_map.find(&conn); + if (it != sent_ctx->live_conn.alive_map.end()) + { + alive = it->second; + } + } + if (!alive) + { + LogErr("[server/handler] alive token missing for request_id=" + std::to_string(request_id) + + " — connection not found in alive_map (unexpected)"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(ENOENT)); + } + + // Enqueue the full work for the pool worker before acknowledging the + // request. A successful acknowledgement therefore means the work is + // admitted to the application queue. + std::vector bytes(message.begin(), message.end()); + { + std::lock_guard lk(sent_ctx->queue_mutex); + sent_ctx->work_queue.push({&conn, alive, request_id, std::move(bytes)}); + } + + sent_ctx->queue_cv.notify_one(); + + Log("[server/handler] SEND received, request_id=" + std::to_string(request_id) + " queued"); + return {}; + }; + + // Use the fire-and-forget callback for ingress; completion is sent by workers via Notify(). + auto start_result = server->StartListening(connect_cb, disconnect_cb, sent_cb, /*sent_with_reply_cb=*/{}); + if (!start_result.has_value()) + { + LogErr("[server] StartListening failed"); + stop_workers.store(true); + queue_cv.notify_all(); + for (auto& t : workers) + { + t.join(); + } + return 1; + } + + Log("[server] listening — waiting for all clients to finish..."); + + int overall_status = 0; + for (std::size_t i = 0U; i < child_pids.size(); ++i) + { + int wstatus = 0; + pid_t pid = waitpid(-1, &wstatus, 0); + if (pid > 0) + { + const bool ok = WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0; + std::ostringstream ss; + ss << "[server] child pid=" << pid << (ok ? " exited OK" : " FAILED"); + (ok ? Log : LogErr)(ss.str()); + if (!ok) + { + overall_status = 1; + } + } + } + + stop_workers.store(true); + queue_cv.notify_all(); + for (auto& t : workers) + { + t.join(); + } + + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_no_reply_server_workload", resources_after_thread_creation, resources_after_workload); + server->StopListening(); + Log("[server] shutdown complete."); + return overall_status; +} + +// --------------------------------------------------------------------------- +// Client — per-call pending state for notify-based demultiplexing +// --------------------------------------------------------------------------- + +struct PendingCall +{ + bool ready{false}; + bool ok{false}; + std::string result_value; + std::mutex mutex; + std::condition_variable cv; +}; + +static bool RunClient(const int client_index, const int call_count, const int thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; + const ServiceParams service_params{ + kServiceIdentifier, + /*max_payload_bytes=*/static_cast(sizeof(IpcBuffer)), + }; + const ClientParams client_params{ + /*max_concurrent_threads=*/static_cast(thread_count), + }; + + const auto protocol_config = MakeProtocolConfig(service_params); + const auto client_config = MakeClientConfig(client_params); + + score::message_passing::ClientFactory client_factory; + + // ------------------------------------------------------------------ + // Client-assigned request IDs: pid in the upper 32 bits, per-process + // counter in the lower 32 bits. Unique within this client process and + // distinguishable across processes (different pids), so the server can + // echo them back without any server-side ID assignment. + // + // Pending-call map: each in-flight request inserts its shared PendingCall + // BEFORE calling Send(), keyed by its pre-assigned id. + // NotifyCallback looks up by id — the entry is always present because + // the insert happens before the send, eliminating the gap that previously + // required a generation counter and two-phase wait. + // + // Lock order: always pending_map_mutex before PendingCall::mutex. + // ------------------------------------------------------------------ + const std::uint64_t pid_upper = static_cast(::getpid()) << 32U; + std::atomic call_counter{1U}; + std::mutex pending_map_mutex; + std::unordered_map> pending_map; + + // ------------------------------------------------------------------ + // Create ONE shared connection for the whole process. + // All threads share it — Send() is safe to call concurrently because + // truly_async=true routes all sends through the library's background thread + // without blocking the caller. + // ------------------------------------------------------------------ + decltype(client_factory.Create(protocol_config, client_config)) client; + + // The state callback is stored in a score::cpp::callback<> with a fixed + // 32-byte inline capacity. Capturing four objects (two promises + two + // atomics) exceeds that limit, so we box them onto the heap and capture + // a single pointer — sizeof(void*) == 8 bytes. + struct ConnectionState + { + ConnectionState( + std::mutex& pending_map_mutex_in, + std::unordered_map>& pending_map_in) + : pending_map_mutex{pending_map_mutex_in}, pending_map{pending_map_in} + { + } + + std::promise ready_promise; + std::promise stopped_promise; + std::atomic ready_set{false}; + std::atomic stopped_set{false}; + std::mutex& pending_map_mutex; + std::unordered_map>& pending_map; + }; + auto conn_state = std::make_shared(pending_map_mutex, pending_map); + auto ready_future = conn_state->ready_promise.get_future(); + auto stopped_future = conn_state->stopped_promise.get_future(); + + // Box the NotifyCallback captures into a heap struct so the lambda fits + // in the 32-byte inline capacity of score::cpp::callback<>. + struct NotifyCtx + { + std::mutex& pending_map_mutex; + std::unordered_map>& pending_map; + int client_index; + }; + // notify_ctx is shared across all calls; the per-call generation guard is + // stored inside each PendingCall and checked under pending_map_mutex. + auto notify_ctx = std::make_shared(NotifyCtx{pending_map_mutex, pending_map, client_index}); + + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement connection_scope{ + latency_measurement, "POC::LowLevel::ClientConnectionSetup"}; + client = client_factory.Create(protocol_config, client_config); + if (!client) + { + LogErr("[client " + std::to_string(client_index) + "] Create failed"); + return false; + } + + client->Start( + [conn_state, client_index](score::message_passing::IClientConnection::State state) { + if (state == score::message_passing::IClientConnection::State::kReady) + { + if (!conn_state->ready_set.exchange(true)) + { + Log("[client " + std::to_string(client_index) + "] connection ready"); + conn_state->ready_promise.set_value(); + } + } + else if (state == score::message_passing::IClientConnection::State::kStopped) + { + if (!conn_state->stopped_set.exchange(true)) + { + conn_state->stopped_promise.set_value(); + } + std::vector> pending_calls; + { + std::lock_guard map_lk(conn_state->pending_map_mutex); + for (const auto& entry : conn_state->pending_map) + { + pending_calls.push_back(entry.second); + } + conn_state->pending_map.clear(); + } + for (const auto& pending : pending_calls) + { + { + std::lock_guard call_lk(pending->mutex); + pending->ok = false; + pending->ready = true; + } + pending->cv.notify_one(); + } + } + }, + // NotifyCallback — runs on the library's engine thread. + // The PendingCall is inserted before Send(), so the entry is available + // when a Notify arrives. + // MUST NOT call any blocking message_passing operation (doc §Client + // Connection callbacks). Only parse the payload, look up the pending + // call by request_id, and signal the condition variable. + [notify_ctx](score::cpp::span message) { + auto start_time = std::chrono::system_clock::now(); + + helper::Response parsed_response; + const bool parse_ok = helper::ParseResponseBytes(message.data(), message.size(), parsed_response); + const std::uint64_t request_id = parsed_response.request_id; + std::string result_value = std::move(parsed_response.string_value); + if (!parse_ok) + { + LogErr("[client " + std::to_string(notify_ctx->client_index) + "] NotifyCallback: invalid response"); + } + + // Look up and signal the waiting thread. + // The entry was inserted before Send() so it is always + // present when Notify arrives. A missing entry means the call already + // timed out and was erased by the calling thread — discard silently. + std::lock_guard map_lk(notify_ctx->pending_map_mutex); + auto it = notify_ctx->pending_map.find(request_id); + if (it != notify_ctx->pending_map.end()) + { + const auto pending = it->second; + { + std::lock_guard call_lk(pending->mutex); + pending->result_value = std::move(result_value); + pending->ok = parse_ok; + pending->ready = true; + } + pending->cv.notify_one(); + } + else + { + LogErr("[client " + std::to_string(notify_ctx->client_index) + + "] NotifyCallback: late notify for timed-out request_id=" + std::to_string(request_id) + + " — discarded"); + } + + auto end_time = std::chrono::system_clock::now(); + auto diff = end_time - start_time; + Log("[client " + std::to_string(notify_ctx->client_index) + + "] NotifyCallback request_id=" + std::to_string(request_id) + " took " + + std::to_string(std::chrono::duration_cast(diff).count()) + " us"); + }); + + if (ready_future.wait_for(std::chrono::seconds(120)) != std::future_status::ready) + { + LogErr("[client " + std::to_string(client_index) + "] timed out waiting for connection"); + client->Stop(); + return false; + } + } + + // ------------------------------------------------------------------ + // Spawn all client threads. They all share the single connection. + // ------------------------------------------------------------------ + std::atomic total_failures{0}; + std::vector threads; + threads.reserve(static_cast(thread_count)); + helper::ThreadStartBarrier thread_start_barrier{static_cast(thread_count)}; + + for (int t = 0; t < thread_count; ++t) + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::LowLevel::ClientThreadCreation"}; + threads.emplace_back([&, t]() { + thread_start_barrier.ArriveAndWait(); + int failures = 0; + + for (int c = 0; c < call_count; ++c) + { + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement latency_scope{ + latency_measurement, "POC::LowLevel::RoundTrip"}; + + // Build request + auto workload_request = helper::CreateRequest(client_index, t, c); + + // Assign the request_id before building the FlatBuffer so it can + // be embedded in the payload and inserted into the pending_map — + // all before the send. Upper 32 bits = pid (process-unique prefix), + // lower 32 bits = per-process monotonic counter (thread-unique within + // this process). Zero is never produced (counter starts at 1). + const std::uint64_t request_id = pid_upper | call_counter.fetch_add(1U, std::memory_order_relaxed); + workload_request.request_id = request_id; + + // Persist the shared-codec request so it outlives the async send queue. + auto request_buffer = std::make_shared>( + helper::BuildRequestBytes(workload_request)); + score::cpp::span request_span{request_buffer->data(), request_buffer->size()}; + + // Insert into the pending map BEFORE the send so the entry is + // guaranteed to exist when NotifyCallback fires. No generation + // counter needed: request_ids are never reused within a process. + auto pending = std::make_shared(); + { + std::lock_guard map_lk(pending_map_mutex); + pending_map[request_id] = pending; + } + + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t + << "] -> Send() str=\"" << workload_request.string_value + << "\" uint64=" << workload_request.uint64_value << " request_id=" << request_id; + Log(ss.str()); + } + + auto send_result = client->Send(request_span); + if (!send_result.has_value()) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t + << "] Send() failed: " << send_result.error(); + LogErr(ss.str()); + std::lock_guard map_lk(pending_map_mutex); + pending_map.erase(request_id); + ++failures; + continue; + } + + // Wait for NotifyCallback to signal ready. + constexpr auto kNotifyTimeout = std::chrono::seconds(300); + bool timed_out = false; + { + std::unique_lock call_lk(pending->mutex); + timed_out = !pending->cv.wait_for(call_lk, kNotifyTimeout, [&] { + return pending->ready; + }); + } + + // Retire: erase from map. The shared state remains alive until a + // deferred NotifyCallback releases it. + { + std::lock_guard map_lk(pending_map_mutex); + pending_map.erase(request_id); + } + + if (timed_out) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t + << "] TIMEOUT waiting for Notify request_id=" << request_id; + LogErr(ss.str()); + ++failures; + continue; + } + + if (!pending->ok || + !helper::Matches(workload_request, helper::Response{request_id, pending->result_value})) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] MISMATCH request_id=" << request_id + << ": expected=\"" << helper::ProcessRequest(workload_request).string_value << "\" got=\"" + << (pending->ok ? pending->result_value : "") << "\""; + LogErr(ss.str()); + ++failures; + continue; + } + + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] <- OK request_id=" << request_id + << " result=\"" << pending->result_value << "\" (Phase 2: Notify received, round-trip complete)"; + Log(ss.str()); + } + + // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + helper::WaitAfterCall(g_random_wait); + } + + total_failures.fetch_add(failures, std::memory_order_relaxed); + }); + } + + thread_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_client_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + thread_start_barrier.Release(); + + for (auto& th : threads) + { + th.join(); + } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_no_reply_client_workload", resources_after_thread_creation, resources_after_workload); + + client->Stop(); + if (stopped_future.wait_for(std::chrono::seconds(30)) != std::future_status::ready) + { + LogErr("[client " + std::to_string(client_index) + "] timed out waiting for connection to stop"); + } + + const int total = call_count * thread_count; + const int failures = total_failures.load(); + const int success = total - failures; + std::ostringstream ss; + ss << "[client " << client_index << "] Results: " << success << "/" << total << " calls succeeded, " << failures + << "/" << total << " calls failed (" << thread_count << " thread(s) x " << call_count << " call(s))"; + Log(ss.str()); + + return failures == 0; +} + +} // namespace score::crypto::ipc::control + +// --------------------------------------------------------------------------- +// main — fork before any IPC setup for a clean per-process state +// --------------------------------------------------------------------------- + +int main(int argc, char** argv) +{ + score::crypto::ipc::poc_helper::PocArguments defaults; + defaults.client_count = g_client_count; + defaults.call_count = g_call_count; + defaults.client_threads = g_client_threads; + defaults.server_threads = g_server_threads; + defaults.sleep_milliseconds = g_sleep_milliseconds; + std::string parse_error; + const auto parsed_arguments = score::crypto::ipc::poc_helper::ParseArguments(argc, argv, parse_error, defaults); + if (!parsed_arguments.has_value()) + { + LogErr("[main] " + parse_error); + return 1; + } + g_client_count = parsed_arguments->client_count; + g_call_count = parsed_arguments->call_count; + g_client_threads = parsed_arguments->client_threads; + g_server_threads = parsed_arguments->server_threads; + g_sleep_milliseconds = parsed_arguments->sleep_milliseconds; + g_random_wait = parsed_arguments->random_wait; + + std::cout << score::crypto::ipc::poc_helper::SettingsSummary( + "poc_low_level", + g_client_count, + g_call_count, + g_client_threads, + g_server_threads, + g_random_wait) + << '\n' + << std::flush; + Log("[main] client_count=" + std::to_string(g_client_count) + " call_count=" + std::to_string(g_call_count) + + " client_threads=" + std::to_string(g_client_threads) + " server_threads=" + + std::to_string(g_server_threads) + " sleep_milliseconds=" + std::to_string(g_sleep_milliseconds)); + + std::vector child_pids; + int my_client_index = -1; + + for (int i = 0; i < g_client_count; ++i) + { + pid_t pid = ::fork(); + if (pid < 0) + { + std::perror("[main] fork"); + for (pid_t cpid : child_pids) + { + ::kill(cpid, SIGTERM); + } + return 1; + } + if (pid == 0) + { + my_client_index = i; + break; + } + child_pids.push_back(pid); + } + + if (my_client_index == -1) + { + return score::crypto::ipc::control::RunServer(child_pids); + } + else + { + const bool ok = score::crypto::ipc::control::RunClient(my_client_index, g_call_count, g_client_threads); + return ok ? 0 : 1; + } +} + +// clang-format off + +// ============================================================================= +// Design comparison: per-thread connection (initial) vs. shared connection +// with Send+Notify (this version) +// ============================================================================= +// +// APPROACH A — One IClientConnection per thread, SendWaitReply +// ------------------------------------------------------------- +// Client threads each own a dedicated connection to the server. +// SendWaitReply() blocks the calling thread until the server calls Reply(). +// The server uses sent_with_reply_callback + pool threads that call Reply(). +// +// Max parallel requests from one client process: +// Exactly thread_count. Each connection carries at most one in-flight +// REQUEST at a time (per-connection REQUEST/REPLY serialization). +// Adding a thread automatically adds a connection and one more parallel +// slot — no explicit queue configuration needed. +// +// Memory / configuration (N client processes, T threads each): +// Server allocates N*T connection objects, each with its own receive +// buffer, reply slot, and notify queue. No application-level queue +// sizing is required beyond the connection count itself — the parallelism +// limit is implicit in the number of connections. +// Server max_queued_sends must cover all N*T concurrent sends. +// +// Pros: +// - Simple: no ticket tracking, no response routing, no shared state on the +// client side. The transport itself matches requests to replies by +// connection ordering. +// - Configuration is trivially correct: parallelism = thread count = connection +// count, with no additional parameters to keep in sync. +// - reply_buffer can be stack-allocated per thread. +// +// Cons: +// - One OS connection (socket fd pair) per thread. For N client processes +// each with T threads, the server holds N*T open connections. +// - The REQUEST/REPLY protocol serializes the server's sent_with_reply +// callback per connection (doc §Server callbacks): while Reply() has not +// been called, no further REQUEST from the same connection is processed. +// This is fine with one connection per thread (each thread sends one +// request at a time), but it means the connection count cannot be reduced +// without losing parallelism. +// - SendWaitReply() blocks the calling thread inside the library during the +// entire round-trip; there is no built-in timeout. +// +// APPROACH B — One IClientConnection per process, SendWithCallback + Notify (this file) +// -------------------------------------------------------------------------------------- +// One connection is shared by all threads. Threads call SendWithCallback() with a +// per-call ReplyCallback. The ReplyCallback (engine thread) parses the ack and inserts +// into the pending_map. The NotifyCallback (same engine thread) routes by request_id +// and signals the waiting thread. The caller waits on an application-level CV with a +// timeout — it is never blocked inside the library. +// +// Max parallel requests from one client process: +// Bounded by the minimum of four independently configured sizes: +// client max_async_replies (one slot per in-flight SendWithCallback) +// client max_queued_sends (one slot per concurrent enqueue) +// server max_queued_sends (server-side receive queue, shared across clients) +// server max_queued_notifies (per-connection notify queue) +// All must be set to >= thread_count. If any one is undersized, +// SendWithCallback() returns ENOBUFS or Notify() is dropped — with no +// automatic backpressure to the calling thread. +// +// Memory / configuration (N client processes, T threads each): +// Server allocates N connection objects (not N*T). Total queue capacity: +// +// server max_queued_sends — shared across ALL connections (QNX). +// At most N simultaneous REQUESTs (one per +// connection), so N slots suffice. +// +// server max_queued_notifies — per connection (QNX). +// Must hold T responses for that client's +// threads. Independent of N. +// +// client max_async_replies — per connection, per client process. +// Must hold T concurrent in-flight callbacks. +// +// client max_queued_sends — per connection, per client process. +// Must hold T concurrent enqueues. +// +// All four client sizes must be kept in sync with T; the coupling is +// enforced by convention only, not by the API. +// +// Pros: +// - O(clients) connections instead of O(clients * threads). +// - SendWithCallback() is non-blocking — the calling thread is never held +// inside the IPC layer regardless of server behavior. +// - Application-level wait_for() timeout bounds how long the thread can wait, +// which is a prerequisite for use in safety-relevant contexts. +// - ReplyCallback and NotifyCallback run sequentially on the engine thread, +// so the pending_map needs no early-notify buffer: the entry is always +// present by the time Notify() arrives. +// - NotifyCallback runs on the library's own thread — no dedicated receive +// thread is needed on the client side. +// +// Cons: +// - Requires ticket tracking (request_id map + mutex) on the client side. +// - ReplyCallback and NotifyCallback must not call any blocking message_passing +// operation (doc §Client Connection callbacks); only signal/mutex work is allowed. +// - Four queue sizes must all be kept >= thread_count. Getting any one wrong +// causes ENOBUFS on send or silent response drops rather than a clean error. +// - The response FlatBuffer (ControlResponse) must carry the request_id +// for response routing; a simpler protocol without a correlation id could +// not use this model. +// +// ANALOGY TO poc_async (mw::com): +// poc_async faced the same root constraint at the higher abstraction level: +// mw::com methods also serialize the handler per skeleton instance. The +// workaround there was identical in spirit — use a short-lived method call +// (enqueue only) + a separate event channel (broadcast) for the response. +// The key difference is that mw::com events broadcast to ALL subscribers, +// requiring one skeleton per client to prevent cross-client leakage and +// making the ticket essential for routing. message_passing Notify() is +// point-to-point per connection, so one connection per process suffices and +// the ticket is only needed for intra-process thread response routing. +// +// clang-format on diff --git a/score/tests/ipc_poc/poc_low_level_no_reply_notify_in_thread.cpp b/score/tests/ipc_poc/poc_low_level_no_reply_notify_in_thread.cpp new file mode 100644 index 000000000..6cd6afa27 --- /dev/null +++ b/score/tests/ipc_poc/poc_low_level_no_reply_notify_in_thread.cpp @@ -0,0 +1,1209 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This file exists only to test and understand IPC mechanisms and to guide a +// proper implementation. It is kept temporarily as reference and will be +// removed once confidence has been gained in the proper implementation. +// ============================================================================= + +/// POC: Send + immediate Notify model — one shared connection per client process +/// +/// This replicates the functionality of poc_async.cpp using only the low-level +/// score::message_passing API, without any score::mw elements. +/// +/// Communication model: +/// +/// Phase 1 — Send for non-blocking request submission: +/// The client thread calls Send() with a ControlRequest. The call enqueues +/// the message and returns immediately. The server's sent_callback +/// validates and enqueues the work, then returns without sending a reply. +/// +/// Phase 2 — Immediate Server Notify: +/// The server callback queues the request and immediately calls Notify(). +/// The notification confirms queue admission and carries the response +/// derived from the request. The worker pool then processes the queued +/// request independently; its result is not part of the measured path. +/// +/// Send() avoids the per-connection REQUEST/REPLY serialization window. This +/// variant also removes worker execution from the client wait: NOTIFY confirms +/// admission immediately after queueing rather than completion of the work. +/// +/// Why Send instead of SendWaitReply: +/// SendWaitReply() blocks the calling thread inside the library with no timeout. +/// A misbehaving QM server (slow callback, scheduling starvation without crash) +/// would hold an ASIL-B thread blocked indefinitely. Send() is non-blocking +/// when the client send queue is enabled. The application-level wait_for() +/// provides the terminal notification timeout. +/// +/// Connection model: +/// One IClientConnection per client process (shared by all threads). +/// The request_id in the FlatBuffer payload is used to route each Notify +/// back to the thread that issued the corresponding Send. +/// Because Notify() is point-to-point (reaches only this connection's client), +/// no cross-client leakage occurs — unlike poc_async's broadcast events which +/// required one skeleton instance per client. +/// +/// Process model: same as poc_async — fork before any IPC setup, parent=server, +/// children=clients. +/// +/// Usage: +/// bazel run //score/tests/ipc_poc:poc_low_level_no_reply_notify_in_thread +/// bazel run //score/tests/ipc_poc:poc_low_level_no_reply_notify_in_thread -- --client_count=3 --call_count=5 +/// bazel run //score/tests/ipc_poc:poc_low_level_no_reply_notify_in_thread -- --client_count=3 --call_count=5 --client_threads=4 +/// bazel run //score/tests/ipc_poc:poc_low_level_no_reply_notify_in_thread -- --client_count=3 --call_count=5 --server_threads=2 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flatbuffers/flatbuffers.h" +#include "score/message_passing/client_factory.h" +#include "score/message_passing/i_client_connection.h" +#include "score/message_passing/i_server_connection.h" +#include "score/message_passing/i_server_factory.h" +#include "score/message_passing/server_factory.h" +#include "score/message_passing/service_protocol_config.h" +#include "score/tests/utility/runtime_measurement.hpp" +#include "score/tests/ipc_poc/ipc_buffer.h" +#include "score/tests/ipc_poc/poc_control_generated.h" +#include "score/tests/ipc_poc/poc_helper.hpp" +#include "score/tests/utility/process_resource_measurement.hpp" + +// --------------------------------------------------------------------------- +// Global parameters (set before fork; never mutated after) +// --------------------------------------------------------------------------- + +static int g_client_count = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_call_count = 1000; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_threads = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_server_threads = 8; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static bool g_random_wait = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +// --------------------------------------------------------------------------- +// Shared logging helper +// --------------------------------------------------------------------------- + +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static constexpr bool kEnableVerboseOutput{false}; +static constexpr bool kEnableLatencyVerbose{false}; + +static void Log(const std::string& line) +{ + if (!kEnableVerboseOutput) + { + return; + } + std::lock_guard lk(g_log_mutex); + std::cout << line << "\n"; +} + +static void LogErr(const std::string& line) +{ + std::stringstream ss; + ss << "[ERROR] " << line << "\n"; + std::lock_guard lk(g_log_mutex); + std::cerr << ss.str(); +} + +// --------------------------------------------------------------------------- +// Protocol config — identifier resolves to an abstract Unix domain socket +// --------------------------------------------------------------------------- + +namespace score::crypto::ipc::control +{ + +namespace helper = score::crypto::ipc::poc_helper; + +static constexpr std::string_view kServiceIdentifier{"score_crypto_poc_ll_no_reply_notify_in_thread"}; + +// --------------------------------------------------------------------------- +// Configuration +// +// All queue sizes derive from two logical inputs: +// +// N — number of distinct client processes (== distinct UIDs the server +// accepts). Controls how many connections the server manages and +// sets the capacity of the global server-side receive queue. +// +// T — maximum number of threads per client process that may have +// concurrent in-flight requests at any one time. Controls the +// per-connection queue depths. +// +// Payload size M is fixed by the IPC protocol (sizeof(IpcBuffer)). +// --------------------------------------------------------------------------- + +/// Parameters shared by both server and client sides. +/// Both sides must be constructed from the same ServiceParams values; +/// a mismatch in any size field causes EMSGSIZE on send or silent +/// truncation on receive. +struct ServiceParams +{ + /// Logical identifier of the service. Maps to an abstract Unix domain + /// socket name on Linux and to a QNX resource-manager path on QNX. + std::string_view identifier; + + /// Maximum byte size of a client→server message (ControlRequest). + /// Must be >= sizeof(the largest FlatBuffer payload sent by any client). + std::uint32_t max_payload_bytes; +}; + +/// Parameters that only the server side needs. +struct ServerParams +{ + /// Number of distinct client processes expected to connect. + /// Used to: + /// - size the global server-side receive queue (N * T slots total) + /// - limit accepted connections to at most N (one per UID) + std::uint32_t max_client_processes; // N + + /// Maximum number of threads per client process that may have concurrent + /// in-flight requests. Used to size per-connection notify queues. + /// Under-sizing this causes Notify() to return ENOBUFS on QNX (response + /// lost, client times out) or blocks the engine thread on Linux. + std::uint32_t max_threads_per_client; // T + + /// Number of server-side pool threads processing requests. + std::uint32_t worker_threads; +}; + +/// Parameters that only the client side needs. +struct ClientParams +{ + /// Maximum number of threads in this process that may have concurrent + /// in-flight requests. Sizes max_queued_sends in the client config so that + /// T concurrent Send() calls can + /// be in-flight simultaneously without getting ENOBUFS. + std::uint32_t max_concurrent_threads; // T +}; + +// --------------------------------------------------------------------------- +// Config factory functions +// --------------------------------------------------------------------------- + +static score::message_passing::ServiceProtocolConfig MakeProtocolConfig(const ServiceParams& p) +{ + return score::message_passing::ServiceProtocolConfig{ + p.identifier, + // max_send_size: upper bound for a client→server ControlRequest FlatBuffer. + /*max_send_size=*/p.max_payload_bytes, + // No REQUEST/REPLY messages are used by this POC. + // The ack only carries request_id and an empty operation batch, so it is much + // smaller than max_payload_bytes in practice. Using the same value keeps both + // sides in sync without a second size constant; the slight over-allocation in the + // client receive buffer is acceptable. + /*max_reply_size=*/0U, + // max_notify_size: upper bound for the full ControlResponse sent by Notify(). + // Must be at least as large as the largest response payload the server produces. + /*max_notify_size=*/p.max_payload_bytes, + }; +} + +static score::message_passing::IServerFactory::ServerConfig MakeServerConfig(const ServerParams& p) +{ + const std::uint32_t n = p.max_client_processes; + const std::uint32_t t = p.max_threads_per_client; + + // NOTE: ServerConfig is read only by the QNX implementation; the Linux/Unix-domain + // implementation ignores all three fields and relies on kernel socket buffers instead. + // The values are set correctly here so that the same code works on QNX without changes. + return score::message_passing::IServerFactory::ServerConfig{ + // Server-side ring buffer for incoming SEND and REQUEST messages (QNX). + // The REQUEST/REPLY protocol serializes one REQUEST per connection: the server + // does not accept the next REQUEST on a connection until Reply() has been called. + // With one connection per client process we therefore have at most N simultaneous + // in-flight REQUESTs — one per client — regardless of how many threads each client + // has. N slots are sufficient; N*T would be an over-allocation. + /*max_queued_sends=*/n, + + // Number of ServerConnection objects pre-allocated at startup (QNX). + // Avoids runtime heap allocation when clients connect, which is required for + // monotonic/bounded memory in safety contexts. Set to N (one per expected client). + /*pre_alloc_connections=*/n, + + // Per-connection NOTIFY queue depth on the server side (QNX). + // Each in-flight Send() on the client side will eventually receive one + // Notify() from the server. With T threads sharing one connection, up to T + // Notify() calls may be queued before the client drains them. + // If this queue overflows, Notify() returns ENOBUFS on QNX — the response is + // silently dropped and the client hangs until the 30-second timeout. + // This is the most critical parameter to size correctly: must be >= T. + /*max_queued_notifies=*/t, + }; +} + +static score::message_passing::IClientFactory::ClientConfig MakeClientConfig(const ClientParams& p) +{ + const std::uint32_t t = p.max_concurrent_threads; + + return score::message_passing::IClientFactory::ClientConfig{ + // One async-reply slot per concurrent thread: each in-flight SendWithCallback() + // holds one slot until its ReplyCallback fires. Must be >= T. + /*max_async_replies=*/0U, + + // Send() with truly_async=true always queues into the send queue before + // the engine thread picks it up. One slot per concurrent thread. + // Must be >= T; shared pool with max_async_replies. + /*max_queued_sends=*/t, + + // This POC uses only SEND, so cross-type ordering is irrelevant. + /*fully_ordered=*/false, + + // Route Send() through the engine's background thread so the + // calling thread is never held inside the IPC layer (non-blocking guarantee). + // Required when max_queued_sends > 0. Mandatory for safety clients sending + // to QM servers where the server callback duration is not bounded. + /*truly_async=*/true, + + // Do not block the calling thread on the first connection attempt. + // Start() is called before the server socket exists (child processes start + // 300 ms after the fork); the background engine thread retries until the + // server is ready and fires the kReady state callback. + /*sync_first_connect=*/false, + }; +} + +// --------------------------------------------------------------------------- +// FlatBuffer helpers +// --------------------------------------------------------------------------- + +static std::vector ProcessRequestBytes(const std::uint64_t request_id, + score::cpp::span message) +{ + helper::Response workload_response; + if (!helper::ProcessRequestBytes(message.data(), message.size(), workload_response)) + { + LogErr("[server/worker] FlatBuffer verification failed"); + return {}; + } + + { + std::ostringstream ss; + ss << "[server/worker] request_id=" << workload_response.request_id << " -> combined=\"" + << workload_response.string_value << "\""; + Log(ss.str()); + } + + if (workload_response.request_id != request_id) + { + return {}; + } + return helper::BuildResponseBytes(workload_response); +} + +// --------------------------------------------------------------------------- +// Server +// --------------------------------------------------------------------------- + +struct WorkItem +{ + score::message_passing::IServerConnection* conn; + std::shared_ptr alive; // per-connection lifetime token; set to false by disconnect_cb + std::uint64_t request_id; + std::vector request_bytes; + std::chrono::steady_clock::time_point queued_at; +}; + +static int RunServer(const std::vector& child_pids) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; + const ServiceParams service_params{ + kServiceIdentifier, + /*max_payload_bytes=*/static_cast(sizeof(IpcBuffer)), + }; + const ServerParams server_params{ + /*max_client_processes=*/static_cast(g_client_count), + /*max_threads_per_client=*/static_cast(g_client_threads), + /*worker_threads=*/static_cast(g_server_threads), + }; + + score::message_passing::ServerFactory server_factory; + const auto protocol_config = MakeProtocolConfig(service_params); + const auto server_config = MakeServerConfig(server_params); + + auto server = server_factory.Create(protocol_config, server_config); + if (!server) + { + LogErr("[server] failed to create server"); + return 1; + } + + // ------------------------------------------------------------------ + // Live-connection guard via per-connection lifetime token. + // + // Worker threads hold a shared_ptr (alive token) inside WorkItem, + // captured at enqueue time. The library destroys a ServerConnection as + // soon as the client disconnects. Without coordination, a worker that + // dequeued a WorkItem before the disconnect fires could call Notify() on + // a destroyed object — or on a new connection that reused the same address. + // + // Fix: each connection gets a shared_ptr initialised to true. + // disconnect_cb sets it to false under live_conn->mutex before the library + // destroys the object. The worker checks the flag under the same mutex + // before calling Notify(), so a false flag always wins the race. + // + // Aliasing is impossible: the WorkItem holds its own shared_ptr copy whose + // control block is unique to that connection's lifetime; a new connection + // that reuses the same address gets a brand-new shared_ptr(true). + // + // Notify() is called while holding live_conn->mutex because the transport + // does not expose a connection lifetime lease. This ordering is required: + // disconnect_cb cannot destroy the connection until a worker has finished + // using its raw pointer. It does not make Notify() bounded. The Unix + // backend uses a blocking sendmsg(), while the QNX backend takes its own + // send mutex and can return ENOBUFS when its notify pool is exhausted. + // Consequently, a blocked Notify() can delay disconnect_cb and therefore + // client admission. Production code must provide a bounded/non-blocking + // Notify() operation or a library-owned connection lease before removing + // this lock or claiming a bounded disconnect path. + // + // Lock order: live_conn->mutex must NOT be taken while holding any + // score::message_passing internal lock. Workers take it only around + // the alive check + Notify(); they release it before touching the work + // queue again. + // + // connect_cb and disconnect_cb use [&] capture and access live_conn directly + // by reference — no heap allocation needed. sent_cb accesses it via + // SentCtx, which holds a LiveConnections& into the same frame. + // ------------------------------------------------------------------ + struct LiveConnections + { + std::mutex mutex; + // Was facing pointer-reuse issues, when just using the Connection address for alive checks + // Thus the additional shared_ptr per connection, used in work_items + std::unordered_map> alive_map; + }; + LiveConnections live_conn; + + // ------------------------------------------------------------------ + // Thread pool: workers dequeue requests and do the work. Notify() is sent + // directly from the server callback after queue admission. + // ------------------------------------------------------------------ + std::mutex queue_mutex; + std::condition_variable queue_cv; + std::queue work_queue; + std::atomic stop_workers{false}; + + std::vector workers; + workers.reserve(server_params.worker_threads); + helper::ThreadStartBarrier worker_start_barrier{server_params.worker_threads}; + for (std::uint32_t w = 0U; w < server_params.worker_threads; ++w) + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::LowLevel::ServerThreadCreation"}; + workers.emplace_back([&, w]() { + worker_start_barrier.ArriveAndWait(); + while (true) + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement wait_scope{ + latency_measurement, "POC::LowLevel::ServerQueueWait"}; + std::unique_lock lk(queue_mutex); + + queue_cv.wait(lk, [&] { + return !work_queue.empty() || stop_workers.load(); + }); + + if (stop_workers.load() && work_queue.empty()) + { + break; + } + + WorkItem item = std::move(work_queue.front()); + work_queue.pop(); + lk.unlock(); + + latency_measurement.RecordElapsed("POC::LowLevel::QueueToProcessingStart", item.queued_at); + + if (g_sleep_milliseconds > 0) + { + Log("[server/worker " + std::to_string(w) + "] simulating work, sleeping " + + std::to_string(g_sleep_milliseconds) + " ms"); + std::this_thread::sleep_for(std::chrono::milliseconds(g_sleep_milliseconds)); + } + + ProcessRequestBytes( + item.request_id, + score::cpp::span{item.request_bytes.data(), item.request_bytes.size()}); + } + }); + } + worker_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_server_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + worker_start_barrier.Release(); + Log("[server] started " + std::to_string(server_params.worker_threads) + " worker thread(s)"); + + // ------------------------------------------------------------------ + // UID admission control. + // + // The server enforces at most one active connection per UID. This: + // - prevents a single client from starving others by opening N*T + // connections and consuming the entire server receive queue; + // - binds resource consumption (queue slots, connection objects) to + // the number of authenticated client processes, not to thread count. + // + // All server callbacks for the same IServer instance are called + // sequentially on the library's internal thread (doc §Server callbacks), + // so connected_uids needs no external mutex. + // + // Rejection policy: + // EAGAIN — the UID is already connected; the library will tell the + // client to retry. Used instead of EACCES so that a client + // which reconnects after a crash is not permanently locked out + // while the previous disconnect callback has not yet fired. + // + // Hint: The idea here is not access control as we did it earlier + // but to prevent resource starvation, since we pre-allocate x buffer + // we have a limit on how many simultaneous connections we can handle + // enforcing one connection per UID is a simple way to prevent a + // single client from consuming all resources and starving others. + // ------------------------------------------------------------------ + std::unordered_set connected_uids; + + auto connect_cb = [&](score::message_passing::IServerConnection& conn) + -> score::cpp::expected { + const uid_t uid = conn.GetClientIdentity().uid; +#if ENFORCE_SINGLE_CONNECTION_PER_UID + if (connected_uids.count(uid) != 0U) + { + std::ostringstream ss; + ss << "[server] rejected connection from uid=" << uid << " (already connected) — client will retry"; + Log(ss.str()); + // EAGAIN: instructs the client library to retry the connection + // rather than transitioning to kStopped with kPermission reason. + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EAGAIN)); + } +#endif + connected_uids.insert(uid); + { + std::lock_guard live_lk(live_conn.mutex); + live_conn.alive_map[&conn] = std::make_shared(true); + } + std::ostringstream ss; + ss << "[server] accepted connection from uid=" << uid << " (" << connected_uids.size() << "/" + << server_params.max_client_processes << " slots used)"; + Log(ss.str()); + return score::message_passing::UserData{static_cast(nullptr)}; + }; + + auto disconnect_cb = [&](score::message_passing::IServerConnection& conn) { + const uid_t uid = conn.GetClientIdentity().uid; + connected_uids.erase(uid); + { + // Flip the alive token to false before the library destroys the + // ServerConnection object. Workers hold a shared_ptr copy of the + // same token and check it under live_conn->mutex before Notify(), + // so a false flag always wins the race against pointer reuse. + std::lock_guard live_lk(live_conn.mutex); + auto it = live_conn.alive_map.find(&conn); + if (it != live_conn.alive_map.end()) + { + *it->second = false; + live_conn.alive_map.erase(it); + } + } + std::ostringstream ss; + ss << "[server] client disconnected uid=" << uid << " (" << connected_uids.size() << "/" + << server_params.max_client_processes << " slots used)"; + Log(ss.str()); + }; + + // Box the captured references into a heap struct so the lambda fits + // in the 32-byte inline capacity of score::cpp::callback<>. + struct SentCtx + { + std::mutex& queue_mutex; + std::condition_variable& queue_cv; + std::queue& work_queue; + LiveConnections& live_conn; + }; + auto sent_ctx = std::make_shared(SentCtx{queue_mutex, queue_cv, work_queue, live_conn}); + + auto sent_cb = + [sent_ctx](score::message_passing::IServerConnection& conn, + score::cpp::span message) -> score::cpp::expected_blank { + // Read the client-assigned request_id from the FlatBuffer. + // The client guarantees it is non-zero and unique within its process. + helper::Request workload_request; + if (!helper::ParseRequestBytes(message.data(), message.size(), workload_request)) + { + LogErr("[server/handler] FlatBuffer verification failed — dropping request"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + const std::uint64_t request_id = workload_request.request_id; + if (request_id == 0U) + { + LogErr("[server/handler] received request with zero request_id — dropping"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(EINVAL)); + } + + // Fetch the alive token for this connection. The connection is guaranteed + // live at this point (connect_cb has fired, disconnect_cb has not), so the + // entry must exist in alive_map. + std::shared_ptr alive; + { + std::lock_guard live_lk(sent_ctx->live_conn.mutex); + auto it = sent_ctx->live_conn.alive_map.find(&conn); + if (it != sent_ctx->live_conn.alive_map.end()) + { + alive = it->second; + } + } + if (!alive) + { + LogErr("[server/handler] alive token missing for request_id=" + std::to_string(request_id) + + " — connection not found in alive_map (unexpected)"); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(ENOENT)); + } + + const auto response_bytes = helper::BuildResponseBytes(helper::ProcessRequest(workload_request)); + std::lock_guard live_lk(sent_ctx->live_conn.mutex); + auto notify_result = conn.Notify( + score::cpp::span{response_bytes.data(), response_bytes.size()}); + if (!notify_result.has_value()) + { + LogErr("[server/handler] Notify() failed for request_id=" + std::to_string(request_id) + + " — check max_queued_notifies >= max_threads_per_client"); + return notify_result; + } + + // Enqueue the full work for the pool worker before notifying the client. + std::vector bytes(message.begin(), message.end()); + + { + std::lock_guard lk(sent_ctx->queue_mutex); + sent_ctx->work_queue.push( + {&conn, alive, request_id, std::move(bytes), std::chrono::steady_clock::now()}); + } + + sent_ctx->queue_cv.notify_one(); + + Log("[server/handler] SEND received, request_id=" + std::to_string(request_id) + " queued"); + return {}; + }; + + // Use the fire-and-forget callback for ingress; queue admission is notified + // directly from the callback. + auto start_result = server->StartListening(connect_cb, disconnect_cb, sent_cb, /*sent_with_reply_cb=*/{}); + if (!start_result.has_value()) + { + LogErr("[server] StartListening failed"); + stop_workers.store(true); + queue_cv.notify_all(); + for (auto& t : workers) + { + t.join(); + } + return 1; + } + + Log("[server] listening — waiting for all clients to finish..."); + + int overall_status = 0; + for (std::size_t i = 0U; i < child_pids.size(); ++i) + { + int wstatus = 0; + pid_t pid = waitpid(-1, &wstatus, 0); + if (pid > 0) + { + const bool ok = WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0; + std::ostringstream ss; + ss << "[server] child pid=" << pid << (ok ? " exited OK" : " FAILED"); + (ok ? Log : LogErr)(ss.str()); + if (!ok) + { + overall_status = 1; + } + } + } + + stop_workers.store(true); + queue_cv.notify_all(); + for (auto& t : workers) + { + t.join(); + } + + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_no_reply_notify_in_thread_server_workload", resources_after_thread_creation, resources_after_workload); + server->StopListening(); + Log("[server] shutdown complete."); + return overall_status; +} + +// --------------------------------------------------------------------------- +// Client — per-call pending state for notify-based demultiplexing +// --------------------------------------------------------------------------- + +struct PendingCall +{ + bool ready{false}; + bool ok{false}; + std::string result_value; + std::mutex mutex; + std::condition_variable cv; +}; + +static bool RunClient(const int client_index, const int call_count, const int thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; + const ServiceParams service_params{ + kServiceIdentifier, + /*max_payload_bytes=*/static_cast(sizeof(IpcBuffer)), + }; + const ClientParams client_params{ + /*max_concurrent_threads=*/static_cast(thread_count), + }; + + const auto protocol_config = MakeProtocolConfig(service_params); + const auto client_config = MakeClientConfig(client_params); + + score::message_passing::ClientFactory client_factory; + + // ------------------------------------------------------------------ + // Client-assigned request IDs: pid in the upper 32 bits, per-process + // counter in the lower 32 bits. Unique within this client process and + // distinguishable across processes (different pids), so the server can + // echo them back without any server-side ID assignment. + // + // Pending-call map: each in-flight request inserts its shared PendingCall + // BEFORE calling Send(), keyed by its pre-assigned id. + // NotifyCallback looks up by id — the entry is always present because + // the insert happens before the send, eliminating the gap that previously + // required a generation counter and two-phase wait. + // + // Lock order: always pending_map_mutex before PendingCall::mutex. + // ------------------------------------------------------------------ + const std::uint64_t pid_upper = static_cast(::getpid()) << 32U; + std::atomic call_counter{1U}; + std::mutex pending_map_mutex; + std::unordered_map> pending_map; + + // ------------------------------------------------------------------ + // Create ONE shared connection for the whole process. + // All threads share it — Send() is safe to call concurrently because + // truly_async=true routes all sends through the library's background thread + // without blocking the caller. + // ------------------------------------------------------------------ + decltype(client_factory.Create(protocol_config, client_config)) client; + + // The state callback is stored in a score::cpp::callback<> with a fixed + // 32-byte inline capacity. Capturing four objects (two promises + two + // atomics) exceeds that limit, so we box them onto the heap and capture + // a single pointer — sizeof(void*) == 8 bytes. + struct ConnectionState + { + ConnectionState( + std::mutex& pending_map_mutex_in, + std::unordered_map>& pending_map_in) + : pending_map_mutex{pending_map_mutex_in}, pending_map{pending_map_in} + { + } + + std::promise ready_promise; + std::promise stopped_promise; + std::atomic ready_set{false}; + std::atomic stopped_set{false}; + std::mutex& pending_map_mutex; + std::unordered_map>& pending_map; + }; + auto conn_state = std::make_shared(pending_map_mutex, pending_map); + auto ready_future = conn_state->ready_promise.get_future(); + auto stopped_future = conn_state->stopped_promise.get_future(); + + // Box the NotifyCallback captures into a heap struct so the lambda fits + // in the 32-byte inline capacity of score::cpp::callback<>. + struct NotifyCtx + { + std::mutex& pending_map_mutex; + std::unordered_map>& pending_map; + int client_index; + }; + // notify_ctx is shared across all calls; the per-call generation guard is + // stored inside each PendingCall and checked under pending_map_mutex. + auto notify_ctx = std::make_shared(NotifyCtx{pending_map_mutex, pending_map, client_index}); + + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement connection_scope{ + latency_measurement, "POC::LowLevel::ClientConnectionSetup"}; + client = client_factory.Create(protocol_config, client_config); + if (!client) + { + LogErr("[client " + std::to_string(client_index) + "] Create failed"); + return false; + } + + client->Start( + [conn_state, client_index](score::message_passing::IClientConnection::State state) { + if (state == score::message_passing::IClientConnection::State::kReady) + { + if (!conn_state->ready_set.exchange(true)) + { + Log("[client " + std::to_string(client_index) + "] connection ready"); + conn_state->ready_promise.set_value(); + } + } + else if (state == score::message_passing::IClientConnection::State::kStopped) + { + if (!conn_state->stopped_set.exchange(true)) + { + conn_state->stopped_promise.set_value(); + } + std::vector> pending_calls; + { + std::lock_guard map_lk(conn_state->pending_map_mutex); + for (const auto& entry : conn_state->pending_map) + { + pending_calls.push_back(entry.second); + } + conn_state->pending_map.clear(); + } + for (const auto& pending : pending_calls) + { + { + std::lock_guard call_lk(pending->mutex); + pending->ok = false; + pending->ready = true; + } + pending->cv.notify_one(); + } + } + }, + // NotifyCallback — runs on the library's engine thread. + // The PendingCall is inserted before Send(), so the entry is available + // when a Notify arrives. + // MUST NOT call any blocking message_passing operation (doc §Client + // Connection callbacks). Only parse the payload, look up the pending + // call by request_id, and signal the condition variable. + [notify_ctx](score::cpp::span message) { + auto start_time = std::chrono::system_clock::now(); + + helper::Response parsed_response; + const bool parse_ok = helper::ParseResponseBytes(message.data(), message.size(), parsed_response); + const std::uint64_t request_id = parsed_response.request_id; + std::string result_value = std::move(parsed_response.string_value); + if (!parse_ok) + { + LogErr("[client " + std::to_string(notify_ctx->client_index) + "] NotifyCallback: invalid response"); + } + + // Look up and signal the waiting thread. + // The entry was inserted before Send() so it is always + // present when Notify arrives. A missing entry means the call already + // timed out and was erased by the calling thread — discard silently. + std::lock_guard map_lk(notify_ctx->pending_map_mutex); + auto it = notify_ctx->pending_map.find(request_id); + if (it != notify_ctx->pending_map.end()) + { + const auto pending = it->second; + { + std::lock_guard call_lk(pending->mutex); + pending->result_value = std::move(result_value); + pending->ok = parse_ok; + pending->ready = true; + } + pending->cv.notify_one(); + } + else + { + LogErr("[client " + std::to_string(notify_ctx->client_index) + + "] NotifyCallback: late notify for timed-out request_id=" + std::to_string(request_id) + + " — discarded"); + } + + auto end_time = std::chrono::system_clock::now(); + auto diff = end_time - start_time; + Log("[client " + std::to_string(notify_ctx->client_index) + + "] NotifyCallback request_id=" + std::to_string(request_id) + " took " + + std::to_string(std::chrono::duration_cast(diff).count()) + " us"); + }); + + if (ready_future.wait_for(std::chrono::seconds(120)) != std::future_status::ready) + { + LogErr("[client " + std::to_string(client_index) + "] timed out waiting for connection"); + client->Stop(); + return false; + } + } + + // ------------------------------------------------------------------ + // Spawn all client threads. They all share the single connection. + // ------------------------------------------------------------------ + std::atomic total_failures{0}; + std::vector threads; + threads.reserve(static_cast(thread_count)); + helper::ThreadStartBarrier thread_start_barrier{static_cast(thread_count)}; + + for (int t = 0; t < thread_count; ++t) + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::LowLevel::ClientThreadCreation"}; + threads.emplace_back([&, t]() { + thread_start_barrier.ArriveAndWait(); + int failures = 0; + + for (int c = 0; c < call_count; ++c) + { + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement latency_scope{ + latency_measurement, "POC::LowLevel::RoundTrip"}; + + // Build request + auto workload_request = helper::CreateRequest(client_index, t, c); + + // Assign the request_id before building the FlatBuffer so it can + // be embedded in the payload and inserted into the pending_map — + // all before the send. Upper 32 bits = pid (process-unique prefix), + // lower 32 bits = per-process monotonic counter (thread-unique within + // this process). Zero is never produced (counter starts at 1). + const std::uint64_t request_id = pid_upper | call_counter.fetch_add(1U, std::memory_order_relaxed); + workload_request.request_id = request_id; + + // Persist the shared-codec request so it outlives the async send queue. + auto request_buffer = std::make_shared>( + helper::BuildRequestBytes(workload_request)); + score::cpp::span request_span{request_buffer->data(), request_buffer->size()}; + + // Insert into the pending map BEFORE the send so the entry is + // guaranteed to exist when NotifyCallback fires. No generation + // counter needed: request_ids are never reused within a process. + auto pending = std::make_shared(); + { + std::lock_guard map_lk(pending_map_mutex); + pending_map[request_id] = pending; + } + + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t + << "] -> Send() str=\"" << workload_request.string_value + << "\" uint64=" << workload_request.uint64_value << " request_id=" << request_id; + Log(ss.str()); + } + + auto send_result = client->Send(request_span); + if (!send_result.has_value()) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t + << "] Send() failed: " << send_result.error(); + LogErr(ss.str()); + std::lock_guard map_lk(pending_map_mutex); + pending_map.erase(request_id); + ++failures; + continue; + } + + // Wait for NotifyCallback to signal ready. + constexpr auto kNotifyTimeout = std::chrono::seconds(300); + bool timed_out = false; + { + std::unique_lock call_lk(pending->mutex); + timed_out = !pending->cv.wait_for(call_lk, kNotifyTimeout, [&] { + return pending->ready; + }); + } + + // Retire: erase from map. The shared state remains alive until a + // deferred NotifyCallback releases it. + { + std::lock_guard map_lk(pending_map_mutex); + pending_map.erase(request_id); + } + + if (timed_out) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t + << "] TIMEOUT waiting for Notify request_id=" << request_id; + LogErr(ss.str()); + ++failures; + continue; + } + + if (!pending->ok || + !helper::Matches(workload_request, helper::Response{request_id, pending->result_value})) + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] MISMATCH request_id=" << request_id + << ": expected=\"" << helper::ProcessRequest(workload_request).string_value << "\" got=\"" + << (pending->ok ? pending->result_value : "") << "\""; + LogErr(ss.str()); + ++failures; + continue; + } + + { + std::ostringstream ss; + ss << "[client " << client_index << "/thread " << t << "] <- OK request_id=" << request_id + << " result=\"" << pending->result_value << "\" (Phase 2: Notify received, round-trip complete)"; + Log(ss.str()); + } + + // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + helper::WaitAfterCall(g_random_wait); + } + + total_failures.fetch_add(failures, std::memory_order_relaxed); + }); + } + + thread_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_client_thread_creation", resources_before_thread_creation, resources_after_thread_creation); + thread_start_barrier.Release(); + + for (auto& th : threads) + { + th.join(); + } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_low_level_no_reply_notify_in_thread_client_workload", resources_after_thread_creation, resources_after_workload); + + client->Stop(); + if (stopped_future.wait_for(std::chrono::seconds(30)) != std::future_status::ready) + { + LogErr("[client " + std::to_string(client_index) + "] timed out waiting for connection to stop"); + } + + const int total = call_count * thread_count; + const int failures = total_failures.load(); + const int success = total - failures; + std::ostringstream ss; + ss << "[client " << client_index << "] Results: " << success << "/" << total << " calls succeeded, " << failures + << "/" << total << " calls failed (" << thread_count << " thread(s) x " << call_count << " call(s))"; + Log(ss.str()); + + return failures == 0; +} + +} // namespace score::crypto::ipc::control + +// --------------------------------------------------------------------------- +// main — fork before any IPC setup for a clean per-process state +// --------------------------------------------------------------------------- + +int main(int argc, char** argv) +{ + score::crypto::ipc::poc_helper::PocArguments defaults; + defaults.client_count = g_client_count; + defaults.call_count = g_call_count; + defaults.client_threads = g_client_threads; + defaults.server_threads = g_server_threads; + defaults.sleep_milliseconds = g_sleep_milliseconds; + std::string parse_error; + const auto parsed_arguments = score::crypto::ipc::poc_helper::ParseArguments(argc, argv, parse_error, defaults); + if (!parsed_arguments.has_value()) + { + LogErr("[main] " + parse_error); + return 1; + } + g_client_count = parsed_arguments->client_count; + g_call_count = parsed_arguments->call_count; + g_client_threads = parsed_arguments->client_threads; + g_server_threads = parsed_arguments->server_threads; + g_sleep_milliseconds = parsed_arguments->sleep_milliseconds; + g_random_wait = parsed_arguments->random_wait; + + std::cout << score::crypto::ipc::poc_helper::SettingsSummary( + "poc_low_level_no_reply_notify_in_thread", + g_client_count, + g_call_count, + g_client_threads, + g_server_threads, + g_random_wait) + << '\n' + << std::flush; + Log("[main] client_count=" + std::to_string(g_client_count) + " call_count=" + std::to_string(g_call_count) + + " client_threads=" + std::to_string(g_client_threads) + " server_threads=" + + std::to_string(g_server_threads) + " sleep_milliseconds=" + std::to_string(g_sleep_milliseconds)); + + std::vector child_pids; + int my_client_index = -1; + + for (int i = 0; i < g_client_count; ++i) + { + pid_t pid = ::fork(); + if (pid < 0) + { + std::perror("[main] fork"); + for (pid_t cpid : child_pids) + { + ::kill(cpid, SIGTERM); + } + return 1; + } + if (pid == 0) + { + my_client_index = i; + break; + } + child_pids.push_back(pid); + } + + if (my_client_index == -1) + { + return score::crypto::ipc::control::RunServer(child_pids); + } + else + { + const bool ok = score::crypto::ipc::control::RunClient(my_client_index, g_call_count, g_client_threads); + return ok ? 0 : 1; + } +} + +// clang-format off + +// ============================================================================= +// Design comparison: per-thread connection (initial) vs. shared connection +// with Send+Notify (this version) +// ============================================================================= +// +// APPROACH A — One IClientConnection per thread, SendWaitReply +// ------------------------------------------------------------- +// Client threads each own a dedicated connection to the server. +// SendWaitReply() blocks the calling thread until the server calls Reply(). +// The server uses sent_with_reply_callback + pool threads that call Reply(). +// +// Max parallel requests from one client process: +// Exactly thread_count. Each connection carries at most one in-flight +// REQUEST at a time (per-connection REQUEST/REPLY serialization). +// Adding a thread automatically adds a connection and one more parallel +// slot — no explicit queue configuration needed. +// +// Memory / configuration (N client processes, T threads each): +// Server allocates N*T connection objects, each with its own receive +// buffer, reply slot, and notify queue. No application-level queue +// sizing is required beyond the connection count itself — the parallelism +// limit is implicit in the number of connections. +// Server max_queued_sends must cover all N*T concurrent sends. +// +// Pros: +// - Simple: no ticket tracking, no response routing, no shared state on the +// client side. The transport itself matches requests to replies by +// connection ordering. +// - Configuration is trivially correct: parallelism = thread count = connection +// count, with no additional parameters to keep in sync. +// - reply_buffer can be stack-allocated per thread. +// +// Cons: +// - One OS connection (socket fd pair) per thread. For N client processes +// each with T threads, the server holds N*T open connections. +// - The REQUEST/REPLY protocol serializes the server's sent_with_reply +// callback per connection (doc §Server callbacks): while Reply() has not +// been called, no further REQUEST from the same connection is processed. +// This is fine with one connection per thread (each thread sends one +// request at a time), but it means the connection count cannot be reduced +// without losing parallelism. +// - SendWaitReply() blocks the calling thread inside the library during the +// entire round-trip; there is no built-in timeout. +// +// APPROACH B — One IClientConnection per process, SendWithCallback + Notify (this file) +// -------------------------------------------------------------------------------------- +// One connection is shared by all threads. Threads call SendWithCallback() with a +// per-call ReplyCallback. The ReplyCallback (engine thread) parses the ack and inserts +// into the pending_map. The NotifyCallback (same engine thread) routes by request_id +// and signals the waiting thread. The caller waits on an application-level CV with a +// timeout — it is never blocked inside the library. +// +// Max parallel requests from one client process: +// Bounded by the minimum of four independently configured sizes: +// client max_async_replies (one slot per in-flight SendWithCallback) +// client max_queued_sends (one slot per concurrent enqueue) +// server max_queued_sends (server-side receive queue, shared across clients) +// server max_queued_notifies (per-connection notify queue) +// All must be set to >= thread_count. If any one is undersized, +// SendWithCallback() returns ENOBUFS or Notify() is dropped — with no +// automatic backpressure to the calling thread. +// +// Memory / configuration (N client processes, T threads each): +// Server allocates N connection objects (not N*T). Total queue capacity: +// +// server max_queued_sends — shared across ALL connections (QNX). +// At most N simultaneous REQUESTs (one per +// connection), so N slots suffice. +// +// server max_queued_notifies — per connection (QNX). +// Must hold T responses for that client's +// threads. Independent of N. +// +// client max_async_replies — per connection, per client process. +// Must hold T concurrent in-flight callbacks. +// +// client max_queued_sends — per connection, per client process. +// Must hold T concurrent enqueues. +// +// All four client sizes must be kept in sync with T; the coupling is +// enforced by convention only, not by the API. +// +// Pros: +// - O(clients) connections instead of O(clients * threads). +// - SendWithCallback() is non-blocking — the calling thread is never held +// inside the IPC layer regardless of server behavior. +// - Application-level wait_for() timeout bounds how long the thread can wait, +// which is a prerequisite for use in safety-relevant contexts. +// - ReplyCallback and NotifyCallback run sequentially on the engine thread, +// so the pending_map needs no early-notify buffer: the entry is always +// present by the time Notify() arrives. +// - NotifyCallback runs on the library's own thread — no dedicated receive +// thread is needed on the client side. +// +// Cons: +// - Requires ticket tracking (request_id map + mutex) on the client side. +// - ReplyCallback and NotifyCallback must not call any blocking message_passing +// operation (doc §Client Connection callbacks); only signal/mutex work is allowed. +// - Four queue sizes must all be kept >= thread_count. Getting any one wrong +// causes ENOBUFS on send or silent response drops rather than a clean error. +// - The response FlatBuffer (ControlResponse) must carry the request_id +// for response routing; a simpler protocol without a correlation id could +// not use this model. +// +// ANALOGY TO poc_async (mw::com): +// poc_async faced the same root constraint at the higher abstraction level: +// mw::com methods also serialize the handler per skeleton instance. The +// workaround there was identical in spirit — use a short-lived method call +// (enqueue only) + a separate event channel (broadcast) for the response. +// The key difference is that mw::com events broadcast to ALL subscribers, +// requiring one skeleton per client to prevent cross-client leakage and +// making the ticket essential for routing. message_passing Notify() is +// point-to-point per connection, so one connection per process suffices and +// the ticket is only needed for intra-process thread response routing. +// +// clang-format on diff --git a/score/tests/ipc_poc/poc_qnx_message_passing.cpp b/score/tests/ipc_poc/poc_qnx_message_passing.cpp new file mode 100644 index 000000000..ee2d2831d --- /dev/null +++ b/score/tests/ipc_poc/poc_qnx_message_passing.cpp @@ -0,0 +1,434 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This POC mirrors poc_unix_socket, but uses native QNX message passing: +// ChannelCreate(), ConnectAttach(), MsgSend(), MsgReceive(), and MsgReply(). +// Multiple client threads attach to one channel. The server dispatches each +// sending thread's requests to one worker thread before replying. +// ============================================================================= + +/// Usage: +/// bazel run //tests/score_com_poc:poc_qnx_message_passing --config=x86_64-qnx +/// bazel run //tests/score_com_poc:poc_qnx_message_passing --config=x86_64-qnx -- --call_count=5 +/// bazel run //tests/score_com_poc:poc_qnx_message_passing --config=x86_64-qnx -- --client_threads=3 --call_count=5 +/// bazel run //tests/score_com_poc:poc_qnx_message_passing --config=x86_64-qnx -- --call_count=3 --sleep_milliseconds=10 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "score/tests/utility/runtime_measurement.hpp" +#include "score/tests/ipc_poc/ipc_buffer.h" +#include "score/tests/ipc_poc/poc_helper.hpp" +#include "score/tests/utility/process_resource_measurement.hpp" + +namespace score::crypto::ipc::control +{ + +namespace helper = score::crypto::ipc::poc_helper; + +static int g_call_count = 100; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_threads = 4; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static bool g_random_wait = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static constexpr bool kEnableVerboseOutput{false}; +static constexpr bool kEnableLatencyVerbose{false}; + +static void Log(const std::string& line) +{ + if (!kEnableVerboseOutput) + { + return; + } + std::lock_guard lock{g_log_mutex}; + std::cout << line << "\n"; +} + +static void LogErr(const std::string& line) +{ + std::lock_guard lock{g_log_mutex}; + std::cerr << "[ERROR] " << line << "\n"; +} + +static bool ProcessRequestBytes(const std::uint8_t* message_data, + const std::size_t message_size, + std::uint64_t& request_id, + std::vector& response) +{ + helper::Response workload_response; + if (!helper::ProcessRequestBytes(message_data, message_size, workload_response)) + { + return false; + } + request_id = workload_response.request_id; + response = helper::BuildResponseBytes(workload_response); + return response.size() <= kMaxIpcBufferSize; +} + +static bool GetResponseSize(const std::array& response, + std::size_t& response_size) +{ + std::uint32_t payload_size = 0U; + std::memcpy(&payload_size, response.data(), sizeof(payload_size)); + if (payload_size == 0U || payload_size > kMaxIpcBufferSize - sizeof(payload_size)) + { + return false; + } + response_size = sizeof(payload_size) + static_cast(payload_size); + return true; +} + +using ReceiveId = decltype(::MsgReceive(0, nullptr, 0, nullptr)); + +static bool RunServerRequest(const ReceiveId rcvid, + std::vector request, + const int client_index) +{ + std::uint64_t request_id = 0U; + std::vector response; + if (!ProcessRequestBytes(request.data(), request.size(), request_id, response)) + { + LogErr("[server worker " + std::to_string(client_index) + "] invalid request message"); + if (::MsgError(static_cast(rcvid), EINVAL) == -1) + { + LogErr("[server worker " + std::to_string(client_index) + "] MsgError() failed: " + + std::string{std::strerror(errno)}); + } + return false; + } + + if (g_sleep_milliseconds > 0) + { + Log("[server worker " + std::to_string(client_index) + "] simulating work for " + + std::to_string(g_sleep_milliseconds) + " ms"); + std::this_thread::sleep_for(std::chrono::milliseconds(g_sleep_milliseconds)); + } + + Log("[server worker " + std::to_string(client_index) + "] processed request_id=" + + std::to_string(request_id) + ", sending FlatBuffer response"); + if (::MsgReply(static_cast(rcvid), EOK, response.data(), response.size()) == -1) + { + LogErr("[server worker " + std::to_string(client_index) + "] MsgReply() failed: " + + std::string{std::strerror(errno)}); + return false; + } + return true; +} + +static void RunServerWorker(const int chid, + const int worker_index, + std::atomic& stopping, + std::atomic& received_request_count, + std::atomic& failures) +{ + while (!stopping.load(std::memory_order_acquire)) + { + std::array request_buffer{}; + _msg_info message_info{}; + const auto rcvid = ::MsgReceive(chid, request_buffer.data(), request_buffer.size(), &message_info); + if (rcvid == 0) + { + const auto* pulse = reinterpret_cast(request_buffer.data()); + if (pulse->code == _PULSE_CODE_DISCONNECT) + { + Log("[server worker " + std::to_string(worker_index) + "] client disconnected"); + continue; + } + Log("[server] ignored pulse code=" + std::to_string(pulse->code)); + continue; + } + if (rcvid < 0) + { + if (!stopping.load(std::memory_order_acquire)) + { + LogErr("[server worker " + std::to_string(worker_index) + "] MsgReceive() failed: " + + std::string{std::strerror(errno)}); + failures.fetch_add(1, std::memory_order_relaxed); + } + return; + } + + const auto message_size = static_cast(message_info.msglen); + received_request_count.fetch_add(1U, std::memory_order_relaxed); + if (message_size == 0U || message_size > request_buffer.size()) + { + LogErr("[server worker " + std::to_string(worker_index) + "] invalid request message"); + if (::MsgError(static_cast(rcvid), EINVAL) == -1) + { + LogErr("[server worker " + std::to_string(worker_index) + "] MsgError() failed: " + + std::string{std::strerror(errno)}); + } + failures.fetch_add(1, std::memory_order_relaxed); + continue; + } + + std::vector request{request_buffer.begin(), request_buffer.begin() + message_size}; + if (!RunServerRequest(rcvid, std::move(request), worker_index)) + { + failures.fetch_add(1, std::memory_order_relaxed); + } + } +} + +static bool RunServer(const int chid, const pid_t client_pid, const int server_thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement setup_measurement{kEnableLatencyVerbose}; + Log("[server] waiting for QNX message-passing requests on channel " + std::to_string(chid)); + + const auto expected_request_count = static_cast(g_client_threads) * + static_cast(g_call_count); + std::atomic stopping{false}; + std::atomic received_request_count{0U}; + std::atomic failures{0}; + std::vector server_threads; + server_threads.reserve(static_cast(server_thread_count)); + for (int worker_index = 0; worker_index < server_thread_count; ++worker_index) + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + setup_measurement, "POC::QnxMessagePassing::ServerWorkerThreadCreation"}; + server_threads.emplace_back([&, worker_index] { + RunServerWorker(chid, worker_index, stopping, received_request_count, failures); + }); + } + + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta("poc_qnx_message_passing_server_thread_creation", + resources_before_thread_creation, + resources_after_thread_creation); + + int wait_status = 0; + const auto waited_pid = ::waitpid(client_pid, &wait_status, 0); + const bool client_ok = waited_pid == client_pid && WIFEXITED(wait_status) && WEXITSTATUS(wait_status) == 0; + stopping.store(true, std::memory_order_release); + if (::ChannelDestroy(chid) == -1) + { + LogErr("[server] ChannelDestroy() failed: " + std::string{std::strerror(errno)}); + } + for (auto& server_thread : server_threads) + { + server_thread.join(); + } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_qnx_message_passing_server_workload", resources_after_thread_creation, resources_after_workload); + const bool success = client_ok && received_request_count.load(std::memory_order_relaxed) == expected_request_count && + failures.load(std::memory_order_relaxed) == 0; + if (client_ok && success) + { + Log("[server] client exited OK"); + } + else + { + LogErr("[server] client failed"); + } + Log("[server] shutdown complete"); + return success && client_ok; +} + +static int ConnectToServer(const pid_t server_pid, const int chid) +{ + for (int attempt = 0; attempt < 300; ++attempt) + { + const auto coid = ::ConnectAttach(0, server_pid, chid, _NTO_SIDE_CHANNEL, 0); + if (coid >= 0) + { + return coid; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + return -1; +} + +static bool RunClientThread(const pid_t server_pid, + const int chid, + const int client_index, + score::crypto::daemon::common::RuntimeMeasurement& latency_measurement) +{ + int coid = -1; + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement connection_scope{ + latency_measurement, "POC::QnxMessagePassing::ClientConnectionSetup"}; + coid = ConnectToServer(server_pid, chid); + } + if (coid < 0) + { + LogErr("[client " + std::to_string(client_index) + "] ConnectAttach() failed: " + + std::string{std::strerror(errno)}); + return false; + } + Log("[client " + std::to_string(client_index) + "] connected to QNX message-passing channel " + + std::to_string(chid)); + + bool success = true; + for (int call = 0; call < g_call_count; ++call) + { + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement latency_scope{ + latency_measurement, "POC::QnxMessagePassing::RoundTrip"}; + const auto workload_request = helper::CreateRequest(0, client_index, call); + const auto request_id = workload_request.request_id; + const auto request = helper::BuildRequestBytes(workload_request); + std::array response{}; + + Log("[client " + std::to_string(client_index) + "] sending request_id=" + + std::to_string(request_id)); + const auto send_status = ::MsgSend(coid, + request.data(), + static_cast(request.size()), + response.data(), + static_cast(response.size())); + if (send_status != EOK) + { + LogErr("[client " + std::to_string(client_index) + "] MsgSend() failed with status=" + + std::to_string(send_status)); + success = false; + break; + } + + std::size_t response_size = 0U; + helper::Response parsed_response; + if (!GetResponseSize(response, response_size) || + !helper::ParseResponseBytes(response.data(), response_size, parsed_response) || + !helper::Matches(workload_request, parsed_response)) + { + LogErr("[client " + std::to_string(client_index) + + "] invalid or unexpected response for request_id=" + std::to_string(request_id)); + success = false; + break; + } + } + helper::WaitAfterCall(g_random_wait); + } + + if (::ConnectDetach(coid) == -1) + { + LogErr("[client " + std::to_string(client_index) + "] ConnectDetach() failed: " + + std::string{std::strerror(errno)}); + success = false; + } + return success; +} + +static bool RunClient(const pid_t server_pid, const int chid, const int client_thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; + std::atomic failures{0}; + std::vector client_threads; + client_threads.reserve(static_cast(client_thread_count)); + helper::ThreadStartBarrier thread_start_barrier{static_cast(client_thread_count)}; + + for (int client_index = 0; client_index < client_thread_count; ++client_index) + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::QnxMessagePassing::ClientThreadCreation"}; + client_threads.emplace_back([&, client_index] { + thread_start_barrier.ArriveAndWait(); + if (!RunClientThread(server_pid, chid, client_index, latency_measurement)) + { + failures.fetch_add(1, std::memory_order_relaxed); + } + }); + } + + thread_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta("poc_qnx_message_passing_client_thread_creation", + resources_before_thread_creation, + resources_after_thread_creation); + thread_start_barrier.Release(); + + for (auto& client_thread : client_threads) + { + client_thread.join(); + } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_qnx_message_passing_client_workload", resources_after_thread_creation, resources_after_workload); + return failures.load(std::memory_order_relaxed) == 0; +} + +} // namespace score::crypto::ipc::control + +int main(int argc, char** argv) +{ + score::crypto::ipc::poc_helper::PocArguments defaults; + defaults.call_count = score::crypto::ipc::control::g_call_count; + defaults.client_threads = score::crypto::ipc::control::g_client_threads; + defaults.sleep_milliseconds = score::crypto::ipc::control::g_sleep_milliseconds; + std::string parse_error; + const auto parsed_arguments = score::crypto::ipc::poc_helper::ParseArguments(argc, argv, parse_error, defaults); + if (!parsed_arguments.has_value()) + { + score::crypto::ipc::control::LogErr("[main] " + parse_error); + return 1; + } + score::crypto::ipc::control::g_call_count = parsed_arguments->call_count; + score::crypto::ipc::control::g_client_threads = parsed_arguments->client_threads; + score::crypto::ipc::control::g_sleep_milliseconds = parsed_arguments->sleep_milliseconds; + score::crypto::ipc::control::g_random_wait = parsed_arguments->random_wait; + + std::cout << score::crypto::ipc::poc_helper::SettingsSummary( + "poc_qnx_message_passing", + 1, + score::crypto::ipc::control::g_call_count, + score::crypto::ipc::control::g_client_threads, + score::crypto::ipc::control::g_client_threads, + score::crypto::ipc::control::g_random_wait) + << '\n' + << std::flush; + const auto chid = ::ChannelCreate(0); + if (chid < 0) + { + score::crypto::ipc::control::LogErr("[main] ChannelCreate() failed: " + std::string{std::strerror(errno)}); + return 1; + } + + const auto server_pid = ::fork(); + if (server_pid < 0) + { + score::crypto::ipc::control::LogErr("[main] fork() failed: " + std::string{std::strerror(errno)}); + ::ChannelDestroy(chid); + return 1; + } + if (server_pid == 0) + { + return score::crypto::ipc::control::RunClient(::getppid(), + chid, + score::crypto::ipc::control::g_client_threads) + ? 0 + : 1; + } + return score::crypto::ipc::control::RunServer( + chid, server_pid, score::crypto::ipc::control::g_client_threads) + ? 0 + : 1; +} diff --git a/score/tests/ipc_poc/poc_thread_pool_queue.cpp b/score/tests/ipc_poc/poc_thread_pool_queue.cpp new file mode 100644 index 000000000..1ec44aca6 --- /dev/null +++ b/score/tests/ipc_poc/poc_thread_pool_queue.cpp @@ -0,0 +1,213 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This file measures the local queue and worker scheduling path without IPC. +// ============================================================================= + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "score/tests/utility/runtime_measurement.hpp" + +namespace +{ + +using Clock = std::chrono::steady_clock; + +constexpr int kProducerThreads{1}; +constexpr int kWorkItemsPerProducer{1000}; +constexpr int kWorkerThreads{8}; +constexpr int kSleepMilliseconds{0}; +constexpr bool kRandomWait{false}; + +void WaitAfterEnqueue(const bool random_wait) +{ + if (!random_wait) + { + return; + } + + thread_local std::mt19937 generator{std::random_device{}()}; + std::uniform_int_distribution distribution{0, 5}; + std::this_thread::sleep_for(std::chrono::milliseconds(distribution(generator))); +} + +class ThreadStartBarrier final +{ + public: + explicit ThreadStartBarrier(const std::size_t participant_count) : participant_count_{participant_count} {} + + void ArriveAndWait() + { + std::unique_lock lock{mutex_}; + ++arrived_count_; + condition_.notify_all(); + condition_.wait(lock, [this] { return released_; }); + } + + void WaitForAll() + { + std::unique_lock lock{mutex_}; + condition_.wait(lock, [this] { return arrived_count_ == participant_count_; }); + } + + void Release() + { + { + std::lock_guard lock{mutex_}; + released_ = true; + } + condition_.notify_all(); + } + + private: + const std::size_t participant_count_; + std::size_t arrived_count_{0U}; + bool released_{false}; + std::mutex mutex_; + std::condition_variable condition_; +}; + +struct WorkItem final +{ + std::uint64_t id{0U}; + std::vector request_bytes; + Clock::time_point queued_at; +}; + +struct WorkQueue final +{ + std::mutex mutex; + std::condition_variable condition; + std::queue items; + bool stopping{false}; +}; + +void ProcessWorkItem(const WorkItem& item, const int sleep_milliseconds, std::atomic& processed_items) +{ + static_cast(item); + if (sleep_milliseconds > 0) + { + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_milliseconds)); + } + + processed_items.fetch_add(1U, std::memory_order_relaxed); +} + +int Run() +{ + score::crypto::daemon::common::RuntimeMeasurement latency_measurement; + WorkQueue work_queue; + std::atomic processed_items{0U}; + std::atomic next_work_item_id{1U}; + + const auto participant_count = static_cast(kProducerThreads + kWorkerThreads); + ThreadStartBarrier start_barrier{participant_count}; + std::vector workers; + workers.reserve(static_cast(kWorkerThreads)); + + for (int worker_index = 0; worker_index < kWorkerThreads; ++worker_index) + { + workers.emplace_back([&, worker_index] { + static_cast(worker_index); + start_barrier.ArriveAndWait(); + while (true) + { + std::unique_lock lock{work_queue.mutex}; + work_queue.condition.wait(lock, [&] { + return !work_queue.items.empty() || work_queue.stopping; + }); + + if (work_queue.stopping && work_queue.items.empty()) + { + break; + } + + WorkItem item = std::move(work_queue.items.front()); + work_queue.items.pop(); + lock.unlock(); + + latency_measurement.RecordElapsed("POC::ThreadPool::QueueToProcessingStart", item.queued_at); + ProcessWorkItem(item, kSleepMilliseconds, processed_items); + } + }); + } + + std::vector producers; + producers.reserve(static_cast(kProducerThreads)); + for (int producer_index = 0; producer_index < kProducerThreads; ++producer_index) + { + producers.emplace_back([&, producer_index] { + start_barrier.ArriveAndWait(); + for (int item_index = 0; item_index < kWorkItemsPerProducer; ++item_index) + { + const auto work_item_id = next_work_item_id.fetch_add(1U, std::memory_order_relaxed); + { + std::lock_guard lock{work_queue.mutex}; + const WorkItem item{work_item_id, {}, Clock::now()}; + work_queue.items.push(item); + } + work_queue.condition.notify_one(); + WaitAfterEnqueue(kRandomWait); + } + }); + } + + start_barrier.WaitForAll(); + std::cout << "[Settings] producer_threads=" << kProducerThreads + << " work_items_per_producer=" << kWorkItemsPerProducer + << " worker_threads=" << kWorkerThreads + << " sleep_milliseconds=" << kSleepMilliseconds + << " random_wait=" << (kRandomWait ? "true" : "false") << '\n'; + start_barrier.Release(); + + for (auto& producer : producers) + { + producer.join(); + } + + { + std::lock_guard lock{work_queue.mutex}; + work_queue.stopping = true; + } + work_queue.condition.notify_all(); + + for (auto& worker : workers) + { + worker.join(); + } + + const auto expected_items = static_cast(kProducerThreads) * + static_cast(kWorkItemsPerProducer); + std::cout << "[Result] processed_items=" << expected_items << '\n'; + return processed_items.load(std::memory_order_relaxed) == expected_items ? 0 : 1; +} + +} // namespace + +int main() +{ + return Run(); +} diff --git a/score/tests/ipc_poc/poc_unix_socket.cpp b/score/tests/ipc_poc/poc_unix_socket.cpp new file mode 100644 index 000000000..9be0c3adb --- /dev/null +++ b/score/tests/ipc_poc/poc_unix_socket.cpp @@ -0,0 +1,459 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// ============================================================================= +// WARNING: EXPERIMENTAL REFERENCE CODE - DO NOT USE IN PRODUCTION +// +// This POC mirrors poc_engine, but uses a raw Unix-domain SOCK_STREAM socket +// instead of score::message_passing. The server accepts multiple connections +// on one endpoint and performs each connection's operations on its own thread. +// ============================================================================= + +/// Usage: +/// bazel run //tests/score_com_poc:poc_unix_socket +/// bazel run //tests/score_com_poc:poc_unix_socket -- --call_count=5 +/// bazel run //tests/score_com_poc:poc_unix_socket -- --client_threads=3 --call_count=5 +/// bazel run //tests/score_com_poc:poc_unix_socket -- --call_count=3 --sleep_milliseconds=10 + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "score/tests/utility/runtime_measurement.hpp" +#include "score/tests/ipc_poc/ipc_buffer.h" +#include "score/tests/ipc_poc/poc_helper.hpp" +#include "score/tests/utility/process_resource_measurement.hpp" + +namespace score::crypto::ipc::control +{ + +namespace helper = score::crypto::ipc::poc_helper; + +static int g_call_count = 1000; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_client_threads = 1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static int g_sleep_milliseconds = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static bool g_random_wait = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static std::mutex g_log_mutex; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static constexpr bool kEnableVerboseOutput{false}; +static constexpr bool kEnableLatencyVerbose{false}; + +static void Log(const std::string& line) +{ + if (!kEnableVerboseOutput) + { + return; + } + std::lock_guard lock{g_log_mutex}; + std::cout << line << "\n"; +} + +static void LogErr(const std::string& line) +{ + std::lock_guard lock{g_log_mutex}; + std::cerr << "[ERROR] " << line << "\n"; +} + +static std::string SocketPath(const pid_t server_pid) +{ + return " score_crypto_poc_socket_" + std::to_string(server_pid) + ".sock"; +} + +static bool MakeAddress(const std::string& path, sockaddr_un& address) +{ + if (path.size() >= sizeof(address.sun_path)) + { + return false; + } + address = sockaddr_un{}; + address.sun_family = AF_UNIX; + std::memcpy(address.sun_path, path.c_str(), path.size() + 1U); + return true; +} + +static bool SendAll(const int socket_fd, const std::uint8_t* data, const std::size_t size) +{ + std::size_t sent = 0U; + while (sent < size) + { + const auto result = ::send(socket_fd, data + sent, size - sent, MSG_NOSIGNAL); + if (result <= 0) + { + return false; + } + sent += static_cast(result); + } + return true; +} + +static bool ReceiveExact(const int socket_fd, std::uint8_t* data, const std::size_t size) +{ + std::size_t received = 0U; + while (received < size) + { + const auto result = ::recv(socket_fd, data + received, size - received, 0); + if (result <= 0) + { + return false; + } + received += static_cast(result); + } + return true; +} + +static bool SendFrame(const int socket_fd, const std::vector& frame) +{ + if (frame.empty() || frame.size() > kMaxIpcBufferSize) + { + return false; + } + return SendAll(socket_fd, frame.data(), frame.size()); +} + +static bool ReceiveFrame(const int socket_fd, std::vector& frame) +{ + std::array size_prefix{}; + if (!ReceiveExact(socket_fd, size_prefix.data(), size_prefix.size())) + { + return false; + } + + std::uint32_t payload_size = 0U; + std::memcpy(&payload_size, size_prefix.data(), sizeof(payload_size)); + if (payload_size == 0U || payload_size > kMaxIpcBufferSize - size_prefix.size()) + { + return false; + } + + frame.resize(size_prefix.size() + payload_size); + std::memcpy(frame.data(), size_prefix.data(), size_prefix.size()); + return ReceiveExact(socket_fd, frame.data() + size_prefix.size(), payload_size); +} + +static bool RunServerPerClient(const int client_fd, const int client_index) +{ + bool success = true; + for (int call = 0; call < g_call_count; ++call) + { + std::vector request; + if (!ReceiveFrame(client_fd, request)) + { + LogErr("[server client " + std::to_string(client_index) + "] failed to receive request"); + success = false; + break; + } + + helper::Response response; + if (!helper::ProcessRequestBytes(request.data(), request.size(), response)) + { + LogErr("[server client " + std::to_string(client_index) + "] failed to process request"); + success = false; + break; + } + request = helper::BuildResponseBytes(response); + + if (g_sleep_milliseconds > 0) + { + Log("[server client " + std::to_string(client_index) + "] simulating work for " + + std::to_string(g_sleep_milliseconds) + " ms"); + std::this_thread::sleep_for(std::chrono::milliseconds(g_sleep_milliseconds)); + } + Log("[server client " + std::to_string(client_index) + "] processed request_id=" + + std::to_string(response.request_id) + ", sending FlatBuffer response"); + if (!SendFrame(client_fd, request)) + { + LogErr("[server client " + std::to_string(client_index) + "] failed to send response"); + success = false; + break; + } + } + + ::close(client_fd); + return success; +} + +static bool RunServer(const pid_t client_pid, const int client_thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement setup_measurement{kEnableLatencyVerbose}; + const auto socket_path = SocketPath(::getpid()); + const auto listen_fd = ::socket(AF_UNIX, SOCK_STREAM, 0); + if (listen_fd < 0) + { + LogErr("[server] socket() failed: " + std::string{std::strerror(errno)}); + return false; + } + + sockaddr_un address{}; + if (!MakeAddress(socket_path, address)) + { + LogErr("[server] socket path is too long"); + ::close(listen_fd); + return false; + } + ::unlink(socket_path.c_str()); + if (::bind(listen_fd, reinterpret_cast(&address), sizeof(address)) < 0) + { + LogErr("[server] bind() failed: " + std::string{std::strerror(errno)}); + ::close(listen_fd); + ::unlink(socket_path.c_str()); + return false; + } + if (::listen(listen_fd, std::max(client_thread_count, 1)) < 0) + { + LogErr("[server] listen() failed: " + std::string{std::strerror(errno)}); + ::close(listen_fd); + ::unlink(socket_path.c_str()); + return false; + } + Log("[server] listening on one Unix-domain endpoint: " + socket_path); + + std::vector client_threads; + client_threads.reserve(static_cast(client_thread_count)); + std::atomic failures{0}; + bool success = true; + for (int client_index = 0; client_index < client_thread_count; ++client_index) + { + int client_fd = -1; + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement connection_scope{ + setup_measurement, "POC::UnixSocket::ServerConnectionSetup"}; + client_fd = ::accept(listen_fd, nullptr, nullptr); + } + if (client_fd < 0) + { + LogErr("[server] accept() failed: " + std::string{std::strerror(errno)}); + success = false; + break; + } + Log("[server] accepted client connection " + std::to_string(client_index)); + client_threads.emplace_back([client_fd, client_index, &failures] { + if (!RunServerPerClient(client_fd, client_index)) + { + failures.fetch_add(1, std::memory_order_relaxed); + } + }); + } + + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta("poc_unix_socket_server_thread_creation", + resources_before_thread_creation, + resources_after_thread_creation); + + for (auto& client_thread : client_threads) + { + client_thread.join(); + } + success = success && client_threads.size() == static_cast(client_thread_count) && + failures.load(std::memory_order_relaxed) == 0; + + ::close(listen_fd); + ::unlink(socket_path.c_str()); + + int wait_status = 0; + const auto waited_pid = waitpid(client_pid, &wait_status, 0); + const bool client_ok = waited_pid == client_pid && WIFEXITED(wait_status) && WEXITSTATUS(wait_status) == 0; + if (client_ok && success) + { + Log("[server] client exited OK"); + } + else + { + LogErr("[server] client failed"); + } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_unix_socket_server_workload", resources_after_thread_creation, resources_after_workload); + Log("[server] shutdown complete"); + return success && client_ok; +} + +static int ConnectToServer(const std::string& socket_path) +{ + sockaddr_un address{}; + if (!MakeAddress(socket_path, address)) + { + return -1; + } + + for (int attempt = 0; attempt < 300; ++attempt) + { + const auto socket_fd = ::socket(AF_UNIX, SOCK_STREAM, 0); + if (socket_fd >= 0 && ::connect(socket_fd, reinterpret_cast(&address), sizeof(address)) == 0) + { + return socket_fd; + } + if (socket_fd >= 0) + { + ::close(socket_fd); + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + return -1; +} + +static bool RunClientThread(const pid_t server_pid, + const int client_index, + score::crypto::daemon::common::RuntimeMeasurement& latency_measurement) +{ + int socket_fd = -1; + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement connection_scope{ + latency_measurement, "POC::UnixSocket::ClientConnectionSetup"}; + socket_fd = ConnectToServer(SocketPath(server_pid)); + } + if (socket_fd < 0) + { + LogErr("[client] failed to connect to Unix-domain socket"); + return false; + } + Log("[client " + std::to_string(client_index) + "] connected to the Unix-domain endpoint"); + + bool success = true; + for (int call = 0; call < g_call_count; ++call) + { + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement latency_scope{ + latency_measurement, "POC::UnixSocket::RoundTrip"}; + const auto workload_request = helper::CreateRequest(0, client_index, call); + const auto request_id = workload_request.request_id; + auto request = helper::BuildRequestBytes(workload_request); + + Log("[client " + std::to_string(client_index) + "] sending request_id=" + std::to_string(request_id)); + if (!SendFrame(socket_fd, request)) + { + LogErr("[client " + std::to_string(client_index) + "] failed to send request"); + success = false; + break; + } + + std::vector response; + helper::Response parsed_response; + if (!ReceiveFrame(socket_fd, response) || + !helper::ParseResponseBytes(response.data(), response.size(), parsed_response) || + !helper::Matches(workload_request, parsed_response)) + { + LogErr("[client " + std::to_string(client_index) + "] invalid or unexpected response for request_id=" + + std::to_string(request_id)); + success = false; + break; + } + } + helper::WaitAfterCall(g_random_wait); + } + + ::shutdown(socket_fd, SHUT_RDWR); + ::close(socket_fd); + return success; +} + +static bool RunClient(const pid_t server_pid, const int client_thread_count) +{ + const auto resources_before_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + score::crypto::daemon::common::RuntimeMeasurement latency_measurement{kEnableLatencyVerbose}; + std::atomic failures{0}; + std::vector client_threads; + client_threads.reserve(static_cast(client_thread_count)); + helper::ThreadStartBarrier thread_start_barrier{static_cast(client_thread_count)}; + + for (int client_index = 0; client_index < client_thread_count; ++client_index) + { + score::crypto::daemon::common::RuntimeMeasurement::ScopedMeasurement thread_scope{ + latency_measurement, "POC::UnixSocket::ClientThreadCreation"}; + client_threads.emplace_back([&, client_index] { + thread_start_barrier.ArriveAndWait(); + if (!RunClientThread(server_pid, client_index, latency_measurement)) + { + failures.fetch_add(1, std::memory_order_relaxed); + } + }); + } + + thread_start_barrier.WaitForAll(); + const auto resources_after_thread_creation = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta("poc_unix_socket_client_thread_creation", + resources_before_thread_creation, + resources_after_thread_creation); + thread_start_barrier.Release(); + + for (auto& client_thread : client_threads) + { + client_thread.join(); + } + const auto resources_after_workload = tests::utility::CaptureProcessResourceSnapshot(); + tests::utility::PrintProcessResourceDelta( + "poc_unix_socket_client_workload", resources_after_thread_creation, resources_after_workload); + return failures.load(std::memory_order_relaxed) == 0; +} + +} // namespace score::crypto::ipc::control + +int main(int argc, char** argv) +{ + score::crypto::ipc::poc_helper::PocArguments defaults; + defaults.call_count = score::crypto::ipc::control::g_call_count; + defaults.sleep_milliseconds = score::crypto::ipc::control::g_sleep_milliseconds; + std::string parse_error; + const auto parsed_arguments = score::crypto::ipc::poc_helper::ParseArguments(argc, argv, parse_error, defaults); + if (!parsed_arguments.has_value()) + { + score::crypto::ipc::control::LogErr("[main] " + parse_error); + return 1; + } + score::crypto::ipc::control::g_call_count = parsed_arguments->call_count; + score::crypto::ipc::control::g_client_threads = parsed_arguments->client_threads; + score::crypto::ipc::control::g_sleep_milliseconds = parsed_arguments->sleep_milliseconds; + score::crypto::ipc::control::g_random_wait = parsed_arguments->random_wait; + + std::cout << score::crypto::ipc::poc_helper::SettingsSummary( + "poc_unix_socket", + 1, + score::crypto::ipc::control::g_call_count, + score::crypto::ipc::control::g_client_threads, + score::crypto::ipc::control::g_client_threads, + score::crypto::ipc::control::g_random_wait) + << '\n' + << std::flush; + const auto server_pid = ::fork(); + if (server_pid < 0) + { + std::perror("[main] fork"); + return 1; + } + if (server_pid == 0) + { + return score::crypto::ipc::control::RunClient(::getppid(), + score::crypto::ipc::control::g_client_threads) + ? 0 + : 1; + } + return score::crypto::ipc::control::RunServer(server_pid, score::crypto::ipc::control::g_client_threads) ? 0 : 1; +} diff --git a/score/tests/utility/BUILD b/score/tests/utility/BUILD index 9751c11ce..b68742fad 100644 --- a/score/tests/utility/BUILD +++ b/score/tests/utility/BUILD @@ -13,6 +13,12 @@ load("@rules_cc//cc:defs.bzl", "cc_library") +cc_library( + name = "runtime_measurement", + hdrs = ["runtime_measurement.hpp"], + visibility = ["//score:__subpackages__"], +) + cc_library( name = "utility", srcs = ["test_utility.cpp"], @@ -22,3 +28,10 @@ cc_library( "@googletest//:gtest", ], ) + +cc_library( + name = "process_resource_measurement", + srcs = ["process_resource_measurement.cpp"], + hdrs = ["process_resource_measurement.hpp"], + visibility = ["//score:__subpackages__"], +) diff --git a/score/tests/utility/process_resource_measurement.cpp b/score/tests/utility/process_resource_measurement.cpp new file mode 100644 index 000000000..e09af86ac --- /dev/null +++ b/score/tests/utility/process_resource_measurement.cpp @@ -0,0 +1,311 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#include "score/tests/utility/process_resource_measurement.hpp" + +#include + +#include +#include +#include +#include +#include + +#if defined(__QNXNTO__) +#include +#include +#include +#include +#endif + +namespace tests::utility +{ +namespace +{ + +#if defined(__linux__) +std::uint64_t ReadProcValueBytes(const char* field_name, bool& available) +{ + available = false; + FILE* status_file = std::fopen("/proc/self/status", "r"); + if (status_file == nullptr) + { + return 0U; + } + + char line[256]{}; + while (std::fgets(line, sizeof(line), status_file) != nullptr) + { + char parsed_field[64]{}; + unsigned long long parsed_value = 0U; + char unit[32]{}; + if (std::sscanf(line, "%63[^:]: %llu %31s", parsed_field, &parsed_value, unit) == 3 && + std::strcmp(parsed_field, field_name) == 0) + { + std::fclose(status_file); + available = true; + return static_cast(parsed_value) * 1024U; + } + } + + std::fclose(status_file); + return 0U; +} + +std::uint64_t ReadProcThreadCount(bool& available) +{ + available = false; + FILE* status_file = std::fopen("/proc/self/status", "r"); + if (status_file == nullptr) + { + return 0U; + } + + char line[256]{}; + while (std::fgets(line, sizeof(line), status_file) != nullptr) + { + unsigned long long parsed_value = 0U; + if (std::sscanf(line, "Threads: %llu", &parsed_value) == 1) + { + std::fclose(status_file); + available = true; + return static_cast(parsed_value); + } + } + + std::fclose(status_file); + return 0U; +} +#endif + +#if defined(__QNXNTO__) +void ReadQnxVmStat(ProcessResourceSnapshot& snapshot) +{ + const long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) + { + return; + } + + char vmstat_path[64]{}; + std::snprintf(vmstat_path, sizeof(vmstat_path), "/proc/%d/vmstat", static_cast(getpid())); + FILE* vmstat_file = std::fopen(vmstat_path, "r"); + if (vmstat_file == nullptr) + { + return; + } + + const std::uint64_t page_size_bytes = static_cast(page_size); + char line[256]{}; + while (std::fgets(line, sizeof(line), vmstat_file) != nullptr) + { + char field[64]{}; + unsigned long long page_count = 0U; + if (std::sscanf(line, "%63[^=]=0x%llx", field, &page_count) != 2) + { + continue; + } + + const bool is_resident = std::strcmp(field, "as_stats.rss") == 0; + const bool is_virtual = std::strcmp(field, "as_stats.map_size") == 0; + if ((!is_resident && !is_virtual) || + static_cast(page_count) > + (std::numeric_limits::max() / page_size_bytes)) + { + continue; + } + + const std::uint64_t value_bytes = static_cast(page_count) * page_size_bytes; + if (is_resident) + { + snapshot.resident_bytes = value_bytes; + snapshot.resident_available = true; + } + else + { + snapshot.virtual_bytes = value_bytes; + snapshot.virtual_available = true; + } + } + + std::fclose(vmstat_file); +} + +void ReadQnxThreadCount(ProcessResourceSnapshot& snapshot) +{ + char ctl_path[64]{}; + std::snprintf(ctl_path, sizeof(ctl_path), "/proc/%d/ctl", static_cast(getpid())); + const int ctl_fd = open(ctl_path, O_RDONLY); + if (ctl_fd < 0) + { + return; + } + + std::uint64_t count = 0U; + pthread_t requested_tid = static_cast(1U); + bool enumeration_complete = false; + for (;;) + { + procfs_status status{}; + status.tid = requested_tid; + const int result = devctl(ctl_fd, DCMD_PROC_TIDSTATUS, &status, sizeof(status), nullptr); + if (result == ESRCH) + { + enumeration_complete = count > 0U; + break; + } + if (result != EOK || status.tid < requested_tid) + { + break; + } + + ++count; + requested_tid = status.tid; + ++requested_tid; + } + + close(ctl_fd); + if (enumeration_complete) + { + snapshot.thread_count = count; + snapshot.thread_count_available = true; + } +} +#endif + +ProcessResourceSnapshot CaptureProcessResourceSnapshotImpl() +{ + ProcessResourceSnapshot snapshot; + +#if defined(__linux__) + snapshot.resident_bytes = ReadProcValueBytes("VmRSS", snapshot.resident_available); + snapshot.virtual_bytes = ReadProcValueBytes("VmSize", snapshot.virtual_available); + snapshot.thread_count = ReadProcThreadCount(snapshot.thread_count_available); +#elif defined(__QNXNTO__) + ReadQnxVmStat(snapshot); + ReadQnxThreadCount(snapshot); +#endif + + struct rusage usage + { + }; + if (getrusage(RUSAGE_SELF, &usage) == 0) + { + snapshot.cpu_user_time_ns = static_cast(usage.ru_utime.tv_sec) * 1'000'000'000U + + static_cast(usage.ru_utime.tv_usec) * 1'000U; + snapshot.cpu_system_time_ns = static_cast(usage.ru_stime.tv_sec) * 1'000'000'000U + + static_cast(usage.ru_stime.tv_usec) * 1'000U; + snapshot.cpu_time_available = true; + +#if defined(__linux__) + snapshot.peak_resident_bytes = static_cast(usage.ru_maxrss); + snapshot.peak_resident_bytes *= 1024U; + snapshot.peak_resident_available = true; + + snapshot.voluntary_context_switches = static_cast(usage.ru_nvcsw); + snapshot.involuntary_context_switches = static_cast(usage.ru_nivcsw); + snapshot.context_switches_available = true; +#endif + } + + return snapshot; +} + +void PrintResourceValue(const char* name, bool available, std::uint64_t value) +{ + if (available) + { + std::fprintf(stdout, " %s=%" PRIu64, name, value); + } + else + { + std::fprintf(stdout, " %s=unavailable", name); + } +} + +void PrintProcessResourceDeltaImpl(std::string_view tag, + const ProcessResourceSnapshot& before, + const ProcessResourceSnapshot& after) +{ + std::fprintf(stdout, + "[Resources] %.*s:", + static_cast(tag.size()), + tag.data()); + PrintResourceValue("resident_before_bytes", before.resident_available, before.resident_bytes); + PrintResourceValue("resident_after_bytes", after.resident_available, after.resident_bytes); + PrintResourceValue("peak_resident_before_bytes", before.peak_resident_available, before.peak_resident_bytes); + PrintResourceValue("peak_resident_after_bytes", after.peak_resident_available, after.peak_resident_bytes); + PrintResourceValue("virtual_before_bytes", before.virtual_available, before.virtual_bytes); + PrintResourceValue("virtual_after_bytes", after.virtual_available, after.virtual_bytes); + PrintResourceValue("threads_before", before.thread_count_available, before.thread_count); + PrintResourceValue("threads_after", after.thread_count_available, after.thread_count); + PrintResourceValue("voluntary_cs_before", before.context_switches_available, before.voluntary_context_switches); + PrintResourceValue("voluntary_cs_after", after.context_switches_available, after.voluntary_context_switches); + PrintResourceValue("involuntary_cs_before", before.context_switches_available, before.involuntary_context_switches); + PrintResourceValue("involuntary_cs_after", after.context_switches_available, after.involuntary_context_switches); + PrintResourceValue("cpu_user_time_before_ns", before.cpu_time_available, before.cpu_user_time_ns); + PrintResourceValue("cpu_user_time_after_ns", after.cpu_time_available, after.cpu_user_time_ns); + PrintResourceValue("cpu_system_time_before_ns", before.cpu_time_available, before.cpu_system_time_ns); + PrintResourceValue("cpu_system_time_after_ns", after.cpu_time_available, after.cpu_system_time_ns); + + if (before.resident_available && after.resident_available) + { + std::fprintf(stdout, + " resident_delta_bytes=%" PRId64, + static_cast(after.resident_bytes) - static_cast(before.resident_bytes)); + } + if (before.virtual_available && after.virtual_available) + { + std::fprintf(stdout, + " virtual_delta_bytes=%" PRId64, + static_cast(after.virtual_bytes) - static_cast(before.virtual_bytes)); + } + if (before.context_switches_available && after.context_switches_available) + { + std::fprintf(stdout, + " voluntary_cs_delta=%" PRId64, + static_cast(after.voluntary_context_switches) - + static_cast(before.voluntary_context_switches)); + std::fprintf(stdout, + " involuntary_cs_delta=%" PRId64, + static_cast(after.involuntary_context_switches) - + static_cast(before.involuntary_context_switches)); + } + if (before.cpu_time_available && after.cpu_time_available) + { + const std::int64_t user_delta = static_cast(after.cpu_user_time_ns) - + static_cast(before.cpu_user_time_ns); + const std::int64_t system_delta = static_cast(after.cpu_system_time_ns) - + static_cast(before.cpu_system_time_ns); + std::fprintf(stdout, " cpu_user_time_delta_ns=%" PRId64, user_delta); + std::fprintf(stdout, " cpu_system_time_delta_ns=%" PRId64, system_delta); + std::fprintf(stdout, " cpu_total_time_delta_ns=%" PRId64, user_delta + system_delta); + } + std::fprintf(stdout, "\n"); +} + +} // namespace + +ProcessResourceSnapshot CaptureProcessResourceSnapshot() +{ + return CaptureProcessResourceSnapshotImpl(); +} + +void PrintProcessResourceDelta(std::string_view tag, + const ProcessResourceSnapshot& before, + const ProcessResourceSnapshot& after) +{ + PrintProcessResourceDeltaImpl(tag, before, after); +} + +} // namespace tests::utility diff --git a/score/tests/utility/process_resource_measurement.hpp b/score/tests/utility/process_resource_measurement.hpp new file mode 100644 index 000000000..6bd3f670f --- /dev/null +++ b/score/tests/utility/process_resource_measurement.hpp @@ -0,0 +1,49 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef SCORE_TESTS_UTILITY_PROCESS_RESOURCE_MEASUREMENT_HPP +#define SCORE_TESTS_UTILITY_PROCESS_RESOURCE_MEASUREMENT_HPP + +#include +#include + +namespace tests::utility +{ + +struct ProcessResourceSnapshot +{ + std::uint64_t resident_bytes{0U}; + std::uint64_t peak_resident_bytes{0U}; + std::uint64_t virtual_bytes{0U}; + std::uint64_t thread_count{0U}; + std::uint64_t voluntary_context_switches{0U}; + std::uint64_t involuntary_context_switches{0U}; + std::uint64_t cpu_user_time_ns{0U}; + std::uint64_t cpu_system_time_ns{0U}; + bool resident_available{false}; + bool peak_resident_available{false}; + bool virtual_available{false}; + bool thread_count_available{false}; + bool context_switches_available{false}; + bool cpu_time_available{false}; +}; + +ProcessResourceSnapshot CaptureProcessResourceSnapshot(); + +void PrintProcessResourceDelta(std::string_view tag, + const ProcessResourceSnapshot& before, + const ProcessResourceSnapshot& after); + +} // namespace tests::utility + +#endif // SCORE_TESTS_UTILITY_PROCESS_RESOURCE_MEASUREMENT_HPP diff --git a/score/tests/utility/runtime_measurement.hpp b/score/tests/utility/runtime_measurement.hpp new file mode 100644 index 000000000..357f80430 --- /dev/null +++ b/score/tests/utility/runtime_measurement.hpp @@ -0,0 +1,281 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef TESTS_UTILITY_RUNTIME_MEASUREMENT_HPP +#define TESTS_UTILITY_RUNTIME_MEASUREMENT_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace score::crypto::daemon::common +{ + +class RuntimeMeasurement final +{ + public: + explicit RuntimeMeasurement(const bool enable_individual_output = false) + : enable_individual_output_(enable_individual_output) + { + measurements_.reserve(1024U * 1024U); + } + + class ScopedMeasurement final + { + public: + ScopedMeasurement(RuntimeMeasurement& measurement, std::string_view tag) + : measurement_(measurement), tag_(tag) + { + measurement_.Start(tag_); + } + + ~ScopedMeasurement() noexcept + { + measurement_.End(tag_); + } + + ScopedMeasurement(const ScopedMeasurement&) = delete; + ScopedMeasurement& operator=(const ScopedMeasurement&) = delete; + + private: + RuntimeMeasurement& measurement_; + std::string_view tag_; + }; + + static RuntimeMeasurement& Instance() noexcept + { + static RuntimeMeasurement instance; + return instance; + } + + void Start(std::string_view tag) + { + std::lock_guard lock(mutex_); + measurements_.push_back({tag, Clock::now(), std::this_thread::get_id(), true}); + } + + void End(std::string_view tag) + { + std::lock_guard lock(mutex_); + measurements_.push_back({tag, Clock::now(), std::this_thread::get_id(), false}); + } + + void RecordElapsed(std::string_view tag, const std::chrono::steady_clock::time_point start) + { + const auto end = Clock::now(); + std::lock_guard lock(mutex_); + measurements_.push_back({tag, start, std::this_thread::get_id(), true}); + measurements_.push_back({tag, end, std::this_thread::get_id(), false}); + } + + void Print() noexcept + { + std::lock_guard lock(mutex_); + const auto unmatched = std::numeric_limits::max(); + std::vector end_for_start(measurements_.size(), unmatched); + std::vector matched_end(measurements_.size(), false); + struct Summary + { + std::string_view tag; + std::uint64_t total_nanoseconds{0U}; + std::size_t count{0U}; + std::vector durations; + }; + std::vector summaries; + summaries.reserve(measurements_.size()); + constexpr std::string_view kOpenSslTagPrefix{"OpenSSL::"}; + + for (std::size_t end_index = 0; end_index < measurements_.size(); ++end_index) + { + const auto& end_measurement = measurements_[end_index]; + if (end_measurement.is_start) + { + continue; + } + + for (std::size_t start_index = end_index; start_index > 0U; --start_index) + { + const auto candidate_index = start_index - 1U; + const auto& candidate = measurements_[candidate_index]; + if (candidate.is_start && end_for_start[candidate_index] == unmatched && + candidate.thread_id == end_measurement.thread_id && candidate.tag == end_measurement.tag) + { + end_for_start[candidate_index] = end_index; + matched_end[end_index] = true; + break; + } + } + } + + for (std::size_t start_index = 0; start_index < measurements_.size(); ++start_index) + { + const auto& start_measurement = measurements_[start_index]; + if (!start_measurement.is_start) + { + continue; + } + + if (end_for_start[start_index] == unmatched) + { + std::fprintf(stderr, + "[Timing] incomplete measurement for %.*s\n", + static_cast(start_measurement.tag.size()), + start_measurement.tag.data()); + std::fflush(stderr); + continue; + } + + const auto& end_measurement = measurements_[end_for_start[start_index]]; + const auto elapsed = + std::chrono::duration_cast(end_measurement.time - start_measurement.time); + if (enable_individual_output_) + { + std::fprintf(stdout, + "[Timing] %.*s: %lld ns\n", + static_cast(start_measurement.tag.size()), + start_measurement.tag.data(), + static_cast(elapsed.count())); + std::fflush(stdout); + } + + Summary* summary = nullptr; + for (auto& candidate : summaries) + { + if (candidate.tag == start_measurement.tag) + { + summary = &candidate; + break; + } + } + if (summary == nullptr) + { + summaries.push_back({start_measurement.tag, 0U, 0U, {}}); + summary = &summaries.back(); + } + summary->total_nanoseconds += static_cast(elapsed.count()); + ++summary->count; + summary->durations.push_back(static_cast(elapsed.count())); + } + + for (std::size_t end_index = 0; end_index < measurements_.size(); ++end_index) + { + if (!measurements_[end_index].is_start && !matched_end[end_index]) + { + std::fprintf(stderr, + "[Timing] incomplete end measurement for %.*s\n", + static_cast(measurements_[end_index].tag.size()), + measurements_[end_index].tag.data()); + std::fflush(stderr); + } + } + + std::uint64_t open_ssl_average_sum_nanoseconds{0U}; + std::size_t open_ssl_subpart_count{0U}; + for (const auto& summary : summaries) + { + const auto average_nanoseconds = summary.total_nanoseconds / summary.count; + auto sorted_durations = summary.durations; + std::sort(sorted_durations.begin(), sorted_durations.end()); + const auto nearest_rank = [&sorted_durations](const std::size_t numerator, const std::size_t denominator) { + const auto rank = (sorted_durations.size() * numerator + denominator - 1U) / denominator; + return sorted_durations[rank - 1U]; + }; + const auto p50_nanoseconds = nearest_rank(1U, 2U); + const auto p90_nanoseconds = nearest_rank(9U, 10U); + const auto p99_nanoseconds = nearest_rank(99U, 100U); + std::fprintf(stdout, + "[Timing] average %.*s: %llu ns over %zu measurement(s)\n", + static_cast(summary.tag.size()), + summary.tag.data(), + static_cast(average_nanoseconds), + summary.count); + std::fprintf(stdout, + "[Timing] p50 %.*s: %llu ns\n", + static_cast(summary.tag.size()), + summary.tag.data(), + static_cast(p50_nanoseconds)); + std::fprintf(stdout, + "[Timing] p90 %.*s: %llu ns\n", + static_cast(summary.tag.size()), + summary.tag.data(), + static_cast(p90_nanoseconds)); + std::fprintf(stdout, + "[Timing] p99 %.*s: %llu ns\n", + static_cast(summary.tag.size()), + summary.tag.data(), + static_cast(p99_nanoseconds)); + std::fflush(stdout); + + if (summary.tag.size() >= kOpenSslTagPrefix.size() && + summary.tag.compare(0U, kOpenSslTagPrefix.size(), kOpenSslTagPrefix) == 0) + { + open_ssl_average_sum_nanoseconds += average_nanoseconds; + ++open_ssl_subpart_count; + } + } + + if (open_ssl_subpart_count > 0U) + { + std::fprintf(stdout, + "[Timing] sum of OpenSSL subpart averages: %llu ns over %zu subpart(s)\n", + static_cast(open_ssl_average_sum_nanoseconds), + open_ssl_subpart_count); + std::fflush(stdout); + } + + std::fflush(stdout); + } + + ~RuntimeMeasurement() noexcept + { + Print(); + } + + RuntimeMeasurement(const RuntimeMeasurement&) = delete; + RuntimeMeasurement& operator=(const RuntimeMeasurement&) = delete; + + private: + using Clock = std::chrono::steady_clock; + using TimePoint = Clock::time_point; + static_assert(Clock::is_steady, "Runtime measurements require a monotonic clock"); + + struct Measurement + { + std::string_view tag; + TimePoint time; + std::thread::id thread_id; + bool is_start; + }; + + const bool enable_individual_output_; + std::mutex mutex_; + std::vector measurements_; +}; + +} // namespace score::crypto::daemon::common + +namespace tests::utility +{ + +using RuntimeMeasurement = ::score::crypto::daemon::common::RuntimeMeasurement; + +} // namespace tests::utility + +#endif // TESTS_UTILITY_RUNTIME_MEASUREMENT_HPP From f3a0cd6c86fe8c542ddc2761404614a2107f654e Mon Sep 17 00:00:00 2001 From: Lobsinger Sebastian Date: Wed, 9 Sep 2026 09:17:37 +0000 Subject: [PATCH 4/4] Added POC code to coverage exclusion list --- MODULE.bazel.lock | 1 - tools/coverage/scope_exclusions.txt | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c6a28ca73..7906f379c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -969,7 +969,6 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_toolchains_rust/0.10.0/source.json": "8bda773be264da16d2a82a03ebb737421dd4a35855f1e9a5d03d9722d84c1df5", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/1.0.2/MODULE.bazel": "e70f396375b9d612b4f41ebceff7f18f68ab423b14625c138a354cc01bc62a10", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/1.1.2/MODULE.bazel": "56d08309931cfad67c2b6691207bb5f761a3946830d620c630d2436630e6b499", - "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/1.2.0/MODULE.bazel": "982db97a4f8440356ea935f103204f644fb2b84d8188df63533c7312c82afc37", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/1.3.1/MODULE.bazel": "4ce18de0fdfb447ba750b98febb24e12471862cbbd5d08f5f683f27886767367", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/2.0.2/MODULE.bazel": "edf7730454fc1c7951d903895524d57efbe059edbf2572cf9bdc0e7a20d6cc55", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_tooling/2.2.0/MODULE.bazel": "178ba4862246b6ba2bbcd96b7e9e728299b19fb94bcf23d315dc2299aabf7178", diff --git a/tools/coverage/scope_exclusions.txt b/tools/coverage/scope_exclusions.txt index b0cce4a64..4806e496f 100644 --- a/tools/coverage/scope_exclusions.txt +++ b/tools/coverage/scope_exclusions.txt @@ -18,3 +18,26 @@ # --- test scaffolding (non-testonly, but test infrastructure only) --- //score/crypto/tests/softhsm:softhsm_test_fixture # SoftHSM fixture for backend tests //score/tests/utility:utility # shared helpers for demo/integration tests + +# --- POCs for the selection of IPC mechanism --- +# Kept as reference till full migration to new IPC is done +# will then be removed. Code will is not meant to be in target +# Helper utility is unly used for runtime_measurement in POCs +//score/tests/ipc_poc:async_control_plane_service +//score/tests/ipc_poc:direct_socket_poc_deps +//score/tests/ipc_poc:generated_poc_control +//score/tests/ipc_poc:low_level_poc_deps +//score/tests/ipc_poc:poc_async +//score/tests/ipc_poc:poc_engine +//score/tests/ipc_poc:poc_engine_sync +//score/tests/ipc_poc:poc_grpc +//score/tests/ipc_poc:poc_grpc_protocol +//score/tests/ipc_poc:poc_helper +//score/tests/ipc_poc:poc_low_level +//score/tests/ipc_poc:poc_low_level_no_reply +//score/tests/ipc_poc:poc_low_level_no_reply_notify_in_thread +//score/tests/ipc_poc:poc_qnx_message_passing +//score/tests/ipc_poc:poc_thread_pool_queue +//score/tests/ipc_poc:poc_unix_socket +//score/tests/utility:process_resource_measurement +//score/tests/utility:runtime_measurement