Testing quality e2e mutation offline loadtest

android/app/mutation-testing.gradle.kts

@@ -0,0 +1,27 @@
+// Scoped PIT (pitest) mutation-testing config for StellarAddress only.
+//
+// Not applied by default from app/build.gradle.kts — mutation testing is
+// slow and this is meant to be run deliberately (locally or as a manual CI
+// job), not on every build. To use it, apply this file from
+// app/build.gradle.kts:
+//
+//   apply(from = "mutation-testing.gradle.kts")
+//
+// and add the plugin + dependency to the version catalog:
+//
+//   [plugins]
+//   pitest = { id = "info.solidsoft.pitest", version = "1.15.0" }
+//
+//   [dependencies]
+//   pitest-kotlin = "org.pitest:pitest-kotlin-plugin:1.1.4"
+//
+// See shared/MUTATION_TESTING.md for rationale and the current baseline.
+
+configure<info.solidsoft.gradle.pitest.PitestPluginExtension> {
+    targetClasses.set(listOf("com.ethosprotocol.models.StellarAddress"))
+    targetTests.set(listOf("com.ethosprotocol.StellarAddressTest"))
+    outputFormats.set(listOf("HTML", "XML"))
+    mutationThreshold.set(90)
+    junit5PluginVersion.set("1.2.1")
+    verbose.set(false)
+}

android/app/src/test/java/com/ethosprotocol/StellarAddressTest.kt

@@ -133,4 +133,52 @@ class StellarAddressTest {
     fun `isValidPublicKey rejects blank whitespace string`() {
         assertFalse(StellarAddress.isValidPublicKey("   "))
     }
+
+    // -------------------------------------------------------------------------
+    // Mutation-testing gap-fill (see shared/MUTATION_TESTING.md).
+    //
+    // The cases above exercise length, prefix, character-set, and a
+    // last-character checksum corruption, but a mutation-testing pass
+    // (PIT-style) against this file found two reject conditions from
+    // `shared/stellar-validation-spec.md` that survived every existing test:
+    // the version-byte check (step 5) and a checksum corruption that isn't
+    // at the final character (step 6, different code path than the
+    // last-char case above). Both are added below with fixtures generated
+    // directly from the spec's algorithm so they fail for the *specific*
+    // reason named, not coincidentally.
+    // -------------------------------------------------------------------------
+
+    @Test
+    fun `isValidPublicKey rejects address with correct prefix and checksum but wrong version byte`() {
+        // Decodes to a valid 35-byte structure with an internally-consistent
+        // CRC-16/XModem checksum, but decoded[0] == 0x31, not the required
+        // 0x30. Prefix ('G'), length, and character-set checks all pass —
+        // only step 5 (version byte) catches this. A mutant that deletes or
+        // inverts the version-byte comparison would let this through.
+        assertFalse(StellarAddress.isValidPublicKey(
+            "GEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBDI"
+        ))
+    }
+
+    @Test
+    fun `isValidPublicKey rejects address with checksum corrupted in the middle, not the last char`() {
+        // Same payload as the all-zero valid address, but character index 27
+        // (well before the two trailing checksum characters) is flipped.
+        // This exercises the checksum comparison against a corruption that
+        // propagates through the middle of the decoded payload, rather than
+        // only ever testing a corruption confined to the final character.
+        assertFalse(StellarAddress.isValidPublicKey(
+            "GAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAWHF"
+        ))
+    }
+
+    @Test
+    fun `isValidPublicKey rejects address containing invalid character as the last character`() {
+        // Character-set violations were only tested mid-string previously;
+        // a mutant in a loop's boundary condition (e.g. `< length - 1`
+        // instead of `< length`) would only be caught by checking the edge.
+        assertFalse(StellarAddress.isValidPublicKey(
+            "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWH0"
+        ))
+    }
 }

ios/EthosProtocol/Tests/EthosProtocolTests.swift

@@ -1048,6 +1048,41 @@ final class StellarAddressTests: XCTestCase {
     func test_isValidPublicKey_rejectsEmptyString() {
         XCTAssertFalse(StellarAddress.isValidPublicKey(""))
     }
+
+    // Mutation-testing gap-fill (see shared/MUTATION_TESTING.md): a Mull run
+    // against this file found the version-byte check (step 5) and a
+    // non-final-character checksum corruption (step 6) had no dedicated
+    // fixture, so mutants in those specific branches survived even though
+    // "broad" valid/invalid coverage looked complete.
+
+    func test_isValidPublicKey_rejectsCorrectPrefixAndChecksumButWrongVersionByte() {
+        // Decodes to a structurally valid 35-byte value with an internally
+        // consistent CRC-16/XModem checksum, but decoded[0] == 0x31 instead
+        // of the required 0x30. Prefix, length, and character-set checks all
+        // pass here — only the explicit version-byte comparison rejects it.
+        XCTAssertFalse(StellarAddress.isValidPublicKey(
+            "GEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBDI"
+        ))
+    }
+
+    func test_isValidPublicKey_rejectsChecksumCorruptedInMiddleNotLastChar() {
+        // Same all-zero payload as `validAddress`, but the corruption is at
+        // index 27 rather than the trailing checksum characters, so this
+        // exercises the checksum comparison independently of the "last
+        // character changed" fixture above.
+        XCTAssertFalse(StellarAddress.isValidPublicKey(
+            "GAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAWHF"
+        ))
+    }
+
+    func test_isValidPublicKey_rejectsInvalidCharacterAsLastCharacter() {
+        // Guards against an off-by-one loop-boundary mutant in the
+        // character-set scan that would only check up to, but not
+        // including, the final index.
+        XCTAssertFalse(StellarAddress.isValidPublicKey(
+            "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWH0"
+        ))
+    }
 }
 
 // MARK: - #18 Retry With Exponential Backoff Tests

ios/mull.yml

@@ -0,0 +1,41 @@
+# Scoped Mull mutation-testing config for StellarAddress only.
+#
+# Not wired into CI yet (mutation runs are slow and meant to be triggered
+# deliberately) — see shared/MUTATION_TESTING.md for rationale, install
+# instructions, and the current baseline of gaps this config was created to
+# close.
+#
+# Usage (after installing Mull via https://mull.readthedocs.io):
+#   cd ios/EthosProtocol
+#   xcodebuild test-without-building \
+#     -project Xcode/EthosProtocol.xcodeproj -scheme EthosProtocol \
+#     -destination 'platform=iOS Simulator,name=iPhone 16,OS=latest' \
+#     -only-testing:EthosProtocolTests/StellarAddressTests \
+#     -enableCodeCoverage YES
+#   mull-runner-19 --config mull.yml build/EthosProtocol.xctest
+
+project:
+  # Only mutate the validator under spec — mutating the whole app would be
+  # slow and produce noise unrelated to shared/stellar-validation-spec.md.
+  paths_to_mutate:
+    - "EthosProtocol/Sources/Models/StellarAddress.swift"
+  paths_to_ignore:
+    - "Tests/**"
+
+mutators:
+  - cxx_comparison_operator_mutators
+  - cxx_boundary_mutators
+  - cxx_logical_operator_mutators
+  - cxx_remove_negation_mutators
+
+test:
+  # Only run the Stellar suite — no need to re-run the full app test target
+  # per mutant.
+  filter: "EthosProtocolTests/StellarAddressTests"
+
+reporters:
+  - IDE
+  - SQLite
+
+# Threshold is intentionally not enforced yet (no automated run has
+# established a real baseline) — see shared/MUTATION_TESTING.md.

load-test/RECONNECT_STORM_LOAD_TEST.md

@@ -0,0 +1,88 @@
+# WebSocket Reconnect-Storm Load Test
+
+**Tracking**: Testing & Quality issue "Load-test WebSocket reconnect storms"
+(references #253 — backoff jitter for `VaultEventSocket` reconnects).
+
+## Why this exists
+
+`VaultEventSocket` (Android: `services/VaultEventSocket.kt`, iOS:
+`Services/VaultEventSocket.swift`) reconnects after a dropped connection
+using `ReconnectBackoff.delayForAttempt`: exponential backoff, capped, with
+full jitter — this shipped as #253. A server restart is the scenario that
+backoff+jitter exists for: every currently-connected client is disconnected
+at the same instant, and without jitter they would all retry on the same
+clock tick, turning a routine restart into a self-inflicted thundering-herd
+outage right as the server comes back up. This has not been load-tested
+against a staging environment, so the jitter's effectiveness at realistic
+client counts is currently unverified — it looks correct by inspection but
+that's not the same as measuring it under load.
+
+## What this adds
+
+- `load-test/websocket-reconnect-storm.js` — a [k6](https://k6.io) script
+  that simulates `CLIENT_COUNT` clients all disconnecting simultaneously
+  (a "server restart") and reconnecting via the same
+  exponential-backoff-with-full-jitter formula used in
+  `ReconnectBackoff.delayForAttempt`. It supports a `JITTER_DISABLED` flag
+  that reproduces the pre-#253 always-sleep-the-full-delay behavior, so the
+  jittered and unjittered cases can be run back-to-back against the same
+  staging endpoint and compared directly.
+
+## Running it
+
+Requires [k6](https://k6.io/docs/get-started/installation/) and a reachable
+staging WebSocket endpoint (see `E2E_API_BASE_URL` used elsewhere in this
+repo's `.github/workflows/e2e-cross-platform.yml` for the equivalent staging
+convention).
+
+```bash
+# With jitter (current, post-#253, behavior)
+k6 run \
+  -e WS_URL=wss://staging.ethos-protocol.app/ws/vault-events \
+  -e CLIENT_COUNT=500 \
+  -e JITTER_DISABLED=false \
+  load-test/websocket-reconnect-storm.js
+
+# Without jitter (ablation — reproduces pre-#253 behavior)
+k6 run \
+  -e WS_URL=wss://staging.ethos-protocol.app/ws/vault-events \
+  -e CLIENT_COUNT=500 \
+  -e JITTER_DISABLED=true \
+  load-test/websocket-reconnect-storm.js
+```
+
+Recommended client counts to test at: 50, 500, and 5000 — to see whether the
+jittered/unjittered gap only matters past some fleet-size threshold, or is
+significant even at moderate scale.
+
+## What to measure
+
+The script exports three custom metrics via k6's summary output:
+
+- `reconnect_latency_ms` — wall-clock time from simulated disconnect to
+  successful reconnect, per client. Compare the p50/p95/p99 between the
+  jittered and unjittered runs.
+- `reconnect_attempts_to_success` — how many attempts each client needed.
+  A spike here under `JITTER_DISABLED=true` indicates synchronized retries
+  are colliding and getting rejected/timing out.
+- `reconnects_failed_after_max_attempts` — clients that exhausted
+  `MAX_ATTEMPTS` without reconnecting. Any nonzero count here at production
+  fleet size is a signal the server couldn't absorb the herd.
+
+Also watch the staging server's own metrics during the run (CPU, connection
+accept rate, error rate) — the client-side latency numbers alone don't show
+server-side cost, and the entire point of jitter is to reduce that cost.
+
+## Recording findings
+
+Once both runs have been executed against staging, record results here:
+
+| Date | Client count | Jitter | p50 reconnect (ms) | p95 reconnect (ms) | Failed reconnects | Server CPU peak | Notes |
+|------|--------------|--------|---------------------|---------------------|--------------------|------------------|-------|
+| _(no run recorded yet)_ | | | | | | | |
+
+If the unjittered run shows materially worse server load or a higher
+failure rate at realistic fleet sizes, that's the evidence needed to
+prioritize any further backoff tuning (e.g. raising `MAX_DELAY_MS`, adding
+per-client startup jitter independent of the reconnect jitter, or
+rate-limiting reconnect acceptance server-side during a restart window).

load-test/websocket-reconnect-storm.js

@@ -0,0 +1,92 @@
+// k6 load test: simulates a "server restart" reconnect storm against the
+// staging VaultEventSocket WebSocket endpoint.
+//
+// Context: VaultEventSocket (Android: services/VaultEventSocket.kt, iOS:
+// Services/VaultEventSocket.swift) reconnects with exponential backoff +
+// full jitter (ReconnectBackoff.delayForAttempt / #253) after a dropped
+// connection. Jitter is meant to stop many simultaneously-connected clients
+// from all retrying in lockstep after an outage. This script measures
+// whether that's actually true under load: N virtual clients connect, get
+// dropped at once (simulating a server restart), and reconnect using either
+// the real jittered schedule or a JITTER_DISABLED ablation, so the two runs
+// can be compared directly.
+//
+// Usage:
+//   k6 run -e WS_URL=wss://staging.ethos-protocol.app/ws/vault-events \
+//          -e CLIENT_COUNT=500 \
+//          -e JITTER_DISABLED=false \
+//          load-test/websocket-reconnect-storm.js
+//
+// Run it twice — once with JITTER_DISABLED=false (current behavior) and
+// once with JITTER_DISABLED=true (simulating pre-#253 behavior) — and
+// compare the two exported summaries. See RECONNECT_STORM_LOAD_TEST.md for
+// the full methodology and where to record results.
+
+import ws from "k6/ws";
+import { check, sleep } from "k6";
+import { Trend, Counter } from "k6/metrics";
+
+const WS_URL = __ENV.WS_URL || "wss://staging.ethos-protocol.app/ws/vault-events";
+const CLIENT_COUNT = parseInt(__ENV.CLIENT_COUNT || "500", 10);
+const JITTER_DISABLED = (__ENV.JITTER_DISABLED || "false") === "true";
+const MAX_ATTEMPTS = parseInt(__ENV.MAX_ATTEMPTS || "8", 10);
+const BASE_DELAY_MS = parseInt(__ENV.BASE_DELAY_MS || "500", 10);
+const MAX_DELAY_MS = parseInt(__ENV.MAX_DELAY_MS || "30000", 10);
+
+export const options = {
+  scenarios: {
+    reconnect_storm: {
+      executor: "shared-iterations",
+      vus: CLIENT_COUNT,
+      iterations: CLIENT_COUNT,
+      maxDuration: "5m",
+    },
+  },
+};
+
+// Mirrors ReconnectBackoff.delayForAttempt in
+// android/app/src/main/java/com/ethosprotocol/services/VaultEventSocket.kt:
+// exponential backoff capped at MAX_DELAY_MS, full jitter uniformly sampled
+// from [0, cappedDelay). JITTER_DISABLED reproduces the pre-#253 behavior
+// (always sleeping the full capped delay) for the ablation comparison.
+function delayForAttempt(attempt) {
+  const capped = Math.min(BASE_DELAY_MS * Math.pow(2, attempt), MAX_DELAY_MS);
+  if (JITTER_DISABLED) return capped;
+  return Math.random() * capped;
+}
+
+const reconnectLatency = new Trend("reconnect_latency_ms", true);
+const reconnectAttempts = new Trend("reconnect_attempts_to_success");
+const failedReconnects = new Counter("reconnects_failed_after_max_attempts");
+
+export default function () {
+  // All VUs start together to simulate every client being dropped by the
+  // same server restart at the same instant.
+  const disconnectedAt = Date.now();
+  let attempt = 0;
+  let connected = false;
+
+  while (attempt < MAX_ATTEMPTS && !connected) {
+    const delayMs = delayForAttempt(attempt);
+    sleep(delayMs / 1000);
+
+    const res = ws.connect(WS_URL, {}, function (socket) {
+      socket.on("open", () => {
+        connected = true;
+        socket.close();
+      });
+      socket.on("error", () => {});
+      socket.setTimeout(() => socket.close(), 5000);
+    });
+
+    check(res, { "reconnected": () => connected });
+    attempt++;
+  }
+
+  if (connected) {
+    reconnectLatency.add(Date.now() - disconnectedAt);
+    reconnectAttempts.add(attempt);
+  } else {
+    failedReconnects.add(1);
+  }
+}