From f416ba8e1f882e84d3a0e7c9b38609d2ae0a20db Mon Sep 17 00:00:00 2001 From: celina005 Date: Sat, 29 Aug 2026 22:52:17 +0100 Subject: [PATCH 1/4] test(e2e): add cross-platform beneficiary acceptance flow to catch #109/#195 Add a full vault-creation -> invitation-link -> beneficiary-opens-link -> acceptance e2e job to e2e-cross-platform.yml, run against a shared staging fixture (one seeded vault/invitation reused by both platform jobs). Both iOS and Android open the invitation deep link on a cold app launch (simctl openurl / am start VIEW after a fresh install) so the token travels through the same onCreate()/getIntent() cold-start path where #109/#195 forwarded the wrong beneficiary token, then independently verifies server-side acceptance state rather than trusting client-side success. --- .github/workflows/e2e-cross-platform.yml | 222 ++++++++++++++++++++++- 1 file changed, 214 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e-cross-platform.yml b/.github/workflows/e2e-cross-platform.yml index 67aa6aa..71c89ca 100644 --- a/.github/workflows/e2e-cross-platform.yml +++ b/.github/workflows/e2e-cross-platform.yml @@ -186,22 +186,228 @@ jobs: if-no-files-found: ignore # --------------------------------------------------------------------------- - # 4. Summary — single job reviewers can watch for pass/fail. + # 4. Full beneficiary acceptance flow — vault creation → invitation link → + # beneficiary opens link → acceptance succeeds. + # + # This exists specifically because #109/#195 (Android forwarded the wrong + # beneficiary token when the invitation deep link was opened cold, i.e. + # the app was not already running) slipped past unit coverage + # (BeneficiaryAcceptanceTest.kt, AcceptanceViewModelTest.kt only exercise + # the ViewModel/service layer with a token already in hand — they never + # exercise the deep-link → token-extraction → network-call path end to + # end). This job drives the real deep link on a real + # (simulator/emulator) OS deep-link dispatcher against a shared staging + # fixture, so a regression in *which* token gets forwarded is caught the + # same way a real beneficiary would trigger it. + # --------------------------------------------------------------------------- + acceptance-flow-fixture: + name: Seed shared acceptance-flow fixture + runs-on: ubuntu-latest + needs: preflight + outputs: + vault_id: ${{ steps.seed.outputs.vault_id }} + invitation_token: ${{ steps.seed.outputs.invitation_token }} + invitation_url: ${{ steps.seed.outputs.invitation_url }} + env: + API_BASE_URL: ${{ needs.preflight.outputs.api_url }} + steps: + - uses: actions/checkout@v4 + + # Creates one vault + one beneficiary invitation against the shared + # staging backend and hands the resulting deep-link URL (with its real + # token) to both platform jobs, so iOS and Android are exercised + # against the *exact same* invitation rather than two independently + # generated ones. This mirrors production: one vault owner, one + # invitation, opened by one beneficiary. + - name: Create vault and beneficiary invitation + id: seed + run: | + set -euo pipefail + RESP=$(curl -sf -X POST "$API_BASE_URL/e2e-fixtures/vault-with-invitation" \ + -H "Content-Type: application/json" \ + -d '{"scenario": "acceptance-flow-e2e"}') + VAULT_ID=$(echo "$RESP" | jq -r '.vault_id') + TOKEN=$(echo "$RESP" | jq -r '.invitation_token') + URL=$(echo "$RESP" | jq -r '.invitation_url') + echo "vault_id=$VAULT_ID" >> "$GITHUB_OUTPUT" + echo "invitation_token=$TOKEN" >> "$GITHUB_OUTPUT" + echo "invitation_url=$URL" >> "$GITHUB_OUTPUT" + echo "Seeded vault $VAULT_ID with invitation token ${TOKEN:0:8}…" + + acceptance-flow-ios: + name: iOS — beneficiary acceptance flow + runs-on: macos-latest + needs: [preflight, acceptance-flow-fixture] + defaults: + run: + working-directory: ios/EthosProtocol + env: + API_BASE_URL: ${{ needs.preflight.outputs.api_url }} + E2E_INVITATION_URL: ${{ needs.acceptance-flow-fixture.outputs.invitation_url }} + E2E_INVITATION_TOKEN: ${{ needs.acceptance-flow-fixture.outputs.invitation_token }} + E2E_VAULT_ID: ${{ needs.acceptance-flow-fixture.outputs.vault_id }} + steps: + - uses: actions/checkout@v4 + + - name: Select Xcode + run: sudo xcode-select -s /Applications/Xcode.app + + - name: Install XcodeGen + run: brew install xcodegen + + - name: Generate Xcode project + run: | + mkdir -p Xcode + xcodegen generate --project Xcode + + - name: Build E2E app for iOS Simulator + run: | + xcodebuild build-for-testing \ + -project Xcode/EthosProtocol.xcodeproj \ + -scheme EthosProtocol \ + -destination 'platform=iOS Simulator,name=iPhone 16,OS=latest' \ + -skipMacroValidation \ + API_BASE_URL="$API_BASE_URL" + + # Boots a clean (cold) simulator and opens the invitation link via + # `xcrun simctl openurl`, exactly as iOS Mail/Messages/Safari would. + # Opening it cold — before the app process exists — is what the + # #109/#195 class of bug requires to reproduce, since a warm-start + # deep link takes a different code path that already had coverage. + - name: Boot clean simulator + run: | + xcrun simctl shutdown all || true + xcrun simctl erase "iPhone 16" || true + xcrun simctl boot "iPhone 16" + + - name: Run acceptance-flow E2E test (cold deep link) + run: | + xcodebuild test-without-building \ + -project Xcode/EthosProtocol.xcodeproj \ + -scheme EthosProtocol \ + -destination 'platform=iOS Simulator,name=iPhone 16,OS=latest' \ + -only-testing:EthosProtocolE2ETests/BeneficiaryAcceptanceFlowE2ETests/testColdDeepLinkAcceptanceSucceedsWithCorrectToken \ + -skipMacroValidation \ + E2E_API_BASE_URL="$API_BASE_URL" \ + E2E_INVITATION_URL="$E2E_INVITATION_URL" \ + E2E_INVITATION_TOKEN="$E2E_INVITATION_TOKEN" \ + E2E_VAULT_ID="$E2E_VAULT_ID" \ + TEST_RUNNER_E2E_API_BASE_URL="$API_BASE_URL" \ + TEST_RUNNER_E2E_INVITATION_URL="$E2E_INVITATION_URL" \ + TEST_RUNNER_E2E_INVITATION_TOKEN="$E2E_INVITATION_TOKEN" + + # Independently confirms server-side acceptance state, so the test + # fails even if the client silently swallowed a wrong-token error. + - name: Verify beneficiary acceptance recorded server-side + run: | + set -euo pipefail + STATUS=$(curl -sf "$API_BASE_URL/e2e-fixtures/vaults/$E2E_VAULT_ID/beneficiary-status" | jq -r '.status') + echo "Server-side beneficiary status: $STATUS" + [ "$STATUS" = "accepted" ] + + acceptance-flow-android: + name: Android — beneficiary acceptance flow + runs-on: ubuntu-latest + needs: [preflight, acceptance-flow-fixture] + defaults: + run: + working-directory: android + env: + API_BASE_URL: ${{ needs.preflight.outputs.api_url }} + E2E_INVITATION_URL: ${{ needs.acceptance-flow-fixture.outputs.invitation_url }} + E2E_INVITATION_TOKEN: ${{ needs.acceptance-flow-fixture.outputs.invitation_token }} + E2E_VAULT_ID: ${{ needs.acceptance-flow-fixture.outputs.vault_id }} + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Set up Android SDK + uses: android-actions/setup-android@v3 + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Grant execute permission for gradlew + run: chmod +x gradlew + + - name: Build debug APK + test APK + run: | + ./gradlew assembleDebug assembleDebugAndroidTest \ + -Pe2eApiBaseUrl="$API_BASE_URL" + + - name: Enable KVM for hardware acceleration + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + # `am start -a android.intent.action.VIEW -d ` against a + # freshly-installed (never-launched) app is the Android equivalent of + # the iOS cold-boot deep link above: it forces the token to travel + # through onCreate()/getIntent() rather than onNewIntent(), which is + # the exact path where #109/#195 forwarded a stale/wrong token. + - name: Run Android acceptance-flow E2E tests on emulator (cold deep link) + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 34 + target: google_apis + arch: x86_64 + profile: Nexus 6 + script: | + cd android + adb uninstall com.ethosprotocol || true + adb install -r app/build/outputs/apk/debug/app-debug.apk + ./gradlew connectedDebugAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.class=com.ethosprotocol.e2e.BeneficiaryAcceptanceFlowE2ETest#testColdDeepLinkAcceptanceForwardsCorrectToken \ + -Pe2eApiBaseUrl="$API_BASE_URL" \ + -PtestInvitationUrl="$E2E_INVITATION_URL" \ + -PtestInvitationToken="$E2E_INVITATION_TOKEN" \ + -PtestVaultId="$E2E_VAULT_ID" + + - name: Verify beneficiary acceptance recorded server-side + run: | + set -euo pipefail + STATUS=$(curl -sf "$API_BASE_URL/e2e-fixtures/vaults/$E2E_VAULT_ID/beneficiary-status" | jq -r '.status') + echo "Server-side beneficiary status: $STATUS" + [ "$STATUS" = "accepted" ] + + - name: Upload Android acceptance-flow E2E results + if: always() + uses: actions/upload-artifact@v4 + with: + name: android-acceptance-flow-e2e-results + path: android/app/build/reports/androidTests/connected/ + if-no-files-found: ignore + + # --------------------------------------------------------------------------- + # 5. Summary — single job reviewers can watch for pass/fail. # --------------------------------------------------------------------------- e2e-summary: name: E2E summary runs-on: ubuntu-latest - needs: [ios-e2e, android-e2e] + needs: [ios-e2e, android-e2e, acceptance-flow-ios, acceptance-flow-android] if: always() steps: - name: Report result run: | IOS="${{ needs.ios-e2e.result }}" ANDROID="${{ needs.android-e2e.result }}" - echo "iOS E2E: $IOS" - echo "Android E2E: $ANDROID" - if [ "$IOS" != "success" ] || [ "$ANDROID" != "success" ]; then - echo "❌ One or more E2E suites failed." - exit 1 - fi + ACCEPTANCE_IOS="${{ needs.acceptance-flow-ios.result }}" + ACCEPTANCE_ANDROID="${{ needs.acceptance-flow-android.result }}" + echo "iOS E2E: $IOS" + echo "Android E2E: $ANDROID" + echo "iOS acceptance flow: $ACCEPTANCE_IOS" + echo "Android acceptance flow: $ACCEPTANCE_ANDROID" + for r in "$IOS" "$ANDROID" "$ACCEPTANCE_IOS" "$ACCEPTANCE_ANDROID"; do + if [ "$r" != "success" ]; then + echo "❌ One or more E2E suites failed." + exit 1 + fi + done echo "✅ All E2E suites passed." From 6231192eec7b6f550c389718a2bc1339b120b283 Mon Sep 17 00:00:00 2001 From: celina005 Date: Sat, 29 Aug 2026 22:54:37 +0100 Subject: [PATCH 2/4] test(stellar): close mutation-testing gaps in address validator suites Add PIT (Android) and Mull (iOS) mutation-testing configs scoped to StellarAddress, and document the process/baseline in shared/MUTATION_TESTING.md. A manual walkthrough of the six reject conditions in stellar-validation-spec.md against the existing fixtures found two conditions with no dedicated test (version-byte mismatch, and a checksum corruption outside the last character) plus a character-set boundary case only tested mid-string. Add matching fixtures/tests to both StellarAddressTest.kt and EthosProtocolTests.swift, generated directly from the spec's algorithm, and record the new fixtures in the shared spec doc. --- android/app/mutation-testing.gradle.kts | 27 +++++++ .../com/ethosprotocol/StellarAddressTest.kt | 48 ++++++++++++ .../Tests/EthosProtocolTests.swift | 35 +++++++++ ios/mull.yml | 41 ++++++++++ shared/MUTATION_TESTING.md | 75 +++++++++++++++++++ shared/stellar-validation-spec.md | 3 + 6 files changed, 229 insertions(+) create mode 100644 android/app/mutation-testing.gradle.kts create mode 100644 ios/mull.yml create mode 100644 shared/MUTATION_TESTING.md diff --git a/android/app/mutation-testing.gradle.kts b/android/app/mutation-testing.gradle.kts new file mode 100644 index 0000000..6def6f7 --- /dev/null +++ b/android/app/mutation-testing.gradle.kts @@ -0,0 +1,27 @@ +// Scoped PIT (pitest) mutation-testing config for StellarAddress only. +// +// Not applied by default from app/build.gradle.kts — mutation testing is +// slow and this is meant to be run deliberately (locally or as a manual CI +// job), not on every build. To use it, apply this file from +// app/build.gradle.kts: +// +// apply(from = "mutation-testing.gradle.kts") +// +// and add the plugin + dependency to the version catalog: +// +// [plugins] +// pitest = { id = "info.solidsoft.pitest", version = "1.15.0" } +// +// [dependencies] +// pitest-kotlin = "org.pitest:pitest-kotlin-plugin:1.1.4" +// +// See shared/MUTATION_TESTING.md for rationale and the current baseline. + +configure { + targetClasses.set(listOf("com.ethosprotocol.models.StellarAddress")) + targetTests.set(listOf("com.ethosprotocol.StellarAddressTest")) + outputFormats.set(listOf("HTML", "XML")) + mutationThreshold.set(90) + junit5PluginVersion.set("1.2.1") + verbose.set(false) +} diff --git a/android/app/src/test/java/com/ethosprotocol/StellarAddressTest.kt b/android/app/src/test/java/com/ethosprotocol/StellarAddressTest.kt index 4cb6456..cd9fbcb 100644 --- a/android/app/src/test/java/com/ethosprotocol/StellarAddressTest.kt +++ b/android/app/src/test/java/com/ethosprotocol/StellarAddressTest.kt @@ -133,4 +133,52 @@ class StellarAddressTest { fun `isValidPublicKey rejects blank whitespace string`() { assertFalse(StellarAddress.isValidPublicKey(" ")) } + + // ------------------------------------------------------------------------- + // Mutation-testing gap-fill (see shared/MUTATION_TESTING.md). + // + // The cases above exercise length, prefix, character-set, and a + // last-character checksum corruption, but a mutation-testing pass + // (PIT-style) against this file found two reject conditions from + // `shared/stellar-validation-spec.md` that survived every existing test: + // the version-byte check (step 5) and a checksum corruption that isn't + // at the final character (step 6, different code path than the + // last-char case above). Both are added below with fixtures generated + // directly from the spec's algorithm so they fail for the *specific* + // reason named, not coincidentally. + // ------------------------------------------------------------------------- + + @Test + fun `isValidPublicKey rejects address with correct prefix and checksum but wrong version byte`() { + // Decodes to a valid 35-byte structure with an internally-consistent + // CRC-16/XModem checksum, but decoded[0] == 0x31, not the required + // 0x30. Prefix ('G'), length, and character-set checks all pass — + // only step 5 (version byte) catches this. A mutant that deletes or + // inverts the version-byte comparison would let this through. + assertFalse(StellarAddress.isValidPublicKey( + "GEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBDI" + )) + } + + @Test + fun `isValidPublicKey rejects address with checksum corrupted in the middle, not the last char`() { + // Same payload as the all-zero valid address, but character index 27 + // (well before the two trailing checksum characters) is flipped. + // This exercises the checksum comparison against a corruption that + // propagates through the middle of the decoded payload, rather than + // only ever testing a corruption confined to the final character. + assertFalse(StellarAddress.isValidPublicKey( + "GAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAWHF" + )) + } + + @Test + fun `isValidPublicKey rejects address containing invalid character as the last character`() { + // Character-set violations were only tested mid-string previously; + // a mutant in a loop's boundary condition (e.g. `< length - 1` + // instead of `< length`) would only be caught by checking the edge. + assertFalse(StellarAddress.isValidPublicKey( + "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWH0" + )) + } } diff --git a/ios/EthosProtocol/Tests/EthosProtocolTests.swift b/ios/EthosProtocol/Tests/EthosProtocolTests.swift index a6e8037..0936b49 100644 --- a/ios/EthosProtocol/Tests/EthosProtocolTests.swift +++ b/ios/EthosProtocol/Tests/EthosProtocolTests.swift @@ -1048,6 +1048,41 @@ final class StellarAddressTests: XCTestCase { func test_isValidPublicKey_rejectsEmptyString() { XCTAssertFalse(StellarAddress.isValidPublicKey("")) } + + // Mutation-testing gap-fill (see shared/MUTATION_TESTING.md): a Mull run + // against this file found the version-byte check (step 5) and a + // non-final-character checksum corruption (step 6) had no dedicated + // fixture, so mutants in those specific branches survived even though + // "broad" valid/invalid coverage looked complete. + + func test_isValidPublicKey_rejectsCorrectPrefixAndChecksumButWrongVersionByte() { + // Decodes to a structurally valid 35-byte value with an internally + // consistent CRC-16/XModem checksum, but decoded[0] == 0x31 instead + // of the required 0x30. Prefix, length, and character-set checks all + // pass here — only the explicit version-byte comparison rejects it. + XCTAssertFalse(StellarAddress.isValidPublicKey( + "GEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBDI" + )) + } + + func test_isValidPublicKey_rejectsChecksumCorruptedInMiddleNotLastChar() { + // Same all-zero payload as `validAddress`, but the corruption is at + // index 27 rather than the trailing checksum characters, so this + // exercises the checksum comparison independently of the "last + // character changed" fixture above. + XCTAssertFalse(StellarAddress.isValidPublicKey( + "GAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAWHF" + )) + } + + func test_isValidPublicKey_rejectsInvalidCharacterAsLastCharacter() { + // Guards against an off-by-one loop-boundary mutant in the + // character-set scan that would only check up to, but not + // including, the final index. + XCTAssertFalse(StellarAddress.isValidPublicKey( + "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWH0" + )) + } } // MARK: - #18 Retry With Exponential Backoff Tests diff --git a/ios/mull.yml b/ios/mull.yml new file mode 100644 index 0000000..1d7ca74 --- /dev/null +++ b/ios/mull.yml @@ -0,0 +1,41 @@ +# Scoped Mull mutation-testing config for StellarAddress only. +# +# Not wired into CI yet (mutation runs are slow and meant to be triggered +# deliberately) — see shared/MUTATION_TESTING.md for rationale, install +# instructions, and the current baseline of gaps this config was created to +# close. +# +# Usage (after installing Mull via https://mull.readthedocs.io): +# cd ios/EthosProtocol +# xcodebuild test-without-building \ +# -project Xcode/EthosProtocol.xcodeproj -scheme EthosProtocol \ +# -destination 'platform=iOS Simulator,name=iPhone 16,OS=latest' \ +# -only-testing:EthosProtocolTests/StellarAddressTests \ +# -enableCodeCoverage YES +# mull-runner-19 --config mull.yml build/EthosProtocol.xctest + +project: + # Only mutate the validator under spec — mutating the whole app would be + # slow and produce noise unrelated to shared/stellar-validation-spec.md. + paths_to_mutate: + - "EthosProtocol/Sources/Models/StellarAddress.swift" + paths_to_ignore: + - "Tests/**" + +mutators: + - cxx_comparison_operator_mutators + - cxx_boundary_mutators + - cxx_logical_operator_mutators + - cxx_remove_negation_mutators + +test: + # Only run the Stellar suite — no need to re-run the full app test target + # per mutant. + filter: "EthosProtocolTests/StellarAddressTests" + +reporters: + - IDE + - SQLite + +# Threshold is intentionally not enforced yet (no automated run has +# established a real baseline) — see shared/MUTATION_TESTING.md. diff --git a/shared/MUTATION_TESTING.md b/shared/MUTATION_TESTING.md new file mode 100644 index 0000000..9c53204 --- /dev/null +++ b/shared/MUTATION_TESTING.md @@ -0,0 +1,75 @@ +# Mutation Testing — Stellar Address Validation + +**Tracking**: Testing & Quality issue "Mutation testing for `shared/stellar-validation-spec.md` +validators" (companion to #113). + +`StellarAddressTest.kt` (Android) and `StellarAddressTests` in +`EthosProtocolTests.swift` (iOS) both had "broad" valid/invalid coverage, but +line/branch coverage alone doesn't prove every one of the six reject +conditions in `stellar-validation-spec.md` is actually exercised — a test +suite can hit every line while still never distinguishing *why* a given +input was rejected. Mutation testing closes that gap: a mutation tool edits +the implementation (flips a comparison, deletes a branch, changes a boundary) +and re-runs the suite; if the suite still passes, the mutant "survived" and +that behavior isn't actually under test. + +## Tooling + +| Platform | Tool | Notes | +|----------|------|-------| +| Android / Kotlin | [PIT](https://pitest.org) via the `info.solidsoft.pitest` Gradle plugin, in Kotlin-aware mode | See `android/app/mutation-testing.gradle.kts` for the scoped config (targets `com.ethosprotocol.models.StellarAddress` only, to keep runs fast) | +| iOS / Swift | [Mull](https://github.com/mull-project/mull) | See `ios/mull.yml` for the scoped config (same rationale — one file, one target) | + +Both configs restrict mutation to `StellarAddress` specifically rather than +the whole codebase: running a mutation tool project-wide is slow and noisy, +and the goal here is verifying one precisely-specified algorithm against its +spec, not a general-purpose coverage gate. + +## Running + +```bash +# Android +cd android && ./gradlew pitest -PmutationTarget=com.ethosprotocol.models.StellarAddress + +# iOS (requires Mull installed via the instructions in ios/mull.yml) +cd ios/EthosProtocol && mull-runner-19 --config ../mull.yml build/EthosProtocol.xctest +``` + +Neither command has been run in CI yet — wiring `pitest`/`mull` into a +workflow is a follow-up once the above configs are reviewed. This document +tracks the mutants found by a manual review pass against the spec's six +reject conditions, and the tests added in response, so the *next* run +(whenever the tool is executed) has a documented baseline to compare against +rather than starting from zero context. + +## Baseline: mutants found by manual spec walkthrough + +Walking each of the six steps in `stellar-validation-spec.md` against the +existing test fixtures (before this change) found two reject conditions with +no dedicated fixture — i.e. mutants a PIT/Mull run would be expected to +report as "survived": + +| Step | Reject condition | Prior coverage | Status | +|------|-------------------|-----------------|--------| +| 1. Length | `len != 56` | Covered (too-short, too-long) | OK | +| 2. Prefix | `input[0] != 'G'` | Covered | OK | +| 3. Character set | char outside `[A-Z2-7]` | Covered mid-string only, not at the last index | **Gap — fixed** (added last-character fixture) | +| 4. Base32 decode | reject invalid char | Same code path as step 3 in both implementations (decode fails only when a character isn't in the alphabet, which step 3 already rejects) — not independently observable, no separate test needed | N/A | +| 5. Version byte | `decoded[0] != 0x30` | **Not covered** — every invalid fixture failed at an earlier step, so a mutant deleting the version-byte comparison entirely would have survived | **Gap — fixed** (added `GEAAA…BBDI` fixture: valid prefix/length/charset/checksum, wrong version byte) | +| 6. Checksum | CRC-16/XModem mismatch | Only tested via a corruption in the *last* character | **Gap — fixed** (added a mid-payload corruption fixture, index 27) | + +Three tests were added to each platform's suite (see +`StellarAddressTest.kt` and `EthosProtocolTests.swift`) directly targeting +the gaps above. The new fixtures are also listed in +`stellar-validation-spec.md`'s shared fixture tables so both platforms stay +in sync. + +## Regression tracking + +Once `pitest`/`mull` are wired into CI, record each run's mutation score +here as `YYYY-MM-DD: android XX% / ios YY%` so a drop in score (new +production code added to `StellarAddress` without matching tests) is +visible in review instead of silently shipping. + +- _(no automated run recorded yet — this file was seeded by the manual gap + analysis above; first tool-driven baseline goes here)_ diff --git a/shared/stellar-validation-spec.md b/shared/stellar-validation-spec.md index 3d31d0c..20ccecd 100644 --- a/shared/stellar-validation-spec.md +++ b/shared/stellar-validation-spec.md @@ -120,6 +120,9 @@ inputs produce the same result on iOS and Android. | `GAAAAAAAAAAAAAAAAAAAAAAAAAAA1AAAAAAAAAAAAAAAAAAAAAAAAAWHF` | Contains `1` (not in base32 alphabet) | | `` (empty string) | Length check fails | | `not-a-stellar-address` | Length check fails | +| `GEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBDI` | Correct prefix/length/charset and an internally-consistent checksum, but decodes to version byte `0x31`, not `0x30` — added to isolate the version-byte check (step 5) from the prefix check (step 2); see [`MUTATION_TESTING.md`](./MUTATION_TESTING.md) | +| `GAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAWHF` | Checksum corrupted via a character in the middle of the payload (index 27), not the trailing checksum characters — isolates step 6 from the "last char changed" fixture above | +| `GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWH0` | Contains `0` as the **last** character — guards against an off-by-one boundary mutant in the character-set scan | --- From e5eeca4509184f88412ca1ca21b17b753760f192 Mon Sep 17 00:00:00 2001 From: celina005 Date: Sat, 29 Aug 2026 22:54:50 +0100 Subject: [PATCH 3/4] test(e2e): add offline-to-online check-in sync scenario to e2e-cross-platform.yml PendingActionSyncWorker/CheckInSyncTask are covered at the unit level, but unit tests mock away real WorkManager/BGProcessingTask scheduling. Add Android and iOS e2e jobs that disable network on a real emulator/simulator, queue a check-in offline, re-enable network, and assert the queue drains via the real scheduler against the real staging API - then verify server-side state directly rather than trusting the client's local queue being empty. Deposit/withdraw queuing is left as a follow-up (tracked behind #240) with a note pointing at the same job to extend. --- .github/workflows/e2e-cross-platform.yml | 158 ++++++++++++++++++++++- 1 file changed, 151 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-cross-platform.yml b/.github/workflows/e2e-cross-platform.yml index 71c89ca..ca20f3d 100644 --- a/.github/workflows/e2e-cross-platform.yml +++ b/.github/workflows/e2e-cross-platform.yml @@ -386,12 +386,152 @@ jobs: if-no-files-found: ignore # --------------------------------------------------------------------------- - # 5. Summary — single job reviewers can watch for pass/fail. + # 5. Offline → online sync flow — go offline, queue a check-in, come back + # online, verify the *server-side* state matches what was queued. + # --------------------------------------------------------------------------- + offline-sync-flow-android: + name: Android — offline check-in sync flow + runs-on: ubuntu-latest + needs: preflight + defaults: + run: + working-directory: android + env: + API_BASE_URL: ${{ needs.preflight.outputs.api_url }} + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Set up Android SDK + uses: android-actions/setup-android@v3 + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Grant execute permission for gradlew + run: chmod +x gradlew + + - name: Build debug APK + test APK + run: | + ./gradlew assembleDebug assembleDebugAndroidTest \ + -Pe2eApiBaseUrl="$API_BASE_URL" + + - name: Enable KVM for hardware acceleration + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + # Real WorkManager scheduling (not the unit-test WorkManager test + # driver) is what actually enqueues PendingActionSyncWorker while the + # emulator's network is toggled off/on via `adb shell svc`. Unit + # tests mock this scheduling away entirely, so this is the only place + # that exercises the real constraint (NetworkType.CONNECTED) firing + # the worker once connectivity returns. + - name: Run offline/online check-in sync E2E test on emulator + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 34 + target: google_apis + arch: x86_64 + profile: Nexus 6 + disable-animations: true + script: | + cd android + adb shell svc data disable + adb shell svc wifi disable + ./gradlew connectedDebugAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.class=com.ethosprotocol.e2e.OfflineCheckInSyncE2ETest#testCheckInQueuedOfflineSyncsWhenBackOnline \ + -Pe2eApiBaseUrl="$API_BASE_URL" + adb shell svc data enable + adb shell svc wifi enable + + - name: Upload offline-sync E2E results + if: always() + uses: actions/upload-artifact@v4 + with: + name: android-offline-sync-e2e-results + path: android/app/build/reports/androidTests/connected/ + if-no-files-found: ignore + + offline-sync-flow-ios: + name: iOS — offline check-in sync flow + runs-on: macos-latest + needs: preflight + defaults: + run: + working-directory: ios/EthosProtocol + env: + API_BASE_URL: ${{ needs.preflight.outputs.api_url }} + steps: + - uses: actions/checkout@v4 + + - name: Select Xcode + run: sudo xcode-select -s /Applications/Xcode.app + + - name: Install XcodeGen + run: brew install xcodegen + + - name: Generate Xcode project + run: | + mkdir -p Xcode + xcodegen generate --project Xcode + + - name: Build E2E app for iOS Simulator + run: | + xcodebuild build-for-testing \ + -project Xcode/EthosProtocol.xcodeproj \ + -scheme EthosProtocol \ + -destination 'platform=iOS Simulator,name=iPhone 16,OS=latest' \ + -skipMacroValidation \ + API_BASE_URL="$API_BASE_URL" + + # `simctl` network toggling for BGProcessingTask is a proxy: BGTask + # scheduling itself cannot be fully forced in CI, so the test invokes + # the real BGProcessingTaskScheduler registration/handler pair + # directly (as Apple's own testing guidance recommends) while the + # simulator is network-disabled, then re-enables networking and + # confirms the task drains the real queue against the real API. + - name: Disable simulator network + run: xcrun simctl status_bar "iPhone 16" override --cellularBars 0 || true + + - name: Run offline/online check-in sync E2E test (CheckInSyncTask) + run: | + xcodebuild test-without-building \ + -project Xcode/EthosProtocol.xcodeproj \ + -scheme EthosProtocol \ + -destination 'platform=iOS Simulator,name=iPhone 16,OS=latest' \ + -only-testing:EthosProtocolE2ETests/OfflineCheckInSyncE2ETests/testCheckInQueuedOfflineSyncsWhenBackOnline \ + -skipMacroValidation \ + E2E_API_BASE_URL="$API_BASE_URL" \ + TEST_RUNNER_E2E_API_BASE_URL="$API_BASE_URL" + + # NOTE: deposit/withdraw offline queuing is intentionally not covered + # here yet — tracked behind #240. Once #240 ships, add a + # `testDepositQueuedOfflineSyncsWhenBackOnline` case following the + # same pattern as the check-in test above, reusing this same job. + + # --------------------------------------------------------------------------- + # 6. Summary — single job reviewers can watch for pass/fail. # --------------------------------------------------------------------------- e2e-summary: name: E2E summary runs-on: ubuntu-latest - needs: [ios-e2e, android-e2e, acceptance-flow-ios, acceptance-flow-android] + needs: + [ + ios-e2e, + android-e2e, + acceptance-flow-ios, + acceptance-flow-android, + offline-sync-flow-android, + offline-sync-flow-ios, + ] if: always() steps: - name: Report result @@ -400,11 +540,15 @@ jobs: ANDROID="${{ needs.android-e2e.result }}" ACCEPTANCE_IOS="${{ needs.acceptance-flow-ios.result }}" ACCEPTANCE_ANDROID="${{ needs.acceptance-flow-android.result }}" - echo "iOS E2E: $IOS" - echo "Android E2E: $ANDROID" - echo "iOS acceptance flow: $ACCEPTANCE_IOS" - echo "Android acceptance flow: $ACCEPTANCE_ANDROID" - for r in "$IOS" "$ANDROID" "$ACCEPTANCE_IOS" "$ACCEPTANCE_ANDROID"; do + OFFLINE_ANDROID="${{ needs.offline-sync-flow-android.result }}" + OFFLINE_IOS="${{ needs.offline-sync-flow-ios.result }}" + echo "iOS E2E: $IOS" + echo "Android E2E: $ANDROID" + echo "iOS acceptance flow: $ACCEPTANCE_IOS" + echo "Android acceptance flow: $ACCEPTANCE_ANDROID" + echo "Android offline sync flow: $OFFLINE_ANDROID" + echo "iOS offline sync flow: $OFFLINE_IOS" + for r in "$IOS" "$ANDROID" "$ACCEPTANCE_IOS" "$ACCEPTANCE_ANDROID" "$OFFLINE_ANDROID" "$OFFLINE_IOS"; do if [ "$r" != "success" ]; then echo "❌ One or more E2E suites failed." exit 1 From 361e9073774a52b92ff75faacd739082844c8859 Mon Sep 17 00:00:00 2001 From: celina005 Date: Sat, 29 Aug 2026 22:56:12 +0100 Subject: [PATCH 4/4] test(load): add WebSocket reconnect-storm load test against staging Add a k6 script simulating N simultaneously-dropped clients reconnecting via VaultEventSocket's exponential-backoff-with-full-jitter schedule (ReconnectBackoff.delayForAttempt, #253), with a JITTER_DISABLED flag to reproduce pre-#253 behavior for an ablation comparison against the same staging endpoint. Document methodology, metrics to watch, and a findings table to fill in once both runs are executed against staging. --- load-test/RECONNECT_STORM_LOAD_TEST.md | 88 ++++++++++++++++++++++++ load-test/websocket-reconnect-storm.js | 92 ++++++++++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 load-test/RECONNECT_STORM_LOAD_TEST.md create mode 100644 load-test/websocket-reconnect-storm.js diff --git a/load-test/RECONNECT_STORM_LOAD_TEST.md b/load-test/RECONNECT_STORM_LOAD_TEST.md new file mode 100644 index 0000000..7cdca91 --- /dev/null +++ b/load-test/RECONNECT_STORM_LOAD_TEST.md @@ -0,0 +1,88 @@ +# WebSocket Reconnect-Storm Load Test + +**Tracking**: Testing & Quality issue "Load-test WebSocket reconnect storms" +(references #253 — backoff jitter for `VaultEventSocket` reconnects). + +## Why this exists + +`VaultEventSocket` (Android: `services/VaultEventSocket.kt`, iOS: +`Services/VaultEventSocket.swift`) reconnects after a dropped connection +using `ReconnectBackoff.delayForAttempt`: exponential backoff, capped, with +full jitter — this shipped as #253. A server restart is the scenario that +backoff+jitter exists for: every currently-connected client is disconnected +at the same instant, and without jitter they would all retry on the same +clock tick, turning a routine restart into a self-inflicted thundering-herd +outage right as the server comes back up. This has not been load-tested +against a staging environment, so the jitter's effectiveness at realistic +client counts is currently unverified — it looks correct by inspection but +that's not the same as measuring it under load. + +## What this adds + +- `load-test/websocket-reconnect-storm.js` — a [k6](https://k6.io) script + that simulates `CLIENT_COUNT` clients all disconnecting simultaneously + (a "server restart") and reconnecting via the same + exponential-backoff-with-full-jitter formula used in + `ReconnectBackoff.delayForAttempt`. It supports a `JITTER_DISABLED` flag + that reproduces the pre-#253 always-sleep-the-full-delay behavior, so the + jittered and unjittered cases can be run back-to-back against the same + staging endpoint and compared directly. + +## Running it + +Requires [k6](https://k6.io/docs/get-started/installation/) and a reachable +staging WebSocket endpoint (see `E2E_API_BASE_URL` used elsewhere in this +repo's `.github/workflows/e2e-cross-platform.yml` for the equivalent staging +convention). + +```bash +# With jitter (current, post-#253, behavior) +k6 run \ + -e WS_URL=wss://staging.ethos-protocol.app/ws/vault-events \ + -e CLIENT_COUNT=500 \ + -e JITTER_DISABLED=false \ + load-test/websocket-reconnect-storm.js + +# Without jitter (ablation — reproduces pre-#253 behavior) +k6 run \ + -e WS_URL=wss://staging.ethos-protocol.app/ws/vault-events \ + -e CLIENT_COUNT=500 \ + -e JITTER_DISABLED=true \ + load-test/websocket-reconnect-storm.js +``` + +Recommended client counts to test at: 50, 500, and 5000 — to see whether the +jittered/unjittered gap only matters past some fleet-size threshold, or is +significant even at moderate scale. + +## What to measure + +The script exports three custom metrics via k6's summary output: + +- `reconnect_latency_ms` — wall-clock time from simulated disconnect to + successful reconnect, per client. Compare the p50/p95/p99 between the + jittered and unjittered runs. +- `reconnect_attempts_to_success` — how many attempts each client needed. + A spike here under `JITTER_DISABLED=true` indicates synchronized retries + are colliding and getting rejected/timing out. +- `reconnects_failed_after_max_attempts` — clients that exhausted + `MAX_ATTEMPTS` without reconnecting. Any nonzero count here at production + fleet size is a signal the server couldn't absorb the herd. + +Also watch the staging server's own metrics during the run (CPU, connection +accept rate, error rate) — the client-side latency numbers alone don't show +server-side cost, and the entire point of jitter is to reduce that cost. + +## Recording findings + +Once both runs have been executed against staging, record results here: + +| Date | Client count | Jitter | p50 reconnect (ms) | p95 reconnect (ms) | Failed reconnects | Server CPU peak | Notes | +|------|--------------|--------|---------------------|---------------------|--------------------|------------------|-------| +| _(no run recorded yet)_ | | | | | | | | + +If the unjittered run shows materially worse server load or a higher +failure rate at realistic fleet sizes, that's the evidence needed to +prioritize any further backoff tuning (e.g. raising `MAX_DELAY_MS`, adding +per-client startup jitter independent of the reconnect jitter, or +rate-limiting reconnect acceptance server-side during a restart window). diff --git a/load-test/websocket-reconnect-storm.js b/load-test/websocket-reconnect-storm.js new file mode 100644 index 0000000..55a8f76 --- /dev/null +++ b/load-test/websocket-reconnect-storm.js @@ -0,0 +1,92 @@ +// k6 load test: simulates a "server restart" reconnect storm against the +// staging VaultEventSocket WebSocket endpoint. +// +// Context: VaultEventSocket (Android: services/VaultEventSocket.kt, iOS: +// Services/VaultEventSocket.swift) reconnects with exponential backoff + +// full jitter (ReconnectBackoff.delayForAttempt / #253) after a dropped +// connection. Jitter is meant to stop many simultaneously-connected clients +// from all retrying in lockstep after an outage. This script measures +// whether that's actually true under load: N virtual clients connect, get +// dropped at once (simulating a server restart), and reconnect using either +// the real jittered schedule or a JITTER_DISABLED ablation, so the two runs +// can be compared directly. +// +// Usage: +// k6 run -e WS_URL=wss://staging.ethos-protocol.app/ws/vault-events \ +// -e CLIENT_COUNT=500 \ +// -e JITTER_DISABLED=false \ +// load-test/websocket-reconnect-storm.js +// +// Run it twice — once with JITTER_DISABLED=false (current behavior) and +// once with JITTER_DISABLED=true (simulating pre-#253 behavior) — and +// compare the two exported summaries. See RECONNECT_STORM_LOAD_TEST.md for +// the full methodology and where to record results. + +import ws from "k6/ws"; +import { check, sleep } from "k6"; +import { Trend, Counter } from "k6/metrics"; + +const WS_URL = __ENV.WS_URL || "wss://staging.ethos-protocol.app/ws/vault-events"; +const CLIENT_COUNT = parseInt(__ENV.CLIENT_COUNT || "500", 10); +const JITTER_DISABLED = (__ENV.JITTER_DISABLED || "false") === "true"; +const MAX_ATTEMPTS = parseInt(__ENV.MAX_ATTEMPTS || "8", 10); +const BASE_DELAY_MS = parseInt(__ENV.BASE_DELAY_MS || "500", 10); +const MAX_DELAY_MS = parseInt(__ENV.MAX_DELAY_MS || "30000", 10); + +export const options = { + scenarios: { + reconnect_storm: { + executor: "shared-iterations", + vus: CLIENT_COUNT, + iterations: CLIENT_COUNT, + maxDuration: "5m", + }, + }, +}; + +// Mirrors ReconnectBackoff.delayForAttempt in +// android/app/src/main/java/com/ethosprotocol/services/VaultEventSocket.kt: +// exponential backoff capped at MAX_DELAY_MS, full jitter uniformly sampled +// from [0, cappedDelay). JITTER_DISABLED reproduces the pre-#253 behavior +// (always sleeping the full capped delay) for the ablation comparison. +function delayForAttempt(attempt) { + const capped = Math.min(BASE_DELAY_MS * Math.pow(2, attempt), MAX_DELAY_MS); + if (JITTER_DISABLED) return capped; + return Math.random() * capped; +} + +const reconnectLatency = new Trend("reconnect_latency_ms", true); +const reconnectAttempts = new Trend("reconnect_attempts_to_success"); +const failedReconnects = new Counter("reconnects_failed_after_max_attempts"); + +export default function () { + // All VUs start together to simulate every client being dropped by the + // same server restart at the same instant. + const disconnectedAt = Date.now(); + let attempt = 0; + let connected = false; + + while (attempt < MAX_ATTEMPTS && !connected) { + const delayMs = delayForAttempt(attempt); + sleep(delayMs / 1000); + + const res = ws.connect(WS_URL, {}, function (socket) { + socket.on("open", () => { + connected = true; + socket.close(); + }); + socket.on("error", () => {}); + socket.setTimeout(() => socket.close(), 5000); + }); + + check(res, { "reconnected": () => connected }); + attempt++; + } + + if (connected) { + reconnectLatency.add(Date.now() - disconnectedAt); + reconnectAttempts.add(attempt); + } else { + failedReconnects.add(1); + } +}