diff --git a/README.md b/README.md index 2479d20..52b14af 100644 --- a/README.md +++ b/README.md @@ -164,3 +164,13 @@ UI-delivered, or rendered-frame counters stop. ## Discord launch links FrameRelay registers `framerelay://open/share/{token}` and `framerelay://open/watch/{token}` for the current Windows user. Launches are forwarded to the running app; share links wait for the user to choose a capture target and start sharing. Tokens are redeemed with the device identity and are not written to application logs. + +### Discord Activity media option + +`FRAMERELAY_WEBSOCKET_MEDIA_ENABLED` is a process environment variable, disabled unless its value parses as `true`. Set it before launching FrameRelay and restart the app. It is independent of preferences and does not load a desktop `.env` file. + +The optional upload sends existing encoded H.264/Opus samples to a separate WebSocket forwarding service using a short-lived, source-bound RelayControl grant. When RTC uses AV1, an independent H.264 branch owns copied capture buffers; it does not downgrade RTC. Upload failures retry with bounded backoff and do not stop the existing RTC media stack. + +RelayControl must enable `FeatureManagement__DiscordWebSocketMedia` (false by default), deploy its isolated `media-relay` container and set `MediaRelay__PublicBaseUrl` plus a distinct internal `MediaRelay__ServiceToken`. The bot does not need that service secret. Discord Activity URL Mapping must include `/media` to the relay and `/relay` to the API. See [wire protocol](docs/protocol/discord-media-v1.md). + +Local validation checks copies, queue bounds, default-off configuration and RTC pipeline regressions. Browser fixtures in the Activity/API repositories verify actual decoded pixels and PCM over the forwarding service. A live Discord/Windows publisher test is still required before rollout. diff --git a/docs/protocol/discord-media-v1-vectors.json b/docs/protocol/discord-media-v1-vectors.json new file mode 100644 index 0000000..5769c2a --- /dev/null +++ b/docs/protocol/discord-media-v1-vectors.json @@ -0,0 +1,61 @@ +[ + { + "name": "config", + "hex": "46524d31010100000100000000000000850000000000000040420f00000000001b410000000000007b22766964656f223a7b22636f646563223a22617663312e343265303166222c227769647468223a3634302c22686569676874223a3336302c22666f726d6174223a22616e6e657862227d2c22617564696f223a7b22636f646563223a226f707573222c2273616d706c6552617465223a34383030302c226368616e6e656c73223a327d7d", + "type": 1, + "timestampUs": 1000000 + }, + { + "name": "video", + "hex": "46524d310102010001000000010000000e0000000000000040420f00000000001b41000000000000000000016742e01f000000016588", + "type": 2, + "timestampUs": 1000000 + }, + { + "name": "audio", + "hex": "46524d31010300000100000002000000030000000000000040420f00000000001b41000000000000f8fffe", + "type": 3, + "timestampUs": 1000000 + }, + { + "name": "keyframe-request", + "hex": "46524d31010400000100000003000000000000000000000040420f00000000001b41000000000000", + "type": 4, + "timestampUs": 1000000 + }, + { + "name": "bad-version", + "hex": "46524d310202010001000000010000000e0000000000000040420f00000000001b41000000000000000000016742e01f000000016588", + "invalid": true + }, + { + "name": "bad-type", + "hex": "46524d310109010001000000010000000e0000000000000040420f00000000001b41000000000000000000016742e01f000000016588", + "invalid": true + }, + { + "name": "bad-flags", + "hex": "46524d310102020001000000010000000e0000000000000040420f00000000001b41000000000000000000016742e01f000000016588", + "invalid": true + }, + { + "name": "zero-generation", + "hex": "46524d310102010000000000010000000e0000000000000040420f00000000001b41000000000000000000016742e01f000000016588", + "invalid": true + }, + { + "name": "reserved", + "hex": "46524d310102010001000000010000000e0000000100000040420f00000000001b41000000000000000000016742e01f000000016588", + "invalid": true + }, + { + "name": "bad-length", + "hex": "46524d31010201000100000001000000630000000000000040420f00000000001b41000000000000000000016742e01f000000016588", + "invalid": true + }, + { + "name": "truncated", + "hex": "46524d31", + "invalid": true + } +] diff --git a/docs/protocol/discord-media-v1.md b/docs/protocol/discord-media-v1.md new file mode 100644 index 0000000..798b295 --- /dev/null +++ b/docs/protocol/discord-media-v1.md @@ -0,0 +1,9 @@ +# Discord media v1 + +WebSocket subprotocol: `framerelay-media-v1`. First complete text message is `{grant}`; every subsequent message is binary. Credentials never appear in URLs. The relay validates grants with RelayControl and renews 30-second leases every 10 seconds. + +Header: 40 bytes, little endian. FRM1 at 0, version 1 at 4, type at 5 (1 configuration, 2 video, 3 audio, 4 keyframe request), flags u16 at 6 (bit 0 video keyframe only), generation u32 at 8, sequence u32 at 12, payload length u32 at 16, reserved zero u32 at 20, timestamp i64 microseconds at 24, duration i64 microseconds at 32. Whole message limit 8 MiB. Reject zero generation, invalid types/flags, negative or JavaScript-unsafe times, trailing or missing bytes. + +Configuration payload is UTF-8 JSON `{video:{codec,width,height,format:"annexb"},audio:{codec:"opus",sampleRate:48000,channels:2}|null}`. H.264 codec string comes from SPS profile/compatibility/level. Video contains complete Annex-B access units, parameter sets on recovery keyframes. Audio contains complete Opus packets. Control has empty payload. No base64 media. + +Each generation starts with configuration and a fresh keyframe. Sequences increase strictly; gaps, backlog, reconnect and codec/size changes require recovery. Queues bound both 64 messages and 16 MiB. A viewer may request one keyframe per second. Supported browser codecs do not imply Discord support; live validation is required before rollout. diff --git a/docs/superpowers/implementation/2026-09-30-discord-websocket-media-verification.md b/docs/superpowers/implementation/2026-09-30-discord-websocket-media-verification.md new file mode 100644 index 0000000..1b03f78 --- /dev/null +++ b/docs/superpowers/implementation/2026-09-30-discord-websocket-media-verification.md @@ -0,0 +1,37 @@ +# Discord WebSocket media implementation and verification + +Approved design: [spec](../specs/2026-09-30-discord-websocket-media-design.md). Execution covered RelayControl, desktop publisher and Go repository Activity. Original dirty checkouts were preserved; changes are local and not deployed. + +## Changes + +| Repository | Main files | Result | +| --- | --- | --- | +| RelayControl | API Features, Session/PublicRoom/Activity endpoints, MediaRelayGrantService, capability cleanup, Program | Official Microsoft.FeatureManagement with existing-feature defaults true; optional media default false; scoped grants and fenced leases | +| RelayControl | services/SonicRelay.MediaRelay, src/SonicRelay.MediaProtocol, Dockerfile/Compose/CI, protocol fixtures | Separate single-instance forwarder; no transcode; 8 MiB messages and 64-message/16 MiB queues; lease renewal; persistent slow-viewer disconnect | +| Desktop | AppComposition, RtcVideoPublishHost, MediaRelayApiClient, Media/WebSocket | Process opt-in upload; shared H264/Opus samples; independent copied-frame H264 branch when RTC uses AV1 | +| Go/Activity | media-protocol/client/player/audio-playback, main, canvas, tests, CI | WebSocket first-message grant; WebCodecs capability checks; bounded canvas/PCM playback and disposal; retry network failures with fresh admission | + +## Feature boundaries + +`FeatureManagement__PublicRooms=true`, `FeatureManagement__DiscordActivity=true`, `FeatureManagement__DuplexAudio=true` and `FeatureManagement__ScreenShare=true` preserve existing behavior when missing. Explicit false overrides defaults. Public-room HTTP and background startup are gated. Activity authorization/admission and bot Activity-only launch routes return404 when disabled; desktop completion/watch fallback routes remain. Disabled duplex creation and screen-share creation/code join/ID join/Activity admission reject before mutation. Existing session end, device identity, auth, retention, TURN and generic signaling stay available. + +The new API `FeatureManagement__DiscordWebSocketMedia=false` and publisher `FRAMERELAY_WEBSOCKET_MEDIA_ENABLED=false` require explicit opt-in. Desktop reads a process variable; it does not load .env. Docker env changes require container recreation. Separate `MediaRelay__ServiceToken` is shared only by API/forwarder, not bot/Activity bundle. `MediaRelay__PublicBaseUrl` configures desktop upload; Discord `/media` mapping serves Activity sockets while `/relay` remains the API mapping. + +## Evidence + +- API solution restore/build verified with zero warnings/errors; full suite: 339 pass (274 API integration, 12 protocol, 6 forwarder, 40 virtual publisher, 7 signaling). +- Desktop focused media tests:50 pass, including fragmented keyframe control and real ClientWebSocket upload, raw/encoded memory ownership, default off, bounds and existing pipeline regression tests. API client/session tests:9 pass. +- Activity:26 Node tests pass; TypeScript and Vite production build pass. Tests include shared binary vectors, invalid headers, support check, keyframe recovery, stale epochs, closed decoder replacement, cleanup and no RTC/TURN access. +- Protocol fixture SHA256 identical in all three repositories: `D91E1E99F4E20C296E7F48FF3328C5D9D0A7D98E993C599BB277909C9D6A237A`. +- Real headless Edge fixture, explicit Run button click: native generated H264/Opus through the actual forwarding registry and sockets,3 rendered video frames,9 decoded audio blocks, green canvas pixel `[0,255,1,255]`, nonzero PCM energy154.314, actual codec `avc1.42c00b`. Audio was measured as PCM; audible output was not claimed. +- Separate forwarder Release publish succeeds. Deployment and infra Compose configurations validate with nonsecret fixtures. +- Repeated release of a removed admission is idempotent; the regression test passes. +- Fresh independent review identified closed decoder recovery and persistent slow viewers; both reproduced and corrected with regression tests. Slow viewer disconnects after three queue overflows within ten seconds; another viewer continues receiving. + +## Validation limits + +Docker daemon unavailable on this machine: image build/run and production deployment were not verified. No live Discord Activity, real Windows capture opt-in, audible A/V synchronization, portal mapping or simultaneous two-machine RTC viewer was exercised. The browser fixture verifies native codec decoding and transport locally; it is not Discord runtime proof. Keep both new flags false until the documented live rollout checks pass. No dependency upgrades were made; only the requested official Microsoft.FeatureManagement4.7.0 package was added to the API. + +## Local fixture + +Run API repository `dotnet run --project tests/SonicRelay.MediaRelay.BrowserFixture` (localhost5174), start Activity Vite on5173, open `/tests/browser-media-fixture.html` and click Run. Fixture fake credentials are local test data; this project is not included in the production images. The optional dependency-free `activity/tests/run-browser-media-fixture.mjs` runs against an owned Chromium debugging instance on9227, opens/closes its fixture tab and supplies the button click. diff --git a/docs/superpowers/plans/2026-09-30-discord-websocket-media.md b/docs/superpowers/plans/2026-09-30-discord-websocket-media.md new file mode 100644 index 0000000..889d526 --- /dev/null +++ b/docs/superpowers/plans/2026-09-30-discord-websocket-media.md @@ -0,0 +1,132 @@ +# Discord WebSocket Media Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Deliver publisher-originated encoded video/audio to Discord through an isolated WebSocket forwarder, with opt-in flags and WebCodecs playback. + +**Architecture:** Desktop keeps its RTC publisher and adds an independent encoded-media upload. RelayControl issues scoped grants and authorizes short leases; a separate forwarding process fans out binary frames. Activity retains Discord identity/session admission and renders decoded media without RTCPeerConnection. + +**Tech Stack:** .NET 10, ClientWebSocket, ASP.NET Core, existing H.264/Opus encoders, TypeScript/Vite, WebCodecs, canvas, Web Audio, Docker. + +**Spec:** [Approved design](../specs/2026-09-30-discord-websocket-media-design.md). Dependency: [RelayControl flags plan](2026-09-30-relaycontrol-feature-flags.md). + +## Global Constraints + +- Use fresh isolated branches from fetched main for RelayControl and Go repo; retain the current desktop worktree/spec commit. Preserve original dirty files and the existing Activity error-summary changes. No forced checkout/pull. +- Confirmed baselines: API main `8c8ad7a`, Go main `4ec5dfd`. Desktop includes the approved spec at `2f56b13`; compare against fetched main before execution and resolve drift without discarding user changes. +- API `FeatureManagement__DiscordWebSocketMedia=false` by default; desktop `FRAMERELAY_WEBSOCKET_MEDIA_ENABLED=false` by default. Four existing API feature flags remain true by default. +- No RTC/TURN/signaling transport replacement. No API-side transcode, persistent media store or cross-instance forwarding. One forwarder instance owns each upload/fanout. +- Maximum whole WebSocket message 8 MiB, per-connection queue maximum 64 messages and 16 MiB. Authorization leases revalidate within 15 seconds and fail closed on expiry. +- Reuse H.264/Opus samples when compatible. When RTC uses AV1, Activity receives an independent H.264 branch; failure of that branch does not downgrade or stop RTC. +- Use CodeGraph before existing-code edits. No full desktop test suite; run the relevant projects/classes. RelayControl full solution check is expressly requested by the user. +- Product dependencies stay unchanged except Microsoft.FeatureManagement in the preceding plan. Use existing test frameworks, browser tooling and native TypeScript DOM declarations; validate any needed WebCodecs type additions locally rather than adding packages automatically. + +## Review Focus + +1. Expired upload grants must never disconnect a publisher's RTC participant; Task 2 cleanup tests. +2. Reconnect/release races must not revoke a newly active viewer lease or leak capacity; Tasks 2 and 3 fencing tests. +3. Native capture buffers may be reused after event callbacks; Task 4 copied-memory tests. +4. Codec changes or oversize/backlogged frames cannot allow undecodable delta frames through; Tasks 3–5 recovery tests. +5. Supported WebCodecs in an ordinary browser is not proof of support inside Discord; Tasks 5 and 7 explicit live checks. + +## File Map + +RelayControl creates `src/SonicRelay.MediaProtocol/{MediaMessage,MediaWireCodec}.cs`, its csproj and tests; API `Endpoints/MediaRelayEndpoints.cs`, `Contracts/MediaRelayContracts.cs`, `Services/MediaRelayGrantService.cs`, `Authorization/MediaRelayServiceAuthenticationHandler.cs`; and a separate `services/SonicRelay.MediaRelay/` web project with endpoints, registry, queue and lease client. API modifies Program, RelayFeatures, Activity admission and capability cleanup only at their defined new boundaries. Add tests and project references to SonicRelay.sln. + +Desktop creates `src/SonicDesktopRelay.Media/WebSocket/` protocol/options/queue/publisher/optional-H264-branch files and `src/SonicDesktopRelay.ApiClient/MediaRelayApiClient.cs`; modifies AppComposition/RtcVideoPublishHost and only the minimal capture handoff required by the isolated branch. Add focused Media, ApiClient and App tests. Do not change preferences/UI just to add the process switch. + +Activity creates `activity/src/{media-protocol,media-client,media-player,audio-playback}.ts` and corresponding `.test.mjs` files; modifies main.ts, index.html/style.css for canvas, package scripts, README and existing deployment workflow as necessary. Keep safeErrorSummary from current main. + +Each repository carries `docs/protocol/discord-media-v1.md` and identical `docs/protocol/discord-media-v1-vectors.json` fixtures. Production deploy adds a separate media image/container and `/media` mapping guidance. + +### Task 1: Freeze and test the cross-language wire protocol + +**Files:** API protocol project/tests; desktop WebSocket/MediaMessage.cs and MediaWireCodec.cs; Activity media-protocol.ts/tests; protocol documents/fixtures in all three repos. +**Interfaces:** C# `MediaMessage(byte Type, ushort Flags, uint Generation, uint Sequence, long TimestampUs, long DurationUs, ReadOnlyMemory Payload)`; `MediaWireCodec.Encode(MediaMessage) -> byte[]`, `Decode(ReadOnlyMemory) -> MediaMessage`. TypeScript exports corresponding `MediaMessage`, `encodeMediaMessage(message): Uint8Array`, `decodeMediaMessage(bytes: Uint8Array): MediaMessage`, and `MediaConfiguration` matching the config payload below. API protocol test project lives at `tests/SonicRelay.MediaProtocol.Tests/SonicRelay.MediaProtocol.Tests.csproj`; desktop tests at `tests/SonicDesktopRelay.Media.Tests/MediaWireCodecTests.cs`. + +Wire header is 40 bytes, little endian: bytes 0–3 ASCII FRM1; byte 4 version=1; byte 5 type (1=config, 2=video, 3=audio, 4=keyframe-request); bytes 6–7 flags (bit0=keyframe); generation u32 at 8; sequence u32 at 12; payload length u32 at 16; reserved zero u32 at 20; timestamp i64 at 24; duration i64 at 32. Reject unknown type/version/flags, nonzero reserved, generation zero, negative times, unsafe JavaScript integers and lengths not exactly matching the bounded whole message. Config is UTF-8 JSON metadata, never JSON/base64 media. Keyframe-request has an empty payload. + +Config payload: `{video:{codec,width,height,format:"annexb"},audio:{codec:"opus",sampleRate,channels}|null}`. Derive H.264 codec string from actual SPS rather than hardcoding encoder profile. Audio format must match actual Opus output. A generation begins with configuration and a keyframe containing necessary parameter sets; sequence restarts at zero and strictly increases within a generation. On reconnect or codec/size change, increment generation; do not wrap. + +- [ ] Write C#/TS tests `Roundtrip_matches_shared_vectors`, `Rejects_truncated_or_oversized_message`, `Rejects_invalid_header_and_unsafe_timestamp`. Assert Decode(Encode(vector)) fields and payload equality; changing payloadLength or version rejects. Use one keyframe vector at timestamp 1,000,000 us and duration 16,667 us, plus audio/config/control and invalid vectors. +- [ ] Run `dotnet test tests/SonicRelay.MediaProtocol.Tests/SonicRelay.MediaProtocol.Tests.csproj`, `dotnet test tests/SonicDesktopRelay.Media.Tests/SonicDesktopRelay.Media.Tests.csproj --filter FullyQualifiedName~MediaWireCodecTests`, and `node --experimental-strip-types --test tests/media-protocol.test.mjs` in Activity; confirm failures precede implementations. +- [ ] Implement codecs without external protocol dependencies. Check SHA256 of fixture files agrees across repos; copied fixtures, not cross-repo build references. Re-run all three focused tests to pass. Commit per repository: `feat: define versioned Discord media wire protocol`. + +### Task 2: API grants, feature opt-in and media participant lifecycle + +**Files:** API MediaRelayContracts/Endpoints/GrantService/ServiceAuthenticationHandler; Program, RelayFeatures, DiscordActivityEndpoints, LaunchCapabilityCleanupService; API `MediaRelayGrantTests.cs`, `MediaRelayPresenceTests.cs` and existing Activity tests. +**Interfaces:** +- `POST /api/sessions/{sessionId}/media-grants` with DeviceBearer creates upload grant only for current source device and live screen_share session. +- Extend `/api/discord/activity/viewer-grants/redeem` request with optional `transport="websocket"`. Preserve absent-transport behavior. New branch returns MediaAdmission and creates a media viewer capability instead of an unused signaling capability/TURN credentials. +- `MediaAdmission(Guid AdmissionId, Guid SessionId, Guid ParticipantId, string Grant, DateTimeOffset ExpiresAt, string MediaUrl)`; AdmissionId identifies the issued capability, and media URL contains no credential. Upload uses the existing source participant ID in the response while its cleanup capability leaves ParticipantId null as specified below. +- `POST /api/discord/activity/media-admissions/{admissionId}/release` accepts the existing Activity identity bearer and releases only that user's/instance's current media-view capability. A replaced/expired admission returns success without disconnecting a newer lease. This supports cleanup before the socket has redeemed its grant. +- Internal authenticated `POST /api/internal/media-relay/redeem {grant,connectionId}`, `/renew {leaseId,connectionId}`, `/release {leaseId,connectionId}`. `MediaLease(Guid LeaseId, Guid SessionId, Guid? ParticipantId, string Role, DateTimeOffset ExpiresAt)`; role exactly upload or view. Service policy reads `MediaRelay:ServiceToken`, never accepts DeviceBearer or bot launch credentials as service auth. +- `MediaRelayGrantService.IssueUploadAsync(Guid sessionId, ClaimsPrincipal user, CancellationToken ct) -> Task`, `RedeemAsync(string grant, Guid connectionId, CancellationToken ct) -> Task`, `RenewAsync(Guid leaseId, Guid connectionId, CancellationToken ct) -> Task`, `ReleaseAsync(Guid leaseId, Guid connectionId, CancellationToken ct) -> Task` and `ReleaseAdmissionAsync(Guid admissionId, LaunchCapability identity, CancellationToken ct) -> Task`; use existing API-style IResult at endpoints. + +- [ ] Write `Missing_media_flag_rejects_upload_without_mutation`, `Wrong_source_and_non_screen_share_rejected`, `Grant_is_single_use_and_role_bound`, `Unknown_or_expired_lease_rejected`, `Flag_disable_revokes_renewal`. Use fake time and real authorization fixtures; replay returns 401 and no second lease. +- [ ] Write `Upload_expiry_leaves_rtc_publisher_connected`, `Media_viewer_disconnect_releases_capacity`, `Old_connection_release_does_not_revoke_reconnected_viewer`, `Unredeemed_admission_can_be_released_by_its_owner`, `Identity_or_activity_revocation_ends_view_lease`, `Admission_without_transport_preserves_existing_contract`. Assert participant statuses and capacity after each race, not just response codes. A different Activity identity cannot release another user's admission. +- [ ] Run API tests filtered to MediaRelay and LaunchIntentActivity before implementation; verify failure for absent media routes/contracts. +- [ ] Add DiscordWebSocketMedia false default to the official feature manager. Require it plus ScreenShare for uploads and additionally DiscordActivity for viewers. Extend existing LaunchCapability kinds with media-upload/media-view; store only token hashes and use existing ConsumedAt concurrency guard for redemption. Grant lifetime 30 seconds; redeemed lease lifetime 30 seconds, renewal every 10 seconds. Original session/identity/Activity authorization remains authoritative at every renewal. +- [ ] Set upload capability ParticipantId=null so cleanup cannot mark the RTC source disconnected. Media-view capability owns only its Activity participant. Fence lease operations with connectionId persisted in the existing MessageId context field for these new kinds; use admission locks/concurrency checks. Admission reconnect reuses the matching Activity media participant and atomically revokes prior leases instead of allocating duplicate capacity. Existing Activity caps/cleanup retain their semantics; no flag tables or schema change solely for flags. +- [ ] Re-run focused API tests to pass; verify expired unreclaimed grants cannot leak viewer slots. Commit: `feat: authorize scoped Discord media grants and leases`. + +### Task 3: Isolated forwarder with bounded fanout + +**Files:** new MediaRelay project: Program.cs, `MediaRelayOptions.cs`, `MediaSocketEndpoint.cs`, `MediaRelayRegistry.cs`, `MediaConnectionQueue.cs`, `RelayControlLeaseClient.cs`; `tests/SonicRelay.MediaRelay.Tests/`, solution/project references. +**Interfaces:** `RelayControlLeaseClient.RedeemAsync(string grant, Guid connectionId, CancellationToken ct) -> Task`, `RenewAsync(...)`, `ReleaseAsync(...)`; `MediaRelayRegistry.AttachAsync(MediaLease lease, Guid connectionId, WebSocket socket, CancellationToken ct) -> Task`; `MediaConnectionQueue.TryEnqueue(MediaMessage message) -> bool` with messages/bytes counters. + +- [ ] Write `Only_one_upload_per_session`, `Viewers_receive_identical_complete_samples`, `Unknown_grant_cannot_subscribe`, `Lease_expiry_closes_socket`, `Slow_viewer_does_not_block_upload_or_other_viewer`, `Partial_frame_cannot_allocate_above_8MiB`, `Generation_gap_requires_keyframe`, `Stale_lease_release_is_fenced`. Use bounded real loopback WebSocket tests with completion signals, cancellation and fake API responses. +- [ ] Run `dotnet test tests/SonicRelay.MediaRelay.Tests/SonicRelay.MediaRelay.Tests.csproj`; expect new implementation absent. +- [ ] Implement `/ws/media` with subprotocol `framerelay-media-v1`; authenticate first text message `{grant}` within 5 seconds, then use binary protocol. Upload and viewer roles come only from the redeemed lease. Never echo/log grant. Single receive/send owner per socket; maximum one upload per session, configured global defaults 64 upload sessions and 256 viewers. +- [ ] Apply 8 MiB whole-message and 64-message/16-MiB queue limits. Disconnect queue-overflow viewers and send timeouts after 5 seconds; do not block other viewers. Publisher upload congestion breaks only media connection. New viewer gets current config, requests a keyframe and skips delta/audio until stream recovery. Coalesce keyframe requests to at most one per second per upload. Reject viewer-originated media/config and upload-originated role changes. +- [ ] Renew every 10 seconds; close no later than lease expiry if API unavailable, with no indefinite grace. Release on close with connection fencing. Bound assembly, heartbeat and shutdown. Configuration: `RelayControl__BaseUrl`, `RelayControl__ServiceToken`; default process memory ceiling declared in deployment is separate from queue ceilings. +- [ ] Re-run forwarder tests to pass, then API lease regression tests. Commit: `feat: add isolated WebSocket media forwarding service`. + +### Task 4: Opt-in desktop upload and codec-isolated H.264 output + +**Files:** desktop MediaRelayApiClient.cs/tests; Media/WebSocket `WebSocketMediaOptions.cs`, `WebSocketMediaPublisher.cs`, `OwnedMediaQueue.cs`, `ActivityH264Output.cs`; AppComposition/RtcVideoPublishHost; capture handoff/ScreenPublishPipeline only if needed; focused Media/App tests. +**Interfaces:** `WebSocketMediaOptions.FromEnvironment(Func getEnvironment) -> WebSocketMediaOptions` with Enabled=false unless value parses true. `MediaRelayApiClient.CreateUploadGrantAsync(Guid sessionId, CancellationToken ct) -> Task`. `WebSocketMediaPublisher.StartAsync(Guid sessionId, CancellationToken ct)`, `TryPublishVideo(EncodedVideoSample)`, `TryPublishAudio(EncodedAudioSample)`, `StopAsync()`, `DisposeAsync()`; injected media API client and socket factory. `ActivityH264Output.TryCapture(VideoFrame frame)` copies into owned bounded encode queue; independent RequestKeyFrame and disposal. + +- [ ] Write `Disabled_has_zero_api_socket_encoder_side_effects`, `Upload_owns_samples_after_callback`, `Socket_failure_does_not_fail_rtc`, `Av1_rtc_stays_av1_with_h264_activity_output`, `Stop_cancels_grant_and_prevents_late_reconnect`, `Resize_emits_new_config_then_keyframe`. Assert call counters, mutated source memory independence, unchanged RTC codec and no post-stop sends. +- [ ] Run desktop tests filtered to WebSocketMedia/ActivityH264Output and existing RtcVideoPublishHostCodecFallbackTests before code. Confirm new behavior is absent. +- [ ] Read flag in AppComposition and inject options plus existing authenticated session HTTP client. Subscribe to encoded H.264 and Opus events with nonblocking owned-memory enqueue. Reuse MediaSessionClock. The send loop emits config/generation and requests keyframes; implement reconnect backoff 1, 2, 4, 8 seconds, always with new grants and cancellation generation guards. No upload work for flag false. +- [ ] For AV1, attach an ownership-safe raw-frame tap to the current capture source with a distinct bounded H.264 encoder queue; do not start another capture or change the shared encoder. Set its quality ceiling from the active publishing profile. Copy BGRA during the callback before native buffers can be reused. If no H.264 branch can start, log/display a media diagnostic while RTC continues. Keep new diagnostics separate from StartFailure/VideoPipelineFailure that stop RTC. +- [ ] Dispose extra subscriptions/branch/upload before shared capture disposal. Run new tests plus existing ScreenPublishPipelineTests, VideoFrameEncodeQueueTests, VideoSampleSendQueueTests, RTC fanout tests and ApiClient grant tests using targeted class filters. Commit: `feat: add flag-controlled Activity media upload without RTC regression`. + +### Task 5: WebCodecs player and timestamp-based synchronization + +**Files:** Activity media-player.ts, audio-playback.ts and `.test.mjs` tests; index.html/style.css canvas changes. +**Interfaces:** `checkDecoderSupport(config: MediaConfiguration): Promise`; `MediaPlayer.configure(config, generation): Promise`, `accept(message: MediaMessage): void`, `resumeAudio(): Promise`, `close(): void`. Inject clock/decoder/output adapters for unit tests. `AudioPlayback.enqueue(data: AudioData, timestampUs: number)`, `resume()`, `reset()`, `close()` owns a bounded Web Audio schedule. + +- [ ] Write `Unsupported_configuration_is_reported_before_decode`, `Delta_before_keyframe_is_ignored`, `Old_generation_is_discarded`, `Decoder_backlog_resets_and_requests_keyframe`, `Audio_video_use_shared_media_clock`, `Close_releases_frames_audio_and_context`, `Blocked_autoplay_requires_user_gesture`. Assert decoder calls and closed resources using fake adapters; no fake-only playback success claim. +- [ ] Run `node --experimental-strip-types --test tests/media-player.test.mjs tests/audio-playback.test.mjs`; expect new modules absent. +- [ ] Implement actual VideoDecoder/AudioDecoder.isConfigSupported checks. Submit EncodedVideoChunk/EncodedAudioChunk with microsecond timestamps. Decode H.264 Annex B only after config+keyframe. Render/close VideoFrames on canvas; copy decoded PCM into bounded AudioBuffers and close AudioData. Use AudioContext time as presentation clock once audio runs, performance.now when video-only; establish 100 ms initial buffering, discard video over 150 ms late, keep decoder backlog <=8 video and <=32 audio chunks and scheduled audio <=500 ms. +- [ ] Audio gap/overflow resets the schedule; decoder error or invalid video dependency requests a coalesced keyframe and clears stale generations. Avoid unbounded Promise chains/arrays. A resize reconfigures only after releasing old frames/decoders. No requirement for SharedArrayBuffer or cross-origin isolation. +- [ ] Run player tests and `npm run build`; expect actual TypeScript checks pass. Commit: `feat: decode and synchronize Activity media with WebCodecs`. + +### Task 6: Activity integration and capacity-safe reconnect + +**Files:** Activity media-client.ts/tests, main.ts, package.json test script, existing error-summary tests; API Activity/media integration tests as needed. +**Interfaces:** `MediaClient.connect(admission: MediaAdmission, player: MediaPlayer, signal: AbortSignal): Promise`, `close(): Promise`; existing authorize/viewer-grant requests retained, redemption sends transport websocket. + +- [ ] Write `No_rtc_constructor_or_ice_request_in_media_flow`, `Reconnect_uses_new_grant_and_generation`, `Stop_releases_admission_and_prevents_retry`, `Capability_failure_cleans_up_allocated_viewer`, `Session_end_closes_player`, `Feature_disabled_is_clear_to_user`. Assert zero RTCPeerConnection/RTCRtpReceiver accesses, participant release and no leaked decoder/socket after stop. +- [ ] Run `node --experimental-strip-types --test tests/media-client.test.mjs tests/media-player.test.mjs tests/error-summary.test.mjs`; expect flow still RTC before implementation. +- [ ] Wire canvas/player to existing Discord SDK identity flow and websocket admission. Check decoder API presence before admission and actual config on arrival; configuration failure and client stop call the admission-release endpoint from Task 2 with the identity bearer and AdmissionId. Derive viewer WSS URL using `/media/ws/media` through Discord mapping, not the direct upload origin returned for desktop. Authenticate first-message grant, parse binary with bounded assembly, forward keyframe controls. +- [ ] Replace Activity-only RTC setup, offer/ICE handling and video-track rendering. Keep error sanitization and status stages. Handle manual retry, stop, closed session and reconnect with fresh admission/media lease under cancellation generation guard. Do not change bot command authentication or generic desktop launch semantics. +- [ ] Re-run new tests, existing routing/error-summary tests and `npm run build`. Commit: `feat: connect Discord Activity using WebSocket media admission`. + +### Task 7: Deployment integration, fixtures and live validation + +**Files:** API Dockerfile or new `services/SonicRelay.MediaRelay/Dockerfile`; `.github/workflows/vps-ci-cd.yml`, deploy/deploy.sh, deploy/docker-compose.prod.yml, appropriate infra Compose/env templates, API README; desktop README and Activity README/workflow when required. Add API/forwarder end-to-end test and Activity browser fixture page/test. + +- [ ] Write `Protocol_fixture_plays_decoded_video_and_audio` with a known H.264/Opus fixture, browser canvas pixel/frame evidence and nonzero decoded PCM/RMS audio assertion. Write an API+forwarder test proving disabled flags prevent connections, authorized upload fans out to a real client, and API session-end closes leases without affecting a concurrent signaling client. Fixtures use generated content, not third-party copyrighted media. +- [ ] Run those tests to expose integration gaps, then configure fixtures/host endpoints to pass using available browser tooling; do not install product/test dependencies without a concrete need and authorization. Browser autoplay test uses an explicit gesture. +- [ ] Build/publish a separate media image and deploy service on infra_network with 512 MiB memory ceiling, global connection ceilings from Task 3, internal API/service credential and exposed WSS proxy path. Preserve existing API image/SHA selection, TURN container and generic signaling routes. Verify env_file passes FeatureManagement__DiscordWebSocketMedia=false by default and MediaRelay__PublicBaseUrl/ServiceToken only to appropriate services. Never dump resolved secrets. +- [ ] Document `/media` Discord URL Mapping, desktop process variable/restart, API container recreation, both opt-ins and no WebRTC fallback in Activity. Example env: existing four flags true, new API media false, publisher false. Changing defaults must be an explicit rollout action, not an incidental deploy side effect. +- [ ] Run `dotnet restore SonicRelay.sln`, `dotnet build SonicRelay.sln --no-restore`, `dotnet test SonicRelay.sln --no-build --logger "console;verbosity=minimal"` in RelayControl. Desktop: build solution and run only affected projects/classes. Activity: `npm run build` plus all directly affected Node tests. Compose: config --quiet with nonsecret fixture env. Record actual outputs and unresolved runtime requirements. +- [ ] Commit per repository: `chore: deploy and document isolated Discord media forwarding`. Review diffs for unrelated API/RTC changes; push/PR only under the agreed delivery scope. +- [ ] Live verification is a separate check: deploy media/API/Activity builds and desktop opt-in, configure mapping, run `/framerelay watch`, verify decoded video, audible synchronized audio, late viewer/keyframe recovery, reconnect, session end, flag-off behavior and simultaneous desktop RTC viewer. Do not claim this passed without real evidence. If server access, portal access or a live publisher is missing, report exactly that boundary and keep flags opt-in. + +## Handoff + +Review both plans before execution. Recommend native implementation in this chat because grants, byte layout, memory ownership and decoder lifecycle depend closely on each other. If delegation is selected, obey the user's allowed GPT-6 Luna/Sol low-or-medium configurations and prohibit recursive subagents; the standard header does not override those limits. diff --git a/docs/superpowers/plans/2026-09-30-relaycontrol-feature-flags.md b/docs/superpowers/plans/2026-09-30-relaycontrol-feature-flags.md new file mode 100644 index 0000000..869b5de --- /dev/null +++ b/docs/superpowers/plans/2026-09-30-relaycontrol-feature-flags.md @@ -0,0 +1,80 @@ +# RelayControl Feature Flags Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add independent kill switches while preserving every existing feature when configuration is absent. + +**Architecture:** Register Microsoft's feature manager over existing .NET configuration and gate only optional feature entry points. Background public-room work checks the same feature before starting. Core authorization, cleanup, signaling and TURN remain operational. + +**Tech Stack:** .NET 10, ASP.NET Core Minimal APIs, Microsoft.FeatureManagement 4.7.0, EF Core, xUnit, Docker Compose. + +**Spec:** [Approved design](../specs/2026-09-30-discord-websocket-media-design.md). This plan delivers existing-feature flags independently; the companion media plan adds the transport-specific switch. + +## Global Constraints + +- Repository: `vitorhugo-dotnet/dotnet_RelayControl`; inspected main `8c8ad7a`. Fetch again before execution and preserve dirty original checkouts. +- Missing PublicRooms, DiscordActivity, DuplexAudio and ScreenShare flags mean true. Explicit false always wins. +- Use FeatureManagement, AddFeatureManagement(), IVariantFeatureManager and centralized RelayFeatures; no alternate flag framework or Azure configuration service. +- Do not flag device identity, authentication/authorization, retention, PostgreSQL, Redis, TURN or generic signaling. +- Honor existing PublicRoom:Enabled in addition to PublicRooms. Docker environment changes require container recreation. +- Before editing existing code, use targeted CodeGraph symbol/edit-context/impact/test queries; index only files needed. Run targeted tests per task; the user expressly requested full RelayControl solution verification at delivery. +- No unrelated refactoring or dependency updates. Add only the requested Microsoft package. Package source: https://www.nuget.org/packages/Microsoft.FeatureManagement/4.7.0 ; API reference: https://learn.microsoft.com/en-us/azure/azure-app-configuration/feature-management-dotnet-reference + +## Review Focus + +1. Explicit Docker false must override enabled defaults; Task 1 configuration tests. +2. Disabled public-room work must not seed a session or connect to signaling; Task 2 background tests. +3. Shared Discord launch routes must retain unrelated desktop recovery/completion behavior; Task 3 branch-specific tests. +4. Both code-based and paired screen-share joins must reject new admission, while existing sessions can end; Task 4 tests. +5. A disabled duplex feature must leave normal broadcast and its existing permission behavior intact; Task 4 tests. + +## File Map + +Create `services/SonicRelay.Api/Features/RelayFeatures.cs` (names/defaults registration) and `FeatureEndpointFilter.cs` (HTTP boundary using the official manager, not a custom evaluator). Modify API `Program.cs`, `SonicRelay.Api.csproj`, feature endpoints and `PublicRoomPublisherService.cs`. Create `tests/SonicRelay.Api.IntegrationTests/FeatureFlagConfigurationTests.cs` and `FeatureFlagEndpointsTests.cs`; extend existing public-room publisher tests. Update `infra/.env.example`, `infra/.env.prod.example`, `infra/compose.yml`, `infra/compose.prod.yml`, `deploy/docker-compose.prod.yml` and README deployment guidance only where each Compose path requires propagation. No root `.env.example` exists in the inspected API revision. + +### Task 1: Official manager and backwards-compatible defaults + +**Files:** `RelayFeatures.cs`, `FeatureEndpointFilter.cs`, `Program.cs`, API csproj and `FeatureFlagConfigurationTests.cs`. +**Interfaces:** `RelayFeatures.PublicRooms/DiscordActivity/DuplexAudio/ScreenShare` string constants; `RelayFeatures.AddDefaults(ConfigurationManager configuration)`; `FeatureEndpointFilter(string featureName) : IEndpointFilter`. + +- [ ] Write tests `Missing_flags_are_enabled`, `Explicit_false_overrides_defaults` and `Authentication_is_not_disabled_by_feature_flags`. Resolve IVariantFeatureManager from SonicRelayApiFactory; assert `await manager.IsEnabledAsync(RelayFeatures.ScreenShare)` is true without configuration and false with `FeatureManagement:ScreenShare=false`. Anonymous protected requests still return 401. +- [ ] Run `dotnet test tests/SonicRelay.Api.IntegrationTests/SonicRelay.Api.IntegrationTests.csproj --filter FullyQualifiedName~FeatureFlagConfigurationTests`; expect missing manager/names before implementation. +- [ ] Add Microsoft.FeatureManagement 4.7.0. Supply an in-memory configuration source at the lowest precedence with four true defaults; do not append true defaults above env/test providers. Register `builder.Services.AddFeatureManagement();`. Implement the filter using `IsEnabledAsync`; disabled feature endpoints return 404 and do not invoke their handler. +- [ ] Re-run the focused test command; expect all configuration assertions passing. Commit only these files: `feat: register RelayControl feature flags with compatible defaults`. + +### Task 2: Public room endpoint and background boundary + +**Files:** `Endpoints/PublicRoomEndpoints.cs`, `Services/PublicRoomPublisherService.cs`, `FeatureFlagEndpointsTests.cs`, `PublicRoomPublisherServiceTests.cs`. +**Interfaces:** Task 1 manager/filter; preserve `PublicRoomSeeder.EnsureSeededAsync(AppDbContext, TimeProvider, CancellationToken)`. + +- [ ] Write `PublicRooms_false_hides_endpoint_without_seeding` (authenticated GET `/api/public-room` is 404, no public session created) and `PublicRooms_false_starts_no_publisher_work` (PublicRoom:Enabled=true but no seeding, signaling connect or playlist reads). Also assert missing feature flag preserves existing PublicRoom:Enabled=false response. +- [ ] Run API integration tests filtered to `FeatureFlagEndpointsTests|PublicRoomPublisherServiceTests`; verify new disabled tests fail. +- [ ] Add the feature filter to public-room HTTP routes. Check the official feature manager before public-room background work and each new RunOnce attempt. Preserve existing options and cancellation/error recovery. Do not change PublicRoomSeeder's general signature or gate cleanup. +- [ ] Re-run that filter; expect new and existing public-room tests pass. Commit: `feat: gate public room HTTP and publisher work`. + +### Task 3: Discord Activity boundaries, including shared launch flows + +**Files:** `Endpoints/DiscordActivityEndpoints.cs`, `Endpoints/DiscordActivityLaunchIntentEndpoints.cs`, `Endpoints/LaunchIntentEndpoints.cs` only at shared Discord branches, `FeatureFlagEndpointsTests.cs`, `LaunchIntentActivityTests.cs` and `LaunchIntentTests.cs`. +**Interfaces:** Task 1 filter on Activity endpoints; manager injection for mixed handlers, retaining their existing public request types. + +- [ ] Write `DiscordActivity_false_disables_authorization_and_activity_launch` asserting 404 for `/api/discord/activity/authorize`, viewer-grants/redeem and bot Activity launch routes with their proper authentication. Write `Discord_disable_preserves_existing_session_signaling_and_desktop_completion` and a test for desktop watch fallback from a ready announcement. +- [ ] Run `dotnet test tests/SonicRelay.Api.IntegrationTests/SonicRelay.Api.IntegrationTests.csproj --filter "FullyQualifiedName~FeatureFlagEndpointsTests|FullyQualifiedName~LaunchIntentActivityTests|FullyQualifiedName~LaunchIntentTests"`; expect the new disable assertions to fail. +- [ ] Apply the official-manager filter to Activity-only routes. For `/redeem`, `/bind`, `/watch` and generic launch-intent routes, identify actual Discord-only behavior and gate that branch rather than removing unrelated desktop handlers. Retain auth/rate limiting, revoked-capability cleanup and session-end operations. +- [ ] Re-run the same focused tests; expect existing desktop launch tests and new disabled Activity tests pass. Commit: `feat: gate Discord Activity launch and admission boundaries`. + +### Task 4: Session mode boundaries and operational integration + +**Files:** `Endpoints/SessionEndpoints.cs`, `Endpoints/DiscordActivityEndpoints.cs` screen-share admission branch, `FeatureFlagEndpointsTests.cs`, existing `DuplexAudioTests.cs` and `ScreenShareSessionTests.cs`; configuration/deployment files in File Map and README. +**Interfaces:** Keep existing session request/response signatures; inject IVariantFeatureManager into CreateAsync and shared join/admission handlers. Error convention: 409 with `{ error: "The requested feature is disabled.", code: "feature_disabled", feature: RelayFeatures. }` for disabled session behavior, distinct from hidden optional HTTP endpoints. + +- [ ] Write `Duplex_false_rejects_duplex_and_preserves_broadcast`: authenticated session create `mode=duplex` returns 409/feature_disabled while `mode=broadcast` returns 200. Assert no duplex row or participant is inserted for rejection. +- [ ] Write `ScreenShare_false_rejects_create_and_both_join_paths`: disabled create returns 409; pre-existing screen_share code join and paired ID join reject without consuming capacity. Activity admission rejects too. Broadcast/audio create/join and ending existing screen-share sessions still work. +- [ ] Inspect StreamSession.Mode: it is immutable and the API has no mode transition endpoint. Do not invent one or gate ordinary broadcast permission behavior as though it were a duplex transition. Test existing audio-permission and TURN/signaling routes remain available for permitted normal sessions. +- [ ] Run `dotnet test tests/SonicRelay.Api.IntegrationTests/SonicRelay.Api.IntegrationTests.csproj --filter "FullyQualifiedName~FeatureFlag|FullyQualifiedName~DuplexAudioTests|FullyQualifiedName~ScreenShareSessionTests|FullyQualifiedName~WebRtcEndpointsTests|FullyQualifiedName~SignalingGrantEndpointsTests"`; verify new disabled assertions fail, then add the guards before create/admission mutations and re-run to pass. +- [ ] Add four true env examples and ensure each supported Compose invocation passes explicit false into API configuration (env_file on deployment path; `${FeatureManagement__:-true}` environment entries where the infra path needs them). Document Docker recreation, defaults and disabled responses. Validate with `docker compose ... config --quiet` using a temporary nonsecret test env; do not dump resolved secret values. +- [ ] Run the user's requested API delivery checks: `dotnet restore SonicRelay.sln`, `dotnet build SonicRelay.sln --no-restore`, `dotnet test SonicRelay.sln --no-build --logger "console;verbosity=minimal"`. Capture actual failures/hangs with bounded waits; no success claim before completion. +- [ ] Commit: `feat: gate duplex and screen share with documented Docker switches`. Review diff for generic infrastructure changes. Report modified files, all gated boundaries, test results and compatibility limits. Do not deploy without the authorized delivery scope. + +## Handoff + +Review this plan together with the companion media plan before implementation. Execution may be native in this chat or subagent-driven under the user's model whitelist. No delegation is authorized merely by this document's standard header. Prefer native execution because transport/security changes share contracts across repositories. diff --git a/docs/superpowers/specs/2026-09-30-discord-websocket-media-design.md b/docs/superpowers/specs/2026-09-30-discord-websocket-media-design.md new file mode 100644 index 0000000..811f936 --- /dev/null +++ b/docs/superpowers/specs/2026-09-30-discord-websocket-media-design.md @@ -0,0 +1,90 @@ +# Discord WebSocket media and feature flags + +Date: 2026-09-30 +Status: design for review; product implementation has not started. + +## Intent and approved approach + +The user approved publisher-originated encoded media over WebSocket, with a separate forwarding service and the existing WebRTC path preserved. The user also requested a publisher feature flag and the four RelayControl flags described in the supplied requirements. The Activity will use WebCodecs for decoding. This is an architectural change spanning three repositories, not a replacement for generic signaling. + +Repositories: `vitorhugo-dotnet/dotnet_FrameRelay`, `vitorhugo-dotnet/dotnet_RelayControl`, and `vitorhugo-dotnet/go_discord_FrameRelay`. The older `vitorhugo-java/dotnet_SonicRelay` URL redirects to RelayControl; implementation must use the current repository and fetched main. Preserve existing dirty checkouts and the current Activity diagnostic changes. + +Success means a live publisher can deliver synchronized screen video and audio to an authorized Discord Activity without RTCPeerConnection, while existing desktop viewers retain their WebRTC behavior. Build/test success alone does not establish live Discord playback. + +## Platform constraints + +Discord Activities explicitly do not support WebRTC. WebSocket is supported through the Activity proxy and configured URL mappings. Source: https://docs.discord.com/developers/activities/development-guides/networking + +WebCodecs exposes VideoDecoder and AudioDecoder, but codecs and configurations are implementation-dependent. Use isConfigSupported with the actual stream configuration; API presence alone is insufficient. Source: https://www.w3.org/TR/webcodecs/ + +Decoded video renders to canvas. Decoded audio feeds a bounded Web Audio playback path with an explicit user gesture when autoplay is blocked. WebCodecs does not itself provide media playback or audio/video synchronization. No fallback to RTCPeerConnection inside the Activity. + +## Feature flags and defaults + +RelayControl uses the official Microsoft.FeatureManagement package, AddFeatureManagement(), the FeatureManagement configuration section, and centralized RelayFeatures names. Use the current supported manager interface and runtime checks at feature boundaries. Preserve explicitly configured false values when supplying defaults. + +| Flag | Environment variable | Default | Boundary | +| --- | --- | --- | --- | +| PublicRooms | FeatureManagement__PublicRooms | true | Public-room endpoints and public-room publisher/background work | +| DiscordActivity | FeatureManagement__DiscordActivity | true | Activity identity, grant, admission and Activity launch flows | +| DuplexAudio | FeatureManagement__DuplexAudio | true | Creating or transitioning into duplex sessions | +| ScreenShare | FeatureManagement__ScreenShare | true | Creating or joining screen_share sessions | +| DiscordWebSocketMedia | FeatureManagement__DiscordWebSocketMedia | false | New media-grant issuance and authorization/renewal for forwarding | + +Desktop publisher opt-in: `FRAMERELAY_WEBSOCKET_MEDIA_ENABLED=true`, default false. Read once in AppComposition and inject an immutable option into the additional publishing component. This is a process environment variable; the desktop does not automatically load Docker .env files. Do not add a settings UI or Microsoft.FeatureManagement dependency to the desktop solely for this switch. + +With publisher opt-in false, there is no media-grant request, WebSocket connection, additional encoder or media subscription. With it true, enable the additional path only for eligible screen_share publishing sessions. API opt-in is also required. Generic desktop WebRTC remains available with either new switch disabled. + +The four existing API features remain enabled when their flags are missing. The new transport remains opt-in. PublicRooms preserves its existing configuration conditions in addition to the feature flag. DuplexAudio leaves broadcast audio unaffected. ScreenShare does not gate audio sessions or generic signaling. DiscordActivity does not indiscriminately disable unrelated desktop launch flows; inspect the existing shared launch endpoints and gate only the Discord feature behavior inside them. + +Environment changes require container recreation or desktop process restart. Document this rather than implying Docker environment variables update dynamically. A service-side feature disable must reject new grants and close media authorization leases on their next validation; existing core session cleanup remains active. + +## Publisher and codec isolation + +RtcVideoPublishHost currently creates one ScreenPublishPipeline and one AudioPublishPipeline per publishing session. Both expose SampleEncoded events. New transport subscriptions must copy or retain sample memory under an explicit ownership contract before enqueueing; they must never await network I/O on capture/encode callbacks. + +Reuse compatible H.264 samples and encoded Opus audio. The existing publisher can negotiate AV1, so an Activity must not force the shared RTC session to H.264. When the RTC encoder is AV1, the additional Activity output requires an independent H.264 encoding branch over ownership-safe captured frames. If that branch cannot start or becomes overloaded, fail only the Activity output and report why; do not silently downgrade or stop RTC viewers. No additional H.264 encoding occurs while the opt-in is false. + +The same principle applies to codec transitions, capture resize and screen/window switches. Start a new media generation and supply fresh decoder configuration and a keyframe. WebSocket failures and reconnection must not propagate into the existing VideoPublisher failure path. Dispose subscriptions, queues, socket and any extra encoder independently at stop/session change. + +## Forwarding service and authorization + +Add a separate ASP.NET Core media forwarding project/container in the RelayControl repository. It accepts one authorized publisher upload per session and fans out encoded data to authorized Activity viewers. It does not terminate WebRTC or transcode. RelayControl remains the authority for session membership, roles, capacity, Activity presence and lifetime. + +Add narrowly scoped media-grant endpoints. The source device obtains an upload grant using its existing authenticated API client and session ownership. An Activity obtains a view grant after existing identity/admission checks. Use expiring, single-use opaque grants scoped to session, participant, role and media transport; do not reuse consumed signaling tokens as media credentials. + +The forwarding service validates/redeems grants through an internal authenticated RelayControl operation and holds short authorization leases, revalidated at most every 15 seconds. Session end, revoked membership, disabled feature or expired lease closes the corresponding media connections. An API outage cannot extend a lease indefinitely. No authentication bypass, trusting browser-supplied Discord IDs, or generic authorization feature flag. + +Use HTTPS/WSS. Browser credentials travel via WebSocket subprotocol or an authenticated first-message handshake, never URL query strings. Internal service credentials stay server-side. Redact grants and tokens from logs. Bound unauthenticated handshake duration and message sizes. Reuse existing API error conventions; feature unavailability must be distinguishable from invalid identity. + +A single forwarding instance owns all publisher/viewer sockets in this first version. Do not claim horizontal scale across instances: no Redis pub/sub, cross-instance routing or persistent media storage is introduced. Shared-memory fanout cannot span multiple containers. + +## Media protocol and flow control + +Use a versioned binary protocol, not JSON/base64 media. Each complete message carries type, generation, sequence, timestamp in microseconds, duration, keyframe status and payload length. Decoder configuration messages identify the precise codec, dimensions/profile and audio sample rate/channel count. Validate all lengths and values before allocation and decoding. + +Video payloads are complete H.264 access units in Annex B format, with SPS/PPS available for decoder initialization and keyframe recovery. Audio payloads are complete Opus packets. Maintain one shared MediaSessionClock; use generation plus timestamps to reject stale messages and synchronize tracks. The implementation plan must freeze byte layout and cross-language protocol fixtures before product transport code. + +Initial resource limits: maximum message 8 MiB; each connection queue bounded by both 64 messages and 16 MiB; no unbounded AudioDecoder/VideoDecoder backlog. Disconnect persistently slow viewers rather than accumulating delay. Dropped dependent video invalidates that viewer's video generation until a fresh keyframe. Keyframe requests are coalesced per publisher, including late join and reconnection. Audio underrun uses bounded silence/resynchronization, not an ever-growing queue. These are initial ceilings, not throughput promises. + +The publisher keeps one upload connection per session. Reconnection requires a new grant and generation; canceled or previous-session operations cannot reconnect. The Activity obtains a new media grant on reconnect and starts from decoder configuration plus a fresh keyframe. No server-side decode, encode or MP4 muxing is required. + +## Activity and deployment + +Keep Discord SDK identity and admission. Replace the Activity's RTC/media setup with a media client, binary parser, decoder/rendering module and bounded audio playback. Check VideoDecoder, AudioDecoder and actual supported configurations before obtaining viewer admission wherever the configuration is available; if it arrives after admission, cleanly release the participant on failure. Provide a precise unsupported-client message and desktop-viewer alternative. + +Map `/media` to the new forwarding service in the Discord Developer Portal; retain existing `/` frontend and `/relay` API mappings. RelayControl deploy files add the media container, resource/connection ceilings, lease-validation service credential and internal API address following existing infra_network conventions. Existing TURN and API deployment remain intact. The desktop receives the upload address from authenticated API responses rather than arbitrary browser input. + +Supply .env examples with the four existing flags true and DiscordWebSocketMedia false. Publisher documentation shows its separate process environment opt-in. Production rollout enables both new switches only after deployed media service, mapping and supported-client playback validation. + +## Verification and review boundaries + +RelayControl tests: all four feature disables; explicit false vs missing defaults; broadcast unaffected by DuplexAudio disable; audio sessions unaffected by ScreenShare disable; generic signaling/TURN/device authorization unaffected; media grants role/session binding, replay, expiration, source ownership and feature disable. Follow the attached restore/build/full-solution-test request for RelayControl after implementation. + +Publisher focused tests: opt-in false has zero additional side effects; enabled upload receives encoded samples; socket failure/slow sink leaves RTC working; memory ownership, bounded queues, generation/reconnect cancellation and stop cleanup; simultaneous AV1 RTC plus H.264 Activity output without shared-codec downgrade. + +Forwarder tests: one publisher per session; multiple viewers; rejected grants; lease expiry/revocation; bounded message assembly and slow-viewer cleanup; keyframe recovery; same protocol fixtures as desktop and Activity. + +Activity tests: binary parsing and invalid input, configuration capability checks, decoder lifecycle, audio/video timestamps, bounded backlog, generation changes and cleanup. Browser test with fixture media verifies actual canvas frames and audible decoded audio. Live Discord test separately verifies proxy/mapping, authorization, late join, reconnection, synchronized playback and concurrent existing RTC viewer. + +No tests/build are claimed for this design-only artifact. Product implementation follows reviewed specification and implementation plan. Deliver repository-specific diffs, build/test evidence and explicit live-validation limitations. diff --git a/src/SonicDesktopRelay.ApiClient/MediaRelayApiClient.cs b/src/SonicDesktopRelay.ApiClient/MediaRelayApiClient.cs new file mode 100644 index 0000000..2e2f742 --- /dev/null +++ b/src/SonicDesktopRelay.ApiClient/MediaRelayApiClient.cs @@ -0,0 +1,13 @@ +namespace SonicDesktopRelay.ApiClient; + +public sealed record MediaAdmission(Guid AdmissionId, Guid SessionId, Guid ParticipantId, string Grant, DateTimeOffset ExpiresAt, string MediaUrl); +public sealed class MediaRelayApiClient(HttpClient http) +{ + public async Task CreateUploadGrantAsync(Guid sessionId, CancellationToken ct) + { + using var response = await http.PostAsync($"/api/sessions/{sessionId}/media-grants", null, ct); + var result = await ApiResponse.ReadAsync(response, ct); + var url = new Uri(http.BaseAddress!, result.MediaUrl); + return result with { MediaUrl = url.AbsoluteUri }; + } +} diff --git a/src/SonicDesktopRelay.App/AppComposition.cs b/src/SonicDesktopRelay.App/AppComposition.cs index 047d6a3..e4d45c5 100644 --- a/src/SonicDesktopRelay.App/AppComposition.cs +++ b/src/SonicDesktopRelay.App/AppComposition.cs @@ -1,3 +1,4 @@ +using SonicDesktopRelay.Media.WebSocket; using System.Net; using System.Runtime.Versioning; using Microsoft.Extensions.Logging; @@ -47,7 +48,9 @@ public AppComposition(BackendSettings settings, string deviceName) var loggerFactory = FrameRelayLogging.Current?.LoggerFactory; var connectionLogger = loggerFactory?.CreateLogger(); var runtimeLogger = loggerFactory?.CreateLogger(); - PublishHost = new RtcVideoPublishHost(iceApi, () => current, loggerFactory); + PublishHost = new RtcVideoPublishHost(iceApi, () => current, loggerFactory, + new MediaRelayApiClient(sessionHttp), + WebSocketMediaOptions.FromEnvironment(Environment.GetEnvironmentVariable)); WatchHost = new RtcVideoWatchHost(iceApi, () => current, loggerFactory); Runtime = new SessionRuntime( diff --git a/src/SonicDesktopRelay.App/RtcVideoPublishHost.cs b/src/SonicDesktopRelay.App/RtcVideoPublishHost.cs index da9fdf9..d23580d 100644 --- a/src/SonicDesktopRelay.App/RtcVideoPublishHost.cs +++ b/src/SonicDesktopRelay.App/RtcVideoPublishHost.cs @@ -1,3 +1,4 @@ +using SonicDesktopRelay.Media.WebSocket; using System.Runtime.Versioning; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -19,7 +20,8 @@ namespace SonicDesktopRelay.App; public sealed class RtcVideoPublishHost( IceApiClient iceApi, Func signaling, - ILoggerFactory? loggerFactory = null) : IVideoPublishHost + ILoggerFactory? loggerFactory = null, + MediaRelayApiClient? mediaApi = null, WebSocketMediaOptions? mediaOptions = null) : IVideoPublishHost { private readonly ILogger _logger = loggerFactory?.CreateLogger() ?? NullLogger.Instance; @@ -29,6 +31,8 @@ public sealed class RtcVideoPublishHost( private static readonly IReadOnlyDictionary EmptyTransportDiagnostics = new Dictionary(); + private WebSocketMediaPublisher? _mediaUpload; + private ActivityH264Output? _activityH264; private ScreenPublishPipeline? _pipeline; private IVideoEncoder? _encoder; private VideoCodecCapabilities? _publisherVideoCapabilities; @@ -207,13 +211,10 @@ public async Task SetIgnoreDiscordAudioAsync(bool value) public event Action? CaptureTargetClosed; - public Task StartAsync(CaptureTarget target, VideoPublishProfile profile, CancellationToken ct) => - StartCoreAsync(target, profile, ct); + public Task StartAsync(Guid sessionId, CaptureTarget target, VideoPublishProfile profile, CancellationToken ct) => + StartCoreAsync(sessionId, target, profile, ct); - public async Task StartAsync(MonitorInfo monitor, VideoPublishProfile profile, CancellationToken ct) - => await StartCoreAsync(new CaptureTarget.Monitor(monitor), profile, ct); - - private async Task StartCoreAsync(CaptureTarget target, VideoPublishProfile profile, CancellationToken ct) + private async Task StartCoreAsync(Guid sessionId, CaptureTarget target, VideoPublishProfile profile, CancellationToken ct) { await _gate.WaitAsync(ct); try @@ -347,6 +348,36 @@ private async Task StartCoreAsync(CaptureTarget target, VideoPublishProfile prof _ => VideoDiagnosticsChanged?.Invoke(), null, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)); + if (mediaOptions?.Enabled == true && mediaApi is not null) + { + try + { + _mediaUpload = new WebSocketMediaPublisher(async (id, token) => + { + var grant = await mediaApi.CreateUploadGrantAsync(id, token); + return new MediaUploadGrant(grant.Grant, grant.MediaUrl); + }, () => + { + if (_encoder is MediaFoundationH264Encoder) pipeline.RequestKeyFrame(KeyFrameRequestReason.Manual); + else _activityH264?.RequestKeyFrame(); + }, audioPipeline is not null, loggerFactory?.CreateLogger()); + pipeline.SampleEncoded += OnMediaVideo; + if (audioPipeline is not null) audioPipeline.SampleEncoded += OnMediaAudio; + if (encoder is MediaFoundationAv1Encoder) + { + _activityH264 = new ActivityH264Output(capture, new MediaFoundationH264Encoder(), clock, + () => pipeline.Quality, () => _encoder is not MediaFoundationH264Encoder); + _activityH264.SampleEncoded += OnMediaVideo; + _activityH264.Failed += OnActivityEncodeFailed; + } + await _mediaUpload.StartAsync(sessionId, ct); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning("Optional Discord media output failed to initialize: {ErrorType}", ex.GetType().Name); + await DisposeMediaUploadAsync(); + } + } // Install the publisher failure path before capture can emit the first frame. await pipeline.StartAsync(target, ct); @@ -561,8 +592,24 @@ private async Task LoadIceAsync(CancellationToken ct) return new IceServerSettings(servers, ForceRelay: false); } + private void OnMediaVideo(EncodedVideoSample sample) => _mediaUpload?.PublishVideo(sample); + private void OnMediaAudio(EncodedAudioSample sample) => _mediaUpload?.PublishAudio(sample); + private void OnActivityEncodeFailed(Exception error) => _logger.LogWarning("Independent Activity encoder stopped: {ErrorType}", error.GetType().Name); + private async Task DisposeMediaUploadAsync() + { + if (_pipeline is not null) _pipeline.SampleEncoded -= OnMediaVideo; + if (_audioPipeline is not null) _audioPipeline.SampleEncoded -= OnMediaAudio; + var upload = _mediaUpload; _mediaUpload = null; + if (upload is not null) await upload.DisposeAsync(); + if (_activityH264 is not null) + { + _activityH264.SampleEncoded -= OnMediaVideo; _activityH264.Failed -= OnActivityEncodeFailed; + await _activityH264.DisposeAsync(); _activityH264 = null; + } + } private async Task DisposeStackAsync() { + await DisposeMediaUploadAsync(); if (_diagnosticsTimer is not null) { await _diagnosticsTimer.DisposeAsync(); diff --git a/src/SonicDesktopRelay.Media/WebSocket/ActivityH264Output.cs b/src/SonicDesktopRelay.Media/WebSocket/ActivityH264Output.cs new file mode 100644 index 0000000..27a7d1a --- /dev/null +++ b/src/SonicDesktopRelay.Media/WebSocket/ActivityH264Output.cs @@ -0,0 +1,47 @@ +namespace SonicDesktopRelay.Media.WebSocket; + +/// An independent encoder tap. It owns copied capture frames and never changes the RTC encoder. +public sealed class ActivityH264Output : IAsyncDisposable +{ + private readonly IScreenCaptureSource capture; + private readonly IVideoEncoder encoder; + private readonly MediaSessionClock clock; + private readonly Func quality; + private readonly Func enabled; + private readonly VideoFrameEncodeQueue queue; + private readonly object gate = new(); + private bool failed, disposed; + public event Action? SampleEncoded; + public event Action? Failed; + public ActivityH264Output(IScreenCaptureSource capture, IVideoEncoder encoder, MediaSessionClock clock, + Func quality, Func enabled) + { + this.capture = capture; this.encoder = encoder; this.clock = clock; this.quality = quality; this.enabled = enabled; + queue = new VideoFrameEncodeQueue(Encode); capture.FrameCaptured += OnFrame; + } + private void OnFrame(VideoFrame frame) + { + if (!failed && !disposed && enabled()) queue.Enqueue(frame); // VideoFrameEncodeQueue copies before returning. + } + private void Encode(VideoFrame frame) + { + try + { + EncodedVideoSample? sample; + lock (gate) + { + if (disposed || failed || !enabled()) return; + sample = encoder.Encode(new VideoFrame(frame.Width, frame.Height, frame.Bgra, clock.Now), quality()); + } + if (sample is { } value && enabled()) SampleEncoded?.Invoke(value); + } + catch (Exception ex) { failed = true; Failed?.Invoke(ex); } + } + public void RequestKeyFrame() { lock (gate) { if (!disposed && !failed) encoder.RequestKeyFrame(); } } + public async ValueTask DisposeAsync() + { + lock (gate) { disposed = true; } + capture.FrameCaptured -= OnFrame; await queue.DisposeAsync(); + lock (gate) encoder.Dispose(); + } +} diff --git a/src/SonicDesktopRelay.Media/WebSocket/MediaMessage.cs b/src/SonicDesktopRelay.Media/WebSocket/MediaMessage.cs new file mode 100644 index 0000000..9be89ba --- /dev/null +++ b/src/SonicDesktopRelay.Media/WebSocket/MediaMessage.cs @@ -0,0 +1,4 @@ +namespace SonicDesktopRelay.Media.WebSocket; + +public readonly record struct MediaMessage(byte Type, ushort Flags, uint Generation, uint Sequence, + long TimestampUs, long DurationUs, ReadOnlyMemory Payload); diff --git a/src/SonicDesktopRelay.Media/WebSocket/MediaWireCodec.cs b/src/SonicDesktopRelay.Media/WebSocket/MediaWireCodec.cs new file mode 100644 index 0000000..ae7744f --- /dev/null +++ b/src/SonicDesktopRelay.Media/WebSocket/MediaWireCodec.cs @@ -0,0 +1,47 @@ +using System.Buffers.Binary; + +namespace SonicDesktopRelay.Media.WebSocket; + +public static class MediaWireCodec +{ + public const int HeaderSize = 40; + public const int MaxMessageSize = 8 * 1024 * 1024; + public const long MaxSafeInteger = 9007199254740991; + + public static byte[] Encode(MediaMessage message) + { + Validate(message); + var bytes = new byte[HeaderSize + message.Payload.Length]; + "FRM1"u8.CopyTo(bytes); bytes[4] = 1; bytes[5] = message.Type; + BinaryPrimitives.WriteUInt16LittleEndian(bytes.AsSpan(6), message.Flags); + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(8), message.Generation); + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(12), message.Sequence); + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(16), (uint)message.Payload.Length); + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(24), message.TimestampUs); + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(32), message.DurationUs); + message.Payload.Span.CopyTo(bytes.AsSpan(HeaderSize)); + return bytes; + } + + public static MediaMessage Decode(ReadOnlyMemory bytes) + { + var s = bytes.Span; + if (s.Length < HeaderSize || s.Length > MaxMessageSize || !s[..4].SequenceEqual("FRM1"u8) + || s[4] != 1 || BinaryPrimitives.ReadUInt32LittleEndian(s[20..]) != 0 + || BinaryPrimitives.ReadUInt32LittleEndian(s[16..]) != s.Length - HeaderSize) + throw new InvalidDataException("Invalid media header."); + var message = new MediaMessage(s[5], BinaryPrimitives.ReadUInt16LittleEndian(s[6..]), + BinaryPrimitives.ReadUInt32LittleEndian(s[8..]), BinaryPrimitives.ReadUInt32LittleEndian(s[12..]), + BinaryPrimitives.ReadInt64LittleEndian(s[24..]), BinaryPrimitives.ReadInt64LittleEndian(s[32..]), bytes[HeaderSize..]); + Validate(message); return message; + } + + private static void Validate(MediaMessage message) + { + if (message.Type is < 1 or > 4 || message.Generation == 0 || message.Flags > 1 + || (message.Flags != 0 && message.Type != 2) + || message.TimestampUs is < 0 or > MaxSafeInteger || message.DurationUs is < 0 or > MaxSafeInteger + || message.Payload.Length > MaxMessageSize - HeaderSize || (message.Type == 4 && message.Payload.Length != 0)) + throw new InvalidDataException("Invalid media message."); + } +} diff --git a/src/SonicDesktopRelay.Media/WebSocket/OwnedMediaQueue.cs b/src/SonicDesktopRelay.Media/WebSocket/OwnedMediaQueue.cs new file mode 100644 index 0000000..cf57113 --- /dev/null +++ b/src/SonicDesktopRelay.Media/WebSocket/OwnedMediaQueue.cs @@ -0,0 +1,34 @@ +using System.Threading.Channels; + +namespace SonicDesktopRelay.Media.WebSocket; + +public sealed class OwnedMediaQueue +{ + private readonly Channel channel = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true }); + private readonly object gate = new(); + private int count, bytes; + public bool TryEnqueue(MediaMessage message) + { + lock (gate) + { + var size = MediaWireCodec.HeaderSize + message.Payload.Length; + if (message.Payload.Length > MediaWireCodec.MaxMessageSize - MediaWireCodec.HeaderSize + || count >= 64 || bytes + size > 16 * 1024 * 1024) return false; + if (!channel.Writer.TryWrite(message with { Payload = message.Payload.ToArray() })) return false; + count++; bytes += size; return true; + } + } + public async ValueTask ReadAsync(CancellationToken ct) + { + while (await channel.Reader.WaitToReadAsync(ct)) + { + lock (gate) + { + if (!channel.Reader.TryRead(out var message)) continue; + count--; bytes -= MediaWireCodec.HeaderSize + message.Payload.Length; return message; + } + } + throw new OperationCanceledException(ct); + } + public void Clear() { lock (gate) { while (channel.Reader.TryRead(out _)) { } count = bytes = 0; } } +} diff --git a/src/SonicDesktopRelay.Media/WebSocket/WebSocketMediaOptions.cs b/src/SonicDesktopRelay.Media/WebSocket/WebSocketMediaOptions.cs new file mode 100644 index 0000000..b498acb --- /dev/null +++ b/src/SonicDesktopRelay.Media/WebSocket/WebSocketMediaOptions.cs @@ -0,0 +1,7 @@ +namespace SonicDesktopRelay.Media.WebSocket; + +public sealed record WebSocketMediaOptions(bool Enabled) +{ + public static WebSocketMediaOptions FromEnvironment(Func getEnvironment) => + new(bool.TryParse(getEnvironment("FRAMERELAY_WEBSOCKET_MEDIA_ENABLED"), out var enabled) && enabled); +} diff --git a/src/SonicDesktopRelay.Media/WebSocket/WebSocketMediaPublisher.cs b/src/SonicDesktopRelay.Media/WebSocket/WebSocketMediaPublisher.cs new file mode 100644 index 0000000..0a3b92f --- /dev/null +++ b/src/SonicDesktopRelay.Media/WebSocket/WebSocketMediaPublisher.cs @@ -0,0 +1,176 @@ +using System.Net.WebSockets; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace SonicDesktopRelay.Media.WebSocket; + +public sealed record MediaUploadGrant(string Grant, string MediaUrl); + +public sealed class WebSocketMediaPublisher(Func> grants, + Action requestKeyFrame, bool hasAudio, ILogger? logger = null, + Func? retryWait = null) : IAsyncDisposable +{ + private readonly ILogger log = logger ?? NullLogger.Instance; + private readonly Func waitBeforeRetry = retryWait ?? Task.Delay; + private readonly object gate = new(); + private readonly CancellationTokenSource stop = new(); + private Task? worker; + private OwnedMediaQueue? queue; + private uint generation, sequence; + private bool recovery = true; + private int width, height; + private byte[]? sps, pps; + private string? codec; + private DateTimeOffset lastKeyRequest; + + public Task StartAsync(Guid sessionId, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + worker ??= Task.Run(() => RunAsync(sessionId, stop.Token)); + return Task.CompletedTask; // Optional transport never delays RTC startup. + } + public void PublishVideo(EncodedVideoSample sample) + { + if (sample.Codec != VideoCodec.H264) return; + try + { + lock (gate) + { + if (queue is null) return; + if (width != sample.Width || height != sample.Height) { width = sample.Width; height = sample.Height; sps = pps = null; codec = null; Recover(); } + foreach (var nal in Nals(sample.Data.Span)) + { + if ((nal[0] & 31) == 7 && nal.Length >= 4) + { + var next = $"avc1.{nal[1]:x2}{nal[2]:x2}{nal[3]:x2}"; + if (codec != next || (sps is not null && !sps.AsSpan().SequenceEqual(nal))) Recover(); + codec = next; sps = nal; + } + if ((nal[0] & 31) == 8) pps = nal; + } + if (recovery) + { + if (!sample.IsKeyFrame || sps is null || pps is null || codec is null) { RequestKey(); return; } + if (generation == uint.MaxValue) throw new InvalidDataException("Generation exhausted."); + generation++; sequence = 0; queue.Clear(); + var config = JsonSerializer.SerializeToUtf8Bytes(new { video = new { codec, width, height, format = "annexb" }, + audio = hasAudio ? new { codec = "opus", sampleRate = 48000, channels = 2 } : null }); + queue.TryEnqueue(new MediaMessage(1, 0, generation, sequence++, 0, 0, config)); recovery = false; + } + var data = sample.Data; + if (sample.IsKeyFrame && sps is not null && pps is not null) + { + var complete = new byte[8 + sps.Length + pps.Length + data.Length]; + new byte[] { 0, 0, 0, 1 }.CopyTo(complete, 0); sps.CopyTo(complete, 4); + new byte[] { 0, 0, 0, 1 }.CopyTo(complete, 4 + sps.Length); pps.CopyTo(complete, 8 + sps.Length); + data.Span.CopyTo(complete.AsSpan(8 + sps.Length + pps.Length)); data = complete; + } + Enqueue(2, sample.IsKeyFrame ? (ushort)1 : (ushort)0, sample.Timestamp, sample.Duration, data); + } + } + catch (Exception ex) { log.LogWarning("Discord media sample rejected: {ErrorType}", ex.GetType().Name); lock (gate) Recover(); } + } + public void PublishAudio(EncodedAudioSample sample) + { + try { lock (gate) { if (queue is not null && !recovery && hasAudio) Enqueue(3, 0, sample.Timestamp, sample.Duration, sample.Data); } } + catch (Exception ex) { log.LogWarning("Discord audio sample rejected: {ErrorType}", ex.GetType().Name); lock (gate) Recover(); } + } + private void Enqueue(byte type, ushort flags, TimeSpan timestamp, TimeSpan duration, ReadOnlyMemory data) + { + if (sequence == uint.MaxValue || !queue!.TryEnqueue(new MediaMessage(type, flags, generation, sequence++, timestamp.Ticks / 10, duration.Ticks / 10, data))) Recover(); + } + private void Recover() { recovery = true; queue?.Clear(); RequestKey(); } + private void RequestKey() + { + lock (gate) + { + if (DateTimeOffset.UtcNow - lastKeyRequest < TimeSpan.FromSeconds(1)) return; + lastKeyRequest = DateTimeOffset.UtcNow; + } + try { requestKeyFrame(); } catch (Exception ex) { log.LogWarning("Discord keyframe request failed: {ErrorType}", ex.GetType().Name); } + } + + private async Task RunAsync(Guid sessionId, CancellationToken ct) + { + var delay = 1; + while (!ct.IsCancellationRequested) + { + var transferredMedia = 0; + using var socket = new ClientWebSocket(); using var connection = CancellationTokenSource.CreateLinkedTokenSource(ct); + try + { + using var connect = CancellationTokenSource.CreateLinkedTokenSource(ct); connect.CancelAfter(TimeSpan.FromSeconds(12)); + var grant = await grants(sessionId, connect.Token); + var url = new UriBuilder(grant.MediaUrl); + if (url.Scheme == "https") url.Scheme = "wss"; else if (url.Scheme == "http") url.Scheme = "ws"; + socket.Options.AddSubProtocol("framerelay-media-v1"); + await socket.ConnectAsync(url.Uri, connect.Token); + await socket.SendAsync(JsonSerializer.SerializeToUtf8Bytes(new { grant = grant.Grant }).AsMemory(), WebSocketMessageType.Text, true, connect.Token); + var owned = new OwnedMediaQueue(); lock (gate) { queue = owned; Recover(); } + var sending = SendAsync(socket, owned, connection.Token, + () => Interlocked.Exchange(ref transferredMedia, 1)); + var receiving = ReceiveAsync(socket, connection.Token); + await Task.WhenAny(sending, receiving); connection.Cancel(); socket.Abort(); + try { await Task.WhenAll(sending, receiving); } catch (Exception ex) when (ex is OperationCanceledException or WebSocketException or InvalidDataException) { } + } + catch (Exception ex) when (!ct.IsCancellationRequested) { log.LogWarning("Discord media upload unavailable: {ErrorType}; retrying", ex.GetType().Name); } + finally { connection.Cancel(); socket.Abort(); lock (gate) { queue?.Clear(); queue = null; recovery = true; } } + if (Volatile.Read(ref transferredMedia) != 0) delay = 1; + try { await waitBeforeRetry(TimeSpan.FromSeconds(delay), ct); } catch (OperationCanceledException) { break; } + delay = Math.Min(30, delay * 2); + } + } + private static async Task SendAsync( + ClientWebSocket socket, + OwnedMediaQueue owned, + CancellationToken ct, + Action mediaTransferred) + { + while (true) + { + var sample = await owned.ReadAsync(ct); + using var send = CancellationTokenSource.CreateLinkedTokenSource(ct); send.CancelAfter(TimeSpan.FromSeconds(5)); + await socket.SendAsync(MediaWireCodec.Encode(sample).AsMemory(), WebSocketMessageType.Binary, true, send.Token); + if (sample.Type is 2 or 3) mediaTransferred(); + } + } + private async Task ReceiveAsync(ClientWebSocket socket, CancellationToken ct) + { + var data = new byte[41]; var last = DateTimeOffset.MinValue; + while (true) + { + var length = 0; ValueWebSocketReceiveResult received; + do + { + received = await socket.ReceiveAsync(data.AsMemory(length), ct); + if (received.MessageType == WebSocketMessageType.Close) return; + length += received.Count; + if (received.MessageType != WebSocketMessageType.Binary || length > 40) + throw new InvalidDataException("Invalid media control."); + } while (!received.EndOfMessage); + if (length != 40 || MediaWireCodec.Decode(data.AsMemory(0, length)).Type != 4) + throw new InvalidDataException("Invalid media control."); + if (DateTimeOffset.UtcNow - last >= TimeSpan.FromSeconds(1)) { last = DateTimeOffset.UtcNow; RequestKey(); } + } + } + private static List Nals(ReadOnlySpan data) + { + var output = new List(); int start = -1; + for (var i = 0; i + 3 <= data.Length; i++) + { + var prefix = data[i] == 0 && data[i + 1] == 0 ? (data[i + 2] == 1 ? 3 : i + 4 <= data.Length && data[i + 2] == 0 && data[i + 3] == 1 ? 4 : 0) : 0; + if (prefix == 0) continue; + if (start >= 0 && i > start && (data[start] & 31) is 7 or 8) output.Add(data[start..i].ToArray()); + start = i + prefix; i += prefix - 1; + } + if (start >= 0 && start < data.Length && (data[start] & 31) is 7 or 8) output.Add(data[start..].ToArray()); + return output; + } + public async ValueTask DisposeAsync() + { + stop.Cancel(); + if (worker is not null) { try { await worker; } catch (OperationCanceledException) { } } + stop.Dispose(); + } +} diff --git a/src/SonicDesktopRelay.Presentation/SessionRuntime.cs b/src/SonicDesktopRelay.Presentation/SessionRuntime.cs index 77e4126..d9a0a25 100644 --- a/src/SonicDesktopRelay.Presentation/SessionRuntime.cs +++ b/src/SonicDesktopRelay.Presentation/SessionRuntime.cs @@ -106,7 +106,7 @@ public async Task StartSharingAsync( try { - await publishHost.StartAsync(target, profile, ct); + await publishHost.StartAsync(created.SessionId, target, profile, ct); } catch (Exception e) when (e is InvalidOperationException or PlatformNotSupportedException) { diff --git a/src/SonicDesktopRelay.Presentation/SessionSnapshot.cs b/src/SonicDesktopRelay.Presentation/SessionSnapshot.cs index 3c1190b..09141a6 100644 --- a/src/SonicDesktopRelay.Presentation/SessionSnapshot.cs +++ b/src/SonicDesktopRelay.Presentation/SessionSnapshot.cs @@ -102,7 +102,7 @@ public interface IVideoPublishHost : IAsyncDisposable string? EncoderName { get; } - Task StartAsync(CaptureTarget target, VideoPublishProfile profile, CancellationToken ct); + Task StartAsync(Guid sessionId, CaptureTarget target, VideoPublishProfile profile, CancellationToken ct); Task StopAsync(); diff --git a/tests/SonicDesktopRelay.ApiClient.Tests/MediaRelayApiClientTests.cs b/tests/SonicDesktopRelay.ApiClient.Tests/MediaRelayApiClientTests.cs new file mode 100644 index 0000000..f8e6a4b --- /dev/null +++ b/tests/SonicDesktopRelay.ApiClient.Tests/MediaRelayApiClientTests.cs @@ -0,0 +1,24 @@ +using System.Net; +using Xunit; + +namespace SonicDesktopRelay.ApiClient.Tests; + +public sealed class MediaRelayApiClientTests +{ + [Theory] + [InlineData("/media/ws/media", "https://relay.example/media/ws/media")] + [InlineData("wss://media.example/ws/media", "wss://media.example/ws/media")] + public async Task Upload_grant_uses_authenticated_session_route_and_preserves_media_url(string url, string expected) + { + var sessionId = Guid.NewGuid(); + var handler = new StubHttpMessageHandler().Respond(HttpStatusCode.OK, + System.Text.Json.JsonSerializer.Serialize(new { admissionId = Guid.NewGuid(), sessionId, + participantId = Guid.NewGuid(), grant = "grant", expiresAt = DateTimeOffset.UtcNow.AddSeconds(30), mediaUrl = url })); + using var http = new HttpClient(handler) { BaseAddress = new("https://relay.example/") }; + http.DefaultRequestHeaders.Authorization = new("Bearer", "device-test"); + var result = await new MediaRelayApiClient(http).CreateUploadGrantAsync(sessionId, CancellationToken.None); + Assert.Equal($"/api/sessions/{sessionId}/media-grants", handler.Requests[0].RequestUri!.AbsolutePath); + Assert.Equal("device-test", handler.Requests[0].Headers.Authorization!.Parameter); + Assert.Equal(expected, result.MediaUrl); + } +} diff --git a/tests/SonicDesktopRelay.Media.Tests/MediaWireCodecTests.cs b/tests/SonicDesktopRelay.Media.Tests/MediaWireCodecTests.cs new file mode 100644 index 0000000..91aaf53 --- /dev/null +++ b/tests/SonicDesktopRelay.Media.Tests/MediaWireCodecTests.cs @@ -0,0 +1,48 @@ +using SonicDesktopRelay.Media.WebSocket; + +namespace SonicDesktopRelay.Media.Tests; + +public sealed class MediaWireCodecTests +{ + [Theory] + [InlineData(1)] [InlineData(2)] [InlineData(3)] [InlineData(4)] + public void Roundtrip_matches_header(int type) + { + var payload = type == 4 ? Array.Empty() : new byte[] { 1, 2, 3 }; + var sample = new MediaMessage((byte)type, (ushort)(type == 2 ? 1 : 0), 1, 3, 1000000, 16667, payload); + var wire = MediaWireCodec.Encode(sample); + Assert.Equal("FRM1", System.Text.Encoding.ASCII.GetString(wire, 0, 4)); + Assert.Equal(payload.Length + 40, wire.Length); + var decoded = MediaWireCodec.Decode(wire); + Assert.Equal(sample.TimestampUs, decoded.TimestampUs); + Assert.Equal(sample.Payload.ToArray(), decoded.Payload.ToArray()); + } + + [Theory] + [InlineData(4, 2)] [InlineData(5, 9)] [InlineData(6, 2)] [InlineData(8, 0)] [InlineData(16, 9)] [InlineData(20, 1)] + public void Rejects_invalid_headers(int offset, byte value) + { + var wire = MediaWireCodec.Encode(new MediaMessage(2, 1, 1, 0, 0, 1, new byte[] { 7 })); + wire[offset] = value; + Assert.Throws(() => MediaWireCodec.Decode(wire)); + } + + [Fact] + public void Rejects_unsafe_time_and_oversized_payload() + { + Assert.Throws(() => MediaWireCodec.Decode(new byte[39])); + Assert.Throws(() => MediaWireCodec.Encode(new MediaMessage(2, 0, 1, 0, long.MaxValue, 0, Array.Empty()))); + Assert.Throws(() => MediaWireCodec.Encode(new MediaMessage(2, 0, 1, 0, 0, 0, new byte[8 * 1024 * 1024]))); + } + [Fact] + public void Shared_vectors_decode_and_reencode_exact_bytes() + { + using var json = System.Text.Json.JsonDocument.Parse(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "discord-media-v1-vectors.json"))); + foreach (var vector in json.RootElement.EnumerateArray()) + { + var wire = Convert.FromHexString(vector.GetProperty("hex").GetString()!); + if (vector.TryGetProperty("invalid", out _)) Assert.Throws(() => MediaWireCodec.Decode(wire)); + else Assert.Equal(wire, MediaWireCodec.Encode(MediaWireCodec.Decode(wire))); + } + } +} diff --git a/tests/SonicDesktopRelay.Media.Tests/ScreenPublishPipelineTests.cs b/tests/SonicDesktopRelay.Media.Tests/ScreenPublishPipelineTests.cs index 43b154d..b5c1a67 100644 --- a/tests/SonicDesktopRelay.Media.Tests/ScreenPublishPipelineTests.cs +++ b/tests/SonicDesktopRelay.Media.Tests/ScreenPublishPipelineTests.cs @@ -549,10 +549,10 @@ public async Task Diagnostics_track_capture_encode_keyframes_and_last_activity() capture.Emit(); time.Advance(TimeSpan.FromMilliseconds(40)); - Assert.True(SpinWait.SpinUntil(() => pipeline.EncodedAccessUnits == 1, TimeSpan.FromSeconds(1))); + await WaitUntilAsync(() => pipeline.EncodedAccessUnits == 1); capture.Emit(); - Assert.True(SpinWait.SpinUntil(() => pipeline.EncodedAccessUnits == 2, TimeSpan.FromSeconds(1))); + await WaitUntilAsync(() => pipeline.EncodedAccessUnits == 2); Assert.Equal(2, pipeline.FramesCaptured); Assert.Equal(2, pipeline.EncodedAccessUnits); Assert.Equal(2, pipeline.KeyframesProduced); @@ -613,6 +613,17 @@ public async Task Diagnostics_count_keyframe_requests_sent_to_the_encoder() Assert.Equal(1, pipeline.PliReceived); } + private static async Task WaitUntilAsync(Func condition) + { + var startedAt = TimeProvider.System.GetTimestamp(); + var timeout = TimeSpan.FromSeconds(5); + + while (!condition() && TimeProvider.System.GetElapsedTime(startedAt) < timeout) + await Task.Delay(10); + + Assert.True(condition(), "Condition was not reached within 5 seconds."); + } + private sealed class FakeCapture : IScreenCaptureSource { public MonitorInfo Monitor { get; private set; } diff --git a/tests/SonicDesktopRelay.Media.Tests/SonicDesktopRelay.Media.Tests.csproj b/tests/SonicDesktopRelay.Media.Tests/SonicDesktopRelay.Media.Tests.csproj index 4d53b38..b1bb735 100644 --- a/tests/SonicDesktopRelay.Media.Tests/SonicDesktopRelay.Media.Tests.csproj +++ b/tests/SonicDesktopRelay.Media.Tests/SonicDesktopRelay.Media.Tests.csproj @@ -23,4 +23,5 @@ + \ No newline at end of file diff --git a/tests/SonicDesktopRelay.Media.Tests/WebSocketMediaPublisherTests.cs b/tests/SonicDesktopRelay.Media.Tests/WebSocketMediaPublisherTests.cs new file mode 100644 index 0000000..18489ef --- /dev/null +++ b/tests/SonicDesktopRelay.Media.Tests/WebSocketMediaPublisherTests.cs @@ -0,0 +1,141 @@ +using System.Net; +using System.Net.Sockets; +using System.Net.WebSockets; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using SonicDesktopRelay.Media.WebSocket; + +namespace SonicDesktopRelay.Media.Tests; + +public sealed class WebSocketMediaPublisherTests +{ + [Fact] + public async Task Publisher_keeps_exponential_backoff_when_socket_closes_before_media_transfer() + { + using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var endpoint = (IPEndPoint)listener.LocalEndpoint; + var waits = new List(); + var waitsGate = new object(); + var threeRetries = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var server = Task.Run(async () => + { + for (var i = 0; i < 3; i++) + await AcceptGrantThenCloseAsync(listener, budget.Token); + }); + + await using var publisher = new WebSocketMediaPublisher( + (_, _) => Task.FromResult(new MediaUploadGrant( + "test-grant", $"ws://127.0.0.1:{endpoint.Port}/ws/media")), + () => { }, + false, + retryWait: (delay, ct) => + { + lock (waitsGate) + { + waits.Add(delay); + if (waits.Count >= 3) + { + threeRetries.TrySetResult(); + return Task.Delay(Timeout.InfiniteTimeSpan, ct); + } + } + + return Task.CompletedTask; + }); + + await publisher.StartAsync(Guid.NewGuid(), budget.Token); + await threeRetries.Task.WaitAsync(budget.Token); + await server.WaitAsync(budget.Token); + + lock (waitsGate) + Assert.Equal( + new[] { TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4) }, + waits.Take(3).ToArray()); + } + + [Fact] + public async Task Publisher_uploads_config_fresh_keyframe_and_opus_over_real_socket() + { + using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var listener = new TcpListener(IPAddress.Loopback, 0); listener.Start(); + var endpoint = (IPEndPoint)listener.LocalEndpoint; + var ready = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var keyRequests = 0; + var server = Task.Run(async () => + { + using var tcp = await listener.AcceptTcpClientAsync(budget.Token); using var stream = tcp.GetStream(); + var header = new StringBuilder(); var one = new byte[1]; + while (!header.ToString().EndsWith("\r\n\r\n", StringComparison.Ordinal)) + { + Assert.True(header.Length < 8192); Assert.Equal(1, await stream.ReadAsync(one, budget.Token)); header.Append((char)one[0]); + } + Assert.StartsWith("GET /ws/media HTTP/1.1", header.ToString()); + var key = header.ToString().Split("\r\n").Single(x => x.StartsWith("Sec-WebSocket-Key:", StringComparison.OrdinalIgnoreCase)).Split(':', 2)[1].Trim(); + var accept = Convert.ToBase64String(SHA1.HashData(Encoding.ASCII.GetBytes(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"))); + await stream.WriteAsync(Encoding.ASCII.GetBytes($"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\nSec-WebSocket-Protocol: framerelay-media-v1\r\n\r\n"), budget.Token); + using var socket = System.Net.WebSockets.WebSocket.CreateFromStream(stream, true, "framerelay-media-v1", TimeSpan.FromSeconds(30)); + var buffer = new byte[8192]; var authentication = await socket.ReceiveAsync(buffer.AsMemory(), budget.Token); + Assert.Equal(WebSocketMessageType.Text, authentication.MessageType); + using var grant = JsonDocument.Parse(buffer.AsMemory(0, authentication.Count)); Assert.Equal("test-grant", grant.RootElement.GetProperty("grant").GetString()); + await Task.Delay(1100, budget.Token); + var request = MediaWireCodec.Encode(new MediaMessage(4, 0, 1, 0, 0, 0, ReadOnlyMemory.Empty)); + await socket.SendAsync(request.AsMemory(0, 20), WebSocketMessageType.Binary, false, budget.Token); + await socket.SendAsync(request.AsMemory(20), WebSocketMessageType.Binary, true, budget.Token); + var messages = new List(); + for (var i = 0; i < 3; i++) + { + using var complete = new MemoryStream(); ValueWebSocketReceiveResult receive; + do { receive = await socket.ReceiveAsync(buffer.AsMemory(), budget.Token); complete.Write(buffer, 0, receive.Count); } while (!receive.EndOfMessage); + messages.Add(MediaWireCodec.Decode(complete.ToArray())); + } + return messages; + }); + await using var publisher = new WebSocketMediaPublisher((_, _) => Task.FromResult(new MediaUploadGrant("test-grant", $"ws://127.0.0.1:{endpoint.Port}/ws/media")), () => { if (Interlocked.Increment(ref keyRequests) >= 2) ready.TrySetResult(); }, true); + await publisher.StartAsync(Guid.NewGuid(), budget.Token); await ready.Task.WaitAsync(budget.Token); + publisher.PublishVideo(new EncodedVideoSample(new byte[] { 0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f, 0, 0, 0, 1, 0x68, 1, 0, 0, 0, 1, 0x65, 1 }, TimeSpan.FromSeconds(1), true, 640, 360, TimeSpan.FromMilliseconds(16.667))); + publisher.PublishAudio(new EncodedAudioSample(new byte[] { 0xf8, 0xff, 0xfe }, 960, TimeSpan.FromMilliseconds(20), TimeSpan.FromSeconds(1))); + var messages = await server.WaitAsync(budget.Token); + Assert.Equal(new byte[] { 1, 2, 3 }, messages.Select(x => x.Type).ToArray()); + Assert.Equal(new uint[] { 0, 1, 2 }, messages.Select(x => x.Sequence).ToArray()); + using var config = JsonDocument.Parse(messages[0].Payload); Assert.Equal("avc1.42e01f", config.RootElement.GetProperty("video").GetProperty("codec").GetString()); + Assert.Equal(1, messages[1].Flags); Assert.Equal(1000000, messages[1].TimestampUs); + Assert.Equal(new byte[] { 0xf8, 0xff, 0xfe }, messages[2].Payload.ToArray()); + } + private static async Task AcceptGrantThenCloseAsync(TcpListener listener, CancellationToken ct) + { + using var tcp = await listener.AcceptTcpClientAsync(ct); + using var stream = tcp.GetStream(); + var header = new StringBuilder(); + var one = new byte[1]; + while (!header.ToString().EndsWith("\r\n\r\n", StringComparison.Ordinal)) + { + Assert.True(header.Length < 8192); + Assert.Equal(1, await stream.ReadAsync(one, ct)); + header.Append((char)one[0]); + } + + var key = header.ToString().Split("\r\n") + .Single(x => x.StartsWith("Sec-WebSocket-Key:", StringComparison.OrdinalIgnoreCase)) + .Split(':', 2)[1].Trim(); + var accept = Convert.ToBase64String(SHA1.HashData( + Encoding.ASCII.GetBytes(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"))); + await stream.WriteAsync(Encoding.ASCII.GetBytes( + $"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\nSec-WebSocket-Protocol: framerelay-media-v1\r\n\r\n"), ct); + + using var socket = System.Net.WebSockets.WebSocket.CreateFromStream( + stream, true, "framerelay-media-v1", TimeSpan.FromSeconds(30)); + var buffer = new byte[1024]; + var authentication = await socket.ReceiveAsync(buffer.AsMemory(), ct); + Assert.Equal(WebSocketMessageType.Text, authentication.MessageType); + using var grant = JsonDocument.Parse(buffer.AsMemory(0, authentication.Count)); + Assert.Equal("test-grant", grant.RootElement.GetProperty("grant").GetString()); + + await socket.CloseOutputAsync( + WebSocketCloseStatus.PolicyViolation, "grant rejected", ct); + } + +} diff --git a/tests/SonicDesktopRelay.Media.Tests/WebSocketMediaTests.cs b/tests/SonicDesktopRelay.Media.Tests/WebSocketMediaTests.cs new file mode 100644 index 0000000..0c17a87 --- /dev/null +++ b/tests/SonicDesktopRelay.Media.Tests/WebSocketMediaTests.cs @@ -0,0 +1,56 @@ +using SonicDesktopRelay.Media.WebSocket; + +namespace SonicDesktopRelay.Media.Tests; + +public sealed class WebSocketMediaTests +{ + [Theory] + [InlineData(null, false)] [InlineData("false", false)] [InlineData("1", false)] [InlineData("true", true)] + public void Publisher_is_explicit_opt_in(string? value, bool expected) => + Assert.Equal(expected, WebSocketMediaOptions.FromEnvironment(_ => value).Enabled); + + [Fact] + public async Task Queue_copies_native_sample_memory_and_bounds_backlog() + { + var queue = new OwnedMediaQueue(); var data = new byte[] { 7 }; + var message = new MediaMessage(2, 1, 1, 0, 0, 1, data); + Assert.True(queue.TryEnqueue(message)); data[0] = 99; + using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + Assert.Equal(7, (await queue.ReadAsync(budget.Token)).Payload.Span[0]); + for (var i = 0; i < 64; i++) Assert.True(queue.TryEnqueue(message)); + Assert.False(queue.TryEnqueue(message)); queue.Clear(); + Assert.False(queue.TryEnqueue(message with { Payload = new byte[8 * 1024 * 1024] })); + } + + private sealed class Capture : IScreenCaptureSource + { + public MonitorInfo Monitor => new("test", "test", 2, 2, true); + public event Action? FrameCaptured; + public void Emit(byte[] bytes) => FrameCaptured?.Invoke(new VideoFrame(2, 2, bytes, TimeSpan.Zero)); + public void SetFrameRate(int framesPerSecond) { } + public Task StopAsync() => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + private sealed class Encoder : IVideoEncoder + { + public readonly TaskCompletionSource Started = new(TaskCreationOptions.RunContinuationsAsynchronously); + public readonly TaskCompletionSource Result = new(TaskCreationOptions.RunContinuationsAsynchronously); + public readonly ManualResetEventSlim Continue = new(false); + public string Name => "fake"; + public EncodedVideoSample? Encode(VideoFrame frame, VideoQuality quality) + { + Started.TrySetResult(); Continue.Wait(TimeSpan.FromSeconds(3)); Result.TrySetResult(frame.Bgra.Span[0]); return null; + } + public void RequestKeyFrame() { } + public void Dispose() { Continue.Dispose(); } + } + [Fact] + public async Task Independent_encoder_owns_raw_frame_after_capture_returns() + { + var capture = new Capture(); var encoder = new Encoder(); + await using var branch = new ActivityH264Output(capture, encoder, new MediaSessionClock(TimeProvider.System), () => VideoQuality.Default, () => true); + var data = new byte[16]; data[0] = 7; capture.Emit(data); + await encoder.Started.Task.WaitAsync(TimeSpan.FromSeconds(3)); data[0] = 99; encoder.Continue.Set(); + Assert.Equal(7, await encoder.Result.Task.WaitAsync(TimeSpan.FromSeconds(3))); + } +} diff --git a/tests/SonicDesktopRelay.Presentation.Tests/SessionRuntimeTests.cs b/tests/SonicDesktopRelay.Presentation.Tests/SessionRuntimeTests.cs index 4a5c901..2543302 100644 --- a/tests/SonicDesktopRelay.Presentation.Tests/SessionRuntimeTests.cs +++ b/tests/SonicDesktopRelay.Presentation.Tests/SessionRuntimeTests.cs @@ -297,6 +297,7 @@ public async Task Sharing_starts_publishing_on_the_chosen_monitor() await runtime.StartSharingAsync(monitor, 3, CancellationToken.None); + Assert.Equal(SessionId, host.StartedSessionId); Assert.Equal(monitor, host.StartedOn); Assert.Equal(SessionPhase.Sharing, runtime.Snapshot.Phase); } @@ -733,6 +734,8 @@ private sealed class FakeVideoPublishHost : IVideoPublishHost public VideoPublishProfile? StartedProfile { get; private set; } + public Guid? StartedSessionId { get; private set; } + public bool Stopped { get; private set; } public string? EncoderName { get; init; } @@ -756,9 +759,10 @@ public Task StartAsync(MonitorInfo monitor, VideoPublishProfile profile, Cancell return Task.CompletedTask; } - public Task StartAsync(CaptureTarget target, VideoPublishProfile profile, CancellationToken ct) + public Task StartAsync(Guid sessionId, CaptureTarget target, VideoPublishProfile profile, CancellationToken ct) { if (StartFailure is not null) throw new InvalidOperationException(StartFailure); + StartedSessionId = sessionId; StartedTarget = target; if (target is CaptureTarget.Monitor monitor) StartedOn = monitor.Info; StartedProfile = profile;