diff --git a/config/config.devenv.json b/config/config.devenv.json index df0ff4c18d..d5b2af8d58 100644 --- a/config/config.devenv.json +++ b/config/config.devenv.json @@ -14,6 +14,8 @@ "membership_event_expiry_ms": 180000000, "delayed_leave_event_delay_ms": 18000, "delayed_leave_event_restart_ms": 4000, - "network_error_retry_ms": 100 + "network_error_retry_ms": 100, + "use_key_delay_ms": 1000, + "key_rotation_grace_period_ms": 10000 } } diff --git a/config/config.sample.json b/config/config.sample.json index 126d76265f..04a8eaef99 100644 --- a/config/config.sample.json +++ b/config/config.sample.json @@ -17,6 +17,8 @@ "membership_event_expiry_ms": 180000000, "delayed_leave_event_delay_ms": 18000, "delayed_leave_event_restart_ms": 4000, - "network_error_retry_ms": 100 + "network_error_retry_ms": 100, + "use_key_delay_ms": 1000, + "key_rotation_grace_period_ms": 10000 } } diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index 4da633caf3..cfd85a257d 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -151,6 +151,23 @@ export interface ConfigOptions { * This is what goes into the m.rtc.member event expiry field and is typically set to a number of hours. */ membership_event_expiry_ms?: number; + + /** + * The delay (in milliseconds) between sending a new encryption key and + * starting to encrypt media with it. This gives other participants time + * to receive the key before media is encrypted with it. + * SDK default: 1000 + */ + use_key_delay_ms?: number; + + /** + * The grace period (in milliseconds) after a key rotation during which + * new joiners can reuse the existing key instead of triggering another + * rotation. Higher values reduce redundant rotations when multiple + * participants join in quick succession. + * SDK default: 10000 + */ + key_rotation_grace_period_ms?: number; }; } diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 63a96755fc..153541cceb 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -16,6 +16,7 @@ const logger = rootLogger.getChild("[MatrixKeyProvider]"); export class MatrixKeyProvider extends BaseKeyProvider { private rtcSession?: MatrixRTCSession; + private joinedAt?: number; public constructor() { super({ ratchetWindowSize: 10, keyringSize: 256 }); @@ -30,6 +31,8 @@ export class MatrixKeyProvider extends BaseKeyProvider { } this.rtcSession = rtcSession; + this.joinedAt = Date.now(); + logger.info(`setRTCSession called, recording join timestamp`); this.rtcSession.on( MatrixRTCSessionEvent.EncryptionKeyChanged, @@ -47,6 +50,14 @@ export class MatrixKeyProvider extends BaseKeyProvider { membershipParts: CallMembershipIdentityParts, rtcBackendIdentity: string, ): void => { + const keyReceivedAt = Date.now(); + const timeSinceJoin = this.joinedAt + ? keyReceivedAt - this.joinedAt + : undefined; + logger.info( + `Key received ${timeSinceJoin !== undefined ? `${timeSinceJoin}ms after join` : "(no join timestamp)"} for ${membershipParts.userId}:${membershipParts.deviceId} index=${encryptionKeyIndex}`, + ); + crypto.subtle .importKey("raw", encryptionKey, "HKDF", false, [ "deriveBits", @@ -54,14 +65,15 @@ export class MatrixKeyProvider extends BaseKeyProvider { ]) .then( (keyMaterial) => { + const importDuration = Date.now() - keyReceivedAt; this.onSetEncryptionKey( keyMaterial, rtcBackendIdentity, encryptionKeyIndex, ); - logger.debug( - `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`, + logger.info( + `Key imported in ${importDuration}ms and sent to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`, ); }, (e) => { diff --git a/src/e2ee/usePrewarmOlmSessions.ts b/src/e2ee/usePrewarmOlmSessions.ts new file mode 100644 index 0000000000..5978ac412d --- /dev/null +++ b/src/e2ee/usePrewarmOlmSessions.ts @@ -0,0 +1,52 @@ +/* +Copyright 2026 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { useEffect } from "react"; +import { type MatrixClient, type Room } from "matrix-js-sdk"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; + +const logger = rootLogger.getChild("[OlmPrewarm]"); + +/** + * Pre-warms Olm sessions with current call participants while the user is in + * the lobby. This avoids a `/keys/claim` round-trip when actually joining the + * call and distributing E2EE media encryption keys. + * + * Internally calls `CryptoApi.prepareToEncrypt(room)` which triggers + * `KeyClaimManager.ensureSessionsForUsers()` for all room members, establishing + * Olm sessions with any devices we haven't communicated with before. + * + * @param client - The Matrix client. + * @param room - The room the call is in. + * @param memberships - Current call memberships (used to trigger re-warming + * when new participants join while the user is still in the lobby). + * @param enabled - Pass `false` to disable (e.g. when the user has already + * joined the call or E2EE is not in use). + */ +export function usePrewarmOlmSessions( + client: MatrixClient, + room: Room, + memberships: CallMembership[], + enabled: boolean, +): void { + useEffect(() => { + if (!enabled) return; + if (memberships.length === 0) return; + + const crypto = client.getCrypto(); + if (!crypto) { + logger.debug("No crypto available, skipping Olm session pre-warming"); + return; + } + + logger.info( + `Pre-warming Olm sessions for room ${room.roomId} (${memberships.length} call members)`, + ); + crypto.prepareToEncrypt(room); + }, [client, room, memberships, enabled]); +} diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 10579c1b9e..42e76279d9 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -76,10 +76,9 @@ export function LivekitRoomAudioRenderer({ .filter((ref) => ref.publication.kind === Track.Kind.Audio) // Only keep tracks from participants that are in the validIdentities list .filter((ref) => { + if (ref.participant.isLocal) return false; const isValid = validIdentities.includes(ref.participant.identity); if (!isValid) { - // TODO make sure to also skip the warn logging for the local identity - // Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call. prefixedLogger.warn( `Audio track ${ref.participant.identity} from ${url} has no matching matrix call member`, `current members: ${validIdentities.join()}`, diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index dfd11ff32c..2f220cca30 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -56,6 +56,7 @@ import { useUrlParams, } from "../UrlParams"; import { E2eeType } from "../e2ee/e2eeType"; +import { usePrewarmOlmSessions } from "../e2ee/usePrewarmOlmSessions"; import { useAudioContext } from "../useAudioContext"; import { callEventAudioSounds, @@ -206,6 +207,15 @@ export const GroupCallView: FC = ({ }; }, [client, displayName, avatarUrl, roomName, room, roomAvatar, e2eeSystem]); + // Pre-warm Olm sessions with call participants while in the lobby so that + // E2EE key exchange is faster when the user actually joins. + usePrewarmOlmSessions( + client, + room, + memberships, + !joined && e2eeSystem.kind === E2eeType.PER_PARTICIPANT, + ); + // Count each member only once, regardless of how many devices they use const participantCount = useMemo( () => new Set(memberships.map((m) => m.userId!)).size, diff --git a/src/state/CallViewModel/localMember/HomeserverConnected.ts b/src/state/CallViewModel/localMember/HomeserverConnected.ts index c8bcd02179..33c84755c9 100644 --- a/src/state/CallViewModel/localMember/HomeserverConnected.ts +++ b/src/state/CallViewModel/localMember/HomeserverConnected.ts @@ -12,7 +12,17 @@ import { type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { ClientEvent, type MatrixClient, SyncState } from "matrix-js-sdk"; -import { fromEvent, startWith, map, tap, type Observable } from "rxjs"; +import { + fromEvent, + startWith, + map, + tap, + switchMap, + timer, + of, + distinctUntilChanged, + type Observable, +} from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { type ObservableScope } from "../../ObservableScope"; @@ -45,11 +55,28 @@ export function createHomeserverConnected$( matrixRTCSession: NodeStyleEventEmitter & Pick, ): HomeserverConnected { - const syncing$ = ( + const syncingRaw$ = ( fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]> ).pipe( startWith([client.getSyncState()]), map(([state]) => state === SyncState.Syncing), + tap((v) => logger.info(`syncing$ (raw): ${v}`)), + ); + + // Sync errors are transient — the SDK auto-retries and /sync abort + // recoveries can take several seconds. Debounce the false→true transition + // with an 8s grace period so brief sync interruptions don't cascade. + const syncing$ = syncingRaw$.pipe( + switchMap((syncing) => + syncing + ? of(true) + : timer(8000).pipe( + map(() => false), + startWith(true), + ), + ), + distinctUntilChanged(), + tap((v) => logger.info(`syncing$ (debounced): ${v}`)), ); const rtsSession$ = scope.behavior( @@ -61,6 +88,7 @@ export function createHomeserverConnected$( const membershipConnected$ = rtsSession$.pipe( map((status) => status === Status.Connected), + tap((v) => logger.info(`membershipConnected$: ${v}`)), ); // This is basically notProbablyLeft$ @@ -77,10 +105,23 @@ export function createHomeserverConnected$( ).pipe( startWith(null), map(() => matrixRTCSession.probablyLeft !== true), + tap((v) => logger.info(`certainlyConnected$ (notProbablyLeft): ${v}`)), ); const combined$ = scope.behavior( and$(syncing$, membershipConnected$, certainlyConnected$).pipe( + // Don't immediately report disconnection for brief hiccups. + // Stay "connected" during a 2s grace period so that momentary sync + // errors or delayed-leave-event timeouts don't flap the UI. + switchMap((connected) => + connected + ? of(true) + : timer(2000).pipe( + map(() => false), + startWith(true), + ), + ), + distinctUntilChanged(), tap((connected) => { logger.info(`Homeserver connected update: ${connected}`); }), diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index eb641ca7cd..92482cf653 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -34,6 +34,7 @@ import { startWith, switchMap, tap, + timer, } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { deepCompare } from "matrix-js-sdk/lib/utils"; @@ -238,6 +239,7 @@ export const createLocalMembership$ = ({ return connectionData.getConnectionForTransport(localTransport); }), + distinctUntilChanged(), tap((connection) => { logger.info( `Local connection updated: ${connection?.transport?.livekit_service_url}`, @@ -393,6 +395,7 @@ export const createLocalMembership$ = ({ const localConnectionState$ = localConnection$.pipe( switchMap((connection) => (connection ? connection.state$ : of(null))), + tap((state) => logger.debug(`localConnectionState$: ${state}`)), ); const mediaState$: Behavior = scope.behavior( @@ -488,11 +491,18 @@ export const createLocalMembership$ = ({ /** * Whether we should tell the user that we're reconnecting to the call. + * Debounced so that brief (<1.5s) disconnections don't flash the UI. */ const reconnecting$ = scope.behavior( matrixAndLivekitConnected$.pipe( - pairwise(), - map(([prev, current]) => prev === true && current === false), + switchMap((connected) => + connected + ? of(false) + : timer(1500).pipe( + map(() => true), + startWith(false), + ), + ), ), false, ); @@ -768,6 +778,9 @@ export function enterRTCSession( matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, + useKeyDelay: matrixRtcSessionConfig?.use_key_delay_ms, + keyRotationGracePeriodMs: + matrixRtcSessionConfig?.key_rotation_grace_period_ms, membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, useExperimentalToDeviceTransport: true,