diff --git a/locales/en/app.json b/locales/en/app.json index 0b0ac7b4dc..197809e640 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -61,6 +61,7 @@ "username": "Username", "video": "Video" }, + "deafen_button_label": "Deafen", "developer_mode": { "always_show_iphone_earpiece": "Show iPhone earpiece option on all platforms", "crypto_version": "Crypto version: {{version}}", @@ -245,6 +246,7 @@ "unauthenticated_view_body": "Not registered yet? <2>Create an account", "unauthenticated_view_login_button": "Login to your account", "unauthenticated_view_ssla_caption": "By clicking \"Go\", you agree to our <2>Software and Services License Agreement (SSLA)", + "undeafen_button_label": "Undeafen", "unmute_microphone_button_label": "Unmute microphone", "version": "{{productName}} version: {{version}}", "video_tile": { @@ -252,6 +254,7 @@ "camera_starting": "Video loading...", "change_fit_contain": "Fit to frame", "collapse": "Collapse", + "deafened": "Deafened", "expand": "Expand", "mute_for_me": "Mute for me", "muted_for_me": "Muted for me", diff --git a/sdk/main.ts b/sdk/main.ts index c65bf4a70c..8adb2b1a6b 100644 --- a/sdk/main.ts +++ b/sdk/main.ts @@ -142,6 +142,7 @@ export async function createMatrixRTCSdk( { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT } }, of({}), of({}), + of({}), constant({ supported: false, processor: undefined }), ); logger.info("CallViewModelCreated"); diff --git a/src/button/Button.tsx b/src/button/Button.tsx index 3136e2da26..270b42e7b8 100644 --- a/src/button/Button.tsx +++ b/src/button/Button.tsx @@ -17,6 +17,8 @@ import { EndCallIcon, ShareScreenSolidIcon, SettingsSolidIcon, + HeadphonesSolidIcon, + HeadphonesOffSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import styles from "./Button.module.css"; @@ -45,6 +47,33 @@ export const MicButton: FC = ({ muted, ...props }) => { ); }; +interface DeafenButtonProps extends ComponentPropsWithoutRef<"button"> { + deafened: boolean; +} + +export const DeafenButton: FC = ({ + deafened, + ...props +}) => { + const { t } = useTranslation(); + const Icon = deafened ? HeadphonesOffSolidIcon : HeadphonesSolidIcon; + const label = deafened + ? t("undeafen_button_label") + : t("deafen_button_label"); + + return ( + + + + ); +}; + interface VideoButtonProps extends ComponentPropsWithoutRef<"button"> { muted: boolean; } diff --git a/src/reactions/DeafenReader.ts b/src/reactions/DeafenReader.ts new file mode 100644 index 0000000000..50c593129a --- /dev/null +++ b/src/reactions/DeafenReader.ts @@ -0,0 +1,211 @@ +/* +Copyright 2026 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type CallMembership, + MatrixRTCSessionEvent, + type MatrixRTCSession, +} from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type MatrixEvent, MatrixEventEvent } from "matrix-js-sdk"; +import { type ReactionEventContent } from "matrix-js-sdk/lib/types"; +import { + RelationType, + EventType, + RoomEvent as MatrixRoomEvent, +} from "matrix-js-sdk"; +import { BehaviorSubject } from "rxjs"; + +import { ElementCallDeafenedKey, type DeafenedInfo } from "."; +import { type ObservableScope } from "../state/ObservableScope"; + +/** + * Listens for deafen state reactions from an RTCSession and populates a subject + * for consumption by the CallViewModel. + */ +export class DeafenReader { + private readonly deafenedSubject$ = new BehaviorSubject< + Record + >({}); + + /** + * The latest set of deafened users. + */ + public readonly deafened$ = this.deafenedSubject$.asObservable(); + + public constructor( + private readonly scope: ObservableScope, + private readonly rtcSession: MatrixRTCSession, + ) { + this.rtcSession.room.on(MatrixRoomEvent.Timeline, this.handleEvent); + this.scope.onEnd(() => + this.rtcSession.room.off(MatrixRoomEvent.Timeline, this.handleEvent), + ); + + this.rtcSession.room.on(MatrixRoomEvent.Redaction, this.handleEvent); + this.scope.onEnd(() => + this.rtcSession.room.off(MatrixRoomEvent.Redaction, this.handleEvent), + ); + + this.rtcSession.room.client.on( + MatrixEventEvent.Decrypted, + this.handleEvent, + ); + this.scope.onEnd(() => + this.rtcSession.room.client.off( + MatrixEventEvent.Decrypted, + this.handleEvent, + ), + ); + + this.rtcSession.room.on( + MatrixRoomEvent.LocalEchoUpdated, + this.handleEvent, + ); + this.scope.onEnd(() => + this.rtcSession.room.off( + MatrixRoomEvent.LocalEchoUpdated, + this.handleEvent, + ), + ); + + this.rtcSession.on( + MatrixRTCSessionEvent.MembershipsChanged, + this.onMembershipsChanged, + ); + this.scope.onEnd(() => + this.rtcSession.off( + MatrixRTCSessionEvent.MembershipsChanged, + this.onMembershipsChanged, + ), + ); + + // Run this once to ensure we have fetched the state from the call. + this.onMembershipsChanged([]); + } + + /** + * Fetches any deafen reactions by the given sender on the given + * membership event. + */ + private getLastDeafenEvent( + membershipEventId: string, + expectedSender: string, + ): MatrixEvent | undefined { + const relations = this.rtcSession.room.relations.getChildEventsForEvent( + membershipEventId, + RelationType.Annotation, + EventType.Reaction, + ); + const allEvents = relations?.getRelations() ?? []; + return allEvents.find( + (reaction) => + reaction.event.sender === expectedSender && + reaction.getType() === EventType.Reaction && + reaction.getContent()?.["m.relates_to"]?.key === + ElementCallDeafenedKey, + ); + } + + private onMembershipsChanged = (oldMemberships: CallMembership[]): void => { + // Remove deafen state for users no longer joined to the call. + for (const identifier of Object.keys(this.deafenedSubject$.value).filter( + (id) => oldMemberships.find((u) => u.userId == id), + )) { + this.removeDeafened(identifier); + } + + // For each member in the call, check to see if a deafen reaction exists. + for (const m of this.rtcSession.memberships) { + if (!m.userId || !m.eventId) { + continue; + } + const identifier = `${m.userId}:${m.deviceId}`; + if ( + this.deafenedSubject$.value[identifier] && + this.deafenedSubject$.value[identifier].membershipEventId !== m.eventId + ) { + // Membership event for sender has changed since deafen was set, reset. + this.removeDeafened(identifier); + } + const reaction = this.getLastDeafenEvent(m.eventId, m.userId); + if (reaction) { + const eventId = reaction?.getId(); + if (!eventId) { + continue; + } + this.addDeafened(`${m.userId}:${m.deviceId}`, { + membershipEventId: m.eventId, + reactionEventId: eventId, + }); + } + } + }; + + private addDeafened(identifier: string, info: DeafenedInfo): void { + this.deafenedSubject$.next({ + ...this.deafenedSubject$.value, + [identifier]: info, + }); + } + + private removeDeafened(identifier: string): void { + this.deafenedSubject$.next( + Object.fromEntries( + Object.entries(this.deafenedSubject$.value).filter( + ([uId]) => uId !== identifier, + ), + ), + ); + } + + private handleEvent = (event: MatrixEvent): void => { + const room = this.rtcSession.room; + if (event.getRoomId() !== room.roomId) return; + if (event.isSending()) return; + + const sender = event.getSender(); + const reactionEventId = event.getId(); + if (!sender || !reactionEventId) return; + + room.client + .decryptEventIfNeeded(event) + .catch((e) => logger.warn(`Failed to decrypt ${event.getId()}`, e)); + if (event.isBeingDecrypted() || event.isDecryptionFailure()) return; + + if (event.getType() === EventType.Reaction) { + const content = event.getContent() as ReactionEventContent; + const membershipEventId = content["m.relates_to"].event_id; + + const membershipEvent = this.rtcSession.memberships.find( + (e) => e.eventId === membershipEventId && e.userId === sender, + ); + if (!membershipEvent) { + return; + } + + if (content?.["m.relates_to"].key === ElementCallDeafenedKey) { + this.addDeafened( + `${membershipEvent.userId}:${membershipEvent.deviceId}`, + { + reactionEventId, + membershipEventId, + }, + ); + } + } else if (event.getType() === EventType.RoomRedaction) { + const targetEvent = event.event.redacts; + const targetUser = Object.entries(this.deafenedSubject$.value).find( + ([_u, r]) => r.reactionEventId === targetEvent, + )?.[0]; + if (!targetUser) { + return; + } + this.removeDeafened(targetUser); + } + }; +} diff --git a/src/reactions/index.ts b/src/reactions/index.ts index acf7e18161..a2581c2872 100644 --- a/src/reactions/index.ts +++ b/src/reactions/index.ts @@ -204,6 +204,19 @@ export const ReactionSet: ReactionOption[] = [ }, ]; +export const ElementCallDeafenedKey = "io.element.call.deafened"; + +export interface DeafenedInfo { + /** + * Call membership event that was reacted to. + */ + membershipEventId: string; + /** + * Event ID of the reaction itself. + */ + reactionEventId: string; +} + export interface RaisedHandInfo { /** * Call membership event that was reacted to. diff --git a/src/reactions/useReactionsSender.tsx b/src/reactions/useReactionsSender.tsx index afb9b78976..f82dfbf483 100644 --- a/src/reactions/useReactionsSender.tsx +++ b/src/reactions/useReactionsSender.tsx @@ -19,13 +19,18 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; import { useClientState } from "../ClientContext"; -import { ElementCallReactionEventType, type ReactionOption } from "."; +import { + ElementCallDeafenedKey, + ElementCallReactionEventType, + type ReactionOption, +} from "."; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { useBehavior } from "../useBehavior"; interface ReactionsSenderContextType { supportsReactions: boolean; toggleRaisedHand: () => Promise; + toggleDeafened: () => Promise; sendReaction: (reaction: ReactionOption) => Promise; } @@ -88,6 +93,15 @@ export const ReactionsSenderProvider = ({ [myMembershipIdentifier, handsRaised], ); + const deafened = useBehavior(vm.deafened$); + const myDeafened = useMemo( + () => + myMembershipIdentifier !== undefined + ? deafened[myMembershipIdentifier] + : undefined, + [myMembershipIdentifier, deafened], + ); + const toggleRaisedHand = useCallback(async () => { if (!myMembershipIdentifier) { return; @@ -131,6 +145,56 @@ export const ReactionsSenderProvider = ({ room, ]); + const toggleDeafened = useCallback(async () => { + if (!myMembershipIdentifier) { + return; + } + const myDeafenedReactionId = myDeafened?.reactionEventId; + + if (!myDeafenedReactionId) { + try { + if (!myMembershipEvent) { + throw new Error("Cannot find own membership event"); + } + const reaction = await room.client.sendEvent( + rtcSession.room.roomId, + EventType.Reaction, + { + "m.relates_to": { + rel_type: RelationType.Annotation, + event_id: myMembershipEvent, + key: ElementCallDeafenedKey, + }, + }, + ); + logger.debug("Sent deafen event", reaction.event_id); + } catch (ex) { + logger.error("Failed to send deafen event", ex); + } + } else { + try { + await room.client.redactEvent( + rtcSession.room.roomId, + myDeafenedReactionId, + ); + logger.debug("Redacted deafen event"); + } catch (ex) { + logger.error( + "Failed to redact deafen event", + myDeafenedReactionId, + ex, + ); + throw ex; + } + } + }, [ + myMembershipEvent, + myMembershipIdentifier, + myDeafened, + rtcSession, + room, + ]); + const sendReaction = useCallback( async (reaction: ReactionOption) => { if (!myMembershipIdentifier || myReaction) { @@ -161,6 +225,7 @@ export const ReactionsSenderProvider = ({ value={{ supportsReactions, toggleRaisedHand, + toggleDeafened, sendReaction, }} > diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 135745eb37..e9778a1ccd 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -36,6 +36,7 @@ import LogoType from "../icons/LogoType.svg?react"; import { EndCallButton, MicButton, + DeafenButton, VideoButton, ShareScreenButton, SettingsButton, @@ -90,6 +91,11 @@ import { useSetting, } from "../settings/settings"; import { ReactionsReader } from "../reactions/ReactionsReader"; +import { DeafenReader } from "../reactions/DeafenReader"; +import { + deafened$ as localDeafened$, + toggleDeafen, +} from "../state/DeafenModel"; import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx"; import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; import { useMediaDevices } from "../MediaDevicesContext.ts"; @@ -133,6 +139,7 @@ export const ActiveCall: FC = (props) => { logger.info("START CALL VIEW SCOPE"); const scope = new ObservableScope(); const reactionsReader = new ReactionsReader(scope, props.rtcSession); + const deafenReader = new DeafenReader(scope, props.rtcSession); const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = urlParams; const vm = createCallViewModel$( @@ -149,6 +156,7 @@ export const ActiveCall: FC = (props) => { }, reactionsReader.raisedHands$, reactionsReader.reactions$, + deafenReader.deafened$, scope.behavior(trackProcessorState$), ); // TODO move this somewhere else once we use the callViewModel in the lobby as well! @@ -203,8 +211,12 @@ export const InCallView: FC = ({ onShareClick, }) => { const { t } = useTranslation(); - const { supportsReactions, sendReaction, toggleRaisedHand } = - useReactionsSender(); + const { + supportsReactions, + sendReaction, + toggleRaisedHand, + toggleDeafened: broadcastToggleDeafened, + } = useReactionsSender(); useWakeLock(); // TODO-MULTI-SFU This is unused now?? @@ -248,15 +260,36 @@ export const InCallView: FC = ({ const toggleVideo = useBehavior(muteStates.video.toggle$); const setAudioEnabled = useBehavior(muteStates.audio.setEnabled$); + const deafened = useBehavior(localDeafened$); + const prevAudioEnabledRef = useRef(audioEnabled); + + const handleToggleDeafen = useCallback(() => { + if (!deafened) { + // About to deafen: save current mic state, then mute mic + prevAudioEnabledRef.current = audioEnabled; + if (audioEnabled) { + setAudioEnabled?.(false); + } + } else { + // About to un-deafen: restore previous mic state + if (prevAudioEnabledRef.current) { + setAudioEnabled?.(true); + } + } + toggleDeafen(); + void broadcastToggleDeafened(); + }, [deafened, audioEnabled, setAudioEnabled, broadcastToggleDeafened]); + // This function incorrectly assumes that there is a camera and microphone, which is not always the case. // TODO: Make sure that this module is resilient when it comes to camera/microphone availability! useCallViewKeyboardShortcuts( containerRef1, - toggleAudio, + deafened ? null : toggleAudio, toggleVideo, - setAudioEnabled, + deafened ? null : setAudioEnabled, (reaction) => void sendReaction(reaction), () => void toggleRaisedHand(), + handleToggleDeafen, ); const audioParticipants = useBehavior(vm.livekitRoomItems$); @@ -665,12 +698,19 @@ export const InCallView: FC = ({ buttons.push( , + , rendering > renders 1`] = ` /> +