From 1d3b9774f010ef57a8bb40bb0ebb2cfa949b7ba7 Mon Sep 17 00:00:00 2001 From: tototomate123 Date: Sun, 19 Apr 2026 19:03:19 +0200 Subject: [PATCH] Add experimental microphone denoise --- locales/de/app.json | 2 + locales/en/app.json | 2 + package.json | 5 +- src/livekit/MicrophoneDenoiseController.ts | 208 ++++++++++++++++++ src/settings/PreferencesSettingsTab.tsx | 17 ++ src/settings/settings.ts | 5 + src/state/CallViewModel/CallViewModel.ts | 2 + .../localMember/Publisher.test.ts | 162 ++++++++++++++ .../CallViewModel/localMember/Publisher.ts | 137 ++++++++++++ yarn.lock | 8 + 10 files changed, 547 insertions(+), 1 deletion(-) create mode 100644 src/livekit/MicrophoneDenoiseController.ts diff --git a/locales/de/app.json b/locales/de/app.json index 95cc49a59a..2db12b90d1 100644 --- a/locales/de/app.json +++ b/locales/de/app.json @@ -196,6 +196,8 @@ "developer_mode_label": "Entwickler-Modus", "developer_mode_label_description": "Aktivieren Sie den Entwicklermodus und zeigen Sie die Registerkarte mit den Entwicklereinstellungen an.", "introduction": "Hier können zusätzliche Optionen für individuelle Anforderungen eingestellt werden.", + "experimental_microphone_denoise_description": "Reduziert Hintergrundgeräusche vom Mikrofon.", + "experimental_microphone_denoise_label": "Geräuschunterdrückung", "reactions_play_sound_description": "Spielen Sie einen Soundeffekt ab, wenn jemand eine Reaktion auf einen Anruf sendet.", "reactions_play_sound_label": "Reaktionstöne abspielen", "reactions_show_description": "Zeige eine Animation, wenn jemand eine Reaktion sendet.", diff --git a/locales/en/app.json b/locales/en/app.json index 5398930f2b..5119c79dd5 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -220,6 +220,8 @@ "developer_mode_label": "Developer mode", "developer_mode_label_description": "Enable developer mode and show developer settings tab.", "introduction": "Here you can configure extra options for an improved experience.", + "experimental_microphone_denoise_description": "Reduce background noise from your microphone.", + "experimental_microphone_denoise_label": "Background noise reduction", "reactions_play_sound_description": "Play a sound effect when anyone sends a reaction into a call.", "reactions_play_sound_label": "Play reaction sounds", "reactions_show_description": "Show an animation when anyone sends a reaction.", diff --git a/package.json b/package.json index e4abd723e3..ffc5548309 100644 --- a/package.json +++ b/package.json @@ -151,5 +151,8 @@ "qs": "^6.14.1", "js-yaml": "^4.1.1" }, - "packageManager": "yarn@4.7.0" + "packageManager": "yarn@4.7.0", + "dependencies": { + "@sapphi-red/web-noise-suppressor": "^0.3.5" + } } diff --git a/src/livekit/MicrophoneDenoiseController.ts b/src/livekit/MicrophoneDenoiseController.ts new file mode 100644 index 0000000000..12001e32b5 --- /dev/null +++ b/src/livekit/MicrophoneDenoiseController.ts @@ -0,0 +1,208 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + loadRnnoise, + RnnoiseWorkletNode, +} from "@sapphi-red/web-noise-suppressor"; +import rnnoiseWorkletPath from "@sapphi-red/web-noise-suppressor/rnnoiseWorklet.js?url"; +import rnnoiseWasmPath from "@sapphi-red/web-noise-suppressor/rnnoise.wasm?url"; +import rnnoiseSimdWasmPath from "@sapphi-red/web-noise-suppressor/rnnoise_simd.wasm?url"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; + +const logger = rootLogger.getChild("[MicrophoneDenoiseController]"); + +export interface MicrophoneTrackProcessor { + connect( + input: MediaStreamAudioSourceNode, + destination: MediaStreamAudioDestinationNode, + ): void; + disconnect(): void; + destroy?(): void; +} + +export interface MicrophoneTrackProcessorFactory { + create(context: AudioContext): Promise; +} + +let rnnoiseWasmBinaryPromise: Promise | undefined; +const rnnoiseWorkletModules = new WeakMap>(); + +async function loadRnnoiseBinary(): Promise { + rnnoiseWasmBinaryPromise ??= loadRnnoise({ + url: rnnoiseWasmPath, + simdUrl: rnnoiseSimdWasmPath, + }); + return rnnoiseWasmBinaryPromise; +} + +async function ensureRnnoiseWorklet(context: AudioContext): Promise { + let modulePromise = rnnoiseWorkletModules.get(context); + if (modulePromise === undefined) { + modulePromise = context.audioWorklet.addModule(rnnoiseWorkletPath); + rnnoiseWorkletModules.set(context, modulePromise); + } + return modulePromise; +} + +class RnnoiseMicrophoneTrackProcessor implements MicrophoneTrackProcessor { + private readonly preFilter: BiquadFilterNode; + + public constructor( + context: AudioContext, + private readonly rnnoiseNode: RnnoiseWorkletNode, + ) { + // Trim subsonic rumble before RNNoise sees the signal. + this.preFilter = context.createBiquadFilter(); + this.preFilter.type = "highpass"; + this.preFilter.frequency.value = 80; + this.preFilter.Q.value = 0.7; + } + + public connect( + input: MediaStreamAudioSourceNode, + destination: MediaStreamAudioDestinationNode, + ): void { + input.connect(this.preFilter); + this.preFilter.connect(this.rnnoiseNode); + this.rnnoiseNode.connect(destination); + } + + public disconnect(): void { + this.preFilter.disconnect(); + this.rnnoiseNode.disconnect(); + } + + public destroy(): void { + this.rnnoiseNode.destroy(); + } +} + +export class RnnoiseMicrophoneTrackProcessorFactory implements MicrophoneTrackProcessorFactory { + public async create( + context: AudioContext, + ): Promise { + if ( + context.audioWorklet === undefined || + typeof AudioWorkletNode === "undefined" + ) { + throw new Error("AudioWorklet is not available"); + } + + const wasmBinary = await loadRnnoiseBinary(); + await ensureRnnoiseWorklet(context); + + const rnnoiseNode = new RnnoiseWorkletNode(context, { + wasmBinary, + maxChannels: 1, + }); + + return new RnnoiseMicrophoneTrackProcessor(context, rnnoiseNode); + } +} + +interface ActivePipeline { + readonly sourceTrack: MediaStreamTrack; + readonly processedTrack: MediaStreamTrack; + readonly context: AudioContext; + readonly sourceNode: MediaStreamAudioSourceNode; + readonly destinationNode: MediaStreamAudioDestinationNode; + readonly processor: MicrophoneTrackProcessor; +} + +/** + * Experimental client-side microphone processing hook. + * + * When enabled, we wrap the captured mic track in a Web Audio + * graph and hand LiveKit a processed output track instead of the raw one. + */ +export class MicrophoneDenoiseController { + private activePipeline?: ActivePipeline; + + public constructor( + private readonly processorFactory: MicrophoneTrackProcessorFactory = new RnnoiseMicrophoneTrackProcessorFactory(), + ) {} + + public get sourceTrack(): MediaStreamTrack | undefined { + return this.activePipeline?.sourceTrack; + } + + public get processedTrack(): MediaStreamTrack | undefined { + return this.activePipeline?.processedTrack; + } + + public async rebuild( + sourceTrack: MediaStreamTrack, + ): Promise { + this.destroy(); + + if (typeof AudioContext === "undefined") { + throw new Error("AudioContext is not available"); + } + + const context = new AudioContext({ + latencyHint: "interactive", + sampleRate: 48000, + }); + + try { + const stream = new MediaStream([sourceTrack]); + const sourceNode = context.createMediaStreamSource(stream); + const destinationNode = context.createMediaStreamDestination(); + const processor = await this.processorFactory.create(context); + + processor.connect(sourceNode, destinationNode); + + const processedTrack = destinationNode.stream.getAudioTracks()[0]; + if (processedTrack === undefined) { + throw new Error( + "Processed microphone stream did not expose an audio track", + ); + } + + if (context.state === "suspended") { + await context.resume(); + } + + this.activePipeline = { + sourceTrack, + processedTrack, + context, + sourceNode, + destinationNode, + processor, + }; + + return processedTrack; + } catch (error) { + try { + await context.close(); + } catch (closeError) { + logger.warn( + "Failed to close microphone denoise audio context", + closeError, + ); + } + throw error; + } + } + + public destroy(): void { + const pipeline = this.activePipeline; + if (pipeline === undefined) return; + + this.activePipeline = undefined; + pipeline.processor.disconnect(); + pipeline.processor.destroy?.(); + pipeline.sourceNode.disconnect(); + pipeline.destinationNode.disconnect(); + pipeline.processedTrack.stop(); + void pipeline.context.close().catch((error) => { + logger.warn("Failed to close microphone denoise audio context", error); + }); + } +} diff --git a/src/settings/PreferencesSettingsTab.tsx b/src/settings/PreferencesSettingsTab.tsx index 82306e7b7c..2a17458a12 100644 --- a/src/settings/PreferencesSettingsTab.tsx +++ b/src/settings/PreferencesSettingsTab.tsx @@ -11,6 +11,7 @@ import { Text } from "@vector-im/compound-web"; import { FieldRow, InputField } from "../input/Input"; import { + experimentalMicrophoneDenoise as experimentalMicrophoneDenoiseSetting, showHandRaisedTimer as showHandRaisedTimerSetting, showReactions as showReactionsSetting, playReactionsSound as playReactionsSoundSetting, @@ -29,6 +30,8 @@ export const PreferencesSettingsTab: FC = () => { const [playReactionsSound, setPlayReactionSound] = useSetting( playReactionsSoundSetting, ); + const [experimentalMicrophoneDenoise, setExperimentalMicrophoneDenoise] = + useSetting(experimentalMicrophoneDenoiseSetting); const onChangeSetting = ( e: ChangeEvent, @@ -76,6 +79,20 @@ export const PreferencesSettingsTab: FC = () => { onChange={(e) => onChangeSetting(e, setPlayReactionSound)} /> + + onChangeSetting(e, setExperimentalMicrophoneDenoise)} + /> + ( export const backgroundBlur = new Setting("background-blur", false); +export const experimentalMicrophoneDenoise = new Setting( + "experimental-microphone-denoise", + false, +); + export const showHandRaisedTimer = new Setting( "hand-raised-show-timer", false, diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index e298bcfdb7..e2e4204309 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -59,6 +59,7 @@ import { } from "../../utils/observable"; import { duplicateTiles, + experimentalMicrophoneDenoise, MatrixRTCMode, playReactionsSound, showReactions, @@ -552,6 +553,7 @@ export function createCallViewModel$( mediaDevices, muteStates, trackProcessorState$, + experimentalMicrophoneDenoise.value$, logger.getChild( "[Publisher " + connection.transport.livekit_service_url + "]", ), diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index e54e706ff9..7305dd6229 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -28,19 +28,57 @@ import { Publisher } from "./Publisher"; import { type Connection } from "../remoteMembers/Connection"; import { type MuteStates } from "../../MuteStates"; +const { rnnoiseNodeMocks, loadRnnoiseMock } = vi.hoisted(() => ({ + rnnoiseNodeMocks: [] as { + connect: ReturnType; + disconnect: ReturnType; + destroy: ReturnType; + }[], + loadRnnoiseMock: vi.fn(), +})); + +vi.mock("@sapphi-red/web-noise-suppressor", () => ({ + loadRnnoise: loadRnnoiseMock, + RnnoiseWorkletNode: class MockRnnoiseWorkletNode { + public connect = vi.fn(); + public disconnect = vi.fn(); + public destroy = vi.fn(); + + public constructor() { + rnnoiseNodeMocks.push({ + connect: this.connect, + disconnect: this.disconnect, + destroy: this.destroy, + }); + } + }, +})); + let scope: ObservableScope; beforeEach(() => { scope = new ObservableScope(); + rnnoiseNodeMocks.length = 0; + loadRnnoiseMock.mockReset(); + loadRnnoiseMock.mockResolvedValue(new ArrayBuffer(8)); }); afterEach(() => scope.end()); +afterEach(() => vi.unstubAllGlobals()); function createMockLocalTrack(source: Track.Source): LocalTrack { + const mediaStreamTrack = { + kind: source === Track.Source.Microphone ? "audio" : "video", + readyState: "live", + stop: vi.fn(), + } as Partial as MediaStreamTrack; const track = { source, isMuted: false, isUpstreamPaused: false, + mediaStreamTrack, + on: vi.fn(), + off: vi.fn(), } as Partial as LocalTrack; vi.mocked(track).mute = vi.fn().mockImplementation(() => { @@ -57,6 +95,11 @@ function createMockLocalTrack(source: Track.Source): LocalTrack { // @ts-expect-error - for that test we want to set isUpstreamPaused directly track.isUpstreamPaused = false; }); + ( + track as LocalTrack & { + replaceTrack: (track: MediaStreamTrack) => Promise; + } + ).replaceTrack = vi.fn().mockResolvedValue(undefined); return track; } @@ -154,6 +197,8 @@ beforeEach(() => { const pub = { track, source: track.source, + audioTrack: + track.source === Track.Source.Microphone ? track : undefined, mute: track.mute, unmute: track.unmute, } as Partial as LocalTrackPublication; @@ -188,6 +233,7 @@ describe("Publisher", () => { mockMediaDevices({}), muteStates, constant({ supported: false, processor: undefined }), + constant(false), logger, ); }); @@ -297,6 +343,120 @@ describe("Publisher", () => { expect(track!.isUpstreamPaused).toBe(false); }); + it("replaces the published microphone track when experimental denoise is enabled", async () => { + const processedTrack = { + kind: "audio", + readyState: "live", + stop: vi.fn(), + } as Partial as MediaStreamTrack; + + const sourceNode = { connect: vi.fn(), disconnect: vi.fn() }; + const preFilterNode = { + connect: vi.fn(), + disconnect: vi.fn(), + frequency: { value: 0 }, + Q: { value: 0 }, + type: "lowpass", + }; + const destinationNode = { + disconnect: vi.fn(), + stream: { + getAudioTracks: (): MediaStreamTrack[] => [processedTrack], + }, + }; + const audioContext = { + state: "running", + createMediaStreamSource: vi.fn().mockReturnValue(sourceNode), + createMediaStreamDestination: vi.fn().mockReturnValue(destinationNode), + createBiquadFilter: vi.fn().mockReturnValue(preFilterNode), + resume: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined), + audioWorklet: { + addModule: vi.fn().mockResolvedValue(undefined), + }, + }; + + class MockAudioContext { + public state = audioContext.state; + public createMediaStreamSource = audioContext.createMediaStreamSource; + public createMediaStreamDestination = + audioContext.createMediaStreamDestination; + public createBiquadFilter = audioContext.createBiquadFilter; + public resume = audioContext.resume; + public close = audioContext.close; + public audioWorklet = audioContext.audioWorklet; + } + + vi.stubGlobal("AudioContext", MockAudioContext); + vi.stubGlobal("AudioWorkletNode", class MockAudioWorkletNode {}); + vi.stubGlobal( + "MediaStream", + vi.fn(function MediaStream(this: { tracks: MediaStreamTrack[] }, tracks) { + this.tracks = tracks; + }), + ); + vi.stubGlobal("MediaStreamTrack", class MockMediaStreamTrack {}); + + const enabledPublisher = new Publisher( + connection, + mockMediaDevices({}), + muteStates, + constant({ supported: false, processor: undefined }), + constant(true), + logger, + ); + + audioEnabled$.next(true); + await enabledPublisher.createAndSetupTracks(); + await flushPromises(); + + const microphoneTrack = localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.audioTrack as unknown as LocalTrack & { + replaceTrack: ReturnType; + }; + + expect(microphoneTrack.replaceTrack).toHaveBeenCalledWith( + processedTrack, + true, + ); + expect(loadRnnoiseMock).toHaveBeenCalledOnce(); + expect(audioContext.audioWorklet.addModule).toHaveBeenCalledOnce(); + expect(rnnoiseNodeMocks).toHaveLength(1); + + await enabledPublisher.destroy(); + vi.unstubAllGlobals(); + }); + + it("falls back to the raw microphone track when denoise setup is unavailable", async () => { + vi.stubGlobal("AudioContext", undefined); + vi.stubGlobal("MediaStreamTrack", class MockMediaStreamTrack {}); + + const enabledPublisher = new Publisher( + connection, + mockMediaDevices({}), + muteStates, + constant({ supported: false, processor: undefined }), + constant(true), + logger, + ); + + audioEnabled$.next(true); + await enabledPublisher.createAndSetupTracks(); + await flushPromises(); + + const microphoneTrack = localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.audioTrack as unknown as LocalTrack & { + replaceTrack: ReturnType; + }; + + expect(microphoneTrack.replaceTrack).not.toHaveBeenCalled(); + + await enabledPublisher.destroy(); + vi.unstubAllGlobals(); + }); + describe("Mute states", () => { let publisher: Publisher; beforeEach(() => { @@ -305,6 +465,7 @@ describe("Publisher", () => { mockMediaDevices({}), muteStates, constant({ supported: false, processor: undefined }), + constant(false), logger, ); }); @@ -360,6 +521,7 @@ describe("Bug fix", () => { mockMediaDevices({}), muteStates, constant({ supported: false, processor: undefined }), + constant(false), logger, ); audioEnabled$.next(true); diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index b7841c498b..a9e4743d0a 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -7,13 +7,16 @@ Please see LICENSE in the repository root for full details. */ import { ConnectionState as LivekitConnectionState, + type LocalAudioTrack, type LocalTrackPublication, LocalVideoTrack, ParticipantEvent, type Room as LivekitRoom, Track, + TrackEvent, } from "livekit-client"; import { + distinctUntilChanged, map, NEVER, type Observable, @@ -29,11 +32,16 @@ import { type ProcessorState, trackProcessorSync, } from "../../../livekit/TrackProcessorContext.tsx"; +import { MicrophoneDenoiseController } from "../../../livekit/MicrophoneDenoiseController.ts"; import { getUrlParams } from "../../../UrlParams.ts"; import { observeTrackReference$ } from "../../observeTrackReference"; import { type Connection } from "../remoteMembers/Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; +interface ReplaceableLocalAudioTrack extends LocalAudioTrack { + mediaStreamTrack: MediaStreamTrack; +} + /** * A wrapper for a Connection object. * This wrapper will manage the connection used to publish to the LiveKit room. @@ -48,6 +56,11 @@ export class Publisher { public shouldPublish = false; private readonly scope = new ObservableScope(); + private readonly microphoneDenoiseController = + new MicrophoneDenoiseController(); + private watchedMicrophoneTrack?: LocalAudioTrack; + private syncingMicrophoneDenoise?: Promise; + private pendingMicrophoneDenoiseResync = false; /** * Creates a new Publisher. @@ -62,6 +75,7 @@ export class Publisher { devices: MediaDevices, private readonly muteStates: MuteStates, trackerProcessorState$: Behavior, + private readonly experimentalMicrophoneDenoise$: Behavior, private logger: Logger, ) { const { controlledAudioDevices } = getUrlParams(); @@ -75,6 +89,7 @@ export class Publisher { this.observeTrackProcessors(this.scope, room, trackerProcessorState$); // Observe media device changes and update LiveKit active devices accordingly this.observeMediaDevices(this.scope, devices, controlledAudioDevices); + this.observeExperimentalMicrophoneDenoise(this.scope); this.workaroundRestartAudioInputTrackChrome(devices, this.scope); @@ -86,6 +101,8 @@ export class Publisher { public async destroy(): Promise { this.scope.end(); + this.unwatchMicrophoneTrack(); + this.microphoneDenoiseController.destroy(); this.logger.info("Scope ended -> unset handler"); this.muteStates.audio.unsetHandler(); this.muteStates.video.unsetHandler(); @@ -118,6 +135,8 @@ export class Publisher { } // also check the mute state and apply it if (localTrackPublication.source === Track.Source.Microphone) { + this.watchMicrophoneTrack(localTrackPublication); + void this.syncMicrophoneDenoisePipeline(); const enabled = this.muteStates.audio.enabled$.value; lkRoom.localParticipant.setMicrophoneEnabled(enabled).catch((e) => { this.logger.error( @@ -256,6 +275,8 @@ export class Publisher { } public async stopTracks(): Promise { + this.unwatchMicrophoneTrack(); + this.microphoneDenoiseController.destroy(); const lkRoom = this.connection.livekitRoom; for (const source of [ Track.Source.Microphone, @@ -314,6 +335,7 @@ export class Publisher { lkRoom.localParticipant .getTrackPublication(Track.Source.Microphone) ?.audioTrack?.restartTrack() + .then(async () => this.syncMicrophoneDenoisePipeline()) .catch((e) => { this.logger.error(`Failed to restart audio device track`, e); }); @@ -374,6 +396,9 @@ export class Publisher { `handler: Setting LiveKit microphone enabled: ${enable}`, ); await lkRoom.localParticipant.setMicrophoneEnabled(enable); + if (enable) { + await this.syncMicrophoneDenoisePipeline(); + } // Unmute will restart the track if it was paused upstream, // but until explicitly requested, we want to keep it paused. if (!this.shouldPublish && enable) { @@ -416,4 +441,116 @@ export class Publisher { ); trackProcessorSync(scope, track$, trackerProcessorState$); } + + private observeExperimentalMicrophoneDenoise(scope: ObservableScope): void { + this.experimentalMicrophoneDenoise$ + .pipe(distinctUntilChanged(), scope.bind()) + .subscribe(() => { + void this.syncMicrophoneDenoisePipeline(); + }); + } + + private watchMicrophoneTrack( + localTrackPublication: LocalTrackPublication, + ): void { + this.unwatchMicrophoneTrack(); + const audioTrack = localTrackPublication.audioTrack; + if (audioTrack === undefined) return; + + this.watchedMicrophoneTrack = audioTrack; + audioTrack.on(TrackEvent.Restarted, this.onMicrophoneTrackRestarted); + } + + private async syncMicrophoneDenoisePipeline(): Promise { + if (this.syncingMicrophoneDenoise !== undefined) { + this.pendingMicrophoneDenoiseResync = true; + return this.syncingMicrophoneDenoise; + } + + const sync = this.syncMicrophoneDenoisePipelineInner().finally(() => { + if (this.syncingMicrophoneDenoise === sync) { + this.syncingMicrophoneDenoise = undefined; + } + if (this.pendingMicrophoneDenoiseResync) { + this.pendingMicrophoneDenoiseResync = false; + void this.syncMicrophoneDenoisePipeline(); + } + }); + this.syncingMicrophoneDenoise = sync; + return sync; + } + + private async syncMicrophoneDenoisePipelineInner(): Promise { + // LiveKit still owns microphone capture, including the original + // getUserMedia constraints. We only swap the published track after capture. + const audioTrack = + this.connection.livekitRoom.localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.audioTrack as ReplaceableLocalAudioTrack | undefined; + + if (audioTrack === undefined) { + this.microphoneDenoiseController.destroy(); + return; + } + + if ( + this.experimentalMicrophoneDenoise$.value === false || + audioTrack.mediaStreamTrack.readyState === "ended" + ) { + await this.restoreRawMicrophoneTrack(audioTrack); + return; + } + + const currentRawTrack = audioTrack.mediaStreamTrack; + if ( + this.microphoneDenoiseController.sourceTrack === currentRawTrack && + this.microphoneDenoiseController.processedTrack !== undefined + ) { + return; + } + + try { + const processedTrack = + await this.microphoneDenoiseController.rebuild(currentRawTrack); + await audioTrack.replaceTrack(processedTrack, true); + this.logger.info( + "Experimental microphone denoise pipeline enabled with RNNoise", + ); + } catch (error) { + this.logger.warn( + "Failed to enable experimental microphone denoise pipeline, falling back to raw microphone track", + error, + ); + this.microphoneDenoiseController.destroy(); + } + } + + private async restoreRawMicrophoneTrack( + audioTrack: ReplaceableLocalAudioTrack, + ): Promise { + const rawTrack = this.microphoneDenoiseController.sourceTrack; + if (rawTrack !== undefined) { + try { + await audioTrack.replaceTrack(rawTrack, true); + } catch (error) { + this.logger.warn( + "Failed to restore raw microphone track after denoise pipeline teardown", + error, + ); + } + } + this.microphoneDenoiseController.destroy(); + } + + private readonly onMicrophoneTrackRestarted = (): void => { + void this.syncMicrophoneDenoisePipeline(); + }; + + private unwatchMicrophoneTrack(): void { + this.watchedMicrophoneTrack?.off( + TrackEvent.Restarted, + this.onMicrophoneTrackRestarted, + ); + this.watchedMicrophoneTrack = undefined; + } } diff --git a/yarn.lock b/yarn.lock index 484f052ab2..204231487f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5220,6 +5220,13 @@ __metadata: languageName: node linkType: hard +"@sapphi-red/web-noise-suppressor@npm:^0.3.5": + version: 0.3.5 + resolution: "@sapphi-red/web-noise-suppressor@npm:0.3.5" + checksum: 10c0/b8959a3bfb4acd899866a02eec0588e6f06dc49461fdc48810e9bf9992aeda262d06c3f8e22d41d33d5eb52710931cf3ad8b8350286efc35154419dc3437b879 + languageName: node + linkType: hard + "@sentry-internal/browser-utils@npm:8.55.1": version: 8.55.1 resolution: "@sentry-internal/browser-utils@npm:8.55.1" @@ -8634,6 +8641,7 @@ __metadata: "@radix-ui/react-slider": "npm:^1.1.2" "@radix-ui/react-visually-hidden": "npm:^1.0.3" "@react-spring/web": "npm:^10.0.0" + "@sapphi-red/web-noise-suppressor": "npm:^0.3.5" "@sentry/react": "npm:^8.0.0" "@sentry/vite-plugin": "npm:^3.0.0" "@storybook/addon-docs": "npm:^10.3.3"