diff --git a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/IWebRTCClient.java b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/IWebRTCClient.java index 0139d26d..14a4814f 100644 --- a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/IWebRTCClient.java +++ b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/IWebRTCClient.java @@ -1,5 +1,7 @@ package io.antmedia.webrtcandroidframework.api; +import android.media.projection.MediaProjection; + import org.json.JSONArray; import org.json.JSONObject; import org.webrtc.DataChannel; @@ -331,4 +333,20 @@ void publish(String streamId, String token, boolean videoCallEnabled, boolean au * Returns true if SDK resources are released and its shutdown, false otherwise. */ boolean isShutdown(); + + /** + * Send system audio on screen share during call. + */ + void switchToSystemAudioRecordingOnScreenShareDuringCall(); + + /** + * Send microphone audio on screen share during call. + */ + void switchToMicrophoneAudioRecordingOnScreenShareDuringCall(); + + /** + * Set audio device module media projection. + * If null passed, audio device module will use microphone audio on screen share. + */ + void setAudioDeviceModuleMediaProjection(MediaProjection mediaProjection); } diff --git a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/WebRTCClientBuilder.java b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/WebRTCClientBuilder.java index c022b3a8..0cc66b5e 100644 --- a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/WebRTCClientBuilder.java +++ b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/WebRTCClientBuilder.java @@ -8,6 +8,7 @@ import java.util.Arrays; import io.antmedia.webrtcandroidframework.core.WebRTCClient; +import io.antmedia.webrtcandroidframework.core.model.ScreenShareAudioSource; public class WebRTCClientBuilder { @@ -156,4 +157,9 @@ public WebRTCClientBuilder setBluetoothEnabled(boolean bluetoothEnabled) { webRTCClientConfig.bluetoothEnabled = bluetoothEnabled; return this; } + + public WebRTCClientBuilder setScreenShareAudioSource(ScreenShareAudioSource screenShareAudioSource) { + webRTCClientConfig.screenShareAudioSource = screenShareAudioSource; + return this; + } } diff --git a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/WebRTCClientConfig.java b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/WebRTCClientConfig.java index 531f6abb..33938a0b 100644 --- a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/WebRTCClientConfig.java +++ b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/api/WebRTCClientConfig.java @@ -9,6 +9,8 @@ import java.util.ArrayList; +import io.antmedia.webrtcandroidframework.core.model.ScreenShareAudioSource; + public class WebRTCClientConfig { @@ -183,4 +185,10 @@ public class WebRTCClientConfig { * Flag for connecting bluetooth headphones. */ public boolean bluetoothEnabled = false; + + /* + * Audio source during screen share. Possible values are MICROPHONE or SYSTEM + * MICROPHONE by default. + */ + public ScreenShareAudioSource screenShareAudioSource = ScreenShareAudioSource.MICROPHONE; } diff --git a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/CustomMediaProjectionCallback.java b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/CustomMediaProjectionCallback.java new file mode 100644 index 00000000..74a022c7 --- /dev/null +++ b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/CustomMediaProjectionCallback.java @@ -0,0 +1,30 @@ +package io.antmedia.webrtcandroidframework.core; + +import android.media.projection.MediaProjection; +import android.util.Log; + +public abstract class CustomMediaProjectionCallback extends MediaProjection.Callback { + + private static final String TAG = "CustomMediaProjectionCallback"; + + public CustomMediaProjectionCallback() { + super(); + } + + public abstract void onMediaProjection(MediaProjection mediaProjection); + + @Override + public void onStop() { + super.onStop(); + } + + @Override + public void onCapturedContentResize(int width, int height) { + super.onCapturedContentResize(width, height); + } + + @Override + public void onCapturedContentVisibilityChanged(boolean isVisible) { + super.onCapturedContentVisibilityChanged(isVisible); + } +} \ No newline at end of file diff --git a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/WebRTCClient.java b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/WebRTCClient.java index 4da274cf..437707cc 100644 --- a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/WebRTCClient.java +++ b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/WebRTCClient.java @@ -17,13 +17,14 @@ import android.util.Log; import android.view.WindowManager; import android.widget.Toast; - import androidx.annotation.NonNull; import org.json.JSONArray; import org.json.JSONObject; import org.webrtc.AddIceObserver; import org.webrtc.AudioSource; +import io.antmedia.webrtcandroidframework.core.model.ScreenShareAudioSource; + import org.webrtc.AudioTrack; import org.webrtc.Camera2Enumerator; import org.webrtc.CameraEnumerator; @@ -836,7 +837,15 @@ public DisplayMetrics getDisplayMetrics() { } public @Nullable VideoCapturer createScreenCapturer() { - return new ScreenCapturerAndroid(config.mediaProjectionIntent, new MediaProjection.Callback() { + return new ScreenCapturerAndroid(config.mediaProjectionIntent, new CustomMediaProjectionCallback() { + @Override + public void onMediaProjection(MediaProjection mediaProjection) { + config.mediaProjection = mediaProjection; + if(adm != null && config.screenShareAudioSource == ScreenShareAudioSource.SYSTEM){ + adm.setMediaProjection(mediaProjection); + } + } + @Override public void onStop() { reportError(getPublishStreamId(), USER_REVOKED_CAPTURE_SCREEN_PERMISSION); @@ -974,7 +983,7 @@ private void publishPlayIfRequested() { public void publish(String streamId) { publish(streamId, null, true, true, - null, null, streamId, "qdadsas"); + null, null, streamId, null); } @@ -1230,7 +1239,7 @@ public void reportError(String streamId, final String description) { public void changeVideoSource(StreamSource newSource) { if (!config.videoSource.equals(newSource)) { - if (newSource.equals(StreamSource.SCREEN) && adm != null) { + if (newSource.equals(StreamSource.SCREEN) && adm != null && config.screenShareAudioSource == ScreenShareAudioSource.SYSTEM) { adm.setMediaProjection(config.mediaProjection); } @@ -2778,4 +2787,66 @@ public boolean isShutdown() { return released; } + @androidx.annotation.Nullable + public AudioDeviceModule getAdm() { + return adm; + } + + + public void createAudioRecord(){ + if(adm != null){ + adm.createAudioRecord(); + } + } + + public void switchToSystemAudioRecordingOnScreenShareDuringCall(){ + if(config.mediaProjection == null){ + Log.i(TAG,"Config media projection is null. Cannot switch system audio on screen share."); + return; + } + if(adm == null){ + return; + } + + stopAdmRecording(); + adm.setMediaProjection(config.mediaProjection); + createAudioRecord(); + startRecording(); + } + + public void switchToMicrophoneAudioRecordingOnScreenShareDuringCall(){ + if(adm == null){ + return; + } + + stopAdmRecording(); + //if media projection is null, microphone will be used to record audio. + adm.setMediaProjection(null); + //media projection is set to null thus it will capture microphone. + createAudioRecord(); + startRecording(); + } + + public void startRecording(){ + if(adm != null){ + adm.startRecording(); + } + } + + public void stopAdmRecording(){ + if(adm != null){ + adm.stopRecording(); + } + } + + public void setAudioDeviceModuleMediaProjection(MediaProjection mediaProjection){ + if(adm != null){ + adm.setMediaProjection(mediaProjection); + } + } + + public void setAdm(JavaAudioDeviceModule adm){ + this.adm = adm; + } + } \ No newline at end of file diff --git a/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/model/ScreenShareAudioSource.java b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/model/ScreenShareAudioSource.java new file mode 100644 index 00000000..69522c40 --- /dev/null +++ b/webrtc-android-framework/src/main/java/io/antmedia/webrtcandroidframework/core/model/ScreenShareAudioSource.java @@ -0,0 +1,6 @@ +package io.antmedia.webrtcandroidframework.core.model; + +public enum ScreenShareAudioSource { + MICROPHONE, + SYSTEM; +} diff --git a/webrtc-android-framework/src/main/java/org/webrtc/ScreenCapturerAndroid.java b/webrtc-android-framework/src/main/java/org/webrtc/ScreenCapturerAndroid.java index b18e8e4a..ec78f9bc 100644 --- a/webrtc-android-framework/src/main/java/org/webrtc/ScreenCapturerAndroid.java +++ b/webrtc-android-framework/src/main/java/org/webrtc/ScreenCapturerAndroid.java @@ -24,6 +24,8 @@ import android.os.Looper; import android.os.Handler; +import io.antmedia.webrtcandroidframework.core.CustomMediaProjectionCallback; + /** * An copy of ScreenCapturerAndroid to capture the screen content while being aware of device orientation */ @@ -127,6 +129,10 @@ public synchronized void startCapture( mediaProjection = mediaProjectionManager.getMediaProjection( Activity.RESULT_OK, mediaProjectionPermissionResultData); + if(mediaProjectionCallback != null){ + ((CustomMediaProjectionCallback) mediaProjectionCallback).onMediaProjection(mediaProjection); + } + // Let MediaProjection callback use the SurfaceTextureHelper thread. mediaProjection.registerCallback(mediaProjectionCallback, surfaceTextureHelper.getHandler()); @@ -251,10 +257,6 @@ public MediaProjection getMediaProjection() { return mediaProjection; } - public void setMediaProjection(MediaProjection mediaProjection) { - this.mediaProjection = mediaProjection; - } - public MediaProjectionManager getMediaProjectionManager() { return mediaProjectionManager; } diff --git a/webrtc-android-framework/src/main/java/org/webrtc/audio/AudioDeviceModule.java b/webrtc-android-framework/src/main/java/org/webrtc/audio/AudioDeviceModule.java index 4f4687bc..088b66e2 100644 --- a/webrtc-android-framework/src/main/java/org/webrtc/audio/AudioDeviceModule.java +++ b/webrtc-android-framework/src/main/java/org/webrtc/audio/AudioDeviceModule.java @@ -41,4 +41,10 @@ public interface AudioDeviceModule { /** Set media projection for the audio record. */ void setMediaProjection(MediaProjection mediaProjection); + void createAudioRecord(); + + void startRecording(); + + void stopRecording(); + } diff --git a/webrtc-android-framework/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java b/webrtc-android-framework/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java index 58546b1f..e8340028 100644 --- a/webrtc-android-framework/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java +++ b/webrtc-android-framework/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java @@ -14,10 +14,13 @@ import android.media.AudioAttributes; import android.media.AudioDeviceInfo; import android.media.AudioManager; +import android.media.MediaRecorder; import android.media.projection.MediaProjection; import android.os.Build; import androidx.annotation.RequiresApi; import java.util.concurrent.ScheduledExecutorService; + +import org.webrtc.AudioSource; import org.webrtc.JniCommon; import org.webrtc.Logging; import android.media.AudioRecord; @@ -406,6 +409,7 @@ public CustomWebRtcAudioRecord getAudioInput() { return (CustomWebRtcAudioRecord)audioInput; } + @Override public long getNativeAudioDeviceModulePointer() { synchronized (nativeLock) { @@ -460,4 +464,20 @@ public void setMediaProjection(MediaProjection mediaProjection){ audioInput.setMediaProjection(mediaProjection); } + @Override + public void createAudioRecord() { + audioInput.createAudioRecord(); + } + + @Override + public void startRecording() { + audioInput.startRecording(); + } + + @Override + public void stopRecording() { + audioInput.stopRecording(); + } + + } diff --git a/webrtc-android-framework/src/main/java/org/webrtc/audio/LegacyAudioDeviceModule.java b/webrtc-android-framework/src/main/java/org/webrtc/audio/LegacyAudioDeviceModule.java index d071e48d..9a6d575f 100644 --- a/webrtc-android-framework/src/main/java/org/webrtc/audio/LegacyAudioDeviceModule.java +++ b/webrtc-android-framework/src/main/java/org/webrtc/audio/LegacyAudioDeviceModule.java @@ -47,4 +47,19 @@ public void setMicrophoneMute(boolean mute) { public void setMediaProjection(MediaProjection mediaProjection) { } + + @Override + public void createAudioRecord() { + + } + + @Override + public void startRecording() { + + } + + @Override + public void stopRecording() { + + } } diff --git a/webrtc-android-framework/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java b/webrtc-android-framework/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java index 3af5dd15..4a378c4f 100644 --- a/webrtc-android-framework/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java +++ b/webrtc-android-framework/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java @@ -25,6 +25,8 @@ import android.media.projection.MediaProjection; import android.os.Build; import android.os.Process; +import android.util.Log; + import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import java.lang.System; @@ -51,6 +53,8 @@ public class WebRtcAudioRecord { private static final String TAG = "WebRtcAudioRecordExternal"; + private static final int DELAY_BEFORE_AUDIO_START_MS = 6000; + // Requested size of each recorded buffer provided to the client. static final int CALLBACK_BUFFER_SIZE_MS = 10; @@ -66,6 +70,7 @@ public class WebRtcAudioRecord { // but the wait times out afther this amount of time. static final long AUDIO_RECORD_THREAD_JOIN_TIMEOUT_MS = 2000; + public static final int DEFAULT_AUDIO_SOURCE = AudioSource.VOICE_COMMUNICATION; // Default audio data format is PCM 16 bit per sample. @@ -108,11 +113,16 @@ public class WebRtcAudioRecord { new AtomicReference<>(); byte[] emptyBytes; - final @Nullable AudioRecordErrorCallback errorCallback; - final @Nullable AudioRecordStateCallback stateCallback; - final @Nullable SamplesReadyCallback audioSamplesReadyCallback; - final boolean isAcousticEchoCancelerSupported; - final boolean isNoiseSuppressorSupported; + private final @Nullable AudioRecordErrorCallback errorCallback; + private final @Nullable AudioRecordStateCallback stateCallback; + private final @Nullable SamplesReadyCallback audioSamplesReadyCallback; + private final boolean isAcousticEchoCancelerSupported; + private final boolean isNoiseSuppressorSupported; + private int channels; + private int sampleRate; + private int bufferSizeInBytes; + private int channelConfig; + /** * Audio thread which keeps calling ByteBuffer.read() waiting for audio @@ -280,7 +290,10 @@ boolean enableBuiltInNS(boolean enable) { } @CalledByNative - int initRecording(int sampleRate, int channels) { + private int initRecording(int sampleRate, int channels) { + this.sampleRate = sampleRate; + this.channels = channels; + Logging.d(TAG, "initRecording(sampleRate=" + sampleRate + ", channels=" + channels + ")"); if (audioRecord != null) { reportWebRtcAudioRecordInitError("InitRecording called twice without StopRecording."); @@ -303,8 +316,8 @@ int initRecording(int sampleRate, int channels) { // Get the minimum buffer size required for the successful creation of // an AudioRecord object, in byte units. // Note that this size doesn't guarantee a smooth recording under load. - final int channelConfig = channelCountToConfiguration(channels); - int minBufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat); + this.channelConfig = channelCountToConfiguration(channels); + int minBufferSize = AudioRecord.getMinBufferSize(sampleRate, this.channelConfig, audioFormat); if (minBufferSize == AudioRecord.ERROR || minBufferSize == AudioRecord.ERROR_BAD_VALUE) { reportWebRtcAudioRecordInitError("AudioRecord.getMinBufferSize failed: " + minBufferSize); return -1; @@ -315,32 +328,9 @@ int initRecording(int sampleRate, int channels) { // AudioRecord instance to ensure smooth recording under load. It has been // verified that it does not increase the actual recording latency. int bufferSizeInBytes = Math.max(BUFFER_SIZE_FACTOR * minBufferSize, byteBuffer.capacity()); + this.bufferSizeInBytes = bufferSizeInBytes; Logging.d(TAG, "bufferSizeInBytes: " + bufferSizeInBytes); - try { - if(mediaProjection != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q){ - audioRecord = createAudioRecordOnQOrHigher( - audioSource, sampleRate, channelConfig, audioFormat, bufferSizeInBytes,mediaProjection); - } - else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - // Use the AudioRecord.Builder class on Android M (23) and above. - // Throws IllegalArgumentException. - audioRecord = createAudioRecordOnMOrHigher( - audioSource, sampleRate, channelConfig, audioFormat, bufferSizeInBytes); - audioSourceMatchesRecordingSessionRef.set(null); - if (preferredDevice != null) { - setPreferredDevice(preferredDevice); - } - } else { - // Use the old AudioRecord constructor for API levels below 23. - // Throws UnsupportedOperationException. - audioRecord = createAudioRecordOnLowerThanM( - audioSource, sampleRate, channelConfig, audioFormat, bufferSizeInBytes); - audioSourceMatchesRecordingSessionRef.set(null); - } - } catch (IllegalArgumentException | UnsupportedOperationException e) { - // Report of exception message is sufficient. Example: "Cannot create AudioRecord". - reportWebRtcAudioRecordInitError(e.getMessage()); - releaseAudioResources(); + if(createAudioRecord() == -1){ return -1; } if (audioRecord == null || audioRecord.getState() != AudioRecord.STATE_INITIALIZED) { @@ -349,6 +339,7 @@ else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { return -1; } effects.enable(audioRecord.getAudioSessionId()); + logMainParameters(); logMainParametersExtended(); // Check number of active recording sessions. Should be zero but we have seen conflict cases @@ -381,31 +372,50 @@ void setPreferredDevice(@Nullable AudioDeviceInfo preferredDevice) { } } - @CalledByNative - protected boolean startRecording() { - Logging.d(TAG, "startRecording"); - assertTrue(audioRecord != null); - assertTrue(audioThread == null); + private boolean attemptStartRecording() { try { audioRecord.startRecording(); } catch (IllegalStateException e) { - reportWebRtcAudioRecordStartError(AudioRecordStartErrorCode.AUDIO_RECORD_START_EXCEPTION, - "AudioRecord.startRecording failed: " + e.getMessage()); + reportWebRtcAudioRecordStartError( + AudioRecordStartErrorCode.AUDIO_RECORD_START_EXCEPTION, + "AudioRecord.startRecording failed: " + e.getMessage() + ); return false; } + if (audioRecord.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) { - reportWebRtcAudioRecordStartError(AudioRecordStartErrorCode.AUDIO_RECORD_START_STATE_MISMATCH, - "AudioRecord.startRecording failed - incorrect state: " - + audioRecord.getRecordingState()); + reportWebRtcAudioRecordStartError( + AudioRecordStartErrorCode.AUDIO_RECORD_START_STATE_MISMATCH, + "AudioRecord.startRecording failed - incorrect state: " + + audioRecord.getRecordingState() + ); return false; } + audioThread = new AudioRecordThread("AudioRecordJavaThread"); audioThread.start(); scheduleLogRecordingConfigurationsTask(audioRecord); return true; } - @CalledByNative + protected boolean startRecording() { + Logging.d(TAG, "startRecording"); + assertTrue(audioRecord != null); + assertTrue(audioThread == null); + + // If mediaProjection is not null here, it means record system audio on screen share. + // Wait asynchronously for a few seconds before starting the recording. + // This requirement of waiting might be related to device(?) + if (mediaProjection != null) { + executor.schedule(this::attemptStartRecording, DELAY_BEFORE_AUDIO_START_MS, TimeUnit.MILLISECONDS); + } else { + // If no mediaProjection, attempt to start immediately. Will record microphone. + return attemptStartRecording(); + } + + return true; + } + protected boolean stopRecording() { Logging.d(TAG, "stopRecording"); assertTrue(audioThread != null); @@ -432,6 +442,7 @@ protected boolean stopRecording() { private AudioRecord createAudioRecordOnQOrHigher( int audioSource, int sampleRate, int channelConfig, int audioFormat, int bufferSizeInBytes, MediaProjection mediaProjection) { Logging.d(TAG, "createAudioRecordOnQOrHigher"); + AudioPlaybackCaptureConfiguration audioConfig = new AudioPlaybackCaptureConfiguration.Builder(mediaProjection) .addMatchingUsage(AudioAttributes.USAGE_MEDIA) @@ -546,6 +557,37 @@ public static void setMicrophoneMute(boolean mute) { microphoneMute = mute; } + public int createAudioRecord(){ + try { + if(mediaProjection != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q){ + audioRecord = createAudioRecordOnQOrHigher( + audioSource, this.sampleRate, channelConfig, audioFormat, this.bufferSizeInBytes, mediaProjection); + } + else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + // Use the AudioRecord.Builder class on Android M (23) and above. + // Throws IllegalArgumentException. + audioRecord = createAudioRecordOnMOrHigher( + audioSource, this.sampleRate, channelConfig, audioFormat, this.bufferSizeInBytes); + audioSourceMatchesRecordingSessionRef.set(null); + if (preferredDevice != null) { + setPreferredDevice(preferredDevice); + } + } else { + // Use the old AudioRecord constructor for API levels below 23. + // Throws UnsupportedOperationException. + audioRecord = createAudioRecordOnLowerThanM( + audioSource, sampleRate, channelConfig, audioFormat, bufferSizeInBytes); + audioSourceMatchesRecordingSessionRef.set(null); + } + } catch (IllegalArgumentException | UnsupportedOperationException e) { + // Report of exception message is sufficient. Example: "Cannot create AudioRecord". + reportWebRtcAudioRecordInitError(e.getMessage()); + releaseAudioResources(); + return -1; + } + return 0; + } + // Releases the native AudioRecord resources. private void releaseAudioResources() { Logging.d(TAG, "releaseAudioResources"); @@ -560,6 +602,10 @@ public void setMediaProjection(MediaProjection mediaProjection){ this.mediaProjection = mediaProjection; } + public MediaProjection getMediaProjection(){ + return mediaProjection; + } + public void reportWebRtcAudioRecordInitError(String errorMessage) { Logging.e(TAG, "Init recording error: " + errorMessage); WebRtcAudioUtils.logAudioState(TAG, context, audioManager); diff --git a/webrtc-android-framework/src/test/java/io/antmedia/webrtcandroidframework/WebRTCClientTest.java b/webrtc-android-framework/src/test/java/io/antmedia/webrtcandroidframework/WebRTCClientTest.java index 3cee738f..48d3f68d 100644 --- a/webrtc-android-framework/src/test/java/io/antmedia/webrtcandroidframework/WebRTCClientTest.java +++ b/webrtc-android-framework/src/test/java/io/antmedia/webrtcandroidframework/WebRTCClientTest.java @@ -3,6 +3,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; @@ -56,6 +57,7 @@ import org.webrtc.VideoSink; import org.webrtc.VideoTrack; import org.webrtc.audio.AudioDeviceModule; +import org.webrtc.audio.CustomWebRtcAudioRecord; import org.webrtc.audio.JavaAudioDeviceModule; import java.lang.reflect.Field; @@ -70,11 +72,11 @@ import io.antmedia.webrtcandroidframework.api.IDataChannelObserver; import io.antmedia.webrtcandroidframework.api.IWebRTCClient; import io.antmedia.webrtcandroidframework.api.IWebRTCListener; -import io.antmedia.webrtcandroidframework.api.WebRTCClientConfig; import io.antmedia.webrtcandroidframework.apprtc.AppRTCAudioManager; import io.antmedia.webrtcandroidframework.core.ProxyVideoSink; import io.antmedia.webrtcandroidframework.core.StreamInfo; import io.antmedia.webrtcandroidframework.core.WebRTCClient; +import io.antmedia.webrtcandroidframework.core.model.ScreenShareAudioSource; import io.antmedia.webrtcandroidframework.websocket.Broadcast; import io.antmedia.webrtcandroidframework.websocket.WebSocketConstants; import io.antmedia.webrtcandroidframework.websocket.WebSocketHandler; @@ -1298,4 +1300,48 @@ public void testTurnServer(){ ); assertTrue(containsTurnServer); } + + @Test + public void testSwitchAudioSourceOnScreenShare(){ + + MediaProjection mediaProjection = mock(MediaProjection.class); + CustomWebRtcAudioRecord audioRecord = mock(CustomWebRtcAudioRecord.class); + + JavaAudioDeviceModule adm = mock(JavaAudioDeviceModule.class); + doReturn(audioRecord).when(adm).getAudioInput(); + + WebRTCClient webRTCClientReal = IWebRTCClient.builder() + .setActivity(context) + .setScreenShareAudioSource(ScreenShareAudioSource.SYSTEM) + .setWebRTCListener(listener) + .build(); + + webRTCClientReal.setAdm(adm); + + webRTCClientReal.getConfig().mediaProjection = mediaProjection; + + webRTCClientReal.changeVideoSource(IWebRTCClient.StreamSource.SCREEN); + + verify(adm, times(1)).setMediaProjection(mediaProjection); + + webRTCClientReal.switchToMicrophoneAudioRecordingOnScreenShareDuringCall(); + verify(adm, times(1)).stopRecording(); + verify(adm, times(1)).createAudioRecord(); + verify(adm, times(1)).startRecording(); + + assertNull(adm.getAudioInput().mediaProjection); + + webRTCClientReal.getConfig().screenShareAudioSource = ScreenShareAudioSource.SYSTEM; + + webRTCClientReal.switchToSystemAudioRecordingOnScreenShareDuringCall(); + + verify(adm, times(2)).stopRecording(); + verify(adm, times(2)).createAudioRecord(); + verify(adm, times(2)).startRecording(); + + doReturn(mediaProjection).when(audioRecord).getMediaProjection(); + + assertNotNull(adm.getAudioInput().getMediaProjection()); + + } } diff --git a/webrtc-android-sample-app/src/androidTest/java/io/antmedia/webrtc_android_sample_app/ScreenCaptureActivityTest.java b/webrtc-android-sample-app/src/androidTest/java/io/antmedia/webrtc_android_sample_app/ScreenCaptureActivityTest.java index 4b9a244e..cc70f5e5 100644 --- a/webrtc-android-sample-app/src/androidTest/java/io/antmedia/webrtc_android_sample_app/ScreenCaptureActivityTest.java +++ b/webrtc-android-sample-app/src/androidTest/java/io/antmedia/webrtc_android_sample_app/ScreenCaptureActivityTest.java @@ -52,6 +52,9 @@ public class ScreenCaptureActivityTest { private float videoBytesSent = 0; + private float audioBytesSent = 0; + + @Rule public GrantPermissionRule permissionRule = GrantPermissionRule.grant(PermissionHandler.FULL_PERMISSIONS); @@ -87,6 +90,9 @@ public void testScreenCapture() throws InterruptedException { assertNotNull(button); button.click(); + onView(withId(R.id.screen_share_audio_source_system)).perform(click()); + + onView(withId(R.id.start_streaming_button)).check(matches(withText("Start"))); Espresso.closeSoftKeyboard(); onView(withId(R.id.start_streaming_button)).perform(click()); @@ -106,6 +112,35 @@ public void testScreenCapture() throws InterruptedException { videoBytesSent = value; }); + onView(withId(R.id.stats_popup_bytes_sent_audio_textview)).check((view, noViewFoundException) -> { + String text = ((TextView) view).getText().toString(); + float value = Float.parseFloat(text); + assertTrue(value > 0f); + audioBytesSent = value; + }); + + onView(withId(R.id.stats_popup_close_button)).perform(click()); + + Thread.sleep(3000); + + onView(withId(R.id.screen_share_audio_source_microphone)).perform(click()); + + Thread.sleep(5000); + + onView(withId(R.id.broadcasting_text_view)) + .check(matches(withText(R.string.live))); + + onView(withId(R.id.show_stats_button)).perform(click()); + + onView(withId(R.id.stats_popup_bytes_sent_audio_textview)).check((view, noViewFoundException) -> { + String text = ((TextView) view).getText().toString(); + float value = Float.parseFloat(text); + assertTrue(value > 0f); + assertTrue( value != audioBytesSent); + + audioBytesSent = value; + }); + onView(withId(R.id.stats_popup_close_button)).perform(click()); Thread.sleep(3000); diff --git a/webrtc-android-sample-app/src/main/java/io/antmedia/webrtc_android_sample_app/basic/ScreenCaptureActivity.java b/webrtc-android-sample-app/src/main/java/io/antmedia/webrtc_android_sample_app/basic/ScreenCaptureActivity.java index 1c2741e4..0e4ce7f5 100644 --- a/webrtc-android-sample-app/src/main/java/io/antmedia/webrtc_android_sample_app/basic/ScreenCaptureActivity.java +++ b/webrtc-android-sample-app/src/main/java/io/antmedia/webrtc_android_sample_app/basic/ScreenCaptureActivity.java @@ -2,7 +2,6 @@ import static io.antmedia.webrtc_android_sample_app.basic.MediaProjectionService.EXTRA_MEDIA_PROJECTION_DATA; -import android.annotation.TargetApi; import android.app.AlertDialog; import android.content.Intent; import android.content.pm.PackageManager; @@ -13,14 +12,13 @@ import android.view.View; import android.widget.Button; import android.widget.EditText; +import android.widget.LinearLayout; import android.widget.RadioGroup; import android.widget.TextView; import android.widget.Toast; import androidx.annotation.NonNull; -import androidx.annotation.RequiresApi; -import org.w3c.dom.Text; import org.webrtc.SurfaceViewRenderer; import io.antmedia.webrtc_android_sample_app.R; @@ -32,19 +30,18 @@ import io.antmedia.webrtcandroidframework.api.IWebRTCListener; import io.antmedia.webrtcandroidframework.api.WebRTCClientConfig; import io.antmedia.webrtcandroidframework.core.PermissionHandler; -import io.antmedia.webrtcandroidframework.core.StatsCollector; +import io.antmedia.webrtcandroidframework.core.model.ScreenShareAudioSource; import io.antmedia.webrtcandroidframework.core.model.TrackStats; -import android.media.projection.MediaProjection; import android.media.projection.MediaProjectionManager; -import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; public class ScreenCaptureActivity extends TestableActivity { + private LinearLayout screenShareAudioSourceContainer; private TextView statusIndicatorTextView; private SurfaceViewRenderer fullScreenRenderer; private EditText streamIdEditText; @@ -55,6 +52,7 @@ public class ScreenCaptureActivity extends TestableActivity { private boolean bluetoothEnabled = false; private IWebRTCClient webRTCClient; private RadioGroup bg; + private RadioGroup screenShareAudioSourceRadioGroup; public static final int CAPTURE_PERMISSION_REQUEST_CODE = 1234; public MediaProjectionManager mediaProjectionManager; private final static long UPDATE_STATS_INTERVAL_MS = 500L; @@ -65,6 +63,9 @@ public class ScreenCaptureActivity extends TestableActivity { private boolean publishStarted = false; private int lastCheckedId = 0; + private ScreenShareAudioSource screenShareAudioSource = ScreenShareAudioSource.MICROPHONE; + + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -72,7 +73,7 @@ protected void onCreate(Bundle savedInstanceState) { fullScreenRenderer = findViewById(R.id.full_screen_renderer); streamIdEditText = findViewById(R.id.stream_id_edittext); - + screenShareAudioSourceContainer = findViewById(R.id.screen_share_audio_source_container); serverUrl = sharedPreferences.getString(getString(R.string.serverAddress), SettingsActivity.DEFAULT_WEBSOCKET_URL); statusIndicatorTextView = findViewById(R.id.broadcasting_text_view); TextView streamIdEditText = findViewById(R.id.stream_id_edittext); @@ -217,6 +218,7 @@ public void createWebRTCClient(){ webRTCClient = IWebRTCClient.builder() .setLocalVideoRenderer(fullScreenRenderer) .setServerUrl(serverUrl) + .setScreenShareAudioSource(screenShareAudioSource) .setActivity(this) .setInitiateBeforeStream(initBeforeStream) .setBluetoothEnabled(bluetoothEnabled) @@ -247,21 +249,67 @@ public void createWebRTCClient(){ return; } else if(checkedId == R.id.rbFront) { + screenShareAudioSourceContainer.setVisibility(View.GONE); newSource = IWebRTCClient.StreamSource.FRONT_CAMERA; + if(streamId != null && webRTCClient.isStreaming(streamId)){ + webRTCClient.switchToMicrophoneAudioRecordingOnScreenShareDuringCall(); + } } else if(checkedId == R.id.rbRear) { + screenShareAudioSourceContainer.setVisibility(View.GONE); newSource = IWebRTCClient.StreamSource.REAR_CAMERA; + if(streamId != null && webRTCClient.isStreaming(streamId)){ + webRTCClient.switchToMicrophoneAudioRecordingOnScreenShareDuringCall(); + } } - // idlingResource.increment(); webRTCClient.changeVideoSource(newSource); - // decrementIdle(); + }); + + screenShareAudioSourceRadioGroup = findViewById(R.id.screen_share_audio_source_radio_group); + if(screenShareAudioSource == ScreenShareAudioSource.MICROPHONE){ + screenShareAudioSourceRadioGroup.check(R.id.screen_share_audio_source_microphone); + }else if(screenShareAudioSource == ScreenShareAudioSource.SYSTEM){ + screenShareAudioSourceRadioGroup.check(R.id.screen_share_audio_source_system); + } + + screenShareAudioSourceRadioGroup.setOnCheckedChangeListener((group, checkedId) -> { + WebRTCClientConfig config = webRTCClient.getConfig(); + + if(checkedId == R.id.screen_share_audio_source_microphone) { + + if(streamId == null || !webRTCClient.isStreaming(streamId)){ + screenShareAudioSource = ScreenShareAudioSource.MICROPHONE; + config.screenShareAudioSource = screenShareAudioSource; + webRTCClient.setAudioDeviceModuleMediaProjection(null); + return; + } + + if(streamId != null && webRTCClient.isStreaming(streamId) && config.screenShareAudioSource == ScreenShareAudioSource.SYSTEM && config.videoSource == IWebRTCClient.StreamSource.SCREEN){ + screenShareAudioSource = ScreenShareAudioSource.MICROPHONE; + config.screenShareAudioSource = screenShareAudioSource; + webRTCClient.switchToMicrophoneAudioRecordingOnScreenShareDuringCall(); + } + } + else if(checkedId == R.id.screen_share_audio_source_system) { + if(streamId == null || !webRTCClient.isStreaming(streamId)){ + screenShareAudioSource = ScreenShareAudioSource.SYSTEM; + config.screenShareAudioSource = screenShareAudioSource; + return; + } + + if(streamId != null && webRTCClient.isStreaming(streamId) && config.screenShareAudioSource == ScreenShareAudioSource.MICROPHONE && config.videoSource == IWebRTCClient.StreamSource.SCREEN){ + screenShareAudioSource = ScreenShareAudioSource.SYSTEM; + config.screenShareAudioSource = screenShareAudioSource; + webRTCClient.switchToSystemAudioRecordingOnScreenShareDuringCall(); + } + + } + }); } public void startStopStream() { - // incrementIdle(); - if (!PermissionHandler.checkCameraPermissions(this)) { PermissionHandler.requestCameraPermissions(this); return; @@ -302,7 +350,6 @@ public void onPublishStarted(String streamId) { publishStarted = true; statusIndicatorTextView.setTextColor(getResources().getColor(R.color.green)); statusIndicatorTextView.setText(getResources().getString(R.string.live)); - // decrementIdle(); } @Override @@ -322,7 +369,6 @@ public void onReconnectionSuccess() { super.onReconnectionSuccess(); statusIndicatorTextView.setTextColor(getResources().getColor(R.color.green)); statusIndicatorTextView.setText(getResources().getString(R.string.live)); - // decrementIdle(); } @Override @@ -342,7 +388,6 @@ public void onPublishFinished(String streamId) { super.onPublishFinished(streamId); statusIndicatorTextView.setTextColor(getResources().getColor(R.color.red)); statusIndicatorTextView.setText(getResources().getString(R.string.disconnected)); - // decrementIdle(); } }; } @@ -406,6 +451,7 @@ public void onActivityResult(int requestCode, int resultCode, Intent data) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { MediaProjectionService.setListener(() -> { + screenShareAudioSourceContainer.setVisibility(View.VISIBLE); startScreenCapturer(); }); @@ -416,10 +462,12 @@ public void onActivityResult(int requestCode, int resultCode, Intent data) { } else { startScreenCapturer(); } + } private void startScreenCapturer() { webRTCClient.changeVideoSource(IWebRTCClient.StreamSource.SCREEN); decrementIdle(); } + } \ No newline at end of file diff --git a/webrtc-android-sample-app/src/main/res/layout/activity_screenshare.xml b/webrtc-android-sample-app/src/main/res/layout/activity_screenshare.xml index a9b0be20..77274ce8 100644 --- a/webrtc-android-sample-app/src/main/res/layout/activity_screenshare.xml +++ b/webrtc-android-sample-app/src/main/res/layout/activity_screenshare.xml @@ -47,6 +47,50 @@ android:text="Rear"/> + + + + + + + + + + + + + + + +