diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 992fc00a71..ebe7059802 100755 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -3,10 +3,11 @@ import static com.limelight.StartExternalDisplayControlReceiver.requestFocusToExternalDisplayControl; import static com.limelight.binding.input.KeyboardTranslator.getModifier; +import static com.limelight.utils.DisplayUtils.getDisplayInfo; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; +import static com.limelight.utils.DisplayUtils.hasSecondaryDisplay; import static com.limelight.utils.ExternalDisplayControlActivity.SECONDARY_SCREEN_NOTIFICATION_ID; import static com.limelight.utils.ExternalDisplayControlActivity.closeExternalDisplayControl; -import static com.limelight.utils.ServerHelper.getActiveDisplay; -import static com.limelight.utils.ServerHelper.getSecondaryDisplay; import com.limelight.binding.PlatformBinding; import com.limelight.binding.audio.AndroidAudioRenderer; @@ -44,6 +45,7 @@ import com.limelight.ui.GameGestures; import com.limelight.ui.StreamContainer; import com.limelight.utils.Dialog; +import com.limelight.utils.DisplayUtils; import com.limelight.utils.ExternalDisplayControlActivity; import com.limelight.utils.MouseModeOption; import com.limelight.utils.PanZoomHandler; @@ -266,7 +268,6 @@ public void onServiceDisconnected(ComponentName componentName) { public static final String EXTRA_SERVER_CERT = "ServerCert"; public static final String EXTRA_VDISPLAY = "VirtualDisplay"; public static final String EXTRA_SERVER_COMMANDS = "ServerCommands"; - public static final String EXTRA_DISPLAY_ID = "DisplayID"; public static final String CLIPBOARD_IDENTIFIER = "ArtemisStreaming"; @@ -384,30 +385,24 @@ protected void onCreate(Bundle savedInstanceState) { getResources().getString(R.string.conn_establishing_msg), true); - Display currentDisplay = null; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - int displayId = getIntent().getIntExtra(EXTRA_DISPLAY_ID, Display.DEFAULT_DISPLAY); - currentDisplay = getSystemService(DisplayManager.class).getDisplay(displayId); - } + Display currentDisplay = DisplayUtils.getGameStreamDisplay(this); if (currentDisplay == null) { - currentDisplay = getWindowManager().getDefaultDisplay(); + LimeLog.severe("FATAL: getGameStreamDisplay returned null! Cannot continue."); + // Show an error to the user and finish + Toast.makeText(this, "Critical Error: Could not determine target display.", Toast.LENGTH_LONG).show(); + finish(); + return; // Important: Stop further execution in onCreate } - onExternelDisplay = currentDisplay.getDisplayId() != Display.DEFAULT_DISPLAY; + onExternelDisplay = (currentDisplay.getDisplayId() != Display.DEFAULT_DISPLAY); boolean shouldInvertDecoderResolution = false; + matchSettings(currentDisplay); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M - && onExternelDisplay - && prefConfig.renderMode == 0 // For 3D we want to maintain configured resolution - ) { - Display.Mode currentMode = currentDisplay.getMode(); - displayWidth = currentMode.getPhysicalWidth(); - displayHeight = currentMode.getPhysicalHeight(); - prefConfig.width = displayWidth; - prefConfig.height = displayHeight; - prefConfig.fps = currentMode.getRefreshRate(); + if (onExternelDisplay) { + displayWidth = prefConfig.width; + displayHeight = prefConfig.height; prefConfig.videoScaleMode = PreferenceConfiguration.ScaleMode.STRETCH; prefConfig.enableFloatingButton = false; prefConfig.showOverlayZoomToggleButton = false; @@ -415,10 +410,6 @@ protected void onCreate(Bundle savedInstanceState) { currentOrientation = Configuration.ORIENTATION_LANDSCAPE; setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_USER_LANDSCAPE); } else { - if (prefConfig.renderMode != 0) { - prefConfig.videoScaleMode = PreferenceConfiguration.ScaleMode.STRETCH; - } - if (prefConfig.autoOrientation) { currentOrientation = getResources().getConfiguration().orientation; } else { @@ -668,7 +659,8 @@ public void notifyCrash(Exception e) { willStreamHdr, shouldInvertDecoderResolution, glPrefs.glRenderer, - this); + this, + currentDisplay); // --- Force tight thresholds (prefConfig.forceTightThresholds) --- try { @@ -695,12 +687,12 @@ public void notifyCrash(Exception e) { decoderRenderer.setPreferLowerDelaysTimeoutUs(500); // 0.5 ms prefConfig.framePacing = PreferenceConfiguration.FRAME_PACING_BALANCED; LimeLog.info("PreferLowerDelays: preferLowerDelays=true, timeout=500us, pacing=BALANCED"); - } else { - // Balanced default + } else if(prefConfig.framePacing == PreferenceConfiguration.FRAME_PACING_BALANCED && !isOnExternalDisplay()) { decoderRenderer.setPreferLowerDelays(false); decoderRenderer.setPreferLowerDelaysTimeoutUs(2000); // 2 ms - prefConfig.framePacing = PreferenceConfiguration.FRAME_PACING_BALANCED; LimeLog.info("Balanced: preferLowerDelays=false, timeout=2000us, pacing=BALANCED"); + } else { + LimeLog.info("No balance mode selected or on external screen (LFR not working)"); } } catch (Throwable ignored) {} @@ -748,7 +740,12 @@ public void notifyCrash(Exception e) { // Set to the optimal mode for streaming float displayRefreshRate = prepareDisplayForRendering(currentDisplay); - LimeLog.info("Display refresh rate: "+displayRefreshRate); + + // Set WindowAttributes is not working on external screens and received fps were wrong + // leading to weird stream connection with barely 500kbs + if (isOnExternalDisplay() && Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + displayRefreshRate = currentDisplay.getMode().getRefreshRate(); + } // If the user requested frame pacing using a capped FPS, we will need to change our // desired FPS setting here in accordance with the active display refresh rate. @@ -775,6 +772,10 @@ public void notifyCrash(Exception e) { if (prefConfig.framePacingWarpFactor > 0) { chosenFrameRate *= prefConfig.framePacingWarpFactor; } + //As external displays might return 60.004 fps, it seems to cause low quality stream + if(isOnExternalDisplay()) { + chosenFrameRate = (int) chosenFrameRate; + } StreamConfiguration config = new StreamConfiguration.Builder() .setResolution( @@ -934,6 +935,53 @@ public void notifyCrash(Exception e) { } catch (Throwable ignored) {} } + private void matchSettings(Display display) { + DisplayUtils.DisplayInfo displayInfo = getDisplayInfo(display); + + if(isMatchDisplayFPS()) { + prefConfig.fps = (int) displayInfo.refreshRate; + } + if(prefConfig.renderMode != 0) { // 3D Mode selected + float ratio = (float) displayInfo.width / (float) displayInfo.height; + + // A 32:9 aspect ratio is 3.555... + final float SBS_3D_ASPECT_RATIO = 32.0f / 9.0f; + + // Use a small tolerance for floating-point comparison + final float EPSILON = 0.01f; + // User can keep render mode 3 in its setting without the need to switch + // so plug in glasses and turn on 3d should trigger it otherwise 2dmode + if (Math.abs(ratio - SBS_3D_ASPECT_RATIO) < EPSILON) { + // This is a 32:9 SbS 3D mode (like 3840x1080). + // We set the displayWidth to be for a single eye (1920). + if(isMatchDisplayResolution()) { + prefConfig.width = displayInfo.width / 2; + prefConfig.height = displayInfo.height; + } + } else { + // This is a standard 16:9, 4:3, etc. mode. No 3d needed + prefConfig.renderMode = 0; + } + } + if(isMatchDisplayResolution() && prefConfig.renderMode == 0) { + prefConfig.width = displayInfo.width; + prefConfig.height = displayInfo.height; + } + if(isMatchBitrate()) { + prefConfig.bitrate = PreferenceConfiguration.getDefaultBitrate(prefConfig.width+"x"+prefConfig.height, ((int) prefConfig.fps) +"", this); + } + } + + private boolean isMatchDisplayResolution() { + return prefConfig.width == 0 && prefConfig.height == 0; + } + private boolean isMatchBitrate() { + return prefConfig.bitrate == 0; + } + private boolean isMatchDisplayFPS() { + return prefConfig.fps == 0; + } + @SuppressLint("ClickableViewAccessibility") private void setupOverlayToggleButton() { if (overlayToggleButton != null) { @@ -1017,7 +1065,7 @@ public void onDisplayAdded(int displayId) { @Override public void onDisplayRemoved(int displayId) { - if (getSecondaryDisplay(getBaseContext()) == null) { + if (onExternelDisplay) { handleDisplayRemoved(); finish(); } @@ -1142,7 +1190,7 @@ public void toggleVirtualController(){ } private void setPreferredOrientationForActivity() { - Display display = getActiveDisplay(Game.this, prefConfig); + Display display = getGameStreamDisplay(Game.this); // For semi-square displays, we use more complex logic to determine which orientation to use (if any) if (PreferenceConfiguration.isSquarishScreen(display)) { @@ -1429,7 +1477,7 @@ private boolean shouldIgnoreInsetsForResolution(int width, int height) { } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - Display display = getActiveDisplay(Game.this, prefConfig); + Display display = getGameStreamDisplay(Game.this); for (Display.Mode candidate : display.getSupportedModes()) { // Ignore insets if this is an exact match for the display resolution if ((width == candidate.getPhysicalWidth() && height == candidate.getPhysicalHeight()) || @@ -3990,16 +4038,18 @@ public boolean isZoomModeEnabled() { return isPanZoomMode; } public void toggleZoomMode() { - this.isPanZoomMode = !this.isPanZoomMode; - if (this.isPanZoomMode) { - Toast.makeText(this, getString(R.string.pan_zoom_mode_enabled), Toast.LENGTH_SHORT).show(); - } else { - Toast.makeText(this, getString(R.string.pan_zoom_mode_disabled), Toast.LENGTH_SHORT).show(); - } - updateZoomButtonAppearance(); + if(prefConfig.renderMode == 0) { + this.isPanZoomMode = !this.isPanZoomMode; + if (this.isPanZoomMode) { + Toast.makeText(this, getString(R.string.pan_zoom_mode_enabled), Toast.LENGTH_SHORT).show(); + } else { + Toast.makeText(this, getString(R.string.pan_zoom_mode_disabled), Toast.LENGTH_SHORT).show(); + } + updateZoomButtonAppearance(); - if (ExternalDisplayControlActivity.instance != null) { - ExternalDisplayControlActivity.instance.toggleZoomMode(false); + if (ExternalDisplayControlActivity.instance != null) { + ExternalDisplayControlActivity.instance.toggleZoomMode(false); + } } } diff --git a/app/src/main/java/com/limelight/ShortcutTrampoline.java b/app/src/main/java/com/limelight/ShortcutTrampoline.java index e7cd2cfefa..f543d6b6b9 100755 --- a/app/src/main/java/com/limelight/ShortcutTrampoline.java +++ b/app/src/main/java/com/limelight/ShortcutTrampoline.java @@ -1,7 +1,5 @@ package com.limelight; -import static com.limelight.utils.ServerHelper.getSecondaryDisplay; - import android.app.Activity; import android.app.Service; import android.content.ComponentName; diff --git a/app/src/main/java/com/limelight/StartExternalDisplayControlReceiver.java b/app/src/main/java/com/limelight/StartExternalDisplayControlReceiver.java index 4c030fa817..de7e1ab1a9 100644 --- a/app/src/main/java/com/limelight/StartExternalDisplayControlReceiver.java +++ b/app/src/main/java/com/limelight/StartExternalDisplayControlReceiver.java @@ -1,5 +1,7 @@ package com.limelight; +import static com.limelight.utils.DisplayUtils.getControlsDisplay; + import android.app.ActivityManager; import android.app.ActivityOptions; import android.content.BroadcastReceiver; @@ -30,7 +32,7 @@ public static void requestFocusToExternalDisplayControl(Context context) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { Intent intentTouchpad = new Intent(context, ExternalDisplayControlActivity.class); intentTouchpad.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK | Intent.FLAG_ACTIVITY_SINGLE_TOP | Intent.FLAG_ACTIVITY_CLEAR_TOP); - Bundle optionsDefault = ActivityOptions.makeBasic().setLaunchDisplayId(Display.DEFAULT_DISPLAY).toBundle(); + Bundle optionsDefault = ActivityOptions.makeBasic().setLaunchDisplayId(getControlsDisplay(context).getDisplayId()).toBundle(); context.startActivity(intentTouchpad, optionsDefault); } } diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index b131684e2b..74466cc5b1 100755 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -1,5 +1,6 @@ package com.limelight.binding.video; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -8,11 +9,9 @@ import java.util.List; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; - import org.jcodec.codecs.h264.H264Utils; import org.jcodec.codecs.h264.io.model.SeqParameterSet; import org.jcodec.codecs.h264.io.model.VUIParameters; - import com.limelight.BuildConfig; import com.limelight.LimeLog; import com.limelight.R; @@ -21,14 +20,12 @@ import com.limelight.preferences.PreferenceConfiguration; import com.limelight.utils.Stereo3DRenderer; import com.limelight.utils.TrafficStatsHelper; - import android.annotation.SuppressLint; import android.util.LongSparseArray; import android.annotation.TargetApi; import android.app.Activity; import android.content.Context; import android.media.MediaCodec; -import android.os.Bundle; import android.media.MediaCodecInfo; import android.media.MediaFormat; import android.media.MediaCodec.BufferInfo; @@ -41,6 +38,7 @@ import android.os.SystemClock; import android.util.Range; import android.view.Choreographer; +import android.view.Display; import android.view.Surface; public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements Choreographer.FrameCallback { @@ -120,6 +118,7 @@ private void updateDecodeLatencyStats(long presentationTimeUs) { private ByteBuffer nextInputBuffer; private Context context; + private Display display; private Activity activity; private MediaCodec videoDecoder; private Thread rendererThread; @@ -365,9 +364,9 @@ public void setRenderTarget(Surface renderTarget) { public MediaCodecDecoderRenderer(Activity activity, PreferenceConfiguration prefs, CrashListener crashListener, int consecutiveCrashCount, boolean meteredData, boolean requestedHdr, boolean invertResolution, - String glRenderer, PerfOverlayListener perfListener) { + String glRenderer, PerfOverlayListener perfListener, Display display) { //dumpDecoders(); - + this.display = display; this.context = activity; this.activity = activity; this.prefs = prefs; @@ -796,7 +795,14 @@ public void onFrameRendered(MediaCodec mediaCodec, long presentationTimeUs, long @Override public int setup(int format, int width, int height, int redrawRate) { - this.targetFps = (redrawRate > 0 ? redrawRate : 60); + // External displayes occasionally return a redrawRate of zero, so default 60 was wrong. + int fpsTarget = redrawRate; + if(display == null && fpsTarget <= 0) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + fpsTarget = (int) getGameStreamDisplay(activity).getMode().getRefreshRate(); + } + } + this.targetFps = fpsTarget; this.initialWidth = invertResolution ? height : width; this.initialHeight = invertResolution ? width : height; this.videoFormat = format; @@ -1073,7 +1079,10 @@ public void doFrame(long frameTimeNanos) { } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - frameTimeNanos -= activity.getWindowManager().getDefaultDisplay().getAppVsyncOffsetNanos(); + if(display == null) { + display = getGameStreamDisplay(activity); + } + frameTimeNanos -= display.getAppVsyncOffsetNanos(); } // Don't render unless a new frame is due. This prevents microstutter when streaming @@ -1164,11 +1173,14 @@ public void run() { long vsyncPeriodNs; float displayHz = 60f; try { - if (Build.VERSION.SDK_INT >= 17 && context != null) { - android.view.Display d = ((android.view.WindowManager) context.getSystemService(android.content.Context.WINDOW_SERVICE)).getDefaultDisplay(); - if (d != null) displayHz = d.getRefreshRate(); + if (Build.VERSION.SDK_INT >= 17 && activity != null) { + if(display == null) { + display = getGameStreamDisplay(activity); + } + if (display != null) displayHz = display.getRefreshRate(); } } catch (Throwable ignored) {} + if (displayHz <= 0f) displayHz = 60f; vsyncPeriodNs = (long) (1_000_000_000L / displayHz); @@ -1814,9 +1826,9 @@ public int submitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int dec sb.append("\t"); sb.append(context.getString(R.string.perf_overlay_lite_packet_loss) + ": "); sb.append(context.getString(R.string.perf_overlay_lite_netdrops,(float)lastTwo.framesLost / lastTwo.totalFrames * 100)); - sb.append("\t FPS："); - sb.append(context.getString(R.string.perf_overlay_lite_fps, fps.totalFps)); if(Stereo3DRenderer.isActive) { + sb.append("\t FPS："); + sb.append(context.getString(R.string.perf_overlay_lite_fps, Stereo3DRenderer.fps)); sb.append(" "); sb.append(context.getString(R.string.perf_overlay_ai_fps)); sb.append(" "); @@ -1827,10 +1839,13 @@ public int submitDecodeUnit(byte[] decodeUnitData, int decodeUnitLength, int dec sb.append(Stereo3DRenderer.renderer); sb.append(" "); sb.append(context.getString(R.string.perf_overlay_drawdelay, Stereo3DRenderer.drawDelay)); + } else { + sb.append("\t FPS："); + sb.append(context.getString(R.string.perf_overlay_lite_fps, fps.totalFps)); } }else{ if(Stereo3DRenderer.isActive) { - sb.append(context.getString(R.string.perf_overlay_streamdetails, initialWidth + "x" + initialHeight, fps.totalFps)); + sb.append(context.getString(R.string.perf_overlay_streamdetails, initialWidth + "x" + initialHeight, Stereo3DRenderer.fps)); sb.append('\n'); sb.append(" "); sb.append(context.getString(R.string.perf_overlay_ai_fps)); diff --git a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java index 96249d87ac..5261dc1d0c 100755 --- a/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java +++ b/app/src/main/java/com/limelight/preferences/PreferenceConfiguration.java @@ -1,5 +1,8 @@ package com.limelight.preferences; +import static com.limelight.utils.DisplayUtils.getDisplayInfo; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; + import android.content.Context; import android.content.SharedPreferences; import android.content.pm.PackageManager; @@ -8,6 +11,7 @@ import com.limelight.nvstream.jni.MoonBridge; import com.limelight.profiles.ProfilesManager; +import com.limelight.utils.DisplayUtils; public class PreferenceConfiguration { @@ -42,6 +46,8 @@ public enum AnalogStickForScrolling { static final String RESOLUTION_PREF_STRING = "list_resolution"; static final String FPS_PREF_STRING = "list_fps"; static final String BITRATE_PREF_STRING = "seekbar_bitrate_kbps"; + + static final String BITRATE_RECOMMENDATION_STRING = "bitrate_recommendation"; private static final String BITRATE_PREF_OLD_STRING = "seekbar_bitrate"; private static final String METERED_BITRATE_PREF_STRING = "seekbar_metered_bitrate_kbps"; private static final String ENABLE_ULTRA_LOW_LATENCY_PREF_STRING = "checkbox_ultra_low_latency"; @@ -282,7 +288,6 @@ public enum AnalogStickForScrolling { public boolean enableBackMenu; public boolean enableFloatingButton; public boolean showOverlayZoomToggleButton; - //Invert video width/height public boolean autoInvertVideoResolution; public int resolutionScaleFactor; @@ -492,7 +497,16 @@ private static String getResolutionString(int width, int height) { } } - public static int getDefaultBitrate(String resString, String fpsString) { + public static int getDefaultBitrate(String resString, String fpsString, Context context) { + + // If MatchDisplayRes/FPS is selected we need the actual values + DisplayUtils.DisplayInfo displayInfo = getDisplayInfo(getGameStreamDisplay(context)); + if(resString.equals("0x0")) { + resString = displayInfo.width + "x" +displayInfo.height; + } + if(fpsString.equals("0")) { + fpsString = displayInfo.refreshRate + ""; + } int width = getWidthFromResolutionString(resString); int height = getHeightFromResolutionString(resString); int fps = Math.round(Float.parseFloat(fpsString)); @@ -579,7 +593,7 @@ public static int getDefaultBitrate(Context context) { SharedPreferences prefs = ProfilesManager.getInstance().getOverlayingSharedPreferences(context); return getDefaultBitrate( prefs.getString(RESOLUTION_PREF_STRING, DEFAULT_RESOLUTION), - prefs.getString(FPS_PREF_STRING, DEFAULT_FPS)); + prefs.getString(FPS_PREF_STRING, DEFAULT_FPS), context); } private static FormatOption getVideoFormatValue(Context context) { @@ -685,6 +699,7 @@ public static void resetStreamingSettings(Context context) { SharedPreferences prefs = ProfilesManager.getInstance().getOverlayingSharedPreferences(context); prefs.edit() .remove(BITRATE_PREF_STRING) + .remove(BITRATE_RECOMMENDATION_STRING) .remove(BITRATE_PREF_OLD_STRING) .remove(LEGACY_RES_FPS_PREF_STRING) .remove(RESOLUTION_PREF_STRING) @@ -832,9 +847,6 @@ else if (str.equals("4K60")) { // This must happen after the preferences migration to ensure the preferences are populated config.bitrate = prefs.getInt(BITRATE_PREF_STRING, prefs.getInt(BITRATE_PREF_OLD_STRING, 0) * 1000); - if (config.bitrate == 0) { - config.bitrate = getDefaultBitrate(context); - } config.meteredBitrate = prefs.getInt((METERED_BITRATE_PREF_STRING), 0); if (config.meteredBitrate == 0) { diff --git a/app/src/main/java/com/limelight/preferences/StreamSettings.java b/app/src/main/java/com/limelight/preferences/StreamSettings.java index 5b2d20de13..3871b25e64 100755 --- a/app/src/main/java/com/limelight/preferences/StreamSettings.java +++ b/app/src/main/java/com/limelight/preferences/StreamSettings.java @@ -1,6 +1,8 @@ package com.limelight.preferences; -import static com.limelight.utils.ServerHelper.getActiveDisplay; +import static com.limelight.preferences.PreferenceConfiguration.BITRATE_RECOMMENDATION_STRING; +import static com.limelight.preferences.PreferenceConfiguration.DEFAULT_FPS; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; import android.content.Context; import android.content.Intent; @@ -76,7 +78,7 @@ public class StreamSettings extends AppCompatActivity { void reloadSettings() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - Display.Mode mode = getActiveDisplay(StreamSettings.this, previousPrefs).getMode(); + Display.Mode mode = getGameStreamDisplay(StreamSettings.this).getMode(); previousDisplayPixelCount = mode.getPhysicalWidth() * mode.getPhysicalHeight(); } prefsFragment = new SettingsFragment(PreferenceConfiguration.readPreferences( @@ -125,7 +127,7 @@ public void onConfigurationChanged(Configuration newConfig) { super.onConfigurationChanged(newConfig); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - Display.Mode mode = getActiveDisplay(StreamSettings.this, previousPrefs).getMode(); + Display.Mode mode = getGameStreamDisplay(StreamSettings.this).getMode(); // If the display's physical pixel count has changed, we consider that it's a new display // and we should reload our settings (which include display-dependent values). @@ -301,17 +303,16 @@ private void removeValue(String preferenceKey, String value, Runnable onMatched) pref.setEntryValues(entryValues); } - private void resetBitrateToDefault(SharedPreferences prefs, String res, String fps) { + private void recalculateRecommendedBitrate(SharedPreferences prefs, String res, String fps) { if (res == null) { res = prefs.getString(PreferenceConfiguration.RESOLUTION_PREF_STRING, PreferenceConfiguration.DEFAULT_RESOLUTION); } if (fps == null) { - fps = prefs.getString(PreferenceConfiguration.FPS_PREF_STRING, PreferenceConfiguration.DEFAULT_FPS); + fps = prefs.getString(PreferenceConfiguration.FPS_PREF_STRING, DEFAULT_FPS); } - prefs.edit() - .putInt(PreferenceConfiguration.BITRATE_PREF_STRING, - PreferenceConfiguration.getDefaultBitrate(res, fps)) + .putInt(PreferenceConfiguration.BITRATE_RECOMMENDATION_STRING, + PreferenceConfiguration.getDefaultBitrate(res, fps, getContext())) .apply(); } @@ -439,7 +440,7 @@ else if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O || } } } - + // Check custom refresh rate String customRefreshRateStr = prevPrefConfig.customRefreshRate; if (customRefreshRateStr != null && !customRefreshRateStr.isEmpty()) { @@ -696,12 +697,24 @@ public boolean onPreferenceChange(Preference preference, Object newValue) { } // Write the new bitrate value - resetBitrateToDefault(prefs, valueStr, null); + recalculateRecommendedBitrate(prefs, valueStr, null); // Allow the original preference change to take place return true; } }); + findPreference(PreferenceConfiguration.BITRATE_PREF_STRING).setOnPreferenceClickListener(new Preference.OnPreferenceClickListener() { + + @Override + public boolean onPreferenceClick(@NonNull Preference preference) { + int recommendationBitrate = getPrefs().getInt(BITRATE_RECOMMENDATION_STRING, 0); + if(recommendationBitrate != 0) { + Toast.makeText(getContext(), getString(R.string.bitrate_recommendation_toast, recommendationBitrate / 1000), Toast.LENGTH_LONG).show(); + } + return true; + } + } + ); findPreference(PreferenceConfiguration.FPS_PREF_STRING).setOnPreferenceChangeListener(new Preference.OnPreferenceChangeListener() { @Override public boolean onPreferenceChange(Preference preference, Object newValue) { @@ -718,7 +731,7 @@ public boolean onPreferenceChange(Preference preference, Object newValue) { } // Write the new bitrate value - resetBitrateToDefault(prefs, null, valueStr); + recalculateRecommendedBitrate(prefs, null, valueStr); // Allow the original preference change to take place return true; @@ -904,7 +917,7 @@ public boolean onPreferenceClick(@NonNull Preference preference) { try { int width = Integer.parseInt(resolutionSegments[0]); int height = Integer.parseInt(resolutionSegments[1]); - + if (width <= 0 || height <= 0) { Toast.makeText(getActivity(), getString(R.string.pref_error_occurred), Toast.LENGTH_SHORT).show(); return false; @@ -934,14 +947,14 @@ public boolean onPreferenceClick(@NonNull Preference preference) { Toast.makeText(getActivity(), getString(R.string.pref_enter_value_0_9999), Toast.LENGTH_SHORT).show(); return false; } - + try { float refreshRate = Float.parseFloat(value); if (refreshRate <= 0) { Toast.makeText(getActivity(), getString(R.string.pref_enter_value_0_9999), Toast.LENGTH_SHORT).show(); return false; } - + // Format to max 3 decimal places String formattedValue = String.format("%.3f", refreshRate); // Remove trailing zeros @@ -964,7 +977,7 @@ private void removeEntryFromListAndSetValue(String resolutionPrefString, String public void run() { SharedPreferences prefs = getPrefs(); setValue(resolutionPrefString, nextDefault); - resetBitrateToDefault(prefs, null, null); + recalculateRecommendedBitrate(prefs, null, null); } }); } diff --git a/app/src/main/java/com/limelight/utils/DisplayUtils.java b/app/src/main/java/com/limelight/utils/DisplayUtils.java new file mode 100644 index 0000000000..73797d021f --- /dev/null +++ b/app/src/main/java/com/limelight/utils/DisplayUtils.java @@ -0,0 +1,629 @@ +package com.limelight.utils; + +import android.app.AlertDialog; +import android.content.ClipData; +import android.content.ClipboardManager; +import android.content.Context; +import android.graphics.Point; +import android.hardware.display.DisplayManager; +import android.os.Build; +import android.os.Handler; +import android.os.Looper; +import android.view.Display; +import android.annotation.TargetApi; +import android.widget.Toast; + +import com.limelight.LimeLog; +import com.limelight.preferences.PreferenceConfiguration; + +import java.util.ArrayList; +import java.util.List; + +public class DisplayUtils { + + private static AlertDialog openDialog = null; + + public static class DisplayInfo { + public final int width; + public final int height; + public final float refreshRate; + public final long totalPixels; + + public DisplayInfo(int width, int height, float refreshRate) { + this.width = Math.max(width, height); + this.height = Math.min(width, height); + this.refreshRate = refreshRate; + this.totalPixels = (long)this.width * this.height; + } + + @Override + public String toString() { + return String.format("%dx%d @ %.1f Hz", width, height, refreshRate); + } + } + + public static DisplayInfo getDisplayInfo(Display display) { + if (display == null) { + LimeLog.warning("getDisplayInfo called with null display."); + return null; + } + + int axeOneLength = 0; + int axeTwoLength = 0; + float displayRefreshRate = 0f; + + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + Display.Mode currentMode = display.getMode(); + if (currentMode != null) { + axeOneLength = currentMode.getPhysicalWidth(); + axeTwoLength = currentMode.getPhysicalHeight(); + displayRefreshRate = currentMode.getRefreshRate(); + } else { + LimeLog.warning("display.getMode() returned null on API " + Build.VERSION.SDK_INT + ". Falling back to legacy methods."); + getLegacyDisplayInfo(display, sizePoint); + axeOneLength = sizePoint.x; + axeTwoLength = sizePoint.y; + displayRefreshRate = display.getRefreshRate(); + } + } else { + getLegacyDisplayInfo(display, sizePoint); + axeOneLength = sizePoint.x; + axeTwoLength = sizePoint.y; + displayRefreshRate = display.getRefreshRate(); + } + } catch (Exception e) { + LimeLog.severe("Error getting display info for display ID " + display.getDisplayId() + ": " + e.getMessage()); + return null; + } + + + if (axeOneLength <= 0 || axeTwoLength <= 0) { + LimeLog.warning("Retrieved invalid dimensions (" + axeOneLength + "x" + axeTwoLength + ") for display ID " + display.getDisplayId()); + if (sizePoint.x <= 0 || sizePoint.y <= 0) { + try { + axeOneLength = display.getWidth(); + axeTwoLength = display.getHeight(); + } catch (Exception ignored) {} + } else { + axeOneLength = sizePoint.x; + axeTwoLength = sizePoint.y; + } + if (axeOneLength <= 0 || axeTwoLength <= 0) { + LimeLog.severe("Could not retrieve valid dimensions for display ID " + display.getDisplayId()); + return null; + } + } + + int physicalWidth = Math.max(axeOneLength, axeTwoLength); + int physicalHeight = Math.min(axeOneLength, axeTwoLength); + + return new DisplayInfo(physicalWidth, physicalHeight, displayRefreshRate); + } + private static final Point sizePoint = new Point(); + + private static synchronized void getLegacyDisplayInfo(Display display, Point outSize) { + outSize.set(0, 0); + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) { + display.getRealSize(outSize); + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.HONEYCOMB_MR2) { + display.getSize(outSize); + } else { + outSize.x = display.getWidth(); + outSize.y = display.getHeight(); + } + } catch (Exception e) { + LimeLog.severe("Exception in getLegacyDisplayInfo: " + e.getMessage()); + outSize.set(0, 0); + } + } + + private static class CategorizedDisplays { + Display mainDefaultDisplay = null; + Display externalPresentationDisplay = null; + Display secondaryInternalDisplay = null; + } + + private static CategorizedDisplays findAndCategorizeDisplays(Context context) { + CategorizedDisplays info = new CategorizedDisplays(); + DisplayManager displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + + if (displayManager == null) { + LimeLog.warning("DisplayManager service not found. Attempting fallback via WindowManager."); + try { + android.view.WindowManager wm = (android.view.WindowManager) context.getSystemService(Context.WINDOW_SERVICE); + if (wm != null) info.mainDefaultDisplay = wm.getDefaultDisplay(); + else LimeLog.severe("WindowManager service also not found."); + } catch (Exception e) { + LimeLog.severe("Could not get default display via WindowManager: " + e.toString()); + } + if (info.mainDefaultDisplay == null) LimeLog.severe("FATAL: Could not obtain any reference to the main display."); + return info; + } + + Display[] displays = {}; + try { + displays = displayManager.getDisplays(); + } catch (Exception e) { + LimeLog.severe("Error getting displays from DisplayManager: " + e.toString()); + } + + + for (Display display : displays) { + if (display == null) continue; + + if (display.getDisplayId() == Display.DEFAULT_DISPLAY) { + info.mainDefaultDisplay = display; + continue; + } + + if ((display.getFlags() & Display.FLAG_PRESENTATION) != 0) { + if (info.externalPresentationDisplay == null) { + info.externalPresentationDisplay = display; + LimeLog.info("Found external presentation display: " + display.getName() + " (ID: " + display.getDisplayId() + ")"); + } else { + LimeLog.info("Ignoring additional external presentation display: " + display.getName()); + } + continue; + } + + if (info.secondaryInternalDisplay == null) { + info.secondaryInternalDisplay = display; + LimeLog.info("Found secondary internal display: " + display.getName() + " (ID: " + display.getDisplayId() + ")"); + } else { + LimeLog.info("Ignoring additional secondary internal display: " + display.getName()); + } + } + + if (info.mainDefaultDisplay == null) { + try { + info.mainDefaultDisplay = displayManager.getDisplay(Display.DEFAULT_DISPLAY); + LimeLog.warning("Main display (ID 0) not found in displays list, using getDisplay(DEFAULT_DISPLAY)."); + } catch (Exception e) { + LimeLog.severe("FATAL: Could not get display for DEFAULT_DISPLAY ID: " + e.toString()); + if (displays.length > 0 && displays[0] != null) info.mainDefaultDisplay = displays[0]; + } + } + if (info.mainDefaultDisplay == null) { + LimeLog.severe("FATAL: Could not obtain any valid reference to the main display after all fallbacks."); + } + // showCategorizationDialog(context, info); + return info; + } + + // --- getGameStreamDisplay - Contains Full Logic --- + public static Display getGameStreamDisplay(Context context) { + ensureDialogShown(context); // Ensure dialog appears once + CategorizedDisplays info = findAndCategorizeDisplays(context); // Get current displays + PreferenceConfiguration prefs = PreferenceConfiguration.readPreferences(context); + + Display defaultDisplay = info.mainDefaultDisplay; + // Essential fallback + if (defaultDisplay == null) { + LimeLog.severe("Cannot determine game stream display: mainDefaultDisplay is null. Using OS default."); + try { + DisplayManager dm = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + Display fallbackDisplay = (dm != null) ? dm.getDisplay(Display.DEFAULT_DISPLAY) : null; + if (fallbackDisplay == null) { LimeLog.severe("OS default display is also null!"); } + return fallbackDisplay; + } catch (Exception e) { + LimeLog.severe("Error getting OS default display in fallback: " + e.toString()); + return null; + } + } + + // --- Logic Flow --- + boolean treatAsInternal = false; + if (prefs.enableFullExDisplay) { + if (info.externalPresentationDisplay != null) { + boolean potentiallyMismatchedIDs = false; + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { // API 34 + try { + String deviceManufacturer = Build.MANUFACTURER; + android.hardware.display.DeviceProductInfo productInfo = info.externalPresentationDisplay.getDeviceProductInfo(); + String displayManufacturerId = (productInfo != null) ? productInfo.getManufacturerPnpId() : null; + + if (deviceManufacturer != null && displayManufacturerId != null && + displayManufacturerId.toLowerCase().contains(deviceManufacturer.toLowerCase())) + { + LimeLog.info("External display manufacturer ID matches device. Treating as potentially internal."); + treatAsInternal = true; + } else { + LimeLog.info("External display manufacturer ID does NOT match device or info unavailable."); + potentiallyMismatchedIDs = true; + } + } catch (Exception e) { + LimeLog.severe("Error comparing manufacturer IDs:" + e); + potentiallyMismatchedIDs = true; + } + } else { + LimeLog.info("Manufacturer ID check skipped (Requires API 34+)."); + potentiallyMismatchedIDs = true; + } + + if (!treatAsInternal) { + DisplayInfo mainInfo = getDisplayInfo(defaultDisplay); + DisplayInfo externalInfo = getDisplayInfo(info.externalPresentationDisplay); + if (potentiallyMismatchedIDs && + mainInfo != null && externalInfo != null && + // externalInfo.refreshRate < mainInfo.refreshRate - 1.0f && // Refresh rate check removed + externalInfo.totalPixels < mainInfo.totalPixels && + !isCommonMonitorAspectRatio(externalInfo)) + { + LimeLog.info("External display (ID " + info.externalPresentationDisplay.getDisplayId() + ") meets heuristics (smaller, non-monitor ratio). Manufacturer check failed/skipped. Assuming it's secondary internal."); + treatAsInternal = true; + } else { + LimeLog.info("enableFullExDisplay ON: Using true external presentation display (ID: " + info.externalPresentationDisplay.getDisplayId() + "). Heuristics did not apply or failed."); + return info.externalPresentationDisplay; + } + } + } + + Display effectiveSecondary = treatAsInternal ? info.externalPresentationDisplay : info.secondaryInternalDisplay; + + if (effectiveSecondary != null) { + DisplayInfo mainInfo = getDisplayInfo(defaultDisplay); + DisplayInfo secondaryInfo = getDisplayInfo(effectiveSecondary); + Display largerScreen = defaultDisplay; + + if (mainInfo != null && secondaryInfo != null) { + if (secondaryInfo.totalPixels > mainInfo.totalPixels) { + largerScreen = effectiveSecondary; + } + } else { + LimeLog.warning("enableFullExDisplay ON: Could not compare internal/effective-secondary displays; assuming default (ID 0) is larger."); + } + LimeLog.info("enableFullExDisplay ON (treating as internal screens): Using LARGER display (ID: " + largerScreen.getDisplayId() + ") for game stream."); + return largerScreen; + } + else { + LimeLog.info("enableFullExDisplay ON: Using main default display (ID: " + defaultDisplay.getDisplayId() + ") for game stream (only internal screen)."); + return defaultDisplay; + } + } + + if (info.secondaryInternalDisplay != null) { + DisplayInfo mainInfo = getDisplayInfo(defaultDisplay); + DisplayInfo secondaryInfo = getDisplayInfo(info.secondaryInternalDisplay); + Display largerInternal = defaultDisplay; + + if (mainInfo != null && secondaryInfo != null) { + if (secondaryInfo.totalPixels > mainInfo.totalPixels) { + largerInternal = info.secondaryInternalDisplay; + } + } else { + LimeLog.warning("Default OFF: Could not compare internal displays; assuming default (ID 0) is larger."); + } + LimeLog.info("Default OFF: Using LARGER internal display (ID: " + largerInternal.getDisplayId() + ") for game stream."); + return largerInternal; + } + + LimeLog.info("Default OFF: Using main default display (ID: " + defaultDisplay.getDisplayId() + ") for game stream (single screen)."); + return defaultDisplay; + } + + + private static boolean isCommonMonitorAspectRatio(DisplayInfo info) { + if (info == null || info.height <= 0) return false; + + float ratio = (float) info.width / (float) info.height; + final float EPSILON = 0.05f; + + final float RATIO_16_9 = 16.0f / 9.0f; + final float RATIO_16_10 = 16.0f / 10.0f; + final float RATIO_21_9 = 21.0f / 9.0f; + final float RATIO_32_9 = 32.0f / 9.0f; + final float RATIO_4_3 = 4.0f / 3.0f; + + boolean isMonitorRatio = Math.abs(ratio - RATIO_16_9) < EPSILON || + Math.abs(ratio - RATIO_16_10) < EPSILON || + Math.abs(ratio - RATIO_21_9) < EPSILON || + Math.abs(ratio - RATIO_32_9) < EPSILON || + Math.abs(ratio - RATIO_4_3) < EPSILON; + + LimeLog.info("Aspect ratio check: " + info.width + "x" + info.height + " -> ratio=" + String.format("%.3f", ratio) + ", isMonitorRatio=" + isMonitorRatio); + return isMonitorRatio; + } + + + // --- getControlsDisplay - Contains Full Logic --- + public static Display getControlsDisplay(Context context) { + ensureDialogShown(context); // Ensure dialog appears once + CategorizedDisplays info = findAndCategorizeDisplays(context); // Get current displays + // --- Determine where the game WILL be displayed --- + Display gameDisplay = getGameStreamDisplay(context); // Call the public method directly + // --- End Game Display Determination --- + + Display defaultDisplay = info.mainDefaultDisplay; + // Essential Fallbacks + if (defaultDisplay == null || gameDisplay == null) { + LimeLog.severe("Cannot determine controls display: mainDefaultDisplay or gameDisplay is null. Using OS default."); + try { + DisplayManager dm = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + Display fallbackDisplay = (dm != null) ? dm.getDisplay(Display.DEFAULT_DISPLAY) : null; + if (fallbackDisplay == null) { LimeLog.severe("OS default display is also null!"); } + return fallbackDisplay; + } catch (Exception e) { + LimeLog.severe("Error getting OS default display in controls fallback: " + e.toString()); + return null; + } + } + + // --- Identify potential candidates for the controls display --- + List controlCandidates = new ArrayList<>(); + if (info.mainDefaultDisplay != null && info.mainDefaultDisplay.getDisplayId() != gameDisplay.getDisplayId()) { + controlCandidates.add(info.mainDefaultDisplay); + } + if (info.secondaryInternalDisplay != null && info.secondaryInternalDisplay.getDisplayId() != gameDisplay.getDisplayId()) { + controlCandidates.add(info.secondaryInternalDisplay); + } + if (info.externalPresentationDisplay != null && info.externalPresentationDisplay.getDisplayId() != gameDisplay.getDisplayId()) { + controlCandidates.add(info.externalPresentationDisplay); + } + + List validControlCandidates = new ArrayList<>(); + for (Display candidate : controlCandidates) { + DisplayInfo candidateInfo = getDisplayInfo(candidate); + if (candidateInfo != null) { + validControlCandidates.add(candidate); + } else { + LimeLog.warning("Control candidate display (ID: " + ((candidate != null) ? candidate.getDisplayId() : "null") + ") is below minimum size or info unavailable. Ignoring."); + } + } + + // --- Select the best control display --- + Display selectedControlsDisplay = null; + if (validControlCandidates.size() == 1) { + selectedControlsDisplay = validControlCandidates.get(0); + LimeLog.info("Using the only valid secondary display (ID: " + selectedControlsDisplay.getDisplayId() + ") for controls."); + } else if (validControlCandidates.size() > 1) { + Display smallestValid = null; + long smallestPixels = Long.MAX_VALUE; + for (Display validCandidate : validControlCandidates) { + DisplayInfo validInfo = getDisplayInfo(validCandidate); + if (validInfo != null && validInfo.totalPixels < smallestPixels) { + smallestPixels = validInfo.totalPixels; + smallestValid = validCandidate; + } + } + selectedControlsDisplay = smallestValid; + if (selectedControlsDisplay != null) { + LimeLog.info("Multiple valid secondary displays found. Using the smallest (ID: " + selectedControlsDisplay.getDisplayId() + ") for controls."); + } else { + LimeLog.warning("Could not determine smallest among valid control candidates. Falling back to overlay."); + selectedControlsDisplay = gameDisplay; + } + } else { + LimeLog.info("No valid secondary display found for controls. Using game display (ID: " + gameDisplay.getDisplayId() + ") for overlay controls."); + selectedControlsDisplay = gameDisplay; + } + + return selectedControlsDisplay; + } + + + private static String getDisplayDetailsString(Display display) { + if (display == null) { + return "null display object"; + } + StringBuilder details = new StringBuilder(); + DisplayInfo di = getDisplayInfo(display); + + details.append("ID: ").append(display.getDisplayId()); + details.append(", Name: ").append(display.getName()); + details.append(", Res: ").append(di != null ? di.toString() : "N/A"); // Use DisplayInfo.toString() + + try { + int flags = display.getFlags(); + List flagNames = new ArrayList<>(); + if ((flags & Display.FLAG_PRIVATE) != 0) flagNames.add("PRIVATE"); + if ((flags & Display.FLAG_PRESENTATION) != 0) flagNames.add("PRESENTATION"); + if ((flags & Display.FLAG_SECURE) != 0) flagNames.add("SECURE"); + if ((flags & Display.FLAG_SUPPORTS_PROTECTED_BUFFERS) != 0) flagNames.add("PROTECTED"); + if ((flags & Display.FLAG_ROUND) != 0) flagNames.add("ROUND"); + + details.append(", Flags: "); + if (flagNames.isEmpty()) { + details.append("None"); + } else { + details.append("[").append(String.join("|", flagNames)).append("]"); + } + details.append(" (").append(flags).append(")"); + } catch (Exception e) { + details.append(", Flags: Error reading flags (").append(e.getMessage()).append(")"); + } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { // API 34 + try { + android.hardware.display.DeviceProductInfo productInfo = display.getDeviceProductInfo(); + if (productInfo != null) { + details.append(", Product: ["); + details.append("Name: ").append(productInfo.getName()); + String manufId = productInfo.getManufacturerPnpId(); + if (manufId != null && !manufId.isEmpty()) { + details.append(", ManufId: ").append(manufId); + } + String prodId = productInfo.getProductId(); + if (prodId != null && !prodId.isEmpty()) { + details.append(", ProdId: ").append(prodId); + } + details.append("]"); + } else { + details.append(", ProductInfo: null"); + } + } catch (NoSuchMethodError e) { + LimeLog.warning("getProductInfo method not found on API 34+ device?"); + details.append(", ProductInfo: Not Available (API Error)"); + } catch (Exception e) { + LimeLog.severe("Error getting product info: " + e.getMessage()); + details.append(", ProductInfo: Error reading info"); + } + } else { + details.append(", ProductInfo: Not Available (API < 34)"); + } + + return details.toString(); + } + + + private static void showCategorizationDialog(Context context, CategorizedDisplays info) { + if (context == null) { + LimeLog.severe("Cannot show display dialog: Context is null."); + return; + } + + if(openDialog != null && openDialog.isShowing()) { + try { openDialog.dismiss(); } catch (Exception e) { LimeLog.warning("Error dismissing previous dialog: " + e.getMessage()); } + openDialog = null; + } + + final StringBuilder messageBuilder = new StringBuilder(); + boolean displayFound = false; + + messageBuilder.append("--- Displays Found ---\n"); + + if (info.mainDefaultDisplay != null) { + messageBuilder.append("Main (Default): \n ").append(getDisplayDetailsString(info.mainDefaultDisplay)).append("\n\n"); + displayFound = true; + } else { + messageBuilder.append("Main (Default): Not Found!\n\n"); + } + if (info.externalPresentationDisplay != null) { + messageBuilder.append("External (Presentation Flag): \n ").append(getDisplayDetailsString(info.externalPresentationDisplay)).append("\n\n"); + displayFound = true; + } else { + messageBuilder.append("External (Presentation Flag): None\n\n"); + } + if (info.secondaryInternalDisplay != null) { + messageBuilder.append("Secondary Internal (No Pres. Flag): \n ").append(getDisplayDetailsString(info.secondaryInternalDisplay)).append("\n\n"); + displayFound = true; + } else { + messageBuilder.append("Secondary Internal (No Pres. Flag): None\n\n"); + } + + // --- Determine final selections for the dialog --- + // Need to replicate the logic *briefly* here to show the result + Display determinedGameDisplay = null; + Display determinedControlsDisplay = null; + try { + // Replicate getGameStreamDisplay logic (simplified, without dialog call) + PreferenceConfiguration tempPrefs = PreferenceConfiguration.readPreferences(context); + Display tempDefault = info.mainDefaultDisplay; + if (tempDefault != null) { + boolean tempTreatAsInternal = false; + if (tempPrefs.enableFullExDisplay && info.externalPresentationDisplay != null) { + // Simplified heuristic check for dialog display purpose + boolean tempPotentiallyMismatched = true; // Assume mismatch for dialog unless proven otherwise by API 34+ check (not replicated here for brevity) + DisplayInfo tempMainInfo = getDisplayInfo(tempDefault); + DisplayInfo tempExternalInfo = getDisplayInfo(info.externalPresentationDisplay); + if (tempPotentiallyMismatched && tempMainInfo != null && tempExternalInfo != null && + tempExternalInfo.totalPixels < tempMainInfo.totalPixels && !isCommonMonitorAspectRatio(tempExternalInfo)) { + tempTreatAsInternal = true; + } else { + determinedGameDisplay = info.externalPresentationDisplay; // Assume external if heuristics fail/don't apply + } + } + + if (determinedGameDisplay == null) { // If not assigned external + Display tempEffectiveSecondary = tempTreatAsInternal ? info.externalPresentationDisplay : info.secondaryInternalDisplay; + if (tempEffectiveSecondary != null) { + DisplayInfo tempMainInfo = getDisplayInfo(tempDefault); + DisplayInfo tempSecondaryInfo = getDisplayInfo(tempEffectiveSecondary); + determinedGameDisplay = tempDefault; + if (tempMainInfo != null && tempSecondaryInfo != null && tempSecondaryInfo.totalPixels > tempMainInfo.totalPixels) { + determinedGameDisplay = tempEffectiveSecondary; + } + } else { + determinedGameDisplay = tempDefault; + } + } + } + // Replicate getControlsDisplay logic (simplified) + if (determinedGameDisplay != null && tempDefault != null) { + boolean tempGameIsTrulyExternal = info.externalPresentationDisplay != null && determinedGameDisplay.getDisplayId() == info.externalPresentationDisplay.getDisplayId(); + // Additional check needed here to ensure it wasn't treated as internal + // For simplicity in dialog, we might omit perfect replication + if (tempGameIsTrulyExternal) { + determinedControlsDisplay = (info.secondaryInternalDisplay != null) ? + ((getDisplayInfo(info.secondaryInternalDisplay).totalPixels < getDisplayInfo(tempDefault).totalPixels) ? info.secondaryInternalDisplay : tempDefault) + : tempDefault; // Simplified: picks smaller of internals, or default + // Missing MIN_SIZE check here for dialog brevity + } else if (info.secondaryInternalDisplay != null) { + determinedControlsDisplay = (determinedGameDisplay.getDisplayId() == tempDefault.getDisplayId()) ? info.secondaryInternalDisplay : tempDefault; + } else { + determinedControlsDisplay = determinedGameDisplay; // Overlay + } + } + + } catch (Exception e) { + LimeLog.severe("Error determining selections for dialog: " + e.getMessage()); + } + + + messageBuilder.append("--- Final Selection ---\n"); + messageBuilder.append("Game Stream Display: \n "); + messageBuilder.append(determinedGameDisplay != null ? getDisplayDetailsString(determinedGameDisplay) : "ERROR (null)").append("\n\n"); + messageBuilder.append("Controls Display: \n "); + messageBuilder.append(determinedControlsDisplay != null ? getDisplayDetailsString(determinedControlsDisplay) : "ERROR (null)").append("\n"); + + + final String message = displayFound ? messageBuilder.toString().trim() : "Error: No displays were categorized."; + final String title = "Display Selection Info"; + + new Handler(Looper.getMainLooper()).post(() -> { + try { + if (context instanceof android.app.Activity && ((android.app.Activity) context).isFinishing()) { + LimeLog.warning("Activity is finishing, cannot show display dialog."); + return; + } + + openDialog = new AlertDialog.Builder(context) + .setTitle(title) + .setMessage(message) + .setPositiveButton(android.R.string.ok, (dialog, which) -> { + dialog.dismiss(); + openDialog = null; + }) + .setNeutralButton("Copy Info", (dialog, which) -> { + ClipboardManager clipboard = (ClipboardManager) context.getSystemService(Context.CLIPBOARD_SERVICE); + if (clipboard != null) { + ClipData clip = ClipData.newPlainText("Display Info", message); + clipboard.setPrimaryClip(clip); + Toast.makeText(context, "Display info copied to clipboard", Toast.LENGTH_SHORT).show(); + } else { + Toast.makeText(context, "Failed to access clipboard", Toast.LENGTH_SHORT).show(); + } + }) + .setCancelable(false) + .show(); + } catch (Exception e) { + LimeLog.severe("Failed to show display selection dialog: " + e.getMessage()); + openDialog = null; + } + }); + } + + private static volatile boolean dialogShown = false; + private static void ensureDialogShown(Context context) { + if (!dialogShown) { + synchronized (DisplayUtils.class) { + if (!dialogShown) { + CategorizedDisplays info = findAndCategorizeDisplays(context); // This now calls the dialog + // Selections are determined inside the dialog show logic for display + dialogShown = true; + } + } + } + } + + + public static boolean hasSecondaryDisplay(Context context) { + ensureDialogShown(context); // Ensure dialog shows if not already + CategorizedDisplays info = findAndCategorizeDisplays(context); // Find displays again + boolean hasSecondary = info.externalPresentationDisplay != null || info.secondaryInternalDisplay != null; + return hasSecondary; + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/limelight/utils/ExternalDisplayControlActivity.java b/app/src/main/java/com/limelight/utils/ExternalDisplayControlActivity.java index 5bbb0a30b0..78c20f7780 100644 --- a/app/src/main/java/com/limelight/utils/ExternalDisplayControlActivity.java +++ b/app/src/main/java/com/limelight/utils/ExternalDisplayControlActivity.java @@ -1,7 +1,7 @@ package com.limelight.utils; import static com.limelight.StartExternalDisplayControlReceiver.requestFocusToGameActivity; -import static com.limelight.utils.ServerHelper.getSecondaryDisplay; +import static com.limelight.utils.DisplayUtils.getGameStreamDisplay; import android.Manifest; import android.annotation.SuppressLint; @@ -69,6 +69,8 @@ public class ExternalDisplayControlActivity extends AppCompatActivity implements private boolean isKeyboardVisible = false; + private boolean shouldManageBrightness = false; + private final Handler handler = new Handler(Looper.getMainLooper()); private int failCount = 0; private Runnable dimScreenRunnable; @@ -122,15 +124,15 @@ protected void onCreate(Bundle savedInstanceState) { if (gameIntent == null) { finish(); } else { - Display secondaryDisplay = getSecondaryDisplay(this); - if (secondaryDisplay != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Display gameStreamDisplay = getGameStreamDisplay(this); + if (gameStreamDisplay != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { ActivityOptions options = ActivityOptions.makeBasic(); - options.setLaunchDisplayId(secondaryDisplay.getDisplayId()); + options.setLaunchDisplayId(gameStreamDisplay.getDisplayId()); Toast.makeText(this, getString(R.string.external_display_info, - secondaryDisplay.getMode().getPhysicalWidth(), - secondaryDisplay.getMode().getPhysicalHeight(), - secondaryDisplay.getMode().getRefreshRate()), + gameStreamDisplay.getMode().getPhysicalWidth(), + gameStreamDisplay.getMode().getPhysicalHeight(), + gameStreamDisplay.getMode().getRefreshRate()), Toast.LENGTH_LONG).show(); startActivity(gameIntent, options.toBundle()); @@ -175,11 +177,17 @@ private void initViews() { }); } + Display controlsDisplay = DisplayUtils.getControlsDisplay(this); + shouldManageBrightness = (controlsDisplay != null && controlsDisplay.getDisplayId() == Display.DEFAULT_DISPLAY); + LimeLog.info("Brightness management " + (shouldManageBrightness ? "enabled" : "disabled") + " for this display (ID: " + ((controlsDisplay != null) ? controlsDisplay.getDisplayId() : "null") + ")"); + initializeComponents(); createProgrammaticUI(); checkNotificationPermission(); initTouchEventHandling(); - setupInactivityTimeoutForBrightness(); + if (shouldManageBrightness) { + setupInactivityTimeoutForBrightness(); + } requestFocusToGameActivity(false); } @@ -224,12 +232,14 @@ public void onKeyboardControllerVisibilityChange(boolean visible) { @SuppressLint("ClickableViewAccessibility") private void setupInactivityTimeoutForBrightness() { + if (!shouldManageBrightness) return; // Save the original brightness WindowManager.LayoutParams layout = getWindow().getAttributes(); originalBrightness = layout.screenBrightness; // Runnable to dim screen dimScreenRunnable = () -> { + if (!shouldManageBrightness) return; WindowManager.LayoutParams l = getWindow().getAttributes(); l.screenBrightness = 0.0f; getWindow().setAttributes(l); @@ -255,6 +265,7 @@ private void updateKeyboardVisibility(boolean visible) { } private void restoreBrightnessIfNeeded() { + if (!shouldManageBrightness) return; WindowManager.LayoutParams l = getWindow().getAttributes(); if (l.screenBrightness == 0.0f) { l.screenBrightness = originalBrightness; @@ -263,12 +274,14 @@ private void restoreBrightnessIfNeeded() { } private void handleUserActivity() { + if (!shouldManageBrightness) return; // Restore brightness if dimmed restoreBrightnessIfNeeded(); resetInactivityTimer(); } private void resetInactivityTimer() { + if (!shouldManageBrightness) return; handler.removeCallbacks(dimScreenRunnable); if (!isKeyboardVisible) { handler.postDelayed(dimScreenRunnable, INACTIVITY_TIMEOUT_MS); diff --git a/app/src/main/java/com/limelight/utils/PanZoomHandler.java b/app/src/main/java/com/limelight/utils/PanZoomHandler.java index d0d3384ac0..c00dd7c421 100644 --- a/app/src/main/java/com/limelight/utils/PanZoomHandler.java +++ b/app/src/main/java/com/limelight/utils/PanZoomHandler.java @@ -112,7 +112,7 @@ private class ScaleListener extends ScaleGestureDetector.SimpleOnScaleGestureLis @Override public boolean onScale(ScaleGestureDetector detector) { float newScaleFactor = scaleFactor * detector.getScaleFactor(); - newScaleFactor = Math.max(1, Math.min(newScaleFactor, MAX_SCALE)); // Apply minimum scale + newScaleFactor = Math.max(0.5f, Math.min(newScaleFactor, MAX_SCALE)); // Apply minimum scale // Calculate pivot point float focusX = detector.getFocusX(); diff --git a/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java b/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java deleted file mode 100644 index 9e21784616..0000000000 --- a/app/src/main/java/com/limelight/utils/ReflectivePaddingInt8Minimal.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.limelight.utils; - -import org.opencv.core.*; -import org.opencv.imgproc.Imgproc; - -import java.nio.ByteBuffer; - -public class ReflectivePaddingInt8Minimal { - - /** - * In-place Reflected Padding + Feather + Blur auf ByteBuffer (INT8 RGB) - * Minimaler Speicher: kein alpha3 Merge, nur 2 temporäre Mats - */ - public static void applyReflectedPadding(ByteBuffer buffer) { - final int size = 256; - final int band = (int)(size * 0.15); // obere/untere 20% - final int featherPx = 12; - final int blurKsize = 7; - - // --- ByteBuffer -> Mat (CV_8UC3) --- - buffer.rewind(); - byte[] arr = new byte[size * size * 3]; - buffer.get(arr); - Mat mat = new Mat(size, size, CvType.CV_8UC3); - mat.put(0, 0, arr); - - // --- Top-Band --- - Mat topBand = mat.submat(band, 2*band, 0, size); - Mat tmp = new Mat(); - Core.flip(topBand, tmp, 0); - Imgproc.GaussianBlur(tmp, tmp, new Size(blurKsize, blurKsize), 0); - blendInt8Minimal(mat.submat(0, band, 0, size), tmp, band, featherPx); - tmp.release(); - topBand.release(); - - // --- Bottom-Band --- - Mat botBand = mat.submat(size - 2*band, size - band, 0, size); - tmp = new Mat(); - Core.flip(botBand, tmp, 0); - Imgproc.GaussianBlur(tmp, tmp, new Size(blurKsize, blurKsize), 0); - blendInt8Minimal(mat.submat(size - band, size, 0, size), tmp, band, featherPx); - tmp.release(); - botBand.release(); - - // --- Mat -> ByteBuffer zurück --- - Core.flip(mat, mat, 0); - mat.get(0,0,arr); - buffer.rewind(); - buffer.put(arr); - buffer.rewind(); - mat.release(); - } - - /** - * INT8 Alpha-Blend ohne Merge: dst = (alpha*padded + (255-alpha)*dst)/255 - * alpha linear von 0-255 über band Pixel - */ - private static void blendInt8Minimal(Mat dst, Mat padded, int band, int featherPx) { - int width = dst.cols(); - int channels = dst.channels(); - byte[] dstRow = new byte[width * channels]; - byte[] padRow = new byte[width * channels]; - - for(int y=0; y= Build.VERSION_CODES.O && prefConfig.enableFullExDisplay && getSecondaryDisplay(parent) != null) { - Context displayContext = parent.createDisplayContext(getSecondaryDisplay(parent)); // use secondary display + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && prefConfig.enableFullExDisplay && hasSecondaryDisplay(parent)) { + Context displayContext = parent.createDisplayContext(getGameStreamDisplay(parent)); // use secondary display gameIntent = new Intent(displayContext, Game.class); gameIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); } @@ -123,11 +98,9 @@ public static Intent createStartIntent(Activity parent, NvApp app, ComputerDetai e.printStackTrace(); } - if (prefConfig.enableFullExDisplay) { - Display secondaryDisplay = getSecondaryDisplay(parent); - if (secondaryDisplay != null) { - int secondaryDisplayId = secondaryDisplay.getDisplayId(); - gameIntent.putExtra(Game.EXTRA_DISPLAY_ID, secondaryDisplayId); + if (prefConfig.enableFullExDisplay && hasSecondaryDisplay(parent)) { + Display gameStreamDisplay = getGameStreamDisplay(parent); + if (gameStreamDisplay != null) { Intent touchpadIntent = new Intent(parent, ExternalDisplayControlActivity.class); touchpadIntent.putExtra(ExternalDisplayControlActivity.EXTRA_LAUNCH_INTENT, gameIntent); return touchpadIntent; diff --git a/app/src/main/java/com/limelight/utils/ShaderUtils.java b/app/src/main/java/com/limelight/utils/ShaderUtils.java index ffb115d1b9..579058df52 100644 --- a/app/src/main/java/com/limelight/utils/ShaderUtils.java +++ b/app/src/main/java/com/limelight/utils/ShaderUtils.java @@ -22,31 +22,41 @@ public class ShaderUtils { "uniform bool u_debugMode;\n" + "\n" + "void main() {\n" + - " float depth = texture2D(s_DepthTexture, v_TexCoord).r;\n" + + " vec2 depthTexCoord = v_TexCoord;\n" + + " // Wende deinen bestehenden Offset auf die korrigierte Koordinate an.\n" + + " depthTexCoord -= vec2(abs(u_parallax / 2.0), 0);\n" + + " float depth = texture2D(s_DepthTexture, depthTexCoord).r;\n" + "\n" + - " // Remap depth into symmetric range around convergence\n" + - " float depthDiff;\n" + - " if (depth < u_convergence) {\n" + - " depthDiff = (depth - u_convergence) / u_convergence; // [-1,0]\n" + - " } else {\n" + - " depthDiff = (depth - u_convergence) / (1.0 - u_convergence); // [0,1]\n" + - " }\n" + + " const float zone_radius = 0.70; // Breite der neutralen Zone um Konvergenz\n" + + "\n" + + " float depthDiff = depth - u_convergence;\n" + + " // clamp to ensure total width = 1 unit, sliding around convergence\n" + + " depthDiff = clamp(depthDiff, -u_convergence, 1.0 - u_convergence);\n" + + "\n" + + " float dist_from_convergence = abs(depth - u_convergence);\n" + + " float fade_multiplier = smoothstep(0.0, zone_radius, dist_from_convergence);\n" + "\n" + " float parallax_magnitude = abs(u_parallax);\n" + " float ai_shift = parallax_magnitude * depthDiff;\n" + "\n" + - " // --- Dynamische Vignette ---\n" + + " // --- Dynamische Vignette, an Konvergenz angepasst ---\n" + " float edgeWidth = 0.01;\n" + " float depthLeft = texture2D(s_DepthTexture, vec2(edgeWidth, 0.5)).r;\n" + " float depthRight = texture2D(s_DepthTexture, vec2(1.0 - edgeWidth, 0.5)).r;\n" + - " float ai_shift_left = u_parallax * (depthLeft - 0.5);\n" + - " float ai_shift_right = u_parallax * (depthRight - 0.5);\n" + + "\n" + + " // Korrektur: Bezug auf u_convergence statt 0.5\n" + + " float ai_shift_left = u_parallax * (depthLeft - u_convergence);\n" + + " float ai_shift_right = u_parallax * (depthRight - u_convergence);\n" + " float maxEdgeShift = max(abs(ai_shift_left), abs(ai_shift_right));\n" + + "\n" + " bool isLeftEye = (u_parallax < 0.0);\n" + - " float isLeftEyeIndicator = isLeftEye ? -1.0 : 1.0;\n" + - " float vignette_start = mix(0.7, 1.0, clamp(maxEdgeShift / 0.5, 0.0, 1.0));\n" + - " const float vignette_end = 1.0;\n" + + " float isLeftEyeIndicator = isLeftEye ? 1.0 : -1.0;\n" + "\n" + + " // Vignette leicht an Konvergenz koppeln\n" + + " float vignette_bias = (u_convergence - 0.5) * 0.3; // ±0.15 Anpassung\n" + + " float vignette_start = mix(0.7 - vignette_bias, 1.0 - vignette_bias,\n" + + " clamp(maxEdgeShift / 0.5, 0.0, 1.0));\n" + + " const float vignette_end = 1.0;\n" + " if ((depth - u_convergence) < 0.0) {\n" + " ai_shift *= isLeftEye ? u_shift : (1.0-u_shift);\n" + " } else {\n" + @@ -66,7 +76,6 @@ public class ShaderUtils { " float artifactBlendFactor = (1.0 - smoothstep(0.1, 1.0, shiftMagnitude)) * 0.005;\n" + " vec4 finalColor = mix(shiftedColor, originalColor, artifactBlendFactor);\n" + "\n" + - " // ---------------- Rot/Blau Debug -----------------\n" + " if (u_debugMode) {\n" + " float debugDepth = final_shift;\n" + " vec3 debugTint = vec3(0.0);\n" + @@ -78,10 +87,30 @@ public class ShaderUtils { " gl_FragColor = finalColor;\n" + "}\n"; - - - - + public static final String FRAGMENT_SHADER_SEPARABLE_DILATE = + "precision mediump float;\n" + + "varying vec2 v_TexCoord;\n" + + "uniform sampler2D s_InputTexture;\n" + + "uniform vec2 u_texelSize;\n" + + "uniform int u_radius;\n" + + // NEU: Die Richtung (z.B. (1.0, 0.0) für horizontal) + "uniform vec2 u_direction;\n" + + "\n" + + "void main() {\n" + + " if (u_radius <= 0) {\n" + + " gl_FragColor = texture2D(s_InputTexture, v_TexCoord);\n" + + " return;\n" + + " }\n" + + "\n" + + " float maxDepth = texture2D(s_InputTexture, v_TexCoord).r;\n" + + "\n" + + " // Loop in one direction only\n" + + " for (int i = -u_radius; i <= u_radius; i++) {\n" + + " vec2 offset = u_direction * float(i) * u_texelSize;\n" + + " maxDepth = max(maxDepth, texture2D(s_InputTexture, v_TexCoord + offset).r);\n" + + " }\n" + + " gl_FragColor = vec4(vec3(maxDepth), 1.0);\n" + + "}\n"; /** * An optimized, single-pass Gaussian blur shader that works as a drop-in replacement. * It achieves better performance by taking fewer texture samples over the same blur radius. @@ -96,9 +125,12 @@ public class ShaderUtils { "uniform float u_parallax;\n" + "void main() {\n" + - "float blurRadius = 60.0 * u_parallax;\n" + - "float blurStep = 2.0 / u_parallax;\n" + - "float sigma = 50.0 * u_parallax;\n" + + " float minRadius = 10.0;\n" + + " float minSigma = 5.0;\n" + + "\n" + + " float blurRadius = max(minRadius, 60.0 * u_parallax);\n" + + " float sigma = max(minSigma, 50.0 * u_parallax);\n" + + " float blurStep = 1.0; // immer in 1-Pixel-Schritten\n" + " vec4 sum = vec4(0.0);\n" + " float weightSum = 0.0;\n" + @@ -113,90 +145,21 @@ public class ShaderUtils { " gl_FragColor = sum / weightSum;\n" + "}\n"; - public static final String SIMPLE_VERTEX_SHADER = - "attribute vec4 a_Position;\n" + - "attribute vec2 a_TexCoord;\n" + + public static final String SIMPLE_FRAGMENT_SHADER = + "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + "varying vec2 v_TexCoord;\n" + + "uniform samplerExternalOES u_Texture;\n" + "void main() {\n" + - " gl_Position = a_Position;\n" + - " v_TexCoord = a_TexCoord;\n" + - "}\n"; - - public static final String EDGE_AWARE_VERTEX_SHADER = - "attribute vec4 a_Position;\n" + - "attribute vec2 a_TexCoord;\n" + - "varying vec2 v_TexCoord;\n" + - "void main(){\n" + - " v_TexCoord = a_TexCoord;\n" + - " gl_Position = a_Position;\n" + - "}\n"; - - public static final String EDGE_AWARE_DEPTH_BLUR_SHADER = - "precision mediump float;\n" + - "varying vec2 v_TexCoord;\n" + - "uniform sampler2D uDepthMap;\n" + - "uniform vec2 u_texelSize;\n" + - "uniform bool u_debugMode;\n" + - "uniform float u_parallax;\n" + - "\n" + - "void main(){\n" + - "float parallaxFactor = clamp(u_parallax, 0.0, 1.0);\n" + - "int radius = int(30.0 * parallaxFactor);\n"+ - "float sharpness = 1.0 * (1.1 - parallaxFactor); \n" + - "float holeThreshold = 20.0 * parallaxFactor; \n" + - " float centerDepth = texture2D(uDepthMap, v_TexCoord).r;\n" + - " vec4 sum = vec4(0.0);\n" + - " float weightSum = 0.0;\n" + - "\n" + - " // ---------------- Horizontal Blur ----------------\n" + - " for(int i=-radius;i<=radius;i++){\n" + - " vec2 offsetUV = v_TexCoord + vec2(float(i)*u_texelSize.x,0.0);\n" + - " float sampleDepth = texture2D(uDepthMap, offsetUV).r;\n" + - " float w = exp(-abs(sampleDepth-centerDepth)*sharpness);\n" + - " sum += texture2D(uDepthMap, offsetUV) * w;\n" + - " weightSum += w;\n" + - " }\n" + - " vec4 blurred = sum/weightSum;\n" + - "\n" + - " // ---------------- Hole Filling ----------------\n" + - " if(abs(blurred.r - centerDepth) > holeThreshold){\n" + - " vec4 fill = vec4(0.0);\n" + - " float fillWeight = 0.0;\n" + - " vec2 offs[4];\n" + - " offs[0] = vec2(u_texelSize.x,0.0);\n" + - " offs[1] = vec2(-u_texelSize.x,0.0);\n" + - " offs[2] = vec2(0.0,u_texelSize.y);\n" + - " offs[3] = vec2(0.0,-u_texelSize.y);\n" + - " for(int k=0;k<4;k++){\n" + - " float d = texture2D(uDepthMap, v_TexCoord+offs[k]).r;\n" + - " if(abs(d-centerDepth)0.0){ blurred = mix(blurred, fill/fillWeight, 0.7); }\n" + - " }\n" + - "\n" + - " // ---------------- Debug Rot/Blau ----------------\n" + - " if(u_debugMode){\n" + - " vec3 debugTint = vec3(0.0);\n" + - " float diff = blurred.r - centerDepth;\n" + - " if(diff > 0.0) debugTint.r = diff*50.0;\n" + - " else debugTint.b = -diff*50.0;\n" + - " blurred.rgb += debugTint;\n" + - " }\n" + - "\n" + - " gl_FragColor = blurred;\n" + + " gl_FragColor = texture2D(u_Texture, v_TexCoord);\n" + "}\n"; - - - public static final String SIMPLE_FRAGMENT_SHADER = + public static final String FLIPPED_FRAGMENT_SHADER = "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + "varying vec2 v_TexCoord;\n" + "uniform samplerExternalOES u_Texture;\n" + "void main() {\n" + - " gl_FragColor = texture2D(u_Texture, v_TexCoord);\n" + + " gl_FragColor = texture2D(u_Texture, vec2(v_TexCoord.x, 1.0 - v_TexCoord.y));\n" + "}\n"; } \ No newline at end of file diff --git a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java index b815b48153..af3f4410a2 100644 --- a/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java +++ b/app/src/main/java/com/limelight/utils/Stereo3DRenderer.java @@ -1,12 +1,14 @@ package com.limelight.utils; +import static com.limelight.utils.ShaderUtils.FRAGMENT_SHADER_SEPARABLE_DILATE; +import static com.limelight.utils.ShaderUtils.VERTEX_SHADER; + import android.content.Context; import android.content.res.AssetFileDescriptor; import android.graphics.SurfaceTexture; import android.opengl.GLES20; import android.opengl.GLES30; import android.opengl.GLSurfaceView; -import android.os.Build; import android.util.Log; import android.view.Surface; @@ -33,8 +35,10 @@ import java.nio.FloatBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; @@ -59,7 +63,6 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private final int NUM_INPUT_BUFFERS = 10; private final int NUM_SMOOTHED_BUFFERS = 3; private final int[] pboHandles = new int[2]; - private int pboIndex = 0; public static boolean isMovieMode = true; private int PBO_SIZE = modelInputWidth * modelInputHeight * 4; @@ -90,19 +93,30 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private final AtomicBoolean isAiRunning = new AtomicBoolean(false); // OpenGL Handles + private int mDilationProgram; private int bilateralBlurProgram; private int depthMapTextureId; private int dibr3dProgram; - - private final AtomicReference latestDepthMap = new AtomicReference<>(null); + private int simple3dProgram; + private int videoTextureId; private int fboHandle; private int fboTextureId; private int filterFboHandle; private int filteredDepthMapTextureId; private int intermediateFboHandle; private int intermediateTextureId; - private int simple3dProgram; - private int videoTextureId; + private int intermediateDilutionFboHandle; + private int intermediateDilutionTextureId; + private int gaussIntermediateFboHandle; + private int gaussIntermediateTextureId; + + // --- VORGELADENE SHADER-LOCATIONS FÜR PERFORMANCE --- + private int mDilationPosHandle, mDilationTexHandle, mDilationInputTextureHandle, + mDilationTexelSizeHandle, mDilationRadiusHandle, mDilationDirectionHandle; + private int mGaussPosHandle, mGaussTexHandle, mGaussInputTextureHandle, + mGaussTexelSizeHandle, mGaussDirectionHandle, mGaussParallaxHandle; + private int mDibrPosHandle, mDibrTexHandle, mDibrColorTexHandle, mDibrDepthTexHandle, + mDibrParallaxHandle, mDibrConvergenceHandle, mDibrShiftHandle, mDibrDebugModeHandle; // AI & TFLite Variables private GpuDelegate gpuDelegate; @@ -122,12 +136,10 @@ public class Stereo3DRenderer implements GLSurfaceView.Renderer, SurfaceTexture. private BlockingQueue freeSmoothedBuffers; private BlockingQueue inferenceInputQueue = new ArrayBlockingQueue<>(1); private PreferenceConfiguration prefConfig; - private ByteBuffer previousPixelBuffer; private Surface videoSurface; private SurfaceTexture videoSurfaceTexture; - - private float ON_DRAW_CHANGE_TRESHOLD = 2.0f; - + private final AtomicReference latestDepthMap = new AtomicReference<>(null); + private float ON_DRAW_CHANGE_TRESHOLD = 2.5f; public interface OnSurfaceReadyListener { void onStereo3DSurfaceReady(Surface surface); @@ -195,22 +207,24 @@ public void onSurfaceDestroyed() { GLES20.glDeleteProgram(simple3dProgram); GLES20.glDeleteProgram(bilateralBlurProgram); GLES20.glDeleteProgram(dibr3dProgram); + GLES20.glDeleteProgram(mDilationProgram); int[] textures = { videoTextureId, depthMapTextureId, filteredDepthMapTextureId, fboTextureId, - intermediateTextureId + intermediateTextureId, + intermediateDilutionTextureId, + }; GLES20.glDeleteTextures(textures.length, textures, 0); - int[] fbos = {fboHandle, intermediateFboHandle, filterFboHandle}; + int[] fbos = {fboHandle, intermediateFboHandle, filterFboHandle, intermediateDilutionFboHandle}; GLES20.glDeleteFramebuffers(fbos.length, fbos, 0); }); if (filledOutputBuffers != null) filledOutputBuffers.clear(); - previousPixelBuffer = null; currentlyRenderingMap = null; prefConfig = null; drawDelay = 0.0f; @@ -224,6 +238,19 @@ public Surface getVideoSurface() { return videoSurface; } + private void initializeGaussIntermediateFbo() { + gaussIntermediateTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); // Oder passende Textur erstellen + int[] fbos = new int[1]; + GLES20.glGenFramebuffers(1, fbos, 0); + gaussIntermediateFboHandle = fbos[0]; + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, gaussIntermediateFboHandle); + GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, gaussIntermediateTextureId, 0); + if (GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER) != GLES20.GL_FRAMEBUFFER_COMPLETE) { + LimeLog.warning("Gauss Intermediate Framebuffer is not complete."); + } + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + } + @Override public void onFrameAvailable(SurfaceTexture surfaceTexture) { synchronized (frameLock) { @@ -241,12 +268,20 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { depthMapTextureId = createEmptyTexture(modelInputWidth, modelInputHeight); - simple3dProgram = createProgram(ShaderUtils.SIMPLE_VERTEX_SHADER, ShaderUtils.SIMPLE_FRAGMENT_SHADER); - bilateralBlurProgram = createProgram(ShaderUtils.VERTEX_SHADER, ShaderUtils.OPTIMIZED_SINGLE_PASS_GAUSSIAN_BLUR_SHADER); - dibr3dProgram = createProgram(ShaderUtils.VERTEX_SHADER, ShaderUtils.FRAGMENT_SHADER_3D); + simple3dProgram = createProgram(VERTEX_SHADER, ShaderUtils.FLIPPED_FRAGMENT_SHADER); + bilateralBlurProgram = createProgram(VERTEX_SHADER, ShaderUtils.OPTIMIZED_SINGLE_PASS_GAUSSIAN_BLUR_SHADER); + dibr3dProgram = createProgram(VERTEX_SHADER, ShaderUtils.FRAGMENT_SHADER_3D); + mDilationProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER_SEPARABLE_DILATE); + if (mDilationProgram == 0) { + throw new RuntimeException("Konnte Dilation-Shader-Programm nicht erstellen."); + } + + getShaderLocations(); initializeFilterFbo(); initializeIntermediateFbo(); + initializeDilationFbo(); + initializeGaussIntermediateFbo(); initializeTfLite(); initializeFbo(); initBuffer(); @@ -258,10 +293,9 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { freeSmoothedBuffers.offer(ByteBuffer.allocateDirect(mapSize).order(ByteOrder.nativeOrder())); } - int pboSize = modelInputWidth * modelInputHeight * 4; - previousPixelBuffer = ByteBuffer.allocateDirect(pboSize).order(ByteOrder.nativeOrder()); + int pboSize = modelInputWidth * modelInputHeight * 3; previousFrameForComparison = ByteBuffer.allocateDirect(pboSize).order(ByteOrder.nativeOrder()); - int inputPixelSize = modelInputWidth * modelInputHeight * 4; + int inputPixelSize = modelInputWidth * modelInputHeight * 3; freeInputBuffers = new ArrayBlockingQueue<>(NUM_INPUT_BUFFERS); inferenceInputQueue = new ArrayBlockingQueue<>(1); for (int i = 0; i < NUM_INPUT_BUFFERS; i++) { @@ -284,6 +318,31 @@ public void onSurfaceCreated(GL10 gl, EGLConfig config) { isActive = true; } + private void getShaderLocations() { + mDilationPosHandle = GLES20.glGetAttribLocation(mDilationProgram, "a_Position"); + mDilationTexHandle = GLES20.glGetAttribLocation(mDilationProgram, "a_TexCoord"); + mDilationInputTextureHandle = GLES20.glGetUniformLocation(mDilationProgram, "s_InputTexture"); + mDilationTexelSizeHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_texelSize"); + mDilationRadiusHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_radius"); + mDilationDirectionHandle = GLES20.glGetUniformLocation(mDilationProgram, "u_direction"); + + mGaussPosHandle = GLES20.glGetAttribLocation(bilateralBlurProgram, "a_Position"); + mGaussTexHandle = GLES20.glGetAttribLocation(bilateralBlurProgram, "a_TexCoord"); + mGaussInputTextureHandle = GLES20.glGetUniformLocation(bilateralBlurProgram, "s_InputTexture"); + mGaussTexelSizeHandle = GLES20.glGetUniformLocation(bilateralBlurProgram, "u_texelSize"); + mGaussDirectionHandle = GLES20.glGetUniformLocation(bilateralBlurProgram, "u_blurDirection"); + mGaussParallaxHandle = GLES20.glGetUniformLocation(bilateralBlurProgram, "u_parallax"); + + mDibrPosHandle = GLES20.glGetAttribLocation(dibr3dProgram, "a_Position"); + mDibrTexHandle = GLES20.glGetAttribLocation(dibr3dProgram, "a_TexCoord"); + mDibrColorTexHandle = GLES20.glGetUniformLocation(dibr3dProgram, "s_ColorTexture"); + mDibrDepthTexHandle = GLES20.glGetUniformLocation(dibr3dProgram, "s_DepthTexture"); + mDibrParallaxHandle = GLES20.glGetUniformLocation(dibr3dProgram, "u_parallax"); + mDibrConvergenceHandle = GLES20.glGetUniformLocation(dibr3dProgram, "u_convergence"); + mDibrShiftHandle = GLES20.glGetUniformLocation(dibr3dProgram, "u_shift"); + mDibrDebugModeHandle = GLES20.glGetUniformLocation(dibr3dProgram, "u_debugMode"); + } + private void initializeIntermediateFbo() { intermediateTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); int[] fbos = new int[1]; @@ -297,8 +356,51 @@ private void initializeIntermediateFbo() { GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); } + private void initializeDilationFbo() { + intermediateDilutionTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); + int[] fbos = new int[1]; + GLES20.glGenFramebuffers(1, fbos, 0); + intermediateDilutionFboHandle = fbos[0]; + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateDilutionFboHandle); + GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId, 0); + if (GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER) != GLES20.GL_FRAMEBUFFER_COMPLETE) { + LimeLog.warning("Dilation Framebuffer is not complete."); + } + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + } + private float getParallax() { - return prefConfig.parallax_depth * 0.7f; + return prefConfig.parallax_depth * 0.10f; + } + + /** + * Wendet einen performanten, zweistufigen Dilation-Filter korrekt an. + * Liest von 'depthMapTextureId', schreibt das Zwischenergebnis nach 'intermediateDilutionFboHandle' (Tex A) + * und das Endergebnis nach 'intermediateFboHandle' (Tex B). + */ + private void applyTwoPassDilation() { + GLES20.glUseProgram(mDilationProgram); + GLES20.glEnableVertexAttribArray(mDilationPosHandle); + GLES20.glEnableVertexAttribArray(mDilationTexHandle); + GLES20.glVertexAttribPointer(mDilationPosHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); + GLES20.glVertexAttribPointer(mDilationTexHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); + GLES20.glUniform1i(mDilationInputTextureHandle, 0); + GLES20.glUniform2f(mDilationTexelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateDilutionFboHandle); + GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, depthMapTextureId); + GLES20.glUniform1i(mDilationRadiusHandle, 7); + GLES20.glUniform2f(mDilationDirectionHandle, 1.0f, 0.0f); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, gaussIntermediateFboHandle); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateDilutionTextureId); + GLES20.glUniform1i(mDilationRadiusHandle, 7); + GLES20.glUniform2f(mDilationDirectionHandle, 0.0f, 1.0f); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + GLES20.glDisableVertexAttribArray(mDilationPosHandle); + GLES20.glDisableVertexAttribArray(mDilationTexHandle); } private void applyTwoPassGaussianBlur() { @@ -306,50 +408,38 @@ private void applyTwoPassGaussianBlur() { GLES20.glUseProgram(blurProgram); - int posHandle = GLES20.glGetAttribLocation(blurProgram, "a_Position"); - int texHandle = GLES20.glGetAttribLocation(blurProgram, "a_TexCoord"); - int inputTextureHandle = GLES20.glGetUniformLocation(blurProgram, "s_InputTexture"); - int texelSizeHandle = GLES20.glGetUniformLocation(blurProgram, "u_texelSize"); - int directionHandle = GLES20.glGetUniformLocation(blurProgram, "u_blurDirection"); - int parallaxHandle = GLES20.glGetUniformLocation(blurProgram, "u_parallax"); - GLES20.glVertexAttribPointer(posHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); - GLES20.glVertexAttribPointer(texHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); - GLES20.glEnableVertexAttribArray(posHandle); - GLES20.glEnableVertexAttribArray(texHandle); - GLES20.glUniform1f(parallaxHandle, getParallax()); + GLES20.glVertexAttribPointer(mGaussPosHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); + GLES20.glVertexAttribPointer(mGaussTexHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); + GLES20.glEnableVertexAttribArray(mGaussPosHandle); + GLES20.glEnableVertexAttribArray(mGaussTexHandle); + GLES20.glUniform1f(mGaussParallaxHandle, getParallax()); - GLES20.glUniform2f(texelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); + GLES20.glUniform2f(mGaussTexelSizeHandle, 1.0f / modelInputWidth, 1.0f / modelInputHeight); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, intermediateFboHandle); GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); - - GLES20.glUniform2f(directionHandle, 1.0f, 0.0f); - + GLES20.glUniform2f(mGaussDirectionHandle, 1.0f, 0.0f); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, depthMapTextureId); - GLES20.glUniform1i(inputTextureHandle, 0); - + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, gaussIntermediateTextureId); + GLES20.glUniform1i(mGaussInputTextureHandle, 0); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, filterFboHandle); GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); - - GLES20.glUniform2f(directionHandle, 0.0f, 1.0f); - + GLES20.glUniform2f(mGaussDirectionHandle, 0.0f, 1.0f); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, intermediateTextureId); - GLES20.glUniform1i(inputTextureHandle, 0); - + GLES20.glUniform1i(mGaussInputTextureHandle, 0); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); } + private void drawBothEyes(int dualBubble3dProgram, float convergence, float shift) { int viewWidth = glSurfaceView.getWidth(); int viewHeight = glSurfaceView.getHeight(); - - float parallax = getParallax() * 0.06f; + float parallax = getParallax() * 0.2f; GLES20.glViewport(0, 0, viewWidth / 2, viewHeight); drawEye(dualBubble3dProgram, -parallax, convergence, shift); @@ -360,32 +450,24 @@ private void drawBothEyes(int dualBubble3dProgram, float convergence, float shif private void drawEye(int program, float parallax, float convergence, float shift) { GLES20.glUseProgram(program); - int posHandle = GLES20.glGetAttribLocation(program, "a_Position"); - int texHandle = GLES20.glGetAttribLocation(program, "a_TexCoord"); - int colorTexHandle = GLES20.glGetUniformLocation(program, "s_ColorTexture"); - int depthTexHandle = GLES20.glGetUniformLocation(program, "s_DepthTexture"); - int parallaxHandle = GLES20.glGetUniformLocation(program, "u_parallax"); - int convergenceHandle = GLES20.glGetUniformLocation(program, "u_convergence"); - int shiftHandle = GLES20.glGetUniformLocation(program, "u_shift"); - int debugModeHandle = GLES20.glGetUniformLocation(program, "u_debugMode"); - GLES20.glVertexAttribPointer(posHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); - GLES20.glVertexAttribPointer(texHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); - GLES20.glEnableVertexAttribArray(posHandle); - GLES20.glEnableVertexAttribArray(texHandle); + GLES20.glVertexAttribPointer(mDibrPosHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); + GLES20.glVertexAttribPointer(mDibrTexHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); + GLES20.glEnableVertexAttribArray(mDibrPosHandle); + GLES20.glEnableVertexAttribArray(mDibrTexHandle); - GLES20.glUniform1i(debugModeHandle, isDebugMode ? 1 : 0); + GLES20.glUniform1i(mDibrDebugModeHandle, isDebugMode ? 1 : 0); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, videoTextureId); - GLES20.glUniform1i(colorTexHandle, 0); + GLES20.glUniform1i(mDibrColorTexHandle, 0); GLES20.glActiveTexture(GLES20.GL_TEXTURE1); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, filteredDepthMapTextureId); - GLES20.glUniform1i(depthTexHandle, 1); - GLES20.glUniform1f(parallaxHandle, parallax); - GLES20.glUniform1f(convergenceHandle, convergence); - GLES20.glUniform1f(shiftHandle, shift); + GLES20.glUniform1i(mDibrDepthTexHandle, 1); + GLES20.glUniform1f(mDibrParallaxHandle, parallax); + GLES20.glUniform1f(mDibrConvergenceHandle, convergence); + GLES20.glUniform1f(mDibrShiftHandle, shift); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); } @@ -416,13 +498,9 @@ public void onDrawFrame(GL10 gl) { synchronized (frameLock) { if (!frameAvailable.get()) { - if (!isMovieMode) { - glSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_CONTINUOUSLY); - } else { glSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY); return; - } - } else if (isMovieMode) { + } else if (isMovieMode) { block = true; } frameAvailable.set(false); @@ -438,8 +516,6 @@ public void onDrawFrame(GL10 gl) { freeSmoothedBuffers.offer(currentlyRenderingMap); } - long startTimeAi = System.nanoTime(); - long endTimeAi = System.nanoTime(); if (tflite != null) { if (block || !isMovieMode) { ByteBuffer pixelBufferForAI = freeInputBuffers.poll(); @@ -450,8 +526,7 @@ public void onDrawFrame(GL10 gl) { pixelBufferForAI.rewind(); previousFrameForComparison.rewind(); previousFrameForComparison.put(pixelBufferForAI); - - if (inferenceInputQueue.offer(new RenderResult(pixelBufferForAI, difference))) { + if (inferenceInputQueue.offer(new RenderResult(pixelBufferForAI, difference, nextFrameId++))) { Log.d("AiTask", "Success: The AI will now process this buffer."); } else { freeInputBuffers.offer(pixelBufferForAI); @@ -466,6 +541,7 @@ public void onDrawFrame(GL10 gl) { try { Thread.sleep(1); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } } } else { @@ -475,8 +551,6 @@ public void onDrawFrame(GL10 gl) { block = false; currentlyRenderingMap = newMap; depthMapResultCount++; - endTimeAi = System.nanoTime(); - Log.d("Stereo3DRenderer", "DepthMap OutputSpeed " + (endTimeAi - startTimeAi) / 1_000_000 + " ms"); } } @@ -484,6 +558,7 @@ public void onDrawFrame(GL10 gl) { uploadLatestDepthMapToGpu(currentlyRenderingMap); } GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + applyTwoPassDilation(); applyTwoPassGaussianBlur(); drawWithShader(); long endTime = System.nanoTime(); @@ -498,7 +573,7 @@ public void onDrawFrame(GL10 gl) { drawDelay = ((float) totalDrawTime / fps / 1000000000f); } totalDrawTime = 0; - fps = calcFps; + fps = (calcFps +1); calcFps = 0; depthMapResultCount = 0; threeDFps = calcThreeDFps; @@ -527,9 +602,7 @@ private ByteBuffer createFlatDepthMap() { int mapSize = modelInputWidth * modelInputHeight; byte[] flatData = new byte[mapSize]; Arrays.fill(flatData, (byte) 128); - ByteBuffer flatMap = ByteBuffer.allocateDirect(mapSize).order(ByteOrder.nativeOrder()); - flatMap.put(flatData); flatMap.rewind(); return flatMap; @@ -545,102 +618,51 @@ private void uploadLatestDepthMapToGpu(ByteBuffer depthMap) { private void drawQuad(int program, float scale, float offset) { GLES20.glUseProgram(program); - int posHandle = GLES20.glGetAttribLocation(program, "a_Position"); int texHandle = GLES20.glGetAttribLocation(program, "a_TexCoord"); int offsetHandle = GLES20.glGetUniformLocation(program, "u_xOffset"); int scaleHandle = GLES20.glGetUniformLocation(program, "u_xScale"); - GLES20.glVertexAttribPointer(posHandle, 2, GLES20.GL_FLOAT, false, 0, quadVertexBuffer); GLES20.glVertexAttribPointer(texHandle, 2, GLES20.GL_FLOAT, false, 0, textureVertexBuffer); GLES20.glEnableVertexAttribArray(posHandle); GLES20.glEnableVertexAttribArray(texHandle); - GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, videoTextureId); - if (scaleHandle != -1) GLES20.glUniform1f(scaleHandle, scale); if (offsetHandle != -1) GLES20.glUniform1f(offsetHandle, offset); - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); } private static class InferenceResult { final ByteBuffer pixelBuffer; final ByteBuffer rawDepthBuffer; - InferenceResult(ByteBuffer pixelBuffer, ByteBuffer rawDepthBuffer) { this.pixelBuffer = pixelBuffer; this.rawDepthBuffer = rawDepthBuffer; } } - private static class RenderResult { - final ByteBuffer pixelBuffer; - final double imageDifference; + private int nextFrameId = 0; + + public class RenderResult { + public final ByteBuffer pixelBuffer; + public final double imageDifference; + public final int frameId; // new field - RenderResult(ByteBuffer pixelBuffer, double imageDifference) { + public RenderResult(ByteBuffer pixelBuffer, double imageDifference, int frameId) { this.pixelBuffer = pixelBuffer; this.imageDifference = imageDifference; - } - } - - public static void convertRgbaToRgb(ByteBuffer rgbaBuffer, ByteBuffer rgbBuffer, int width, int height) { - Mat rgbaMat = null; - Mat rgbMat = null; - try { - rgbaMat = new Mat(height, width, CvType.CV_8UC4, rgbaBuffer); - rgbMat = new Mat(height, width, CvType.CV_8UC3, rgbBuffer); - Imgproc.cvtColor(rgbaMat, rgbMat, Imgproc.COLOR_RGBA2RGB); - } finally { - if (rgbaMat != null) { - rgbaMat.release(); - } - if (rgbMat != null) { - rgbMat.release(); - } - } - } - - public static double calculateAverageDifferenceOCV(ByteBuffer buffer1, ByteBuffer buffer2, int width, int height) { - if (buffer1 == null || buffer2 == null) { - return 1; - } - - Mat mat1 = null; - Mat mat2 = null; - Mat diffMat = null; - try { - mat1 = new Mat(height, width, CvType.CV_8UC1, buffer1); - mat2 = new Mat(height, width, CvType.CV_8UC1, buffer2); - diffMat = new Mat(); - Core.absdiff(mat1, mat2, diffMat); - Scalar meanDifference = Core.mean(diffMat); - return meanDifference.val[0] / 255.0; - } finally { - if (mat1 != null) { - mat1.release(); - } - if (mat2 != null) { - mat2.release(); - } - if (diffMat != null) { - diffMat.release(); - } + this.frameId = frameId; } } private void initializePBOs() { PBO_SIZE = modelInputWidth * modelInputHeight * 4; - GLES30.glGenBuffers(2, pboHandles, 0); - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, pboHandles[0]); GLES30.glBufferData(GLES30.GL_PIXEL_PACK_BUFFER, PBO_SIZE, null, GLES30.GL_DYNAMIC_READ); - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, pboHandles[1]); GLES30.glBufferData(GLES30.GL_PIXEL_PACK_BUFFER, PBO_SIZE, null, GLES30.GL_DYNAMIC_READ); - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, 0); } @@ -649,46 +671,13 @@ private Boolean readPixelsForAI(ByteBuffer destinationBuffer) { GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); drawQuad(simple3dProgram, 1.0f, 0.0f); destinationBuffer.rewind(); - - GLES20.glReadPixels(0, 0, modelInputWidth, modelInputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, destinationBuffer); - + GLES20.glReadPixels(0, 0, modelInputWidth, modelInputHeight, GLES20.GL_RGB, GLES20.GL_UNSIGNED_BYTE, destinationBuffer); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); - return true; } - private boolean readPixelsForAI_Async(ByteBuffer destinationBuffer) { - GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fboHandle); - GLES20.glViewport(0, 0, modelInputWidth, modelInputHeight); - drawQuad(simple3dProgram, 1.0f, 0.0f); - int writeIndex = pboIndex; - int readIndex = (pboIndex + 1) % 2; - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, pboHandles[writeIndex]); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - GLES30.glReadPixels(0, 0, modelInputWidth, modelInputHeight, GLES30.GL_RGBA, GLES30.GL_UNSIGNED_BYTE, 0); - } - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, pboHandles[readIndex]); - ByteBuffer mappedBuffer = (ByteBuffer) GLES30.glMapBufferRange( - GLES30.GL_PIXEL_PACK_BUFFER, 0, PBO_SIZE, GLES30.GL_MAP_READ_BIT); - boolean success = false; - if (mappedBuffer != null) { - destinationBuffer.rewind(); - mappedBuffer.rewind(); - destinationBuffer.put(mappedBuffer); - GLES30.glUnmapBuffer(GLES30.GL_PIXEL_PACK_BUFFER); - success = true; - } - - GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, 0); - GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); - - pboIndex = readIndex; - return success; - } - private void initializeTfLite() { Interpreter.Options options = new Interpreter.Options(); - try { GpuDelegate.Options gpuOptions = new GpuDelegate.Options(); gpuOptions.setQuantizedModelsAllowed(true); @@ -701,7 +690,7 @@ private void initializeTfLite() { tflite = new Interpreter(loadModelFile(context, AI_MODEL), options); } catch (Exception e) { LimeLog.info("GPU Delegate nicht verfügbar: " + e.getMessage()); - gpuDelegate.close(); + if (gpuDelegate != null) gpuDelegate.close(); try { nnApiDelegate = new NnApiDelegate(); options.addDelegate(nnApiDelegate); @@ -710,10 +699,10 @@ private void initializeTfLite() { renderer = "NNAPI"; } catch (Exception exception) { LimeLog.info("NNAPI Delegate nicht verfügbar: " + e.getMessage()); - nnApiDelegate.close(); + if (nnApiDelegate != null) nnApiDelegate.close(); try { LimeLog.info("Fallback: CPU"); - tflite = new Interpreter(loadModelFile(context, AI_MODEL), options); + tflite = new Interpreter(loadModelFile(context, AI_MODEL), new Interpreter.Options()); renderer = "CPU"; } catch (Exception ex) { reinitializeTfLiteOnCpu(); @@ -731,7 +720,6 @@ private void reinitializeTfLiteOnCpu() { gpuDelegate.close(); gpuDelegate = null; } - try { Interpreter.Options options = new Interpreter.Options(); options.setUseNNAPI(true); @@ -758,7 +746,7 @@ public void onSurfaceChanged(GL10 gl, int width, int height) { } private void initializeFbo() { - fboTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); + fboTextureId = createRgbTexture(modelInputWidth, modelInputHeight); int[] fbos = new int[1]; GLES20.glGenFramebuffers(1, fbos, 0); fboHandle = fbos[0]; @@ -770,6 +758,29 @@ private void initializeFbo() { GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); } + private int createRgbTexture(int width, int height) { + int[] textures = new int[1]; + GLES20.glGenTextures(1, textures, 0); + int textureId = textures[0]; + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId); + GLES20.glTexImage2D( + GLES20.GL_TEXTURE_2D, + 0, + GLES20.GL_RGB, // <- RGB statt RGBA + width, + height, + 0, + GLES20.GL_RGB, // <- RGB statt RGBA + GLES20.GL_UNSIGNED_BYTE, + null + ); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + return textureId; + } + private void initializeFilterFbo() { filteredDepthMapTextureId = createRgbaTexture(modelInputWidth, modelInputHeight); int[] fbos = new int[1]; @@ -821,26 +832,31 @@ private int createRgbaTexture(int width, int height) { return textureId; } - private int loadShader(int type, String shaderCode) { - int shader = GLES20.glCreateShader(type); - GLES20.glShaderSource(shader, shaderCode); - GLES20.glCompileShader(shader); + private int createProgram(String vertex, String fragment) { + int vertexShader = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER); + GLES20.glShaderSource(vertexShader, vertex); + GLES20.glCompileShader(vertexShader); + int[] compiled = new int[1]; - GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); + GLES20.glGetShaderiv(vertexShader, GLES20.GL_COMPILE_STATUS, compiled, 0); if (compiled[0] == 0) { - LimeLog.severe("Could not compile shader " + type + ":"); - LimeLog.severe(GLES20.glGetShaderInfoLog(shader)); - GLES20.glDeleteShader(shader); - shader = 0; + LimeLog.severe("Could not compile vertex shader:"); + LimeLog.severe(GLES20.glGetShaderInfoLog(vertexShader)); + GLES20.glDeleteShader(vertexShader); + return 0; } - return shader; - } - private int createProgram(String vertex, String fragment) { - int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertex); - if (vertexShader == 0) return 0; - int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragment); - if (fragmentShader == 0) return 0; + int fragmentShader = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER); + GLES20.glShaderSource(fragmentShader, fragment); + GLES20.glCompileShader(fragmentShader); + + GLES20.glGetShaderiv(fragmentShader, GLES20.GL_COMPILE_STATUS, compiled, 0); + if (compiled[0] == 0) { + LimeLog.severe("Could not compile fragment shader:"); + LimeLog.severe(GLES20.glGetShaderInfoLog(fragmentShader)); + GLES20.glDeleteShader(fragmentShader); + return 0; + } int program = GLES20.glCreateProgram(); if (program != 0) { @@ -859,84 +875,6 @@ private int createProgram(String vertex, String fragment) { return program; } - private double hasFrameChangedSignificantlyOCV(ByteBuffer newPixelBuffer, ByteBuffer oldPixelBuffer) { - if (newPixelBuffer == null || oldPixelBuffer == null || newPixelBuffer.capacity() != oldPixelBuffer.capacity()) { - return 1.0; // maximal unterschiedliche Frames - } - - Mat mat1 = null, mat2 = null; - Mat gray1 = null, gray2 = null; - Mat edges1 = null, edges2 = null; - Mat histGray1 = null, histGray2 = null; - Mat histEdge1 = null, histEdge2 = null; - - try { - mat1 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, newPixelBuffer); - mat2 = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC4, oldPixelBuffer); - - // Graustufen - gray1 = new Mat(); - gray2 = new Mat(); - Imgproc.cvtColor(mat1, gray1, Imgproc.COLOR_RGBA2GRAY); - Imgproc.cvtColor(mat2, gray2, Imgproc.COLOR_RGBA2GRAY); - - // Kanten (Sobel) - edges1 = new Mat(); - edges2 = new Mat(); - Mat gradX1 = new Mat(), gradY1 = new Mat(); - Mat gradX2 = new Mat(), gradY2 = new Mat(); - Imgproc.Sobel(gray1, gradX1, CvType.CV_16S, 1, 0); - Imgproc.Sobel(gray1, gradY1, CvType.CV_16S, 0, 1); - Core.convertScaleAbs(gradX1, gradX1); - Core.convertScaleAbs(gradY1, gradY1); - Core.addWeighted(gradX1, 0.5, gradY1, 0.5, 0, edges1); - - Imgproc.Sobel(gray2, gradX2, CvType.CV_16S, 1, 0); - Imgproc.Sobel(gray2, gradY2, CvType.CV_16S, 0, 1); - Core.convertScaleAbs(gradX2, gradX2); - Core.convertScaleAbs(gradY2, gradY2); - Core.addWeighted(gradX2, 0.5, gradY2, 0.5, 0, edges2); - - gradX1.release(); - gradY1.release(); - gradX2.release(); - gradY2.release(); - - // Histogramme Graustufen - histGray1 = new Mat(); - histGray2 = new Mat(); - Imgproc.calcHist(Collections.singletonList(gray1), new MatOfInt(0), new Mat(), histGray1, new MatOfInt(256), new MatOfFloat(0f, 256f)); - Imgproc.calcHist(Collections.singletonList(gray2), new MatOfInt(0), new Mat(), histGray2, new MatOfInt(256), new MatOfFloat(0f, 256f)); - - // Histogramme Kanten - histEdge1 = new Mat(); - histEdge2 = new Mat(); - Imgproc.calcHist(Collections.singletonList(edges1), new MatOfInt(0), new Mat(), histEdge1, new MatOfInt(256), new MatOfFloat(0f, 256f)); - Imgproc.calcHist(Collections.singletonList(edges2), new MatOfInt(0), new Mat(), histEdge2, new MatOfInt(256), new MatOfFloat(0f, 256f)); - - // Vergleich: Graustufen + Kanten - double grayDiff = 1.0 - Imgproc.compareHist(histGray1, histGray2, Imgproc.HISTCMP_CORREL); - double edgeDiff = 1.0 - Imgproc.compareHist(histEdge1, histEdge2, Imgproc.HISTCMP_CORREL); - - // Kombiniere beide Differenzen (Gewichtung kann angepasst werden) - double combinedDiff = 0.5 * grayDiff + 0.5 * edgeDiff; - return combinedDiff; - - } finally { - if (mat1 != null) mat1.release(); - if (mat2 != null) mat2.release(); - if (gray1 != null) gray1.release(); - if (gray2 != null) gray2.release(); - if (edges1 != null) edges1.release(); - if (edges2 != null) edges2.release(); - if (histGray1 != null) histGray1.release(); - if (histGray2 != null) histGray2.release(); - if (histEdge1 != null) histEdge1.release(); - if (histEdge2 != null) histEdge2.release(); - } - } - - private double hasSceneChangedFast(ByteBuffer currentFrame, ByteBuffer previousFrame) { if (currentFrame == null || previousFrame == null || currentFrame.capacity() != previousFrame.capacity()) { return 0.0; @@ -948,62 +886,88 @@ private double hasSceneChangedFast(ByteBuffer currentFrame, ByteBuffer previousF long totalDifference = 0; int pixelsSampled = 0; - final int PIXEL_STRIDE = 4; - final int PIXEL_SAMPLE_RATE = 32; - final int ROW_SAMPLE_RATE = 32; + final int PIXEL_STRIDE = 3; // RGB = 3 bytes per pixel + final int PIXEL_SAMPLE_RATE = 32; // horizontal sampling + final int ROW_SAMPLE_RATE = 32; // vertical sampling final int SAMPLE_STRIDE = PIXEL_STRIDE * PIXEL_SAMPLE_RATE; final int ROW_STRIDE = modelInputWidth * PIXEL_STRIDE * ROW_SAMPLE_RATE; + final int frameCapacity = currentFrame.capacity(); - for (int row = 0; row < currentFrame.capacity(); row += ROW_STRIDE) { + for (int row = 0; row < frameCapacity; row += ROW_STRIDE) { for (int col = 0; col < modelInputWidth * PIXEL_STRIDE; col += SAMPLE_STRIDE) { int index = row + col; - if (index + 2 >= currentFrame.capacity()) break; + if (index + 2 >= frameCapacity) break; + + int rDiff = Math.abs((currentFrame.get(index) & 0xFF) - (previousFrame.get(index) & 0xFF)); + int gDiff = Math.abs((currentFrame.get(index + 1) & 0xFF) - (previousFrame.get(index + 1) & 0xFF)); + int bDiff = Math.abs((currentFrame.get(index + 2) & 0xFF) - (previousFrame.get(index + 2) & 0xFF)); - totalDifference += Math.abs((currentFrame.get(index) & 0xFF) - (previousFrame.get(index) & 0xFF)); - totalDifference += Math.abs((currentFrame.get(index + 1) & 0xFF) - (previousFrame.get(index + 1) & 0xFF)); - totalDifference += Math.abs((currentFrame.get(index + 2) & 0xFF) - (previousFrame.get(index + 2) & 0xFF)); + totalDifference += rDiff + gDiff + bDiff; pixelsSampled++; } } if (pixelsSampled == 0) return 0.0; - - double averageDifference = (double) totalDifference / pixelsSampled; - - return averageDifference; + return (double) totalDifference / pixelsSampled; } - private class AiTask implements Runnable { + private class AiTask implements Runnable { private ByteBuffer previousRawMap = null; + private int lastFrameId = -1; // Track last processed frame ID @Override public void run() { ByteBuffer pixelBuffer = null; double difference = 0.0f; + while (!Thread.currentThread().isInterrupted()) { long startTime = System.nanoTime(); long waitTime = System.nanoTime(); + long waitTime_end = System.nanoTime(); long aiTime = System.nanoTime(); long aiTime_end = System.nanoTime(); + try { if (tflite == null) return; + + // --- Take latest frame from queue --- RenderResult result = inferenceInputQueue.take(); + waitTime_end = System.nanoTime(); pixelBuffer = result.pixelBuffer; difference = result.imageDifference; + int currentFrameId = result.frameId; + + // Skip if this frame was already processed + if (currentFrameId == lastFrameId) { + // Reuse previous map + ByteBuffer outputBuffer = freeOutputBuffers.take(); + outputBuffer.clear(); + previousRawMap.rewind(); + outputBuffer.put(previousRawMap); + outputBuffer.rewind(); + filledOutputBuffers.put(new InferenceResult(pixelBuffer, outputBuffer)); + pixelBuffer = null; + Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: lastFrameId"); + continue; + } + + lastFrameId = currentFrameId; + + // --- Process new frame --- ByteBuffer outputBuffer = freeOutputBuffers.take(); - waitTime = System.nanoTime(); outputBuffer.rewind(); if (difference > ON_DRAW_CHANGE_TRESHOLD || previousRawMap == null) { tfliteInputBuffer.rewind(); pixelBuffer.rewind(); - - convertRgbaToRgb(pixelBuffer, tfliteInputBuffer, modelInputWidth, modelInputHeight); - + tfliteInputBuffer.put(pixelBuffer); + tfliteInputBuffer.rewind(); aiTime = System.nanoTime(); - ReflectivePaddingInt8Minimal.applyReflectedPadding(tfliteInputBuffer); tflite.run(tfliteInputBuffer, outputBuffer); + aiTime_end = System.nanoTime(); + + // Store result for reuse if (previousRawMap == null) { previousRawMap = ByteBuffer.allocateDirect(outputBuffer.capacity()); } @@ -1012,138 +976,213 @@ public void run() { previousRawMap.put(outputBuffer); previousRawMap.rewind(); } else { + // No significant change: reuse previous map outputBuffer.clear(); previousRawMap.rewind(); outputBuffer.put(previousRawMap); outputBuffer.rewind(); } calcThreeDFps++; - aiTime_end = System.nanoTime(); filledOutputBuffers.put(new InferenceResult(pixelBuffer, outputBuffer)); pixelBuffer = null; + } catch (InterruptedException e) { - LimeLog.severe("AI inference failed: " + e.getMessage()); + LimeLog.severe("AI inference interrupted: " + e.getMessage()); Thread.currentThread().interrupt(); } catch (Exception e) { LimeLog.severe("AI inference failed: " + e.getMessage()); gpuDelegateFailed.set(true); } finally { - long duration = (System.nanoTime() - startTime) / 1_000_000; - long waitTimeText = (waitTime - startTime) / 1_000_000; - long aitimeText = (aiTime_end - aiTime) / 1_000_000; if (pixelBuffer != null) { freeInputBuffers.offer(pixelBuffer); } - Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: " + duration + " ms " + filledOutputBuffers.remainingCapacity() + " " + waitTimeText + " ms" + "aitime: " + aitimeText); + + long duration = (System.nanoTime() - startTime) / 1_000_000; + long waitTimeText = (waitTime_end - waitTime) / 1_000_000; + long aitimeText = (aiTime_end - aiTime) / 1_000_000; + long restTimeText = ((System.nanoTime() - startTime) - (aiTime_end - aiTime)- (waitTime_end - waitTime)) / 1_000_000; + + Log.d("Stereo3DRenderer", "CalculateTime AiDepthMap: " + duration + + " ms waitTime: " + waitTimeText + + " ms aitime: " + aitimeText + + " otherTime: " + restTimeText); } } isAiRunning.set(false); } } - private class AiResultHandling implements Runnable { - - private static final double IMAGE_DIFFERENCE_MULTIPLIER = 100.0; - private static final double MAX_SMOOTHING_FACTOR = 1; - private static final double MIN_SMOOTHING_FACTOR = 0.005; - - // --- Member Fields --- - private final byte[] processedDataArray = new byte[modelInputWidth * modelInputHeight]; - private Mat previousSmoothedMat; + private class AiResultHandling implements Runnable { + private static final double DEPTH_DIFF_THRESHOLD = 0.1; + private static final double MAX_SMOOTHING = 1.0; + private static final double MIN_SMOOTHING = 0.0; + private Mat previousSmoothedMat; // Bleibt Member, hält Zustand über Frames private boolean isFirstFrame = true; + // --- Wiederverwendbare Mat-Objekte für diesen Thread --- + private Mat reusableRawMat = null; // Wird nur als Header verwendet + private Mat reusableRawFloat = new Mat(); + private Mat reusableDiffMat = new Mat(); + private Mat reusableDiffFloat = new Mat(); + private Mat reusableMeanMat = new Mat(); + private Mat reusableVarianceMat = new Mat(); + private Mat reusableNormalizedForShader = new Mat(); + private Mat reusableOutputMat = new Mat(); + @Override public void run() { ByteBuffer resultBuffer = createFlatDepthMap(); InferenceResult result = null; - while (!Thread.currentThread().isInterrupted()) { + while (isAiResultHandlingRunning.get() && !Thread.currentThread().isInterrupted()) { long startTime = System.nanoTime(); long waitTime = System.nanoTime(); - Mat rawMat = null; - Mat processedMat = null; + long waitTime_end = System.nanoTime(); + long aiTime = System.nanoTime(); + long aiTime_end = System.nanoTime(); try { - result = filledOutputBuffers.take(); - resultBuffer = freeSmoothedBuffers.take(); - waitTime = System.nanoTime(); - + result = filledOutputBuffers.take(); // Blockiert, bis Ergebnis da ist + resultBuffer = freeSmoothedBuffers.take(); // Blockiert, bis Puffer frei ist + waitTime_end = System.nanoTime(); + // Verarbeite nur das letzte verfügbare Ergebnis InferenceResult intermediate; while ((intermediate = filledOutputBuffers.poll()) != null) { - freeInputBuffers.offer(result.pixelBuffer); + freeInputBuffers.offer(result.pixelBuffer); // Gib alte Buffer zurück freeOutputBuffers.offer(result.rawDepthBuffer); - result = intermediate; + result = intermediate; // Behalte das Neueste } - ByteBuffer rawDepthBuffer = result.rawDepthBuffer; - ByteBuffer currentPixelBuffer = result.pixelBuffer; - - currentPixelBuffer.rewind(); - double imageDifference = hasFrameChangedSignificantlyOCV(currentPixelBuffer, previousPixelBuffer) * IMAGE_DIFFERENCE_MULTIPLIER; - rawMat = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC1, rawDepthBuffer); - processedMat = new Mat(); - Core.normalize(rawMat, processedMat, 0, 255, Core.NORM_MINMAX); + // --- Verwende wiederverwendbare Mats --- + // Erstelle nur den Header neu, zeigt auf den Buffer, keine Datenkopie + reusableRawMat = new Mat(modelInputHeight, modelInputWidth, CvType.CV_8UC1, result.rawDepthBuffer); + reusableRawMat.convertTo(reusableRawFloat, CvType.CV_32F); if (isFirstFrame) { - previousSmoothedMat = processedMat.clone(); + // Initialisiere previousSmoothedMat sicher + if (previousSmoothedMat == null) { + previousSmoothedMat = new Mat(); + } + reusableRawFloat.copyTo(previousSmoothedMat); isFirstFrame = false; + } else if (previousSmoothedMat == null || previousSmoothedMat.empty()) { + // Fallback, falls Initialisierung fehlschlug + if (previousSmoothedMat == null) previousSmoothedMat = new Mat(); + reusableRawFloat.copyTo(previousSmoothedMat); + LimeLog.warning("previousSmoothedMat war null/leer, neu initialisiert in Schleife."); } - double smoothing = (imageDifference * 10) / (threeDFps * 3); - smoothing = Math.min(smoothing, MAX_SMOOTHING_FACTOR); - smoothing = Math.max(smoothing, MIN_SMOOTHING_FACTOR); - Mat diff = new Mat(); - Core.absdiff(processedMat, previousSmoothedMat, diff); - Core.MinMaxLocResult mmr = Core.minMaxLoc(diff); - double thresholdValue = Math.max(1, mmr.maxVal * ((1.0 - smoothing)) * 0.1); - Mat validMask = new Mat(); - Imgproc.threshold(diff, validMask, thresholdValue, 255, Imgproc.THRESH_BINARY_INV); - processedMat.copyTo(previousSmoothedMat, validMask); - Mat blended = new Mat(); - Core.addWeighted(processedMat, smoothing, previousSmoothedMat, 1.0 - smoothing, 0.0, blended); - Mat inverseMask = new Mat(); - Core.bitwise_not(validMask, inverseMask); - blended.copyTo(previousSmoothedMat, inverseMask); - diff.release(); - validMask.release(); - inverseMask.release(); - blended.release(); - previousSmoothedMat.get(0, 0, processedDataArray); - rawDepthBuffer.rewind(); - rawDepthBuffer.put(processedDataArray); - - rawDepthBuffer.rewind(); - resultBuffer.rewind(); - resultBuffer.put(rawDepthBuffer); - - rawDepthBuffer.rewind(); - resultBuffer.rewind(); - latestDepthMap.set(resultBuffer); + aiTime = System.nanoTime(); + // --- Differenzberechnung --- + Core.absdiff(reusableRawFloat, previousSmoothedMat, reusableDiffMat); + Scalar sumDiff = Core.sumElems(reusableDiffMat); + // Vermeide Division durch Null, falls Mat leer ist + double totalPixels = reusableDiffMat.total(); + double meanDiff = (totalPixels > 0) ? sumDiff.val[0] / (totalPixels * 255.0) : 0.0; - previousPixelBuffer.rewind(); - previousPixelBuffer.put(currentPixelBuffer); - } catch (Exception e) { - LimeLog.severe("AI exception " + e.getMessage()); - } finally { - if (rawMat != null) { - rawMat.release(); + reusableDiffMat.convertTo(reusableDiffFloat, CvType.CV_32F, 1.0 / 255.0); + Scalar meanVal = Core.mean(reusableDiffFloat); + + // Stelle sicher, dass meanMat die korrekte Größe/Typ hat, bevor setTo verwendet wird + if (reusableMeanMat.empty() || !reusableMeanMat.size().equals(reusableDiffFloat.size()) || reusableMeanMat.type() != reusableDiffFloat.type()) { + reusableMeanMat.create(reusableDiffFloat.size(), reusableDiffFloat.type()); } - if (processedMat != null) { - processedMat.release(); + reusableMeanMat.setTo(new Scalar(meanVal.val[0])); + + Core.subtract(reusableDiffFloat, reusableMeanMat, reusableVarianceMat); // reusableVarianceMat wird überschrieben + Core.multiply(reusableVarianceMat, reusableVarianceMat, reusableVarianceMat); // In-place quadrieren + double varianceSum = Core.sumElems(reusableVarianceMat).val[0]; + double stdDev = (totalPixels > 0) ? Math.sqrt(varianceSum / totalPixels) : 0.0; + + double depthMapDifference = Math.max(meanDiff, stdDev); + + // --- Glättungsfaktor --- + double smoothing; + if (depthMapDifference > DEPTH_DIFF_THRESHOLD) { + smoothing = MAX_SMOOTHING; + } else if (depthMapDifference > 0.01) { + smoothing = depthMapDifference * 2.0; // Evtl. Faktor anpassen + smoothing = Math.max(MIN_SMOOTHING, Math.min(MAX_SMOOTHING, smoothing)); + } else { + smoothing = 0; } - if (resultBuffer != null) { - freeSmoothedBuffers.offer(resultBuffer); + + // --- Glättung anwenden --- + Imgproc.accumulateWeighted(reusableRawFloat, previousSmoothedMat, smoothing); + + // --- Normalisieren für Ausgabe --- + Core.MinMaxLocResult mmr = Core.minMaxLoc(previousSmoothedMat); + double range = mmr.maxVal - mmr.minVal; + if (range < 1e-6) range = 1e-6; // Schutz vor Division durch Null + + Core.subtract(previousSmoothedMat, new Scalar(mmr.minVal), reusableNormalizedForShader); + Core.divide(reusableNormalizedForShader, new Scalar(range), reusableNormalizedForShader); + + reusableNormalizedForShader.convertTo(reusableOutputMat, CvType.CV_8U, 255.0); + + if (reusableOutputMat.isContinuous() && resultBuffer.hasArray()) { + reusableOutputMat.get(0, 0, resultBuffer.array()); + // Wichtig: Limit muss evtl. angepasst werden, wenn Puffer größer ist + resultBuffer.limit(reusableOutputMat.rows() * reusableOutputMat.cols() * (int)reusableOutputMat.elemSize()); + } else { + int bufferSize = modelInputWidth * modelInputHeight; + byte[] data = new byte[bufferSize]; + reusableOutputMat.get(0, 0, data); + resultBuffer.put(data); } + aiTime_end = System.nanoTime(); + resultBuffer.rewind(); // Puffer für den Konsumenten vorbereiten + + latestDepthMap.set(resultBuffer); + resultBuffer = null; // Besitz wurde an latestDepthMap übergeben + + } catch (InterruptedException e) { + LimeLog.warning("AiResultHandling interrupted."); + Thread.currentThread().interrupt(); // Interrupt-Status wiederherstellen + break; // Schleife verlassen + } catch (Exception e) { + LimeLog.severe("AI result handling exception: " + e.getMessage()); + // Hier könnte man überlegen, ob isFirstFrame zurückgesetzt werden soll + } finally { + // Gib nur die Buffer zurück, deren Besitz nicht übertragen wurde + if (resultBuffer != null) freeSmoothedBuffers.offer(resultBuffer); if (result != null) { freeInputBuffers.offer(result.pixelBuffer); freeOutputBuffers.offer(result.rawDepthBuffer); + result = null; // Referenz löschen } long duration = (System.nanoTime() - startTime) / 1_000_000; - long waitTimeText = (waitTime - startTime) / 1_000_000; - Log.d("Stereo3DRenderer", "CalculateTime AiResult: " + duration + " ms" + " " + freeOutputBuffers.remainingCapacity() + " " + waitTimeText + " ms "); + long waitTimeText = (waitTime_end - waitTime) / 1_000_000; + long aitimeText = (aiTime_end - aiTime) / 1_000_000; + long restTimeText = ((System.nanoTime() - startTime) - (aiTime_end - aiTime)- (waitTime_end - waitTime)) / 1_000_000; + + Log.d("Stereo3DRenderer", "CalculateTime AiResult: " + duration + + " ms waitTime: " + waitTimeText + + " ms calc time: " + aitimeText + + " otherTime: " + restTimeText); } + } // Ende while-Schleife + + // --- Aufräumen, wenn der Thread endet --- + releaseMat(previousSmoothedMat); previousSmoothedMat = null; + // releaseMat(reusableRawMat); // Ist nur ein Header, muss nicht freigegeben werden + releaseMat(reusableRawFloat); + releaseMat(reusableDiffMat); + releaseMat(reusableDiffFloat); + releaseMat(reusableMeanMat); + releaseMat(reusableVarianceMat); + releaseMat(reusableNormalizedForShader); + releaseMat(reusableOutputMat); + + isAiResultHandlingRunning.set(false); // Signal setzen, dass der Thread beendet ist + LimeLog.info("AiResultHandling finished."); + } // Ende run() + + // Hilfsmethode zum sicheren Freigeben von Mats + private void releaseMat(Mat mat) { + if (mat != null && !mat.empty()) { + mat.release(); } - isAiResultHandlingRunning.set(false); } } } \ No newline at end of file diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 2eb455f4f7..aef010af5a 100755 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -652,18 +652,6 @@ %1$dмс %1$.2fмс Уменьшает задержку за счёт пропуска кадров в очереди - Графики производительности - График задержки сети - График времени декодирования - График FPS - Включить графики производительности - Вывод графиков производительности в игровой панели - График: задержка сети - График средней задержки сети (мс) - График: время декодирования - График времени декодирования кадра (мс) - График: FPS - График входящих/отрендереных кадров в секунду Монитор производительности LFR (эксперементально) Агрессивная вертикальная синхронизация (эксперементально) diff --git a/app/src/main/res/values/arrays.xml b/app/src/main/res/values/arrays.xml index 763ed88197..8cd1700d91 100755 --- a/app/src/main/res/values/arrays.xml +++ b/app/src/main/res/values/arrays.xml @@ -1,6 +1,7 @@ + @string/resolution_auto @string/resolution_360p @string/resolution_480p @string/resolution_720p @@ -11,6 +12,7 @@ + 0x0 640x360 854x480 1280x720 @@ -20,12 +22,14 @@ + @string/fps_auto @string/fps_30 @string/fps_60 @string/fps_90 @string/fps_120 + 0 30 60 90 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 95f64c92a5..c0a74348e3 100755 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -192,7 +192,8 @@ Video frame rate Increase for a smoother video stream. Decrease for better performance on lower end devices. Video bitrate - Increase for better image quality. Decrease to improve performance on slower connections. + Setting based recommendation: %1$s or 0 Mbps (Automatic) + NEW: 0 for automatically calculated based on fps/resolution on stream start. Increase for better image quality. Decrease to improve performance on slower connections. Video bitrate on metered networks High bitrates consume more data, lower bitrates improve smoothness in metered networks.\nSet 0 for ¼ of the above bitrate settings. Mbps @@ -331,6 +332,7 @@ Gamepad type may be changed due to motion sensor emulation + Match Display Resolution 360p 480p 720p @@ -338,6 +340,7 @@ 1440p 4K + Match Display FPS 30 FPS 60 FPS 90 FPS diff --git a/app/src/main/res/xml/preferences.xml b/app/src/main/res/xml/preferences.xml index 5b009c891f..f90870896d 100755 --- a/app/src/main/res/xml/preferences.xml +++ b/app/src/main/res/xml/preferences.xml @@ -35,7 +35,7 @@ app:iconSpaceReserved="false" seekbar:divisor="1000" seekbar:keyStep="1000" - seekbar:min="500" + seekbar:min="0" seekbar:step="500" /> - + + - - - + - +