From d0d0ece9786792dd19a25808aca64282a3bc6394 Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Mon, 14 Aug 2023 00:28:25 +0800 Subject: [PATCH] Fail-fast camera mirroring on Android 11 and older --- app/src/cli.c | 3 +- .../com/genymobile/scrcpy/AudioCapture.java | 32 ++++++-- .../AudioCaptureForegroundException.java | 7 ++ .../com/genymobile/scrcpy/AudioEncoder.java | 4 +- .../genymobile/scrcpy/AudioRawRecorder.java | 4 +- .../com/genymobile/scrcpy/CameraEncoder.java | 71 ++++++----------- .../scrcpy/CaptureForegroundException.java | 7 -- .../com/genymobile/scrcpy/SurfaceEncoder.java | 10 +-- .../com/genymobile/scrcpy/Workarounds.java | 77 ++++--------------- 9 files changed, 82 insertions(+), 133 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/AudioCaptureForegroundException.java delete mode 100644 server/src/main/java/com/genymobile/scrcpy/CaptureForegroundException.java diff --git a/app/src/cli.c b/app/src/cli.c index e451a28981..d0c5b0c197 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -189,7 +189,8 @@ static const struct sc_option options[] = { .longopt_id = OPT_VIDEO_SOURCE, .longopt = "video-source", .argdesc = "source", - .text = "Select the video source (display or camera).\n" + .text = "Select the video source (display or camera) to mirror.\n" + "Camera mirroring requires Android 12.\n" "Default is display.", }, { diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java index 73c9e59315..5575ffb69c 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java @@ -1,7 +1,11 @@ package com.genymobile.scrcpy; +import com.genymobile.scrcpy.wrappers.ServiceManager; + import android.annotation.SuppressLint; import android.annotation.TargetApi; +import android.content.ComponentName; +import android.content.Intent; import android.media.AudioFormat; import android.media.AudioRecord; import android.media.AudioTimestamp; @@ -60,7 +64,25 @@ private static AudioRecord createAudioRecord(int audioSource) { return builder.build(); } - private void tryStartRecording(int attempts, int delayMs) throws CaptureForegroundException { + private static void startWorkaroundAndroid11() { + // Android 11 requires Apps to be at foreground to record audio. + // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground. + // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android + // shell ("com.android.shell"). + // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the + // foreground. + Intent intent = new Intent(Intent.ACTION_MAIN); + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + intent.addCategory(Intent.CATEGORY_LAUNCHER); + intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity")); + ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent); + } + + private static void stopWorkaroundAndroid11() { + ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME); + } + + private void tryStartRecording(int attempts, int delayMs) throws AudioCaptureForegroundException { while (attempts-- > 0) { // Wait for activity to start SystemClock.sleep(delayMs); @@ -72,7 +94,7 @@ private void tryStartRecording(int attempts, int delayMs) throws CaptureForegrou Ln.e("Failed to start audio capture"); Ln.e("On Android 11, audio capture must be started in the foreground, make sure that the device is unlocked when starting " + "scrcpy."); - throw new CaptureForegroundException(); + throw new AudioCaptureForegroundException(); } else { Ln.d("Failed to start audio capture, retrying..."); } @@ -92,13 +114,13 @@ private void startRecording() { recorder.startRecording(); } - public void start() throws CaptureForegroundException { + public void start() throws AudioCaptureForegroundException { if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { - Workarounds.startForegroundWorkaround(); + startWorkaroundAndroid11(); try { tryStartRecording(5, 100); } finally { - Workarounds.stopForegroundWorkaround(); + stopWorkaroundAndroid11(); } } else { startRecording(); diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCaptureForegroundException.java b/server/src/main/java/com/genymobile/scrcpy/AudioCaptureForegroundException.java new file mode 100644 index 0000000000..baa7d84649 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCaptureForegroundException.java @@ -0,0 +1,7 @@ +package com.genymobile.scrcpy; + +/** + * Exception thrown if audio capture failed on Android 11 specifically because the running App (shell) was not in foreground. + */ +public class AudioCaptureForegroundException extends Exception { +} diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 2ee3666dbd..bec79b0535 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -125,7 +125,7 @@ public void start(TerminationListener listener) { } catch (ConfigurationException e) { // Do not print stack trace, a user-friendly error-message has already been logged fatalError = true; - } catch (CaptureForegroundException e) { + } catch (AudioCaptureForegroundException e) { // Do not print stack trace, a user-friendly error-message has already been logged } catch (IOException e) { Ln.e("Audio encoding error", e); @@ -169,7 +169,7 @@ private synchronized void waitEnded() { } @TargetApi(Build.VERSION_CODES.M) - public void encode() throws IOException, ConfigurationException, CaptureForegroundException { + public void encode() throws IOException, ConfigurationException, AudioCaptureForegroundException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { Ln.w("Audio disabled: it is not supported before Android 11"); streamer.writeDisableStream(false); diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java b/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java index bf59212a8a..7d2adade0b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java @@ -21,7 +21,7 @@ public AudioRawRecorder(AudioCapture capture, Streamer streamer) { this.streamer = streamer; } - private void record() throws IOException, CaptureForegroundException { + private void record() throws IOException, AudioCaptureForegroundException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { Ln.w("Audio disabled: it is not supported before Android 11"); streamer.writeDisableStream(false); @@ -60,7 +60,7 @@ public void start(TerminationListener listener) { boolean fatalError = false; try { record(); - } catch (CaptureForegroundException e) { + } catch (AudioCaptureForegroundException e) { // Do not print stack trace, a user-friendly error-message has already been logged } catch (IOException e) { Ln.e("Audio recording error", e); diff --git a/server/src/main/java/com/genymobile/scrcpy/CameraEncoder.java b/server/src/main/java/com/genymobile/scrcpy/CameraEncoder.java index 7712b24bfd..a517aac521 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CameraEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/CameraEncoder.java @@ -1,6 +1,7 @@ package com.genymobile.scrcpy; import android.annotation.SuppressLint; +import android.annotation.TargetApi; import android.graphics.Rect; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; @@ -13,7 +14,6 @@ import android.os.Build; import android.os.Handler; import android.os.HandlerThread; -import android.os.SystemClock; import android.view.Surface; import java.util.Arrays; @@ -25,30 +25,30 @@ public class CameraEncoder extends SurfaceEncoder { - private int maxSize; - private String cameraId; - private CameraPosition cameraPosition; + private final String cameraId; + private final CameraPosition cameraPosition; + private int maxSize; private String actualCameraId; private CameraDevice cameraDevice; - private HandlerThread cameraThread; - private Handler cameraHandler; + private final Handler cameraHandler; public CameraEncoder(int maxSize, String cameraId, CameraPosition cameraPosition, Streamer streamer, - int videoBitRate, int maxFps, List codecOptions, String encoderName, boolean downsizeOnError) { + int videoBitRate, int maxFps, List codecOptions, String encoderName, boolean downsizeOnError) { super(streamer, videoBitRate, maxFps, codecOptions, encoderName, downsizeOnError); this.maxSize = maxSize; this.cameraId = cameraId; this.cameraPosition = cameraPosition; - cameraThread = new HandlerThread("camera"); + HandlerThread cameraThread = new HandlerThread("camera"); cameraThread.start(); cameraHandler = new Handler(cameraThread.getLooper()); } @SuppressLint("MissingPermission") + @TargetApi(Build.VERSION_CODES.N) private CameraDevice openCamera(String id) throws CameraAccessException, InterruptedException { Ln.v("Open Camera: " + id); @@ -95,7 +95,7 @@ public void onError(CameraDevice camera, int error) { } } - @SuppressWarnings("deprecation") + @TargetApi(Build.VERSION_CODES.N) private CameraCaptureSession createCaptureSession(CameraDevice camera, Surface surface) throws CameraAccessException, InterruptedException { Ln.v("Create Capture Session"); @@ -121,6 +121,7 @@ public void onConfigureFailed(CameraCaptureSession session) { } } + @TargetApi(Build.VERSION_CODES.N) private void setRepeatingRequest(CameraCaptureSession session, CaptureRequest request) throws CameraAccessException, InterruptedException { Ln.v("Set Repeating Request"); @@ -129,13 +130,13 @@ private void setRepeatingRequest(CameraCaptureSession session, CaptureRequest re session.setRepeatingRequest(request, new CameraCaptureSession.CaptureCallback() { @Override public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, - long timestamp, long frameNumber) { + long timestamp, long frameNumber) { future.complete(null); } @Override public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request, - CaptureFailure failure) { + CaptureFailure failure) { future.completeExceptionally(new CameraAccessException(CameraAccessException.CAMERA_ERROR)); } }, cameraHandler); @@ -148,7 +149,11 @@ public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request } @Override - protected void initialize() { + protected void initialize() throws ConfigurationException { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) { + Ln.e("Camera mirroring is not support before Android 12"); + throw new ConfigurationException("Camera mirroring is not supported"); + } } @Override @@ -219,9 +224,6 @@ protected void setSize(int size) { private void setSurfaceInternal(Surface surface) throws CameraAccessException { try { - // openCamera, createCaptureSession and setRepeatingRequest all - // requires foreground workaround on Android 11. - // getCameraIdList and getCameraCharacteristics don't need that. cameraDevice = openCamera(actualCameraId); CameraCaptureSession session = createCaptureSession(cameraDevice, surface); CaptureRequest.Builder requestBuilder = cameraDevice @@ -236,41 +238,12 @@ private void setSurfaceInternal(Surface surface) throws CameraAccessException { } } - private void trySetSurface(int attempts, int delayMs, Surface surface) throws CaptureForegroundException { - while (attempts-- > 0) { - // Wait for activity to start - SystemClock.sleep(delayMs); - try { - setSurfaceInternal(surface); - return; // it worked - } catch (CameraAccessException e) { - if (attempts == 0) { - Ln.e("Failed to start camera capture"); - Ln.e("On Android 11, camera capture must be started in the foreground, make sure that the device is unlocked when starting " - + "scrcpy."); - throw new CaptureForegroundException(); - } else { - Ln.d("Failed to start camera capture, retrying..."); - } - } - } - } - @Override - protected void setSurface(Surface surface) throws CaptureForegroundException { - if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { - Workarounds.startForegroundWorkaround(); - try { - trySetSurface(5, 100, surface); - } finally { - // Workarounds.stopForegroundWorkaround(); - } - } else { - try { - setSurfaceInternal(surface); - } catch (CameraAccessException e) { - throw new RuntimeException(e); - } + protected void setSurface(Surface surface) { + try { + setSurfaceInternal(surface); + } catch (CameraAccessException e) { + throw new RuntimeException(e); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/CaptureForegroundException.java b/server/src/main/java/com/genymobile/scrcpy/CaptureForegroundException.java deleted file mode 100644 index b27cfb40d5..0000000000 --- a/server/src/main/java/com/genymobile/scrcpy/CaptureForegroundException.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.genymobile.scrcpy; - -/** - * Exception thrown if audio/camera capture failed on Android 11 specifically because the running App (shell) was not in foreground. - */ -public class CaptureForegroundException extends Exception { -} diff --git a/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java b/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java index e7baec2ef8..496946db00 100644 --- a/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java @@ -48,13 +48,13 @@ public SurfaceEncoder(Streamer streamer, int videoBitRate, int maxFps, List activityThreadClass; private static Object activityThread; - private static Context systemContext; private static CameraManager cameraManager; private Workarounds() { @@ -96,24 +91,20 @@ private static void prepareMainLooper() { Looper.prepareMainLooper(); } - @SuppressLint("PrivateApi") - private static Object getActivityThread() throws ReflectiveOperationException { + @SuppressLint("PrivateApi,DiscouragedPrivateApi") + private static void fillActivityThread() throws Exception { if (activityThread == null) { // ActivityThread activityThread = new ActivityThread(); activityThreadClass = Class.forName("android.app.ActivityThread"); Constructor activityThreadConstructor = activityThreadClass.getDeclaredConstructor(); activityThreadConstructor.setAccessible(true); activityThread = activityThreadConstructor.newInstance(); - } - return activityThread; - } - private static void fillActivityThread() throws Exception { - // ActivityThread.sCurrentActivityThread = activityThread; - activityThreadClass = Class.forName("android.app.ActivityThread"); - Field sCurrentActivityThreadField = activityThreadClass.getDeclaredField("sCurrentActivityThread"); - sCurrentActivityThreadField.setAccessible(true); - sCurrentActivityThreadField.set(null, getActivityThread()); + // ActivityThread.sCurrentActivityThread = activityThread; + Field sCurrentActivityThreadField = activityThreadClass.getDeclaredField("sCurrentActivityThread"); + sCurrentActivityThreadField.setAccessible(true); + sCurrentActivityThreadField.set(null, activityThread); + } } @SuppressLint("PrivateApi,DiscouragedPrivateApi") @@ -136,7 +127,6 @@ private static void fillAppInfo() { appInfoField.set(appBindData, applicationInfo); // activityThread.mBoundApplication = appBindData; - activityThreadClass = Class.forName("android.app.ActivityThread"); Field mBoundApplicationField = activityThreadClass.getDeclaredField("mBoundApplication"); mBoundApplicationField.setAccessible(true); mBoundApplicationField.set(activityThread, appBindData); @@ -157,7 +147,6 @@ private static void fillAppContext() { baseField.set(app, FakeContext.get()); // activityThread.mInitialApplication = app; - activityThreadClass = Class.forName("android.app.ActivityThread"); Field mInitialApplicationField = activityThreadClass.getDeclaredField("mInitialApplication"); mInitialApplicationField.setAccessible(true); mInitialApplicationField.set(activityThread, app); @@ -167,29 +156,22 @@ private static void fillAppContext() { } } - @SuppressLint("PrivateApi") - public static Context getSystemContext() throws ReflectiveOperationException { - if (systemContext == null) { + public static void fillBaseContext() { + try { + fillActivityThread(); + try { // Hide warnings on XiaoMi devices Class themeManagerStubClass = Class.forName("android.content.res.ThemeManagerStub"); Field sResourceField = themeManagerStubClass.getDeclaredField("sResource"); sResourceField.setAccessible(true); sResourceField.set(null, null); - } catch (ReflectiveOperationException ignore) { } + } catch (ReflectiveOperationException ignore) { + } - Object activityThread = getActivityThread(); Method getSystemContextMethod = activityThreadClass.getDeclaredMethod("getSystemContext"); - systemContext = (Context) getSystemContextMethod.invoke(activityThread); - } - return systemContext; - } - - public static void fillBaseContext() { - try { - fillActivityThread(); - - FakeContext.get().setBaseContext(getSystemContext()); + Context context = (Context) getSystemContextMethod.invoke(activityThread); + FakeContext.get().setBaseContext(context); } catch (Throwable throwable) { // this is a workaround, so failing is not an error Ln.d("Could not fill base context: " + throwable.getMessage()); @@ -345,33 +327,4 @@ public static CameraManager getCameraManager() { return cameraManager; } - private static int foregroundWorkaroundCount = 0; - - public static synchronized void startForegroundWorkaround() { - if (foregroundWorkaroundCount++ == 0) { - Ln.v("Starting Foreground Workaround"); - - // Android 11 requires Apps to be at foreground to record audio. - // Normally, each App has its own user ID, so Android checks whether the - // requesting App has the user ID that's at the foreground. - // But scrcpy server is NOT an App, it's a Java application started from Android - // shell, so it has the same user ID (2000) with Android - // shell ("com.android.shell"). - // If there is an Activity from Android shell running at foreground, then the - // permission system will believe scrcpy is also in the - // foreground. - Intent intent = new Intent(Intent.ACTION_MAIN); - intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); - intent.addCategory(Intent.CATEGORY_LAUNCHER); - intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity")); - ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent); - } - } - - public static synchronized void stopForegroundWorkaround() { - if (--foregroundWorkaroundCount == 0) { - Ln.v("Stopping Foreground Workaround"); - ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME); - } - } }