Skip to content

Commit

Permalink
Extract SurfaceCapture from ScreenEncoder
Browse files Browse the repository at this point in the history
Extract an interface SurfaceCapture from ScreenEncoder, representing a
video source which can be rendered to a Surface for encoding.

Split ScreenEncoder into:
 - ScreenCapture, implementing SurfaceCapture to capture the device
   screen,
 - SurfaceEncoder, to encode any SurfaceCapture.

This separation prepares the introduction of another SurfaceCapture
implementation to capture the camera instead of the device screen.

PR #4213 <#4213>

Co-authored-by: Romain Vimont <[email protected]>
Signed-off-by: Romain Vimont <[email protected]>
  • Loading branch information
yume-chan and rom1v committed Oct 31, 2023
1 parent 41ccb58 commit a2fb1b4
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 71 deletions.
83 changes: 83 additions & 0 deletions server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package com.genymobile.scrcpy;

import com.genymobile.scrcpy.wrappers.SurfaceControl;

import android.graphics.Rect;
import android.os.Build;
import android.os.IBinder;
import android.view.Surface;

public class ScreenCapture extends SurfaceCapture implements Device.RotationListener, Device.FoldListener {

private final Device device;
private IBinder display;

public ScreenCapture(Device device) {
this.device = device;
}

@Override
public void init() {
display = createDisplay();
device.setRotationListener(this);
device.setFoldListener(this);
}

@Override
public void start(Surface surface) {
ScreenInfo screenInfo = device.getScreenInfo();
Rect contentRect = screenInfo.getContentRect();

// does not include the locked video orientation
Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
int videoRotation = screenInfo.getVideoRotation();
int layerStack = device.getLayerStack();
setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
}

@Override
public void release() {
device.setRotationListener(null);
device.setFoldListener(null);
SurfaceControl.destroyDisplay(display);
}

@Override
public Size getSize() {
return device.getScreenInfo().getVideoSize();
}

@Override
public void setMaxSize(int maxSize) {
device.setMaxSize(maxSize);
}

@Override
public void onFoldChanged(int displayId, boolean folded) {
requestReset();
}

@Override
public void onRotationChanged(int rotation) {
requestReset();
}

private static IBinder createDisplay() {
// Since Android 12 (preview), secure displays could not be created with shell permissions anymore.
// On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S".
boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S".equals(
Build.VERSION.CODENAME));
return SurfaceControl.createDisplay("scrcpy", secure);
}

private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) {
SurfaceControl.openTransaction();
try {
SurfaceControl.setDisplaySurface(display, surface);
SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
SurfaceControl.setDisplayLayerStack(display, layerStack);
} finally {
SurfaceControl.closeTransaction();
}
}
}
3 changes: 2 additions & 1 deletion server/src/main/java/com/genymobile/scrcpy/Server.java
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc
if (video) {
Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecMeta(),
options.getSendFrameMeta());
ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
ScreenCapture screenCapture = new ScreenCapture(device);
SurfaceEncoder screenEncoder = new SurfaceEncoder(screenCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError());
asyncProcessors.add(screenEncoder);
}
Expand Down
61 changes: 61 additions & 0 deletions server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package com.genymobile.scrcpy;

import android.view.Surface;

import java.util.concurrent.atomic.AtomicBoolean;

/**
* A video source which can be rendered on a Surface for encoding.
*/
public abstract class SurfaceCapture {

private final AtomicBoolean resetCapture = new AtomicBoolean();

/**
* Request the encoding session to be restarted, for example if the capture implementation detects that the video source size has changed (on
* device rotation for example).
*/
protected void requestReset() {
resetCapture.set(true);
}

/**
* Consume the reset request (intended to be called by the encoder).
*
* @return {@code true} if a reset request was pending, {@code false} otherwise.
*/
public boolean consumeReset() {
return resetCapture.getAndSet(false);
}

/**
* Called once before the capture starts.
*/
public abstract void init();

/**
* Called after the capture ends (if and only if {@link #init()} has been called).
*/
public abstract void release();

/**
* Start the capture to the target surface.
*
* @param surface the surface which will be encoded
*/
public abstract void start(Surface surface);

/**
* Return the video size
*
* @return the video size
*/
public abstract Size getSize();

/**
* Set the maximum capture size (set by the encoder if it does not support the current size).
*
* @param maxSize Maximum size
*/
public abstract void setMaxSize(int maxSize);
}
Original file line number Diff line number Diff line change
@@ -1,13 +1,8 @@
package com.genymobile.scrcpy;

import com.genymobile.scrcpy.wrappers.SurfaceControl;

import android.graphics.Rect;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.os.Build;
import android.os.IBinder;
import android.os.Looper;
import android.os.SystemClock;
import android.view.Surface;
Expand All @@ -17,7 +12,7 @@
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;

public class ScreenEncoder implements Device.RotationListener, Device.FoldListener, AsyncProcessor {
public class SurfaceEncoder implements AsyncProcessor {

private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds
private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms
Expand All @@ -27,9 +22,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800};
private static final int MAX_CONSECUTIVE_ERRORS = 3;

private final AtomicBoolean resetCapture = new AtomicBoolean();

private final Device device;
private final SurfaceCapture capture;
private final Streamer streamer;
private final String encoderName;
private final List<CodecOption> codecOptions;
Expand All @@ -43,9 +36,9 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
private Thread thread;
private final AtomicBoolean stopped = new AtomicBoolean();

public ScreenEncoder(Device device, Streamer streamer, int videoBitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
public SurfaceEncoder(SurfaceCapture capture, Streamer streamer, int videoBitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
boolean downsizeOnError) {
this.device = device;
this.capture = capture;
this.streamer = streamer;
this.videoBitRate = videoBitRate;
this.maxFps = maxFps;
Expand All @@ -54,51 +47,29 @@ public ScreenEncoder(Device device, Streamer streamer, int videoBitRate, int max
this.downsizeOnError = downsizeOnError;
}

@Override
public void onFoldChanged(int displayId, boolean folded) {
resetCapture.set(true);
}

@Override
public void onRotationChanged(int rotation) {
resetCapture.set(true);
}

private boolean consumeResetCapture() {
return resetCapture.getAndSet(false);
}

private void streamScreen() throws IOException, ConfigurationException {
Codec codec = streamer.getCodec();
MediaCodec mediaCodec = createMediaCodec(codec, encoderName);
MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions);
IBinder display = createDisplay();
device.setRotationListener(this);
device.setFoldListener(this);

streamer.writeVideoHeader(device.getScreenInfo().getVideoSize());
capture.init();

boolean alive;
try {
do {
ScreenInfo screenInfo = device.getScreenInfo();
Rect contentRect = screenInfo.getContentRect();
streamer.writeVideoHeader(capture.getSize());

boolean alive;

// include the locked video orientation
Rect videoRect = screenInfo.getVideoSize().toRect();
format.setInteger(MediaFormat.KEY_WIDTH, videoRect.width());
format.setInteger(MediaFormat.KEY_HEIGHT, videoRect.height());
do {
Size size = capture.getSize();
format.setInteger(MediaFormat.KEY_WIDTH, size.getWidth());
format.setInteger(MediaFormat.KEY_HEIGHT, size.getHeight());

Surface surface = null;
try {
mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
surface = mediaCodec.createInputSurface();

// does not include the locked video orientation
Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
int videoRotation = screenInfo.getVideoRotation();
int layerStack = device.getLayerStack();
setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
capture.start(surface);

mediaCodec.start();

Expand All @@ -107,7 +78,7 @@ private void streamScreen() throws IOException, ConfigurationException {
mediaCodec.stop();
} catch (IllegalStateException | IllegalArgumentException e) {
Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage());
if (!prepareRetry(device, screenInfo)) {
if (!prepareRetry(size)) {
throw e;
}
Ln.i("Retrying...");
Expand All @@ -121,13 +92,11 @@ private void streamScreen() throws IOException, ConfigurationException {
} while (alive);
} finally {
mediaCodec.release();
device.setRotationListener(null);
device.setFoldListener(null);
SurfaceControl.destroyDisplay(display);
capture.release();
}
}

private boolean prepareRetry(Device device, ScreenInfo screenInfo) {
private boolean prepareRetry(Size currentSize) {
if (firstFrameSent) {
++consecutiveErrors;
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
Expand All @@ -147,15 +116,15 @@ private boolean prepareRetry(Device device, ScreenInfo screenInfo) {

// Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising)

int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize());
int newMaxSize = chooseMaxSizeFallback(currentSize);
if (newMaxSize == 0) {
// Must definitively fail
return false;
}

// Retry with a smaller device size
Ln.i("Retrying with -m" + newMaxSize + "...");
device.setMaxSize(newMaxSize);
capture.setMaxSize(newMaxSize);
return true;
}

Expand All @@ -176,14 +145,14 @@ private boolean encode(MediaCodec codec, Streamer streamer) throws IOException {
boolean alive = true;
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();

while (!consumeResetCapture() && !eof) {
while (!capture.consumeReset() && !eof) {
if (stopped.get()) {
alive = false;
break;
}
int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
try {
if (consumeResetCapture()) {
if (capture.consumeReset()) {
// must restart encoding with new size
break;
}
Expand Down Expand Up @@ -264,25 +233,6 @@ private static MediaFormat createFormat(String videoMimeType, int bitRate, int m
return format;
}

private static IBinder createDisplay() {
// Since Android 12 (preview), secure displays could not be created with shell permissions anymore.
// On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S".
boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S"
.equals(Build.VERSION.CODENAME));
return SurfaceControl.createDisplay("scrcpy", secure);
}

private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) {
SurfaceControl.openTransaction();
try {
SurfaceControl.setDisplaySurface(display, surface);
SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
SurfaceControl.setDisplayLayerStack(display, layerStack);
} finally {
SurfaceControl.closeTransaction();
}
}

@Override
public void start(TerminationListener listener) {
thread = new Thread(() -> {
Expand Down

0 comments on commit a2fb1b4

Please sign in to comment.