From a4e7a7ef0fdd8db252cf0de0e3eae5c07433ae86 Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Sun, 16 Jul 2023 17:07:19 +0800 Subject: [PATCH] add --video-source, --camera and --list-camera options --- app/src/cli.c | 56 +++ app/src/options.c | 3 + app/src/options.h | 8 + app/src/scrcpy.c | 5 +- app/src/server.c | 14 +- app/src/server.h | 3 + .../com/genymobile/scrcpy/CameraEncoder.java | 363 ++++-------------- .../java/com/genymobile/scrcpy/LogUtils.java | 56 +++ .../java/com/genymobile/scrcpy/Options.java | 28 ++ .../com/genymobile/scrcpy/ScreenEncoder.java | 295 ++------------ .../java/com/genymobile/scrcpy/Server.java | 23 +- .../com/genymobile/scrcpy/SurfaceEncoder.java | 286 ++++++++++++++ .../com/genymobile/scrcpy/VideoSource.java | 22 ++ 13 files changed, 595 insertions(+), 567 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java create mode 100644 server/src/main/java/com/genymobile/scrcpy/VideoSource.java diff --git a/app/src/cli.c b/app/src/cli.c index 09f853f538..fca61d8858 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -33,6 +33,7 @@ enum { OPT_MAX_FPS, OPT_LOCK_VIDEO_ORIENTATION, OPT_DISPLAY_ID, + OPT_CAMERA_ID, OPT_ROTATION, OPT_RENDER_DRIVER, OPT_NO_MIPMAPS, @@ -69,6 +70,7 @@ enum { OPT_AUDIO_ENCODER, OPT_LIST_ENCODERS, OPT_LIST_DISPLAYS, + OPT_LIST_CAMERAS, OPT_REQUIRE_AUDIO, OPT_AUDIO_BUFFER, OPT_AUDIO_OUTPUT_BUFFER, @@ -76,6 +78,7 @@ enum { OPT_NO_VIDEO, OPT_NO_AUDIO_PLAYBACK, OPT_NO_VIDEO_PLAYBACK, + OPT_VIDEO_SOURCE, OPT_AUDIO_SOURCE, OPT_KILL_ADB_ON_CLOSE, OPT_TIME_LIMIT, @@ -181,6 +184,13 @@ static const struct sc_option options[] = { "a higher value (10). Do not change this setting otherwise.\n" "Default is 5.", }, + { + .longopt_id = OPT_VIDEO_SOURCE, + .longopt = "video-source", + .argdesc = "source", + .text = "Select the video source (display or camera).\n" + "Default is display.", + }, { .shortopt = 'b', .longopt = "video-bit-rate", @@ -239,6 +249,15 @@ static const struct sc_option options[] = { " scrcpy --list-displays\n" "Default is 0.", }, + { + .longopt_id = OPT_CAMERA_ID, + .longopt = "camera", + .argdesc = "id", + .text = "Specify the device camera id to mirror when using --video-source camera.\n" + "The available camera ids can be listed by:\n" + " scrcpy --list-cameras\n" + "Default is the first one.", + }, { .longopt_id = OPT_DISPLAY_BUFFER, .longopt = "display-buffer", @@ -317,6 +336,11 @@ static const struct sc_option options[] = { .longopt = "list-displays", .text = "List device displays.", }, + { + .longopt_id = OPT_LIST_CAMERAS, + .longopt = "list-cameras", + .text = "List device cameras.", + }, { .longopt_id = OPT_LIST_ENCODERS, .longopt = "list-encoders", @@ -1625,6 +1649,22 @@ parse_audio_source(const char *optarg, enum sc_audio_source *source) { return false; } +static bool +parse_video_source(const char *optarg, enum sc_video_source *source) { + if (!strcmp(optarg, "display")) { + *source = SC_VIDEO_SOURCE_DISPLAY; + return true; + } + + if (!strcmp(optarg, "camera")) { + *source = SC_VIDEO_SOURCE_CAMERA; + return true; + } + + LOGE("Unsupported video source: %s (expected display or camera)", optarg); + return false; +} + static bool parse_time_limit(const char *s, sc_tick *tick) { long value; @@ -1669,6 +1709,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } break; + case OPT_CAMERA_ID: + opts->camera_id = optarg; + break; case 'd': opts->select_usb = true; break; @@ -1950,6 +1993,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_LIST_DISPLAYS: opts->list_displays = true; break; + case OPT_LIST_CAMERAS: + opts->list_cameras = true; + break; case OPT_REQUIRE_AUDIO: opts->require_audio = true; break; @@ -1969,6 +2015,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } break; + case OPT_VIDEO_SOURCE: + if (!parse_video_source(optarg, &opts->video_source)) { + return false; + } + break; case OPT_KILL_ADB_ON_CLOSE: opts->kill_adb_on_close = true; break; @@ -2124,6 +2175,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } } + if (opts->video_source == SC_VIDEO_SOURCE_CAMERA && opts->control) { + LOGI("--video-source camera disables control"); + opts->control = false; + } + if (!opts->control) { if (opts->turn_screen_off) { LOGE("Could not request to turn screen off if control is disabled"); diff --git a/app/src/options.c b/app/src/options.c index 530e003b5c..b4e21659a8 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -14,6 +14,7 @@ const struct scrcpy_options scrcpy_options_default = { .log_level = SC_LOG_LEVEL_INFO, .video_codec = SC_CODEC_H264, .audio_codec = SC_CODEC_OPUS, + .video_source = SC_VIDEO_SOURCE_DISPLAY, .audio_source = SC_AUDIO_SOURCE_OUTPUT, .record_format = SC_RECORD_FORMAT_AUTO, .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, @@ -39,6 +40,7 @@ const struct scrcpy_options scrcpy_options_default = { .window_width = 0, .window_height = 0, .display_id = 0, + .camera_id = NULL, .display_buffer = 0, .audio_buffer = SC_TICK_FROM_MS(50), .audio_output_buffer = SC_TICK_FROM_MS(5), @@ -81,5 +83,6 @@ const struct scrcpy_options scrcpy_options_default = { .require_audio = false, .list_encoders = false, .list_displays = false, + .list_cameras = false, .kill_adb_on_close = false, }; diff --git a/app/src/options.h b/app/src/options.h index 1f36ad7fdc..f537d48f76 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -49,6 +49,11 @@ enum sc_audio_source { SC_AUDIO_SOURCE_MIC, }; +enum sc_video_source { + SC_VIDEO_SOURCE_DISPLAY, + SC_VIDEO_SOURCE_CAMERA, +}; + enum sc_lock_video_orientation { SC_LOCK_VIDEO_ORIENTATION_UNLOCKED = -1, // lock the current orientation when scrcpy starts @@ -121,6 +126,7 @@ struct scrcpy_options { enum sc_codec video_codec; enum sc_codec audio_codec; enum sc_audio_source audio_source; + enum sc_video_source video_source; enum sc_record_format record_format; enum sc_keyboard_input_mode keyboard_input_mode; enum sc_mouse_input_mode mouse_input_mode; @@ -139,6 +145,7 @@ struct scrcpy_options { uint16_t window_width; uint16_t window_height; uint32_t display_id; + const char* camera_id; sc_tick display_buffer; sc_tick audio_buffer; sc_tick audio_output_buffer; @@ -181,6 +188,7 @@ struct scrcpy_options { bool require_audio; bool list_encoders; bool list_displays; + bool list_cameras; bool kill_adb_on_close; }; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index aabb7c5a14..90529c0714 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -351,6 +351,7 @@ scrcpy(struct scrcpy_options *options) { .log_level = options->log_level, .video_codec = options->video_codec, .audio_codec = options->audio_codec, + .video_source = options->video_source, .audio_source = options->audio_source, .crop = options->crop, .port_range = options->port_range, @@ -363,6 +364,7 @@ scrcpy(struct scrcpy_options *options) { .lock_video_orientation = options->lock_video_orientation, .control = options->control, .display_id = options->display_id, + .camera_id = options->camera_id, .video = options->video, .audio = options->audio, .show_touches = options->show_touches, @@ -381,6 +383,7 @@ scrcpy(struct scrcpy_options *options) { .power_on = options->power_on, .list_encoders = options->list_encoders, .list_displays = options->list_displays, + .list_cameras = options->list_cameras, .kill_adb_on_close = options->kill_adb_on_close, }; @@ -399,7 +402,7 @@ scrcpy(struct scrcpy_options *options) { server_started = true; - if (options->list_encoders || options->list_displays) { + if (options->list_encoders || options->list_displays || options->list_cameras) { bool ok = await_for_server(NULL); ret = ok ? SCRCPY_EXIT_SUCCESS : SCRCPY_EXIT_FAILURE; goto end; diff --git a/app/src/server.c b/app/src/server.c index 4d787ea942..432392d2c7 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -246,6 +246,10 @@ execute_server(struct sc_server *server, ADD_PARAM("audio_codec=%s", sc_server_get_codec_name(params->audio_codec)); } + if (params->video_source != SC_VIDEO_SOURCE_DISPLAY){ + assert(params->audio_source == SC_VIDEO_SOURCE_CAMERA); + ADD_PARAM("video_source=camera"); + } if (params->audio_source != SC_AUDIO_SOURCE_OUTPUT) { assert(params->audio_source == SC_AUDIO_SOURCE_MIC); ADD_PARAM("audio_source=mic"); @@ -270,9 +274,12 @@ execute_server(struct sc_server *server, // By default, control is true ADD_PARAM("control=false"); } - if (params->display_id) { + if (params->video_source == SC_VIDEO_SOURCE_DISPLAY && params->display_id) { ADD_PARAM("display_id=%" PRIu32, params->display_id); } + if (params->video_source == SC_VIDEO_SOURCE_CAMERA && params->camera_id) { + ADD_PARAM("camera_id=%s", params->camera_id); + } if (params->show_touches) { ADD_PARAM("show_touches=true"); } @@ -316,6 +323,9 @@ execute_server(struct sc_server *server, if (params->list_displays) { ADD_PARAM("list_displays=true"); } + if (params->list_cameras) { + ADD_PARAM("list_cameras=true"); + } #undef ADD_PARAM @@ -895,7 +905,7 @@ run_server(void *data) { // If --list-* is passed, then the server just prints the requested data // then exits. - if (params->list_encoders || params->list_displays) { + if (params->list_encoders || params->list_displays || params->list_cameras) { sc_pid pid = execute_server(server, params); if (pid == SC_PROCESS_NONE) { goto error_connection_failed; diff --git a/app/src/server.h b/app/src/server.h index adba265235..d3cf20c64d 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -26,6 +26,7 @@ struct sc_server_params { enum sc_log_level log_level; enum sc_codec video_codec; enum sc_codec audio_codec; + enum sc_video_source video_source; enum sc_audio_source audio_source; const char *crop; const char *video_codec_options; @@ -42,6 +43,7 @@ struct sc_server_params { int8_t lock_video_orientation; bool control; uint32_t display_id; + const char *camera_id; bool video; bool audio; bool show_touches; @@ -58,6 +60,7 @@ struct sc_server_params { bool power_on; bool list_encoders; bool list_displays; + bool list_cameras; bool kill_adb_on_close; }; diff --git a/server/src/main/java/com/genymobile/scrcpy/CameraEncoder.java b/server/src/main/java/com/genymobile/scrcpy/CameraEncoder.java index fa618e6167..e31b5ff78e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CameraEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/CameraEncoder.java @@ -10,57 +10,34 @@ import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.params.StreamConfigurationMap; import android.media.MediaCodec; -import android.media.MediaCodecInfo; -import android.media.MediaFormat; import android.os.Handler; import android.os.HandlerThread; import android.os.Looper; -import android.os.SystemClock; import android.view.Surface; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.concurrent.Semaphore; -import java.util.concurrent.atomic.AtomicBoolean; -public class CameraEncoder implements AsyncProcessor { - - private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds - private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms - private static final String KEY_MAX_FPS_TO_ENCODER = "max-fps-to-encoder"; - - // Keep the values in descending order - private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800}; - private static final int MAX_CONSECUTIVE_ERRORS = 3; +public class CameraEncoder extends SurfaceEncoder { private int maxSize; - private final Streamer streamer; - private final String encoderName; - private final List codecOptions; - private final int videoBitRate; - private final int maxFps; - private final boolean downsizeOnError; - - private boolean firstFrameSent; - private int consecutiveErrors; + private String cameraId; - private Thread thread; - private final AtomicBoolean stopped = new AtomicBoolean(); + private CameraDevice cameraDevice; + private CameraManager cameraManager; private HandlerThread cameraThread; private Handler cameraHandler; - public CameraEncoder(int maxSize, Streamer streamer, int videoBitRate, int maxFps, List codecOptions, String encoderName, boolean downsizeOnError) { + public CameraEncoder(int maxSize, String cameraId, Streamer streamer, int videoBitRate, int maxFps, + List codecOptions, String encoderName, boolean downsizeOnError) { + super(streamer, videoBitRate, maxFps, codecOptions, encoderName, downsizeOnError); + this.maxSize = maxSize; - this.streamer = streamer; - this.videoBitRate = videoBitRate; - this.maxFps = maxFps; - this.codecOptions = codecOptions; - this.encoderName = encoderName; - this.downsizeOnError = downsizeOnError; + this.cameraId = cameraId; cameraThread = new HandlerThread("camera"); cameraThread.start(); @@ -68,10 +45,11 @@ public CameraEncoder(int maxSize, Streamer streamer, int videoBitRate, int maxFp } @SuppressLint("MissingPermission") - private CameraDevice openCamera(CameraManager manager, String id) throws CameraAccessException, InterruptedException { + private CameraDevice openCamera(String id) + throws CameraAccessException, InterruptedException { Semaphore semaphore = new Semaphore(0); final CameraDevice[] result = new CameraDevice[1]; - manager.openCamera(id, new CameraDevice.StateCallback() { + cameraManager.openCamera(id, new CameraDevice.StateCallback() { @Override public void onOpened(CameraDevice camera) { result[0] = camera; @@ -93,7 +71,8 @@ public void onError(CameraDevice camera, int error) { } @SuppressWarnings("deprecation") - private CameraCaptureSession createCaptureSession(CameraDevice camera, Surface surface) throws CameraAccessException, InterruptedException { + private CameraCaptureSession createCaptureSession(CameraDevice camera, Surface surface) + throws CameraAccessException, InterruptedException { Semaphore semaphore = new Semaphore(0); final CameraCaptureSession[] result = new CameraCaptureSession[1]; camera.createCaptureSession(Collections.singletonList(surface), new CameraCaptureSession.StateCallback() { @@ -112,283 +91,81 @@ public void onConfigureFailed(CameraCaptureSession session) { return result[0]; } - @SuppressLint("MissingPermission") - private void streamCamera() throws ConfigurationException, IOException { - Looper.prepare(); - - Codec codec = streamer.getCodec(); - MediaCodec mediaCodec = createMediaCodec(codec, encoderName); - MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions); - + @Override + protected void initialize() { try { - CameraManager manager = (CameraManager) Workarounds.getSystemContext().getSystemService(Context.CAMERA_SERVICE); - String[] ids = manager.getCameraIdList(); - - String backCameraId = null; - CameraCharacteristics backCameraCharacteristics = null; - for (String id : ids) { - CameraCharacteristics characteristics = manager.getCameraCharacteristics(id); - Integer facing = characteristics.get(CameraCharacteristics.LENS_FACING); - String facingName = null; - switch (facing) { - case CameraCharacteristics.LENS_FACING_BACK: - facingName = "Back"; - backCameraId = id; - backCameraCharacteristics = characteristics; - break; - case CameraCharacteristics.LENS_FACING_FRONT: - facingName = "Front"; - break; - case CameraCharacteristics.LENS_FACING_EXTERNAL: - facingName = "External"; - break; - } - Ln.i("Camera " + id + ", Facing: " + facingName); - } - - if (backCameraId == null) { - Ln.e("No back facing camera found."); - return; - } - - CameraDevice camera = openCamera(manager, backCameraId); - - StreamConfigurationMap map = backCameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); - android.util.Size[] sizes = Arrays.stream(map.getOutputSizes(MediaCodec.class)).sorted((a, b) -> b.getWidth() - a.getWidth()).toArray(android.util.Size[]::new); - - for (android.util.Size size : sizes) { - Ln.i("Supported Size: " + size.getWidth() + "x" + size.getHeight()); - } - - streamer.writeVideoHeader(new Size(sizes[0].getWidth(), sizes[0].getHeight())); - - boolean alive; - try { - do { - android.util.Size selectedSize = null; - if (maxSize == 0) { - selectedSize = sizes[0]; - } else { - for (android.util.Size size : sizes) { - if (size.getWidth() < maxSize && size.getHeight() < maxSize) { - selectedSize = size; - break; - } - } - if (selectedSize == null) { - selectedSize = sizes[sizes.length - 1]; - } - } - - Ln.i("Selected Size: " + selectedSize.getWidth() + "x" + selectedSize.getHeight()); - - format.setInteger(MediaFormat.KEY_WIDTH, selectedSize.getWidth()); - format.setInteger(MediaFormat.KEY_HEIGHT, selectedSize.getHeight()); - - Surface surface = null; - try { - mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - surface = mediaCodec.createInputSurface(); - - CameraCaptureSession session = createCaptureSession(camera, surface); - CaptureRequest.Builder requestBuilder = camera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); - requestBuilder.addTarget(surface); - CaptureRequest request = requestBuilder.build(); - session.setRepeatingRequest(request, null, cameraHandler); - - mediaCodec.start(); - - alive = encode(mediaCodec, streamer); - // do not call stop() on exception, it would trigger an IllegalStateException - mediaCodec.stop(); - } catch (IllegalStateException | IllegalArgumentException e) { - Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage()); - if (!prepareRetry(new Size(selectedSize.getWidth(), selectedSize.getHeight()))) { - throw e; - } - Ln.i("Retrying..."); - alive = true; - } finally { - mediaCodec.reset(); - if (surface != null) { - surface.release(); - } - } - } while (alive); - } finally { - mediaCodec.release(); - camera.close(); - } - } catch (ReflectiveOperationException | CameraAccessException | InterruptedException e) { - Ln.e("Can't open camera.", e); + Looper.prepare(); + cameraManager = (CameraManager) Workarounds.getSystemContext() + .getSystemService(Context.CAMERA_SERVICE); + } catch (Exception e) { + throw new RuntimeException("Can't access camera", e); } } - private boolean prepareRetry(Size currentSize) { - if (firstFrameSent) { - ++consecutiveErrors; - if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { - // Definitively fail - return false; - } - - // Wait a bit to increase the probability that retrying will fix the problem - SystemClock.sleep(50); - return true; - } - - if (!downsizeOnError) { - // Must fail immediately - return false; - } - - // Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising) - - int newMaxSize = chooseMaxSizeFallback(currentSize); - Ln.i("newMaxSize = " + newMaxSize); - if (newMaxSize == 0) { - // Must definitively fail - return false; - } - - // Retry with a smaller device size - Ln.i("Retrying with -m" + newMaxSize + "..."); - maxSize = newMaxSize; - return true; - } - - private int chooseMaxSizeFallback(Size failedSize) { - int currentMaxSize = Math.max(failedSize.getWidth(), failedSize.getHeight()); - for (int value : MAX_SIZE_FALLBACK) { - if (value < currentMaxSize) { - // We found a smaller value to reduce the video size - return value; - } - } - // No fallback, fail definitively - return 0; - } - - private boolean encode(MediaCodec codec, Streamer streamer) throws IOException { - boolean eof = false; - boolean alive = true; - MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); - - while (!eof) { - if (stopped.get()) { - alive = false; - break; - } - int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1); - try { - eof = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; - if (outputBufferId >= 0) { - ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId); - - boolean isConfig = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; - if (!isConfig) { - // If this is not a config packet, then it contains a frame - firstFrameSent = true; - consecutiveErrors = 0; + @Override + protected Size getSize() throws ConfigurationException { + try { + if (cameraId != null) { + cameraDevice = openCamera(cameraId); + } else { + String[] cameraIds = cameraManager.getCameraIdList(); + cameraDevice = openCamera(cameraIds[0]); + } + + CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraDevice.getId()); + StreamConfigurationMap map = characteristics + .get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + android.util.Size[] sizes = Arrays.stream(map.getOutputSizes(MediaCodec.class)) + .sorted(Comparator.comparing(android.util.Size::getWidth).reversed()) + .toArray(android.util.Size[]::new); + + android.util.Size selectedSize = null; + if (maxSize == 0) { + selectedSize = sizes[0]; + } else { + for (android.util.Size size : sizes) { + if (size.getWidth() < maxSize && size.getHeight() < maxSize) { + selectedSize = size; + break; } - - streamer.writePacket(codecBuffer, bufferInfo); } - } finally { - if (outputBufferId >= 0) { - codec.releaseOutputBuffer(outputBufferId, false); + if (selectedSize == null) { + selectedSize = sizes[sizes.length - 1]; } } - } - return !eof && alive; - } - - private static MediaCodec createMediaCodec(Codec codec, String encoderName) throws IOException, ConfigurationException { - if (encoderName != null) { - Ln.d("Creating encoder by name: '" + encoderName + "'"); - try { - return MediaCodec.createByCodecName(encoderName); - } catch (IllegalArgumentException e) { - Ln.e("Video encoder '" + encoderName + "' for " + codec.getName() + " not found\n" + LogUtils.buildVideoEncoderListMessage()); - throw new ConfigurationException("Unknown encoder: " + encoderName); - } catch (IOException e) { - Ln.e("Could not create video encoder '" + encoderName + "' for " + codec.getName() + "\n" + LogUtils.buildVideoEncoderListMessage()); - throw e; - } - } - - try { - MediaCodec mediaCodec = MediaCodec.createEncoderByType(codec.getMimeType()); - Ln.d("Using video encoder: '" + mediaCodec.getName() + "'"); - return mediaCodec; - } catch (IOException | IllegalArgumentException e) { - Ln.e("Could not create default video encoder for " + codec.getName() + "\n" + LogUtils.buildVideoEncoderListMessage()); - throw e; + return new Size(selectedSize.getWidth(), selectedSize.getHeight()); + } catch (IllegalArgumentException e) { + Ln.e("Camera " + cameraId + " not found\n" + LogUtils.buildCameraListMessage()); + throw new ConfigurationException("Unknown camera id: " + cameraId); + } catch (Exception e) { + throw new RuntimeException("Can't access camera", e); } } - private static MediaFormat createFormat(String videoMimeType, int bitRate, int maxFps, List codecOptions) { - MediaFormat format = new MediaFormat(); - format.setString(MediaFormat.KEY_MIME, videoMimeType); - format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); - // must be present to configure the encoder, but does not impact the actual frame rate, which is variable - format.setInteger(MediaFormat.KEY_FRAME_RATE, 60); - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL); - // display the very first frame, and recover from bad quality when no new frames - format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, REPEAT_FRAME_DELAY_US); // µs - if (maxFps > 0) { - // The key existed privately before Android 10: - // - // - format.setFloat(KEY_MAX_FPS_TO_ENCODER, maxFps); - } - - if (codecOptions != null) { - for (CodecOption option : codecOptions) { - String key = option.getKey(); - Object value = option.getValue(); - CodecUtils.setCodecOption(format, key, value); - Ln.d("Video codec option set: " + key + " (" + value.getClass().getSimpleName() + ") = " + value); - } - } - - return format; - } - - @Override - public void start(TerminationListener listener) { - thread = new Thread(() -> { - try { - streamCamera(); - } catch (ConfigurationException e) { - // Do not print stack trace, a user-friendly error-message has already been logged - } catch (IOException e) { - // Broken pipe is expected on close, because the socket is closed by the client - if (!IO.isBrokenPipe(e)) { - Ln.e("Video encoding error", e); - } - } finally { - Ln.d("Screen streaming stopped"); - listener.onTerminated(true); - } - }, "video"); - thread.start(); + protected void setSize(int size) { + maxSize = size; } @Override - public void stop() { - if (thread != null) { - stopped.set(true); + protected void setSurface(Surface surface) { + try { + CameraCaptureSession session = createCaptureSession(cameraDevice, surface); + CaptureRequest.Builder requestBuilder = cameraDevice + .createCaptureRequest(CameraDevice.TEMPLATE_RECORD); + requestBuilder.addTarget(surface); + CaptureRequest request = requestBuilder.build(); + session.setRepeatingRequest(request, null, cameraHandler); + } catch (Exception e) { + throw new RuntimeException("Can't access camera", e); } } @Override - public void join() throws InterruptedException { - if (thread != null) { - thread.join(); + protected void dispose() { + if (cameraDevice != null) { + cameraDevice.close(); } } } diff --git a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java index 243a156bab..f8c443cf59 100644 --- a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java +++ b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java @@ -3,6 +3,14 @@ import com.genymobile.scrcpy.wrappers.DisplayManager; import com.genymobile.scrcpy.wrappers.ServiceManager; +import android.content.Context; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraManager; +import android.hardware.camera2.params.StreamConfigurationMap; +import android.media.MediaCodec; +import android.os.Looper; + +import java.util.Arrays; import java.util.List; public final class LogUtils { @@ -60,4 +68,52 @@ public static String buildDisplayListMessage() { } return builder.toString(); } + + public static String buildCameraListMessage() { + try { + Looper.prepare(); + + StringBuilder builder = new StringBuilder("List of cameras:"); + CameraManager cameraManager = (CameraManager) Workarounds.getSystemContext() + .getSystemService(Context.CAMERA_SERVICE); + String[] cameraIds = cameraManager.getCameraIdList(); + if (cameraIds.length == 0) { + builder.append("\n (none)"); + } else { + for (String id : cameraIds) { + builder.append("\n --video-source=camera --camera=").append(id).append(" ("); + CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(id); + Integer facing = characteristics.get(CameraCharacteristics.LENS_FACING); + switch (facing) { + case CameraCharacteristics.LENS_FACING_BACK: + builder.append("Back"); + break; + case CameraCharacteristics.LENS_FACING_FRONT: + builder.append("Front"); + break; + case CameraCharacteristics.LENS_FACING_EXTERNAL: + builder.append("External"); + break; + default: + builder.append("Unknown"); + break; + } + builder.append(", "); + StreamConfigurationMap map = characteristics + .get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + android.util.Size[] sizes = Arrays.stream(map.getOutputSizes(MediaCodec.class)) + .sorted((a, b) -> b.getWidth() - a.getWidth()).toArray(android.util.Size[]::new); + if (sizes.length == 0) { + builder.append("size unknown"); + } else { + builder.append(sizes[0].getWidth()).append("x").append(sizes[0].getHeight()); + } + builder.append(")"); + } + } + return builder.toString(); + } catch (Exception e) { + throw new RuntimeException("Can't access camera", e); + } + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index aab6fce86c..37da19f0f3 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -14,6 +14,7 @@ public class Options { private int maxSize; private VideoCodec videoCodec = VideoCodec.H264; private AudioCodec audioCodec = AudioCodec.OPUS; + private VideoSource videoSource = VideoSource.DISPLAY; private AudioSource audioSource = AudioSource.OUTPUT; private int videoBitRate = 8000000; private int audioBitRate = 128000; @@ -23,6 +24,7 @@ public class Options { private Rect crop; private boolean control = true; private int displayId; + private String cameraId; private boolean showTouches; private boolean stayAwake; private List videoCodecOptions; @@ -38,6 +40,7 @@ public class Options { private boolean listEncoders; private boolean listDisplays; + private boolean listCameras; // Options not used by the scrcpy client, but useful to use scrcpy-server directly private boolean sendDeviceMeta = true; // send device name and size @@ -73,6 +76,10 @@ public AudioCodec getAudioCodec() { return audioCodec; } + public VideoSource getVideoSource() { + return videoSource; + } + public AudioSource getAudioSource() { return audioSource; } @@ -109,6 +116,10 @@ public int getDisplayId() { return displayId; } + public String getCameraId() { + return cameraId; + } + public boolean getShowTouches() { return showTouches; } @@ -161,6 +172,10 @@ public boolean getListDisplays() { return listDisplays; } + public boolean getListCameras() { + return listCameras; + } + public boolean getSendDeviceMeta() { return sendDeviceMeta; } @@ -230,6 +245,13 @@ public static Options parse(String... args) { } options.audioCodec = audioCodec; break; + case "video_source": + VideoSource videoSource = VideoSource.findByName(value); + if (videoSource == null) { + throw new IllegalArgumentException("Video source " + value + " not supported"); + } + options.videoSource = videoSource; + break; case "audio_source": AudioSource audioSource = AudioSource.findByName(value); if (audioSource == null) { @@ -264,6 +286,9 @@ public static Options parse(String... args) { case "display_id": options.displayId = Integer.parseInt(value); break; + case "camera_id": + options.cameraId = value; + break; case "show_touches": options.showTouches = Boolean.parseBoolean(value); break; @@ -306,6 +331,9 @@ public static Options parse(String... args) { case "list_displays": options.listDisplays = Boolean.parseBoolean(value); break; + case "list_cameras": + options.listCameras = Boolean.parseBoolean(value); + break; case "send_device_meta": options.sendDeviceMeta = Boolean.parseBoolean(value); break; diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index 5a9db10d3c..e2addc89d6 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -3,55 +3,22 @@ import com.genymobile.scrcpy.wrappers.SurfaceControl; import android.graphics.Rect; -import android.media.MediaCodec; -import android.media.MediaCodecInfo; -import android.media.MediaFormat; import android.os.Build; import android.os.IBinder; -import android.os.Looper; -import android.os.SystemClock; import android.view.Surface; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -public class ScreenEncoder implements Device.RotationListener, Device.FoldListener, AsyncProcessor { - - private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds - private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms - private static final String KEY_MAX_FPS_TO_ENCODER = "max-fps-to-encoder"; - - // Keep the values in descending order - private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800}; - private static final int MAX_CONSECUTIVE_ERRORS = 3; - - private final AtomicBoolean resetCapture = new AtomicBoolean(); +public class ScreenEncoder extends SurfaceEncoder implements Device.RotationListener, Device.FoldListener { private final Device device; - private final Streamer streamer; - private final String encoderName; - private final List codecOptions; - private final int videoBitRate; - private final int maxFps; - private final boolean downsizeOnError; - - private boolean firstFrameSent; - private int consecutiveErrors; - private Thread thread; - private final AtomicBoolean stopped = new AtomicBoolean(); + private IBinder display; - public ScreenEncoder(Device device, Streamer streamer, int videoBitRate, int maxFps, List codecOptions, String encoderName, - boolean downsizeOnError) { + public ScreenEncoder(Device device, Streamer streamer, int videoBitRate, int maxFps, List codecOptions, + String encoderName, boolean downsizeOnError) { + super(streamer, videoBitRate, maxFps, codecOptions, encoderName, downsizeOnError); this.device = device; - this.streamer = streamer; - this.videoBitRate = videoBitRate; - this.maxFps = maxFps; - this.codecOptions = codecOptions; - this.encoderName = encoderName; - this.downsizeOnError = downsizeOnError; } @Override @@ -64,211 +31,47 @@ public void onRotationChanged(int rotation) { resetCapture.set(true); } - private boolean consumeResetCapture() { - return resetCapture.getAndSet(false); - } - - private void streamScreen() throws IOException, ConfigurationException { - Codec codec = streamer.getCodec(); - MediaCodec mediaCodec = createMediaCodec(codec, encoderName); - MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions); - IBinder display = createDisplay(); + @Override + protected void initialize() { + display = createDisplay(); device.setRotationListener(this); device.setFoldListener(this); - - streamer.writeVideoHeader(device.getScreenInfo().getVideoSize()); - - boolean alive; - try { - do { - ScreenInfo screenInfo = device.getScreenInfo(); - Rect contentRect = screenInfo.getContentRect(); - - // include the locked video orientation - Rect videoRect = screenInfo.getVideoSize().toRect(); - format.setInteger(MediaFormat.KEY_WIDTH, videoRect.width()); - format.setInteger(MediaFormat.KEY_HEIGHT, videoRect.height()); - - Surface surface = null; - try { - mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - surface = mediaCodec.createInputSurface(); - - // does not include the locked video orientation - Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect(); - int videoRotation = screenInfo.getVideoRotation(); - int layerStack = device.getLayerStack(); - setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack); - - mediaCodec.start(); - - alive = encode(mediaCodec, streamer); - // do not call stop() on exception, it would trigger an IllegalStateException - mediaCodec.stop(); - } catch (IllegalStateException | IllegalArgumentException e) { - Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage()); - if (!prepareRetry(device, screenInfo)) { - throw e; - } - Ln.i("Retrying..."); - alive = true; - } finally { - mediaCodec.reset(); - if (surface != null) { - surface.release(); - } - } - } while (alive); - } finally { - mediaCodec.release(); - device.setRotationListener(null); - device.setFoldListener(null); - SurfaceControl.destroyDisplay(display); - } - } - - private boolean prepareRetry(Device device, ScreenInfo screenInfo) { - if (firstFrameSent) { - ++consecutiveErrors; - if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { - // Definitively fail - return false; - } - - // Wait a bit to increase the probability that retrying will fix the problem - SystemClock.sleep(50); - return true; - } - - if (!downsizeOnError) { - // Must fail immediately - return false; - } - - // Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising) - - int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize()); - if (newMaxSize == 0) { - // Must definitively fail - return false; - } - - // Retry with a smaller device size - Ln.i("Retrying with -m" + newMaxSize + "..."); - device.setMaxSize(newMaxSize); - return true; } - private static int chooseMaxSizeFallback(Size failedSize) { - int currentMaxSize = Math.max(failedSize.getWidth(), failedSize.getHeight()); - for (int value : MAX_SIZE_FALLBACK) { - if (value < currentMaxSize) { - // We found a smaller value to reduce the video size - return value; - } - } - // No fallback, fail definitively - return 0; + @Override + protected Size getSize() { + return device.getScreenInfo().getVideoSize(); } - private boolean encode(MediaCodec codec, Streamer streamer) throws IOException { - boolean eof = false; - boolean alive = true; - MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); - - while (!consumeResetCapture() && !eof) { - if (stopped.get()) { - alive = false; - break; - } - int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1); - try { - if (consumeResetCapture()) { - // must restart encoding with new size - break; - } - - eof = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; - if (outputBufferId >= 0) { - ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId); - - boolean isConfig = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; - if (!isConfig) { - // If this is not a config packet, then it contains a frame - firstFrameSent = true; - consecutiveErrors = 0; - } - - streamer.writePacket(codecBuffer, bufferInfo); - } - } finally { - if (outputBufferId >= 0) { - codec.releaseOutputBuffer(outputBufferId, false); - } - } - } - - return !eof && alive; + @Override + protected void setSize(int size) { + device.setMaxSize(size); } - private static MediaCodec createMediaCodec(Codec codec, String encoderName) throws IOException, ConfigurationException { - if (encoderName != null) { - Ln.d("Creating encoder by name: '" + encoderName + "'"); - try { - return MediaCodec.createByCodecName(encoderName); - } catch (IllegalArgumentException e) { - Ln.e("Video encoder '" + encoderName + "' for " + codec.getName() + " not found\n" + LogUtils.buildVideoEncoderListMessage()); - throw new ConfigurationException("Unknown encoder: " + encoderName); - } catch (IOException e) { - Ln.e("Could not create video encoder '" + encoderName + "' for " + codec.getName() + "\n" + LogUtils.buildVideoEncoderListMessage()); - throw e; - } - } - - try { - MediaCodec mediaCodec = MediaCodec.createEncoderByType(codec.getMimeType()); - Ln.d("Using video encoder: '" + mediaCodec.getName() + "'"); - return mediaCodec; - } catch (IOException | IllegalArgumentException e) { - Ln.e("Could not create default video encoder for " + codec.getName() + "\n" + LogUtils.buildVideoEncoderListMessage()); - throw e; - } + @Override + protected void setSurface(Surface surface) { + ScreenInfo screenInfo = device.getScreenInfo(); + Rect contentRect = screenInfo.getContentRect(); + + // does not include the locked video orientation + Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect(); + int videoRotation = screenInfo.getVideoRotation(); + int layerStack = device.getLayerStack(); + setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack); } - private static MediaFormat createFormat(String videoMimeType, int bitRate, int maxFps, List codecOptions) { - MediaFormat format = new MediaFormat(); - format.setString(MediaFormat.KEY_MIME, videoMimeType); - format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); - // must be present to configure the encoder, but does not impact the actual frame rate, which is variable - format.setInteger(MediaFormat.KEY_FRAME_RATE, 60); - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL); - // display the very first frame, and recover from bad quality when no new frames - format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, REPEAT_FRAME_DELAY_US); // µs - if (maxFps > 0) { - // The key existed privately before Android 10: - // - // - format.setFloat(KEY_MAX_FPS_TO_ENCODER, maxFps); - } - - if (codecOptions != null) { - for (CodecOption option : codecOptions) { - String key = option.getKey(); - Object value = option.getValue(); - CodecUtils.setCodecOption(format, key, value); - Ln.d("Video codec option set: " + key + " (" + value.getClass().getSimpleName() + ") = " + value); - } - } - - return format; + @Override + protected void dispose() { + device.setRotationListener(null); + device.setFoldListener(null); + SurfaceControl.destroyDisplay(display); } private static IBinder createDisplay() { // Since Android 12 (preview), secure displays could not be created with shell permissions anymore. // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S". boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S" - .equals(Build.VERSION.CODENAME)); + .equals(Build.VERSION.CODENAME)); return SurfaceControl.createDisplay("scrcpy", secure); } @@ -282,42 +85,4 @@ private static void setDisplaySurface(IBinder display, Surface surface, int orie SurfaceControl.closeTransaction(); } } - - @Override - public void start(TerminationListener listener) { - thread = new Thread(() -> { - // Some devices (Meizu) deadlock if the video encoding thread has no Looper - // - Looper.prepare(); - - try { - streamScreen(); - } catch (ConfigurationException e) { - // Do not print stack trace, a user-friendly error-message has already been logged - } catch (IOException e) { - // Broken pipe is expected on close, because the socket is closed by the client - if (!IO.isBrokenPipe(e)) { - Ln.e("Video encoding error", e); - } - } finally { - Ln.d("Screen streaming stopped"); - listener.onTerminated(true); - } - }, "video"); - thread.start(); - } - - @Override - public void stop() { - if (thread != null) { - stopped.set(true); - } - } - - @Override - public void join() throws InterruptedException { - if (thread != null) { - thread.join(); - } - } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 78eb060cca..fc2d683610 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -130,11 +130,19 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc } if (video) { - Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecMeta(), - options.getSendFrameMeta()); - CameraEncoder screenEncoder = new CameraEncoder(options.getMaxSize(), videoStreamer, options.getVideoBitRate(), options.getMaxFps(), - options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError()); - asyncProcessors.add(screenEncoder); + Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), + options.getSendCodecMeta(), options.getSendFrameMeta()); + if (options.getVideoSource() == VideoSource.DISPLAY) { + ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), + options.getMaxFps(), options.getVideoCodecOptions(), options.getVideoEncoder(), + options.getDownsizeOnError()); + asyncProcessors.add(screenEncoder); + } else { + CameraEncoder cameraEncoder = new CameraEncoder(options.getMaxSize(), options.getCameraId(), + videoStreamer, options.getVideoBitRate(), options.getMaxFps(), + options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError()); + asyncProcessors.add(cameraEncoder); + } } Completion completion = new Completion(asyncProcessors.size()); @@ -179,7 +187,7 @@ public static void main(String... args) throws Exception { Ln.initLogLevel(options.getLogLevel()); - if (options.getListEncoders() || options.getListDisplays()) { + if (options.getListEncoders() || options.getListDisplays() || options.getListCameras()) { if (options.getCleanup()) { CleanUp.unlinkSelf(); } @@ -191,6 +199,9 @@ public static void main(String... args) throws Exception { if (options.getListDisplays()) { Ln.i(LogUtils.buildDisplayListMessage()); } + if (options.getListCameras()) { + Ln.i(LogUtils.buildCameraListMessage()); + } // Just print the requested data, do not mirror return; } diff --git a/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java b/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java new file mode 100644 index 0000000000..c9f2abf0ff --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java @@ -0,0 +1,286 @@ +package com.genymobile.scrcpy; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import android.os.SystemClock; +import android.view.Surface; + +public abstract class SurfaceEncoder implements AsyncProcessor { + + private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds + private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms + private static final String KEY_MAX_FPS_TO_ENCODER = "max-fps-to-encoder"; + + // Keep the values in descending order + private static final int[] MAX_SIZE_FALLBACK = { 2560, 1920, 1600, 1280, 1024, 800 }; + private static final int MAX_CONSECUTIVE_ERRORS = 3; + + protected final AtomicBoolean resetCapture = new AtomicBoolean(); + + private final Streamer streamer; + private final String encoderName; + private final List codecOptions; + private final int videoBitRate; + private final int maxFps; + private final boolean downsizeOnError; + + private boolean firstFrameSent; + private int consecutiveErrors; + + private Thread thread; + private final AtomicBoolean stopped = new AtomicBoolean(); + + public SurfaceEncoder(Streamer streamer, int videoBitRate, int maxFps, List codecOptions, + String encoderName, + boolean downsizeOnError) { + this.streamer = streamer; + this.videoBitRate = videoBitRate; + this.maxFps = maxFps; + this.codecOptions = codecOptions; + this.encoderName = encoderName; + this.downsizeOnError = downsizeOnError; + } + + protected abstract void initialize(); + + protected abstract Size getSize() throws ConfigurationException; + + protected abstract void setSize(int size); + + protected abstract void setSurface(Surface surface); + + protected abstract void dispose(); + + private boolean consumeResetCapture() { + return resetCapture.getAndSet(false); + } + + protected void startStream() throws IOException, ConfigurationException { + initialize(); + + Codec codec = streamer.getCodec(); + MediaCodec mediaCodec = createMediaCodec(codec, encoderName); + MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions); + + streamer.writeVideoHeader(getSize()); + + boolean alive; + try { + do { + Size size = getSize(); + format.setInteger(MediaFormat.KEY_WIDTH, size.getWidth()); + format.setInteger(MediaFormat.KEY_HEIGHT, size.getHeight()); + + Surface surface = null; + try { + mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + surface = mediaCodec.createInputSurface(); + + setSurface(surface); + + mediaCodec.start(); + + alive = encode(mediaCodec, streamer); + // do not call stop() on exception, it would trigger an IllegalStateException + mediaCodec.stop(); + } catch (IllegalStateException | IllegalArgumentException e) { + Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage()); + if (!prepareRetry(size)) { + throw e; + } + Ln.i("Retrying..."); + alive = true; + } finally { + mediaCodec.reset(); + if (surface != null) { + surface.release(); + } + } + } while (alive); + } finally { + mediaCodec.release(); + dispose(); + } + } + + private boolean prepareRetry(Size currentSize) { + if (firstFrameSent) { + ++consecutiveErrors; + if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { + // Definitively fail + return false; + } + + // Wait a bit to increase the probability that retrying will fix the problem + SystemClock.sleep(50); + return true; + } + + if (!downsizeOnError) { + // Must fail immediately + return false; + } + + // Downsizing on error is only enabled if an encoding failure occurs before the + // first frame (downsizing later could be surprising) + + int newMaxSize = chooseMaxSizeFallback(currentSize); + Ln.i("newMaxSize = " + newMaxSize); + if (newMaxSize == 0) { + // Must definitively fail + return false; + } + + // Retry with a smaller device size + Ln.i("Retrying with -m" + newMaxSize + "..."); + setSize(newMaxSize); + return true; + } + + private int chooseMaxSizeFallback(Size failedSize) { + int currentMaxSize = Math.max(failedSize.getWidth(), failedSize.getHeight()); + for (int value : MAX_SIZE_FALLBACK) { + if (value < currentMaxSize) { + // We found a smaller value to reduce the video size + return value; + } + } + // No fallback, fail definitively + return 0; + } + + private boolean encode(MediaCodec codec, Streamer streamer) throws IOException { + boolean eof = false; + boolean alive = true; + MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + + while (!consumeResetCapture() && !eof) { + if (stopped.get()) { + alive = false; + break; + } + int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1); + try { + eof = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; + if (outputBufferId >= 0) { + ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId); + + boolean isConfig = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; + if (!isConfig) { + // If this is not a config packet, then it contains a frame + firstFrameSent = true; + consecutiveErrors = 0; + } + + streamer.writePacket(codecBuffer, bufferInfo); + } + } finally { + if (outputBufferId >= 0) { + codec.releaseOutputBuffer(outputBufferId, false); + } + } + } + + return !eof && alive; + } + + private static MediaCodec createMediaCodec(Codec codec, String encoderName) + throws IOException, ConfigurationException { + if (encoderName != null) { + Ln.d("Creating encoder by name: '" + encoderName + "'"); + try { + return MediaCodec.createByCodecName(encoderName); + } catch (IllegalArgumentException e) { + Ln.e("Video encoder '" + encoderName + "' for " + codec.getName() + " not found\n" + + LogUtils.buildVideoEncoderListMessage()); + throw new ConfigurationException("Unknown encoder: " + encoderName); + } catch (IOException e) { + Ln.e("Could not create video encoder '" + encoderName + "' for " + codec.getName() + "\n" + + LogUtils.buildVideoEncoderListMessage()); + throw e; + } + } + + try { + MediaCodec mediaCodec = MediaCodec.createEncoderByType(codec.getMimeType()); + Ln.d("Using video encoder: '" + mediaCodec.getName() + "'"); + return mediaCodec; + } catch (IOException | IllegalArgumentException e) { + Ln.e("Could not create default video encoder for " + codec.getName() + "\n" + + LogUtils.buildVideoEncoderListMessage()); + throw e; + } + } + + private static MediaFormat createFormat(String videoMimeType, int bitRate, int maxFps, + List codecOptions) { + MediaFormat format = new MediaFormat(); + format.setString(MediaFormat.KEY_MIME, videoMimeType); + format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); + // must be present to configure the encoder, but does not impact the actual + // frame rate, which is variable + format.setInteger(MediaFormat.KEY_FRAME_RATE, 60); + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL); + // display the very first frame, and recover from bad quality when no new frames + format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, REPEAT_FRAME_DELAY_US); // µs + if (maxFps > 0) { + // The key existed privately before Android 10: + // + // + format.setFloat(KEY_MAX_FPS_TO_ENCODER, maxFps); + } + + if (codecOptions != null) { + for (CodecOption option : codecOptions) { + String key = option.getKey(); + Object value = option.getValue(); + CodecUtils.setCodecOption(format, key, value); + Ln.d("Video codec option set: " + key + " (" + value.getClass().getSimpleName() + ") = " + value); + } + } + + return format; + } + + @Override + public void start(TerminationListener listener) { + thread = new Thread(() -> { + try { + startStream(); + } catch (ConfigurationException e) { + // Do not print stack trace, a user-friendly error-message has already been + // logged + } catch (IOException e) { + // Broken pipe is expected on close, because the socket is closed by the client + if (!IO.isBrokenPipe(e)) { + Ln.e("Video encoding error", e); + } + } finally { + Ln.d("Screen streaming stopped"); + listener.onTerminated(true); + } + }, "video"); + thread.start(); + } + + @Override + public void stop() { + if (thread != null) { + stopped.set(true); + } + } + + @Override + public void join() throws InterruptedException { + if (thread != null) { + thread.join(); + } + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/VideoSource.java b/server/src/main/java/com/genymobile/scrcpy/VideoSource.java new file mode 100644 index 0000000000..e454b8ff89 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/VideoSource.java @@ -0,0 +1,22 @@ +package com.genymobile.scrcpy; + +public enum VideoSource { + DISPLAY("display"), + CAMERA("camera"); + + private final String name; + + VideoSource(String name) { + this.name = name; + } + + static VideoSource findByName(String name) { + for (VideoSource audioSource : VideoSource.values()) { + if (name.equals(audioSource.name)) { + return audioSource; + } + } + + return null; + } +}