From e3da97a80fe456b211ad4a0194e2e4651956f071 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 01/30] Write trailer from recorder thread The recorder thread wrote the whole content except the trailer, which was odd. --- app/src/recorder.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 2e3b0c28a5..3f5eb0d09c 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -171,25 +171,8 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { void recorder_close(struct recorder *recorder) { - if (recorder->header_written) { - int ret = av_write_trailer(recorder->ctx); - if (ret < 0) { - LOGE("Failed to write trailer to %s", recorder->filename); - recorder->failed = true; - } - } else { - // the recorded file is empty - recorder->failed = true; - } avio_close(recorder->ctx->pb); avformat_free_context(recorder->ctx); - - if (recorder->failed) { - LOGE("Recording failed to %s", recorder->filename); - } else { - const char *format_name = recorder_get_format_name(recorder->format); - LOGI("Recording complete to %s file: %s", format_name, recorder->filename); - } } static bool @@ -317,7 +300,26 @@ run_recorder(void *data) { sc_mutex_unlock(&recorder->mutex); break; } + } + + if (!recorder->failed) { + if (recorder->header_written) { + int ret = av_write_trailer(recorder->ctx); + if (ret < 0) { + LOGE("Failed to write trailer to %s", recorder->filename); + recorder->failed = true; + } + } else { + // the recorded file is empty + recorder->failed = true; + } + } + if (recorder->failed) { + LOGE("Recording failed to %s", recorder->filename); + } else { + const char *format_name = recorder_get_format_name(recorder->format); + LOGI("Recording complete to %s file: %s", format_name, recorder->filename); } LOGD("Recorder thread ended"); From 8bae1f6b7fc7a82ea2d04c76e21fd127617c97d6 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 02/30] Remove option --render-expired-frames This flag forced the decoder to wait for the previous frame to be consumed by the display. It was initially implemented as a compilation flag for testing, not intended to be exposed at runtime. But to remove ifdefs and to allow users to test this flag easily, it had finally been exposed by commit ebccb9f6cc111e8acfbe10d656cac5c1f1b744a0. In practice, it turned out to be useless: it had no practical impact, and it did not solve or mitigate any performance issues causing frame skipping. But that added some complexity to the codebase: it required an additional condition variable, and made video buffer calls possibly blocking, which in turn required code to interrupt it on exit. To prepare support for multiple sinks plugged to the decoder (display and v4l2 for example), the blocking call used for pacing the decoder output becomes unacceptable, so just remove this useless "feature". --- README.md | 12 ------------ app/scrcpy.1 | 4 ---- app/src/cli.c | 9 ++------- app/src/decoder.c | 5 ----- app/src/decoder.h | 3 --- app/src/scrcpy.c | 9 +++------ app/src/scrcpy.h | 2 -- app/src/stream.c | 7 ------- app/src/stream.h | 3 --- app/src/video_buffer.c | 38 +------------------------------------- app/src/video_buffer.h | 9 +-------- app/tests/test_cli.c | 2 -- 12 files changed, 7 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 6a987a73a3..5f583b8116 100644 --- a/README.md +++ b/README.md @@ -491,18 +491,6 @@ scrcpy -Sw ``` -#### Render expired frames - -By default, to minimize latency, _scrcpy_ always renders the last decoded frame -available, and drops any previous one. - -To force the rendering of all frames (at a cost of a possible increased -latency), use: - -```bash -scrcpy --render-expired-frames -``` - #### Show touches For presentations, it may be useful to show physical touches (on the physical diff --git a/app/scrcpy.1 b/app/scrcpy.1 index bf3bc9cdd3..ea9bce9ba6 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -155,10 +155,6 @@ Supported names are currently "direct3d", "opengl", "opengles2", "opengles", "me .UR https://wiki.libsdl.org/SDL_HINT_RENDER_DRIVER .UE -.TP -.B \-\-render\-expired\-frames -By default, to minimize latency, scrcpy always renders the last available decoded frame, and drops any previous ones. This flag forces to render all frames, at a cost of a possible increased latency. - .TP .BI "\-\-rotation " value Set the initial display rotation. Possibles values are 0, 1, 2 and 3. Each increment adds a 90 degrees rotation counterclockwise. diff --git a/app/src/cli.c b/app/src/cli.c index 042a1f4c38..ec3c12941a 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -143,12 +143,6 @@ scrcpy_print_usage(const char *arg0) { " \"opengles2\", \"opengles\", \"metal\" and \"software\".\n" " \n" "\n" - " --render-expired-frames\n" - " By default, to minimize latency, scrcpy always renders the\n" - " last available decoded frame, and drops any previous ones.\n" - " This flag forces to render all frames, at a cost of a\n" - " possible increased latency.\n" - "\n" " --rotation value\n" " Set the initial display rotation.\n" " Possibles values are 0, 1, 2 and 3. Each increment adds a 90\n" @@ -816,7 +810,8 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { opts->stay_awake = true; break; case OPT_RENDER_EXPIRED_FRAMES: - opts->render_expired_frames = true; + LOGW("Option --render-expired-frames has been removed. This " + "flag has been ignored."); break; case OPT_WINDOW_TITLE: opts->window_title = optarg; diff --git a/app/src/decoder.c b/app/src/decoder.c index a13cf75eb8..b71011941b 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -69,8 +69,3 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) { #endif return true; } - -void -decoder_interrupt(struct decoder *decoder) { - video_buffer_interrupt(decoder->video_buffer); -} diff --git a/app/src/decoder.h b/app/src/decoder.h index bbd7a9a763..ba06583e1d 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -26,7 +26,4 @@ decoder_close(struct decoder *decoder); bool decoder_push(struct decoder *decoder, const AVPacket *packet); -void -decoder_interrupt(struct decoder *decoder); - #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 388bb73dd0..2f8abd64fa 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -305,7 +305,7 @@ scrcpy(const struct scrcpy_options *options) { } fps_counter_initialized = true; - if (!video_buffer_init(&video_buffer, options->render_expired_frames)) { + if (!video_buffer_init(&video_buffer)) { goto end; } video_buffer_initialized = true; @@ -398,11 +398,8 @@ scrcpy(const struct scrcpy_options *options) { LOGD("quit..."); end: - // stop stream and controller so that they don't continue once their socket - // is shutdown - if (stream_started) { - stream_stop(&stream); - } + // The stream is not stopped explicitly, because it will stop by itself on + // end-of-stream if (controller_started) { controller_stop(&controller); } diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index 8ee70f606b..f91cb6b88f 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -72,7 +72,6 @@ struct scrcpy_options { bool control; bool display; bool turn_screen_off; - bool render_expired_frames; bool prefer_text; bool window_borderless; bool mipmaps; @@ -120,7 +119,6 @@ struct scrcpy_options { .control = true, \ .display = true, \ .turn_screen_off = false, \ - .render_expired_frames = false, \ .prefer_text = false, \ .window_borderless = false, \ .mipmaps = true, \ diff --git a/app/src/stream.c b/app/src/stream.c index ba72f1642e..e0a223be3f 100644 --- a/app/src/stream.c +++ b/app/src/stream.c @@ -285,13 +285,6 @@ stream_start(struct stream *stream) { return true; } -void -stream_stop(struct stream *stream) { - if (stream->decoder) { - decoder_interrupt(stream->decoder); - } -} - void stream_join(struct stream *stream) { sc_thread_join(&stream->thread, NULL); diff --git a/app/src/stream.h b/app/src/stream.h index d4a9bd4afb..421c1bf01e 100644 --- a/app/src/stream.h +++ b/app/src/stream.h @@ -31,9 +31,6 @@ stream_init(struct stream *stream, socket_t socket, bool stream_start(struct stream *stream); -void -stream_stop(struct stream *stream); - void stream_join(struct stream *stream); diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index 94619840c2..d4954afcf5 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -7,7 +7,7 @@ #include "util/log.h" bool -video_buffer_init(struct video_buffer *vb, bool wait_consumer) { +video_buffer_init(struct video_buffer *vb) { vb->producer_frame = av_frame_alloc(); if (!vb->producer_frame) { goto error_0; @@ -28,18 +28,6 @@ video_buffer_init(struct video_buffer *vb, bool wait_consumer) { goto error_3; } - vb->wait_consumer = wait_consumer; - if (wait_consumer) { - ok = sc_cond_init(&vb->pending_frame_consumed_cond); - if (!ok) { - sc_mutex_destroy(&vb->mutex); - goto error_2; - } - // interrupted is not used if wait_consumer is disabled since offering - // a frame will never block - vb->interrupted = false; - } - // there is initially no frame, so consider it has already been consumed vb->pending_frame_consumed = true; @@ -61,9 +49,6 @@ video_buffer_init(struct video_buffer *vb, bool wait_consumer) { void video_buffer_destroy(struct video_buffer *vb) { - if (vb->wait_consumer) { - sc_cond_destroy(&vb->pending_frame_consumed_cond); - } sc_mutex_destroy(&vb->mutex); av_frame_free(&vb->consumer_frame); av_frame_free(&vb->pending_frame); @@ -93,12 +78,6 @@ video_buffer_producer_offer_frame(struct video_buffer *vb) { assert(vb->cbs); sc_mutex_lock(&vb->mutex); - if (vb->wait_consumer) { - // wait for the current (expired) frame to be consumed - while (!vb->pending_frame_consumed && !vb->interrupted) { - sc_cond_wait(&vb->pending_frame_consumed_cond, &vb->mutex); - } - } av_frame_unref(vb->pending_frame); swap_frames(&vb->producer_frame, &vb->pending_frame); @@ -125,23 +104,8 @@ video_buffer_consumer_take_frame(struct video_buffer *vb) { swap_frames(&vb->consumer_frame, &vb->pending_frame); av_frame_unref(vb->pending_frame); - if (vb->wait_consumer) { - // unblock video_buffer_offer_decoded_frame() - sc_cond_signal(&vb->pending_frame_consumed_cond); - } sc_mutex_unlock(&vb->mutex); // consumer_frame is only written from this thread, no need to lock return vb->consumer_frame; } - -void -video_buffer_interrupt(struct video_buffer *vb) { - if (vb->wait_consumer) { - sc_mutex_lock(&vb->mutex); - vb->interrupted = true; - sc_mutex_unlock(&vb->mutex); - // wake up blocking wait - sc_cond_signal(&vb->pending_frame_consumed_cond); - } -} diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h index 4d11e3ab37..48e57ff422 100644 --- a/app/src/video_buffer.h +++ b/app/src/video_buffer.h @@ -34,10 +34,7 @@ struct video_buffer { AVFrame *consumer_frame; sc_mutex mutex; - bool wait_consumer; // never overwrite a pending frame if it is not consumed - bool interrupted; - sc_cond pending_frame_consumed_cond; bool pending_frame_consumed; const struct video_buffer_callbacks *cbs; @@ -56,7 +53,7 @@ struct video_buffer_callbacks { }; bool -video_buffer_init(struct video_buffer *vb, bool wait_consumer); +video_buffer_init(struct video_buffer *vb); void video_buffer_destroy(struct video_buffer *vb); @@ -75,8 +72,4 @@ video_buffer_producer_offer_frame(struct video_buffer *vb); const AVFrame * video_buffer_consumer_take_frame(struct video_buffer *vb); -// wake up and avoid any blocking call -void -video_buffer_interrupt(struct video_buffer *vb); - #endif diff --git a/app/tests/test_cli.c b/app/tests/test_cli.c index cd222d6363..3fa9b3d781 100644 --- a/app/tests/test_cli.c +++ b/app/tests/test_cli.c @@ -58,7 +58,6 @@ static void test_options(void) { "--push-target", "/sdcard/Movies", "--record", "file", "--record-format", "mkv", - "--render-expired-frames", "--serial", "0123456789abcdef", "--show-touches", "--turn-screen-off", @@ -87,7 +86,6 @@ static void test_options(void) { assert(!strcmp(opts->push_target, "/sdcard/Movies")); assert(!strcmp(opts->record_filename, "file")); assert(opts->record_format == SC_RECORD_FORMAT_MKV); - assert(opts->render_expired_frames); assert(!strcmp(opts->serial, "0123456789abcdef")); assert(opts->show_touches); assert(opts->turn_screen_off); From adaa88952d4ea0e46c084a751234cb219721c88c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 03/30] Remove compat with old FFmpeg decoding API The new API has been introduced in 2016 in libavcodec 57.xx, it's very old. This will avoid to maintain two code paths for decoding. --- app/src/compat.h | 10 ---------- app/src/decoder.c | 17 ----------------- 2 files changed, 27 deletions(-) diff --git a/app/src/compat.h b/app/src/compat.h index 9a84a4c17d..f3e7bd7a3d 100644 --- a/app/src/compat.h +++ b/app/src/compat.h @@ -8,7 +8,6 @@ # define _DARWIN_C_SOURCE #endif -#include #include #include @@ -33,15 +32,6 @@ # define SCRCPY_LAVF_REQUIRES_REGISTER_ALL #endif -// In ffmpeg/doc/APIchanges: -// 2016-04-21 - 7fc329e - lavc 57.37.100 - avcodec.h -// Add a new audio/video encoding and decoding API with decoupled input -// and output -- avcodec_send_packet(), avcodec_receive_frame(), -// avcodec_send_frame() and avcodec_receive_packet(). -#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100) -# define SCRCPY_LAVF_HAS_NEW_ENCODING_DECODING_API -#endif - #if SDL_VERSION_ATLEAST(2, 0, 5) // # define SCRCPY_SDL_HAS_HINT_MOUSE_FOCUS_CLICKTHROUGH diff --git a/app/src/decoder.c b/app/src/decoder.c index b71011941b..f05303a369 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -36,9 +36,6 @@ decoder_close(struct decoder *decoder) { bool decoder_push(struct decoder *decoder, const AVPacket *packet) { -// the new decoding/encoding API has been introduced by: -// -#ifdef SCRCPY_LAVF_HAS_NEW_ENCODING_DECODING_API int ret; if ((ret = avcodec_send_packet(decoder->codec_ctx, packet)) < 0) { LOGE("Could not send video packet: %d", ret); @@ -53,19 +50,5 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) { LOGE("Could not receive video frame: %d", ret); return false; } -#else - int got_picture; - int len = avcodec_decode_video2(decoder->codec_ctx, - decoder->video_buffer->producer_frame, - &got_picture, - packet); - if (len < 0) { - LOGE("Could not decode video packet: %d", len); - return false; - } - if (got_picture) { - video_buffer_producer_offer_frame(decoder->video_buffer); - } -#endif return true; } From 7bb17e1abc7e78a25a17429a72c1bea6bd6d9660 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 04/30] Remove compat with old FFmpeg codec params API The new API has been introduced in 2016 in libavformat 57.xx, it's very old. This will avoid to maintain two code paths for codec parameters. --- app/src/compat.h | 10 ---------- app/src/recorder.c | 13 ------------- 2 files changed, 23 deletions(-) diff --git a/app/src/compat.h b/app/src/compat.h index f3e7bd7a3d..9d9a78841f 100644 --- a/app/src/compat.h +++ b/app/src/compat.h @@ -11,16 +11,6 @@ #include #include -// In ffmpeg/doc/APIchanges: -// 2016-04-11 - 6f69f7a / 9200514 - lavf 57.33.100 / 57.5.0 - avformat.h -// Add AVStream.codecpar, deprecate AVStream.codec. -#if (LIBAVFORMAT_VERSION_MICRO >= 100 /* FFmpeg */ && \ - LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 33, 100)) \ - || (LIBAVFORMAT_VERSION_MICRO < 100 && /* Libav */ \ - LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 5, 0)) -# define SCRCPY_LAVF_HAS_NEW_CODEC_PARAMS_API -#endif - // In ffmpeg/doc/APIchanges: // 2018-02-06 - 0694d87024 - lavf 58.9.100 - avformat.h // Deprecate use of av_register_input_format(), av_register_output_format(), diff --git a/app/src/recorder.c b/app/src/recorder.c index 3f5eb0d09c..c0d7aed130 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -141,19 +141,11 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { return false; } -#ifdef SCRCPY_LAVF_HAS_NEW_CODEC_PARAMS_API ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; ostream->codecpar->codec_id = input_codec->id; ostream->codecpar->format = AV_PIX_FMT_YUV420P; ostream->codecpar->width = recorder->declared_frame_size.width; ostream->codecpar->height = recorder->declared_frame_size.height; -#else - ostream->codec->codec_type = AVMEDIA_TYPE_VIDEO; - ostream->codec->codec_id = input_codec->id; - ostream->codec->pix_fmt = AV_PIX_FMT_YUV420P; - ostream->codec->width = recorder->declared_frame_size.width; - ostream->codec->height = recorder->declared_frame_size.height; -#endif int ret = avio_open(&recorder->ctx->pb, recorder->filename, AVIO_FLAG_WRITE); @@ -188,13 +180,8 @@ recorder_write_header(struct recorder *recorder, const AVPacket *packet) { // copy the first packet to the extra data memcpy(extradata, packet->data, packet->size); -#ifdef SCRCPY_LAVF_HAS_NEW_CODEC_PARAMS_API ostream->codecpar->extradata = extradata; ostream->codecpar->extradata_size = packet->size; -#else - ostream->codec->extradata = extradata; - ostream->codec->extradata_size = packet->size; -#endif int ret = avformat_write_header(recorder->ctx, NULL); if (ret < 0) { From 85f6f32b9e913db79ffac18e4fc9a183900f0ce9 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 05/30] Make video_buffer more generic The video buffer took ownership of the producer frame (so that it could swap frames quickly). In order to support multiple sinks plugged to the decoder, the decoded frame must not be consumed by the display video buffer. Therefore, move the producer and consumer frames out of the video buffer, and use FFmpeg AVFrame refcounting to share ownership while avoiding copies. --- app/src/decoder.c | 17 ++++++++++-- app/src/decoder.h | 1 + app/src/screen.c | 14 +++++++++- app/src/screen.h | 2 ++ app/src/video_buffer.c | 62 ++++++++++++++++++++---------------------- app/src/video_buffer.h | 41 ++++++++++++---------------- 6 files changed, 77 insertions(+), 60 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index f05303a369..247b459c71 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -25,11 +25,20 @@ decoder_open(struct decoder *decoder, const AVCodec *codec) { return false; } + decoder->frame = av_frame_alloc(); + if (!decoder->frame) { + LOGE("Could not create decoder frame"); + avcodec_close(decoder->codec_ctx); + avcodec_free_context(&decoder->codec_ctx); + return false; + } + return true; } void decoder_close(struct decoder *decoder) { + av_frame_free(&decoder->frame); avcodec_close(decoder->codec_ctx); avcodec_free_context(&decoder->codec_ctx); } @@ -41,11 +50,13 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) { LOGE("Could not send video packet: %d", ret); return false; } - ret = avcodec_receive_frame(decoder->codec_ctx, - decoder->video_buffer->producer_frame); + ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame); if (!ret) { // a frame was received - video_buffer_producer_offer_frame(decoder->video_buffer); + bool ok = video_buffer_push(decoder->video_buffer, decoder->frame); + // A frame lost should not make the whole pipeline fail. The error, if + // any, is already logged. + (void) ok; } else if (ret != AVERROR(EAGAIN)) { LOGE("Could not receive video frame: %d", ret); return false; diff --git a/app/src/decoder.h b/app/src/decoder.h index ba06583e1d..50dd7fe034 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -12,6 +12,7 @@ struct decoder { struct video_buffer *video_buffer; AVCodecContext *codec_ctx; + AVFrame *frame; }; void diff --git a/app/src/screen.c b/app/src/screen.c index 934e418fb6..de73455421 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -378,6 +378,15 @@ screen_init(struct screen *screen, struct video_buffer *vb, return false; } + screen->frame = av_frame_alloc(); + if (!screen->frame) { + LOGC("Could not create screen frame"); + SDL_DestroyTexture(screen->texture); + SDL_DestroyRenderer(screen->renderer); + SDL_DestroyWindow(screen->window); + return false; + } + // Reset the window size to trigger a SIZE_CHANGED event, to workaround // HiDPI issues with some SDL renderers when several displays having // different HiDPI scaling are connected @@ -403,6 +412,7 @@ screen_show_window(struct screen *screen) { void screen_destroy(struct screen *screen) { + av_frame_free(&screen->frame); SDL_DestroyTexture(screen->texture); SDL_DestroyRenderer(screen->renderer); SDL_DestroyWindow(screen->window); @@ -510,7 +520,9 @@ update_texture(struct screen *screen, const AVFrame *frame) { static bool screen_update_frame(struct screen *screen) { - const AVFrame *frame = video_buffer_consumer_take_frame(screen->vb); + av_frame_unref(screen->frame); + video_buffer_consume(screen->vb, screen->frame); + AVFrame *frame = screen->frame; fps_counter_add_rendered_frame(screen->fps_counter); diff --git a/app/src/screen.h b/app/src/screen.h index dca65d4152..cd84977996 100644 --- a/app/src/screen.h +++ b/app/src/screen.h @@ -36,6 +36,8 @@ struct screen { bool fullscreen; bool maximized; bool mipmaps; + + AVFrame *frame; }; struct screen_params { diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index d4954afcf5..18a180fa0b 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -8,24 +8,22 @@ bool video_buffer_init(struct video_buffer *vb) { - vb->producer_frame = av_frame_alloc(); - if (!vb->producer_frame) { - goto error_0; - } - vb->pending_frame = av_frame_alloc(); if (!vb->pending_frame) { - goto error_1; + return false; } - vb->consumer_frame = av_frame_alloc(); - if (!vb->consumer_frame) { - goto error_2; + vb->tmp_frame = av_frame_alloc(); + if (!vb->tmp_frame) { + av_frame_free(&vb->pending_frame); + return false; } bool ok = sc_mutex_init(&vb->mutex); if (!ok) { - goto error_3; + av_frame_free(&vb->pending_frame); + av_frame_free(&vb->tmp_frame); + return false; } // there is initially no frame, so consider it has already been consumed @@ -36,23 +34,13 @@ video_buffer_init(struct video_buffer *vb) { vb->cbs = NULL; return true; - -error_3: - av_frame_free(&vb->consumer_frame); -error_2: - av_frame_free(&vb->pending_frame); -error_1: - av_frame_free(&vb->producer_frame); -error_0: - return false; } void video_buffer_destroy(struct video_buffer *vb) { sc_mutex_destroy(&vb->mutex); - av_frame_free(&vb->consumer_frame); av_frame_free(&vb->pending_frame); - av_frame_free(&vb->producer_frame); + av_frame_free(&vb->tmp_frame); } static inline void @@ -73,14 +61,24 @@ video_buffer_set_consumer_callbacks(struct video_buffer *vb, vb->cbs_userdata = cbs_userdata; } -void -video_buffer_producer_offer_frame(struct video_buffer *vb) { +bool +video_buffer_push(struct video_buffer *vb, const AVFrame *frame) { assert(vb->cbs); sc_mutex_lock(&vb->mutex); - av_frame_unref(vb->pending_frame); - swap_frames(&vb->producer_frame, &vb->pending_frame); + // Use a temporary frame to preserve pending_frame in case of error. + // tmp_frame is an empty frame, no need to call av_frame_unref() beforehand. + int r = av_frame_ref(vb->tmp_frame, frame); + if (r) { + LOGE("Could not ref frame: %d", r); + return false; + } + + // Now that av_frame_ref() succeeded, we can replace the previous + // pending_frame + swap_frames(&vb->pending_frame, &vb->tmp_frame); + av_frame_unref(vb->tmp_frame); bool skipped = !vb->pending_frame_consumed; vb->pending_frame_consumed = false; @@ -93,19 +91,19 @@ video_buffer_producer_offer_frame(struct video_buffer *vb) { } else { vb->cbs->on_frame_available(vb, vb->cbs_userdata); } + + return true; } -const AVFrame * -video_buffer_consumer_take_frame(struct video_buffer *vb) { +void +video_buffer_consume(struct video_buffer *vb, AVFrame *dst) { sc_mutex_lock(&vb->mutex); assert(!vb->pending_frame_consumed); vb->pending_frame_consumed = true; - swap_frames(&vb->consumer_frame, &vb->pending_frame); - av_frame_unref(vb->pending_frame); + av_frame_move_ref(dst, vb->pending_frame); + // av_frame_move_ref() resets its source frame, so no need to call + // av_frame_unref() sc_mutex_unlock(&vb->mutex); - - // consumer_frame is only written from this thread, no need to lock - return vb->consumer_frame; } diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h index 48e57ff422..cdecb25950 100644 --- a/app/src/video_buffer.h +++ b/app/src/video_buffer.h @@ -12,26 +12,23 @@ typedef struct AVFrame AVFrame; /** - * There are 3 frames in memory: - * - one frame is held by the producer (producer_frame) - * - one frame is held by the consumer (consumer_frame) - * - one frame is shared between the producer and the consumer (pending_frame) + * A video buffer holds 1 pending frame, which is the last frame received from + * the producer (typically, the decoder). * - * The producer generates a frame into the producer_frame (it may takes time). + * If a pending frame has not been consumed when the producer pushes a new + * frame, then it is lost. The intent is to always provide access to the very + * last frame to minimize latency. * - * Once the frame is produced, it calls video_buffer_producer_offer_frame(), - * which swaps the producer and pending frames. - * - * When the consumer is notified that a new frame is available, it calls - * video_buffer_consumer_take_frame() to retrieve it, which swaps the pending - * and consumer frames. The frame is valid until the next call, without - * blocking the producer. + * The producer and the consumer typically do not live in the same thread. + * That's the reason why the callback on_frame_available() does not provide the + * frame as parameter: the consumer might post an event to its own thread to + * retrieve the pending frame from there, and that frame may have changed since + * the callback if producer pushed a new one in between. */ struct video_buffer { - AVFrame *producer_frame; AVFrame *pending_frame; - AVFrame *consumer_frame; + AVFrame *tmp_frame; // To preserve the pending frame on error sc_mutex mutex; @@ -42,12 +39,11 @@ struct video_buffer { }; struct video_buffer_callbacks { - // Called when a new frame can be consumed by - // video_buffer_consumer_take_frame(vb) + // Called when a new frame can be consumed. // This callback is mandatory (it must not be NULL). void (*on_frame_available)(struct video_buffer *vb, void *userdata); - // Called when a pending frame has been overwritten by the producer + // Called when a pending frame has been overwritten by the producer. // This callback is optional (it may be NULL). void (*on_frame_skipped)(struct video_buffer *vb, void *userdata); }; @@ -63,13 +59,10 @@ video_buffer_set_consumer_callbacks(struct video_buffer *vb, const struct video_buffer_callbacks *cbs, void *cbs_userdata); -// set the producer frame as ready for consuming -void -video_buffer_producer_offer_frame(struct video_buffer *vb); +bool +video_buffer_push(struct video_buffer *vb, const AVFrame *frame); -// mark the consumer frame as consumed and return it -// the frame is valid until the next call to this function -const AVFrame * -video_buffer_consumer_take_frame(struct video_buffer *vb); +void +video_buffer_consume(struct video_buffer *vb, AVFrame *dst); #endif From 321bf79a0b67b2a9c184967417cae83fe3e166dd Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 06/30] Add container_of() macro This will allow to get the parent of an embedded struct. --- app/src/common.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/src/common.h b/app/src/common.h index 27c8d2fba0..accbc61563 100644 --- a/app/src/common.h +++ b/app/src/common.h @@ -8,4 +8,7 @@ #define MIN(X,Y) (X) < (Y) ? (X) : (Y) #define MAX(X,Y) (X) > (Y) ? (X) : (Y) +#define container_of(ptr, type, member) \ + ((type *) (((char *) (ptr)) - offsetof(type, member))) + #endif From 11165027048fdb40be97190f54abeea5ff7e71cd Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 07/30] Add packet sink trait This trait will allow to abstract the concrete sink types from the packet producer (stream.c). --- app/src/trait/packet_sink.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 app/src/trait/packet_sink.h diff --git a/app/src/trait/packet_sink.h b/app/src/trait/packet_sink.h new file mode 100644 index 0000000000..fe9c137d0a --- /dev/null +++ b/app/src/trait/packet_sink.h @@ -0,0 +1,27 @@ +#ifndef SC_PACKET_SINK +#define SC_PACKET_SINK + +#include "common.h" + +#include +#include + +typedef struct AVCodec AVCodec; +typedef struct AVPacket AVPacket; + +/** + * Packet sink trait. + * + * Component able to receive AVPackets should implement this trait. + */ +struct sc_packet_sink { + const struct sc_packet_sink_ops *ops; +}; + +struct sc_packet_sink_ops { + bool (*open)(struct sc_packet_sink *sink, const AVCodec *codec); + void (*close)(struct sc_packet_sink *sink); + bool (*push)(struct sc_packet_sink *sink, const AVPacket *packet); +}; + +#endif From fb07f4af55d094f7d0651d658c769ed61cf077b7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 08/30] Reorder recorder functions This will make further commits more readable. --- app/src/recorder.c | 202 ++++++++++++++++++++++----------------------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index c0d7aed130..c24393de93 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -57,50 +57,6 @@ recorder_queue_clear(struct recorder_queue *queue) { } } -bool -recorder_init(struct recorder *recorder, - const char *filename, - enum sc_record_format format, - struct size declared_frame_size) { - recorder->filename = strdup(filename); - if (!recorder->filename) { - LOGE("Could not strdup filename"); - return false; - } - - bool ok = sc_mutex_init(&recorder->mutex); - if (!ok) { - LOGC("Could not create mutex"); - free(recorder->filename); - return false; - } - - ok = sc_cond_init(&recorder->queue_cond); - if (!ok) { - LOGC("Could not create cond"); - sc_mutex_destroy(&recorder->mutex); - free(recorder->filename); - return false; - } - - queue_init(&recorder->queue); - recorder->stopped = false; - recorder->failed = false; - recorder->format = format; - recorder->declared_frame_size = declared_frame_size; - recorder->header_written = false; - recorder->previous = NULL; - - return true; -} - -void -recorder_destroy(struct recorder *recorder) { - sc_cond_destroy(&recorder->queue_cond); - sc_mutex_destroy(&recorder->mutex); - free(recorder->filename); -} - static const char * recorder_get_format_name(enum sc_record_format format) { switch (format) { @@ -110,63 +66,6 @@ recorder_get_format_name(enum sc_record_format format) { } } -bool -recorder_open(struct recorder *recorder, const AVCodec *input_codec) { - const char *format_name = recorder_get_format_name(recorder->format); - assert(format_name); - const AVOutputFormat *format = find_muxer(format_name); - if (!format) { - LOGE("Could not find muxer"); - return false; - } - - recorder->ctx = avformat_alloc_context(); - if (!recorder->ctx) { - LOGE("Could not allocate output context"); - return false; - } - - // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate() - // returns (on purpose) a pointer-to-const, but AVFormatContext.oformat - // still expects a pointer-to-non-const (it has not be updated accordingly) - // - recorder->ctx->oformat = (AVOutputFormat *) format; - - av_dict_set(&recorder->ctx->metadata, "comment", - "Recorded by scrcpy " SCRCPY_VERSION, 0); - - AVStream *ostream = avformat_new_stream(recorder->ctx, input_codec); - if (!ostream) { - avformat_free_context(recorder->ctx); - return false; - } - - ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; - ostream->codecpar->codec_id = input_codec->id; - ostream->codecpar->format = AV_PIX_FMT_YUV420P; - ostream->codecpar->width = recorder->declared_frame_size.width; - ostream->codecpar->height = recorder->declared_frame_size.height; - - int ret = avio_open(&recorder->ctx->pb, recorder->filename, - AVIO_FLAG_WRITE); - if (ret < 0) { - LOGE("Failed to open output file: %s", recorder->filename); - // ostream will be cleaned up during context cleaning - avformat_free_context(recorder->ctx); - return false; - } - - LOGI("Recording started to %s file: %s", format_name, recorder->filename); - - return true; -} - -void -recorder_close(struct recorder *recorder) { - avio_close(recorder->ctx->pb); - avformat_free_context(recorder->ctx); -} - static bool recorder_write_header(struct recorder *recorder, const AVPacket *packet) { AVStream *ostream = recorder->ctx->streams[0]; @@ -314,6 +213,63 @@ run_recorder(void *data) { return 0; } +bool +recorder_open(struct recorder *recorder, const AVCodec *input_codec) { + const char *format_name = recorder_get_format_name(recorder->format); + assert(format_name); + const AVOutputFormat *format = find_muxer(format_name); + if (!format) { + LOGE("Could not find muxer"); + return false; + } + + recorder->ctx = avformat_alloc_context(); + if (!recorder->ctx) { + LOGE("Could not allocate output context"); + return false; + } + + // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate() + // returns (on purpose) a pointer-to-const, but AVFormatContext.oformat + // still expects a pointer-to-non-const (it has not be updated accordingly) + // + recorder->ctx->oformat = (AVOutputFormat *) format; + + av_dict_set(&recorder->ctx->metadata, "comment", + "Recorded by scrcpy " SCRCPY_VERSION, 0); + + AVStream *ostream = avformat_new_stream(recorder->ctx, input_codec); + if (!ostream) { + avformat_free_context(recorder->ctx); + return false; + } + + ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + ostream->codecpar->codec_id = input_codec->id; + ostream->codecpar->format = AV_PIX_FMT_YUV420P; + ostream->codecpar->width = recorder->declared_frame_size.width; + ostream->codecpar->height = recorder->declared_frame_size.height; + + int ret = avio_open(&recorder->ctx->pb, recorder->filename, + AVIO_FLAG_WRITE); + if (ret < 0) { + LOGE("Failed to open output file: %s", recorder->filename); + // ostream will be cleaned up during context cleaning + avformat_free_context(recorder->ctx); + return false; + } + + LOGI("Recording started to %s file: %s", format_name, recorder->filename); + + return true; +} + +void +recorder_close(struct recorder *recorder) { + avio_close(recorder->ctx->pb); + avformat_free_context(recorder->ctx); +} + bool recorder_start(struct recorder *recorder) { LOGD("Starting recorder thread"); @@ -365,3 +321,47 @@ recorder_push(struct recorder *recorder, const AVPacket *packet) { sc_mutex_unlock(&recorder->mutex); return true; } + +bool +recorder_init(struct recorder *recorder, + const char *filename, + enum sc_record_format format, + struct size declared_frame_size) { + recorder->filename = strdup(filename); + if (!recorder->filename) { + LOGE("Could not strdup filename"); + return false; + } + + bool ok = sc_mutex_init(&recorder->mutex); + if (!ok) { + LOGC("Could not create mutex"); + free(recorder->filename); + return false; + } + + ok = sc_cond_init(&recorder->queue_cond); + if (!ok) { + LOGC("Could not create cond"); + sc_mutex_destroy(&recorder->mutex); + free(recorder->filename); + return false; + } + + queue_init(&recorder->queue); + recorder->stopped = false; + recorder->failed = false; + recorder->format = format; + recorder->declared_frame_size = declared_frame_size; + recorder->header_written = false; + recorder->previous = NULL; + + return true; +} + +void +recorder_destroy(struct recorder *recorder) { + sc_cond_destroy(&recorder->queue_cond); + sc_mutex_destroy(&recorder->mutex); + free(recorder->filename); +} From fea3f29ffdf1a2cd1d9fe8ff4cf9ebf141ececf6 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 09/30] Privatize recorder threading The fact that the recorder uses a separate thread is an internal detail, so the functions _start(), _stop() and _join() should not be exposed. Instead, start the thread on _open() and _stop()+_join() on close(). This paves the way to expose the recorder as a packet sink trait. --- app/src/recorder.c | 26 ++++++++------------------ app/src/recorder.h | 11 +---------- app/src/stream.c | 13 +------------ 3 files changed, 10 insertions(+), 40 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index c24393de93..387d8f79fc 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -259,42 +259,32 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { return false; } - LOGI("Recording started to %s file: %s", format_name, recorder->filename); - - return true; -} - -void -recorder_close(struct recorder *recorder) { - avio_close(recorder->ctx->pb); - avformat_free_context(recorder->ctx); -} - -bool -recorder_start(struct recorder *recorder) { LOGD("Starting recorder thread"); - bool ok = sc_thread_create(&recorder->thread, run_recorder, "recorder", recorder); if (!ok) { LOGC("Could not start recorder thread"); + avio_close(recorder->ctx->pb); + avformat_free_context(recorder->ctx); return false; } + LOGI("Recording started to %s file: %s", format_name, recorder->filename); + return true; } void -recorder_stop(struct recorder *recorder) { +recorder_close(struct recorder *recorder) { sc_mutex_lock(&recorder->mutex); recorder->stopped = true; sc_cond_signal(&recorder->queue_cond); sc_mutex_unlock(&recorder->mutex); -} -void -recorder_join(struct recorder *recorder) { sc_thread_join(&recorder->thread, NULL); + + avio_close(recorder->ctx->pb); + avformat_free_context(recorder->ctx); } bool diff --git a/app/src/recorder.h b/app/src/recorder.h index be2b2dffdc..97e499bca5 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -28,7 +28,7 @@ struct recorder { sc_thread thread; sc_mutex mutex; sc_cond queue_cond; - bool stopped; // set on recorder_stop() by the stream reader + bool stopped; // set on recorder_close() bool failed; // set on packet write failure struct recorder_queue queue; @@ -52,15 +52,6 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec); void recorder_close(struct recorder *recorder); -bool -recorder_start(struct recorder *recorder); - -void -recorder_stop(struct recorder *recorder); - -void -recorder_join(struct recorder *recorder); - bool recorder_push(struct recorder *recorder, const AVPacket *packet); diff --git a/app/src/stream.c b/app/src/stream.c index e0a223be3f..787e951519 100644 --- a/app/src/stream.c +++ b/app/src/stream.c @@ -203,17 +203,12 @@ run_stream(void *data) { LOGE("Could not open recorder"); goto finally_close_decoder; } - - if (!recorder_start(stream->recorder)) { - LOGE("Could not start recorder"); - goto finally_close_recorder; - } } stream->parser = av_parser_init(AV_CODEC_ID_H264); if (!stream->parser) { LOGE("Could not initialize parser"); - goto finally_stop_and_join_recorder; + goto finally_close_recorder; } // We must only pass complete frames to av_parser_parse2()! @@ -243,12 +238,6 @@ run_stream(void *data) { } av_parser_close(stream->parser); -finally_stop_and_join_recorder: - if (stream->recorder) { - recorder_stop(stream->recorder); - LOGI("Finishing recording..."); - recorder_join(stream->recorder); - } finally_close_recorder: if (stream->recorder) { recorder_close(stream->recorder); From 71327e82cbc2c10a2c84684050303b682435558a Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 10/30] Expose recorder as packet sink Make recorder implement the packet sink trait. This will allow the stream to push packets without depending on the concrete sink type. --- app/src/recorder.c | 29 +++++++++++++++++++++++++++++ app/src/recorder.h | 3 +++ 2 files changed, 32 insertions(+) diff --git a/app/src/recorder.c b/app/src/recorder.c index 387d8f79fc..fabbc95c8b 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -5,6 +5,9 @@ #include "util/log.h" +/** Downcast packet_sink to recorder */ +#define DOWNCAST(SINK) container_of(SINK, struct recorder, packet_sink) + static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us static const AVOutputFormat * @@ -312,6 +315,24 @@ recorder_push(struct recorder *recorder, const AVPacket *packet) { return true; } +static bool +recorder_packet_sink_open(struct sc_packet_sink *sink, const AVCodec *codec) { + struct recorder *recorder = DOWNCAST(sink); + return recorder_open(recorder, codec); +} + +static void +recorder_packet_sink_close(struct sc_packet_sink *sink) { + struct recorder *recorder = DOWNCAST(sink); + recorder_close(recorder); +} + +static bool +recorder_packet_sink_push(struct sc_packet_sink *sink, const AVPacket *packet) { + struct recorder *recorder = DOWNCAST(sink); + return recorder_push(recorder, packet); +} + bool recorder_init(struct recorder *recorder, const char *filename, @@ -346,6 +367,14 @@ recorder_init(struct recorder *recorder, recorder->header_written = false; recorder->previous = NULL; + static const struct sc_packet_sink_ops ops = { + .open = recorder_packet_sink_open, + .close = recorder_packet_sink_close, + .push = recorder_packet_sink_push, + }; + + recorder->packet_sink.ops = &ops; + return true; } diff --git a/app/src/recorder.h b/app/src/recorder.h index 97e499bca5..4991f0cf26 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -8,6 +8,7 @@ #include "coords.h" #include "scrcpy.h" +#include "trait/packet_sink.h" #include "util/queue.h" #include "util/thread.h" @@ -19,6 +20,8 @@ struct record_packet { struct recorder_queue QUEUE(struct record_packet); struct recorder { + struct sc_packet_sink packet_sink; // packet sink trait + char *filename; enum sc_record_format format; AVFormatContext *ctx; From f82aab2057c7c6d9d5c3503743d1f6818ed66fcc Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 11/30] Reorder decoder functions This will make further commits more readable. --- app/src/decoder.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index 247b459c71..3b94afef31 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -6,11 +6,6 @@ #include "video_buffer.h" #include "util/log.h" -void -decoder_init(struct decoder *decoder, struct video_buffer *vb) { - decoder->video_buffer = vb; -} - bool decoder_open(struct decoder *decoder, const AVCodec *codec) { decoder->codec_ctx = avcodec_alloc_context3(codec); @@ -63,3 +58,8 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) { } return true; } + +void +decoder_init(struct decoder *decoder, struct video_buffer *vb) { + decoder->video_buffer = vb; +} From f27403e27f1849febff9e4bcd068c401357d1625 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 12/30] Expose decoder as packet sink Make decoder implement the packet sink trait. This will allow the stream to push packets without depending on the concrete sink type. --- app/src/decoder.c | 29 +++++++++++++++++++++++++++++ app/src/decoder.h | 4 ++++ 2 files changed, 33 insertions(+) diff --git a/app/src/decoder.c b/app/src/decoder.c index 3b94afef31..41e2fe8506 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -6,6 +6,9 @@ #include "video_buffer.h" #include "util/log.h" +/** Downcast packet_sink to decoder */ +#define DOWNCAST(SINK) container_of(SINK, struct decoder, packet_sink) + bool decoder_open(struct decoder *decoder, const AVCodec *codec) { decoder->codec_ctx = avcodec_alloc_context3(codec); @@ -59,7 +62,33 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) { return true; } +static bool +decoder_packet_sink_open(struct sc_packet_sink *sink, const AVCodec *codec) { + struct decoder *decoder = DOWNCAST(sink); + return decoder_open(decoder, codec); +} + +static void +decoder_packet_sink_close(struct sc_packet_sink *sink) { + struct decoder *decoder = DOWNCAST(sink); + decoder_close(decoder); +} + +static bool +decoder_packet_sink_push(struct sc_packet_sink *sink, const AVPacket *packet) { + struct decoder *decoder = DOWNCAST(sink); + return decoder_push(decoder, packet); +} + void decoder_init(struct decoder *decoder, struct video_buffer *vb) { decoder->video_buffer = vb; + + static const struct sc_packet_sink_ops ops = { + .open = decoder_packet_sink_open, + .close = decoder_packet_sink_close, + .push = decoder_packet_sink_push, + }; + + decoder->packet_sink.ops = &ops; } diff --git a/app/src/decoder.h b/app/src/decoder.h index 50dd7fe034..c3f7cb7303 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -3,12 +3,16 @@ #include "common.h" +#include "trait/packet_sink.h" + #include #include struct video_buffer; struct decoder { + struct sc_packet_sink packet_sink; // packet sink trait + struct video_buffer *video_buffer; AVCodecContext *codec_ctx; From eb971390ed28e4cf9a4fd62de48cfac83dfae7da Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 13/30] Make stream push packets to sinks Now that decoder and recorder implement the packet sink trait, make stream push packets to the sinks without depending on the concrete sink types. --- app/src/decoder.c | 8 +++- app/src/decoder.h | 3 -- app/src/recorder.c | 6 +-- app/src/recorder.h | 9 ----- app/src/scrcpy.c | 10 ++++- app/src/stream.c | 96 +++++++++++++++++++++++++--------------------- app/src/stream.h | 15 ++++++-- 7 files changed, 82 insertions(+), 65 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index 41e2fe8506..134ffd3c27 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -41,8 +41,14 @@ decoder_close(struct decoder *decoder) { avcodec_free_context(&decoder->codec_ctx); } -bool +static bool decoder_push(struct decoder *decoder, const AVPacket *packet) { + bool is_config = packet->pts == AV_NOPTS_VALUE; + if (is_config) { + // nothing to do + return true; + } + int ret; if ((ret = avcodec_send_packet(decoder->codec_ctx, packet)) < 0) { LOGE("Could not send video packet: %d", ret); diff --git a/app/src/decoder.h b/app/src/decoder.h index c3f7cb7303..e3730db5e4 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -28,7 +28,4 @@ decoder_open(struct decoder *decoder, const AVCodec *codec); void decoder_close(struct decoder *decoder); -bool -decoder_push(struct decoder *decoder, const AVPacket *packet); - #endif diff --git a/app/src/recorder.c b/app/src/recorder.c index fabbc95c8b..f0ec86dcb0 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -216,7 +216,7 @@ run_recorder(void *data) { return 0; } -bool +static bool recorder_open(struct recorder *recorder, const AVCodec *input_codec) { const char *format_name = recorder_get_format_name(recorder->format); assert(format_name); @@ -277,7 +277,7 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { return true; } -void +static void recorder_close(struct recorder *recorder) { sc_mutex_lock(&recorder->mutex); recorder->stopped = true; @@ -290,7 +290,7 @@ recorder_close(struct recorder *recorder) { avformat_free_context(recorder->ctx); } -bool +static bool recorder_push(struct recorder *recorder, const AVPacket *packet) { sc_mutex_lock(&recorder->mutex); assert(!recorder->stopped); diff --git a/app/src/recorder.h b/app/src/recorder.h index 4991f0cf26..1b2b9284c4 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -49,13 +49,4 @@ recorder_init(struct recorder *recorder, const char *filename, void recorder_destroy(struct recorder *recorder); -bool -recorder_open(struct recorder *recorder, const AVCodec *input_codec); - -void -recorder_close(struct recorder *recorder); - -bool -recorder_push(struct recorder *recorder, const AVPacket *packet); - #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 2f8abd64fa..ad705c4ab7 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -336,7 +336,15 @@ scrcpy(const struct scrcpy_options *options) { av_log_set_callback(av_log_callback); - stream_init(&stream, server.video_socket, dec, rec); + stream_init(&stream, server.video_socket); + + if (dec) { + stream_add_sink(&stream, &dec->packet_sink); + } + + if (rec) { + stream_add_sink(&stream, &rec->packet_sink); + } if (options->display) { if (options->control) { diff --git a/app/src/stream.c b/app/src/stream.c index 787e951519..a11218e3ab 100644 --- a/app/src/stream.c +++ b/app/src/stream.c @@ -66,25 +66,11 @@ notify_stopped(void) { } static bool -process_config_packet(struct stream *stream, AVPacket *packet) { - if (stream->recorder && !recorder_push(stream->recorder, packet)) { - LOGE("Could not send config packet to recorder"); - return false; - } - return true; -} - -static bool -process_frame(struct stream *stream, AVPacket *packet) { - if (stream->decoder && !decoder_push(stream->decoder, packet)) { - return false; - } - - if (stream->recorder) { - packet->dts = packet->pts; - - if (!recorder_push(stream->recorder, packet)) { - LOGE("Could not send packet to recorder"); +push_packet_to_sinks(struct stream *stream, const AVPacket *packet) { + for (unsigned i = 0; i < stream->sink_count; ++i) { + struct sc_packet_sink *sink = stream->sinks[i]; + if (!sink->ops->push(sink, packet)) { + LOGE("Could not send config packet to sink %d", i); return false; } } @@ -111,9 +97,11 @@ stream_parse(struct stream *stream, AVPacket *packet) { packet->flags |= AV_PKT_FLAG_KEY; } - bool ok = process_frame(stream, packet); + packet->dts = packet->pts; + + bool ok = push_packet_to_sinks(stream, packet); if (!ok) { - LOGE("Could not process frame"); + LOGE("Could not process packet"); return false; } @@ -156,7 +144,7 @@ stream_push_packet(struct stream *stream, AVPacket *packet) { if (is_config) { // config packet - bool ok = process_config_packet(stream, packet); + bool ok = push_packet_to_sinks(stream, packet); if (!ok) { return false; } @@ -177,6 +165,33 @@ stream_push_packet(struct stream *stream, AVPacket *packet) { return true; } +static void +stream_close_first_sinks(struct stream *stream, unsigned count) { + while (count) { + struct sc_packet_sink *sink = stream->sinks[--count]; + sink->ops->close(sink); + } +} + +static inline void +stream_close_sinks(struct stream *stream) { + stream_close_first_sinks(stream, stream->sink_count); +} + +static bool +stream_open_sinks(struct stream *stream, const AVCodec *codec) { + for (unsigned i = 0; i < stream->sink_count; ++i) { + struct sc_packet_sink *sink = stream->sinks[i]; + if (!sink->ops->open(sink, codec)) { + LOGE("Could not open packet sink %d", i); + stream_close_first_sinks(stream, i); + return false; + } + } + + return true; +} + static int run_stream(void *data) { struct stream *stream = data; @@ -193,22 +208,15 @@ run_stream(void *data) { goto end; } - if (stream->decoder && !decoder_open(stream->decoder, codec)) { - LOGE("Could not open decoder"); + if (!stream_open_sinks(stream, codec)) { + LOGE("Could not open stream sinks"); goto finally_free_codec_ctx; } - if (stream->recorder) { - if (!recorder_open(stream->recorder, codec)) { - LOGE("Could not open recorder"); - goto finally_close_decoder; - } - } - stream->parser = av_parser_init(AV_CODEC_ID_H264); if (!stream->parser) { LOGE("Could not initialize parser"); - goto finally_close_recorder; + goto finally_close_sinks; } // We must only pass complete frames to av_parser_parse2()! @@ -238,14 +246,8 @@ run_stream(void *data) { } av_parser_close(stream->parser); -finally_close_recorder: - if (stream->recorder) { - recorder_close(stream->recorder); - } -finally_close_decoder: - if (stream->decoder) { - decoder_close(stream->decoder); - } +finally_close_sinks: + stream_close_sinks(stream); finally_free_codec_ctx: avcodec_free_context(&stream->codec_ctx); end: @@ -254,12 +256,18 @@ run_stream(void *data) { } void -stream_init(struct stream *stream, socket_t socket, - struct decoder *decoder, struct recorder *recorder) { +stream_init(struct stream *stream, socket_t socket) { stream->socket = socket; - stream->decoder = decoder, - stream->recorder = recorder; stream->has_pending = false; + stream->sink_count = 0; +} + +void +stream_add_sink(struct stream *stream, struct sc_packet_sink *sink) { + assert(stream->sink_count < STREAM_MAX_SINKS); + assert(sink); + assert(sink->ops); + stream->sinks[stream->sink_count++] = sink; } bool diff --git a/app/src/stream.h b/app/src/stream.h index 421c1bf01e..81175420e8 100644 --- a/app/src/stream.h +++ b/app/src/stream.h @@ -8,14 +8,19 @@ #include #include +#include "trait/packet_sink.h" #include "util/net.h" #include "util/thread.h" +#define STREAM_MAX_SINKS 2 + struct stream { socket_t socket; sc_thread thread; - struct decoder *decoder; - struct recorder *recorder; + + struct sc_packet_sink *sinks[STREAM_MAX_SINKS]; + unsigned sink_count; + AVCodecContext *codec_ctx; AVCodecParserContext *parser; // successive packets may need to be concatenated, until a non-config @@ -25,8 +30,10 @@ struct stream { }; void -stream_init(struct stream *stream, socket_t socket, - struct decoder *decoder, struct recorder *recorder); +stream_init(struct stream *stream, socket_t socket); + +void +stream_add_sink(struct stream *stream, struct sc_packet_sink *sink); bool stream_start(struct stream *stream); From bb6ac2b084a284b6edd97d350425f175e2ee2e76 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:39:00 +0200 Subject: [PATCH 14/30] Add frame sink trait This trait will allow to abstract the concrete sink types from the frame producer (decoder.c). --- app/src/trait/frame_sink.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 app/src/trait/frame_sink.h diff --git a/app/src/trait/frame_sink.h b/app/src/trait/frame_sink.h new file mode 100644 index 0000000000..64ab0de938 --- /dev/null +++ b/app/src/trait/frame_sink.h @@ -0,0 +1,26 @@ +#ifndef SC_FRAME_SINK +#define SC_FRAME_SINK + +#include "common.h" + +#include +#include + +typedef struct AVFrame AVFrame; + +/** + * Frame sink trait. + * + * Component able to receive AVFrames should implement this trait. + */ +struct sc_frame_sink { + const struct sc_frame_sink_ops *ops; +}; + +struct sc_frame_sink_ops { + bool (*open)(struct sc_frame_sink *sink); + void (*close)(struct sc_frame_sink *sink); + bool (*push)(struct sc_frame_sink *sink, const AVFrame *frame); +}; + +#endif From 6d8f0a247f80b7bcdd0621824462ab327db86c31 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 15/30] Expose screen as frame sink Make screen implement the frame sink trait. This will allow the decoder to push frames without depending on the concrete sink type. --- app/src/screen.c | 33 +++++++++++++++++++++++++++++++++ app/src/screen.h | 3 +++ 2 files changed, 36 insertions(+) diff --git a/app/src/screen.c b/app/src/screen.c index de73455421..0aa0b832e5 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -13,6 +13,8 @@ #define DISPLAY_MARGINS 96 +#define DOWNCAST(SINK) container_of(SINK, struct screen, frame_sink) + static inline struct size get_rotated_size(struct size size, int rotation) { struct size rotated_size; @@ -262,6 +264,29 @@ event_watcher(void *data, SDL_Event *event) { } #endif +static bool +screen_frame_sink_open(struct sc_frame_sink *sink) { + struct screen *screen = DOWNCAST(sink); + (void) screen; + + // nothing to do, the screen is already open on the main thread + return true; +} + +static void +screen_frame_sink_close(struct sc_frame_sink *sink) { + struct screen *screen = DOWNCAST(sink); + (void) screen; + + // nothing to do, the screen lifecycle is not managed by the frame producer +} + +static bool +screen_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { + struct screen *screen = DOWNCAST(sink); + return video_buffer_push(screen->vb, frame); +} + bool screen_init(struct screen *screen, struct video_buffer *vb, struct fps_counter *fps_counter, @@ -402,6 +427,14 @@ screen_init(struct screen *screen, struct video_buffer *vb, SDL_AddEventWatch(event_watcher, screen); #endif + static const struct sc_frame_sink_ops ops = { + .open = screen_frame_sink_open, + .close = screen_frame_sink_close, + .push = screen_frame_sink_push, + }; + + screen->frame_sink.ops = &ops; + return true; } diff --git a/app/src/screen.h b/app/src/screen.h index cd84977996..d57b715280 100644 --- a/app/src/screen.h +++ b/app/src/screen.h @@ -9,10 +9,13 @@ #include "coords.h" #include "opengl.h" +#include "trait/frame_sink.h" struct video_buffer; struct screen { + struct sc_frame_sink frame_sink; // frame sink trait + struct video_buffer *vb; struct fps_counter *fps_counter; From cec5bcbe0b9b837f4fcbc01df0596dc711b4d71c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 16/30] Make decoder push frames to sinks Now that screen implements the packet sink trait, make decoder push packets to the sinks without depending on the concrete sink types. --- app/src/decoder.c | 68 ++++++++++++++++++++++++++++++++++++++++++----- app/src/decoder.h | 12 ++++----- app/src/scrcpy.c | 4 ++- 3 files changed, 70 insertions(+), 14 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index 134ffd3c27..34f2a15fd3 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -4,12 +4,40 @@ #include "events.h" #include "video_buffer.h" +#include "trait/frame_sink.h" #include "util/log.h" /** Downcast packet_sink to decoder */ #define DOWNCAST(SINK) container_of(SINK, struct decoder, packet_sink) -bool +static void +decoder_close_first_sinks(struct decoder *decoder, unsigned count) { + while (count) { + struct sc_frame_sink *sink = decoder->sinks[--count]; + sink->ops->close(sink); + } +} + +static inline void +decoder_close_sinks(struct decoder *decoder) { + decoder_close_first_sinks(decoder, decoder->sink_count); +} + +static bool +decoder_open_sinks(struct decoder *decoder) { + for (unsigned i = 0; i < decoder->sink_count; ++i) { + struct sc_frame_sink *sink = decoder->sinks[i]; + if (!sink->ops->open(sink)) { + LOGE("Could not open frame sink %d", i); + decoder_close_first_sinks(decoder, i); + return false; + } + } + + return true; +} + +static bool decoder_open(struct decoder *decoder, const AVCodec *codec) { decoder->codec_ctx = avcodec_alloc_context3(codec); if (!decoder->codec_ctx) { @@ -31,16 +59,38 @@ decoder_open(struct decoder *decoder, const AVCodec *codec) { return false; } + if (!decoder_open_sinks(decoder)) { + LOGE("Could not open decoder sinks"); + av_frame_free(&decoder->frame); + avcodec_close(decoder->codec_ctx); + avcodec_free_context(&decoder->codec_ctx); + return false; + } + return true; } -void +static void decoder_close(struct decoder *decoder) { + decoder_close_sinks(decoder); av_frame_free(&decoder->frame); avcodec_close(decoder->codec_ctx); avcodec_free_context(&decoder->codec_ctx); } +static bool +push_frame_to_sinks(struct decoder *decoder, const AVFrame *frame) { + for (unsigned i = 0; i < decoder->sink_count; ++i) { + struct sc_frame_sink *sink = decoder->sinks[i]; + if (!sink->ops->push(sink, frame)) { + LOGE("Could not send frame to sink %d", i); + return false; + } + } + + return true; +} + static bool decoder_push(struct decoder *decoder, const AVPacket *packet) { bool is_config = packet->pts == AV_NOPTS_VALUE; @@ -57,7 +107,7 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) { ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame); if (!ret) { // a frame was received - bool ok = video_buffer_push(decoder->video_buffer, decoder->frame); + bool ok = push_frame_to_sinks(decoder, decoder->frame); // A frame lost should not make the whole pipeline fail. The error, if // any, is already logged. (void) ok; @@ -87,9 +137,7 @@ decoder_packet_sink_push(struct sc_packet_sink *sink, const AVPacket *packet) { } void -decoder_init(struct decoder *decoder, struct video_buffer *vb) { - decoder->video_buffer = vb; - +decoder_init(struct decoder *decoder) { static const struct sc_packet_sink_ops ops = { .open = decoder_packet_sink_open, .close = decoder_packet_sink_close, @@ -98,3 +146,11 @@ decoder_init(struct decoder *decoder, struct video_buffer *vb) { decoder->packet_sink.ops = &ops; } + +void +decoder_add_sink(struct decoder *decoder, struct sc_frame_sink *sink) { + assert(decoder->sink_count < DECODER_MAX_SINKS); + assert(sink); + assert(sink->ops); + decoder->sinks[decoder->sink_count++] = sink; +} diff --git a/app/src/decoder.h b/app/src/decoder.h index e3730db5e4..bae97869c2 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -8,24 +8,22 @@ #include #include -struct video_buffer; +#define DECODER_MAX_SINKS 1 struct decoder { struct sc_packet_sink packet_sink; // packet sink trait - struct video_buffer *video_buffer; + struct sc_frame_sink *sinks[DECODER_MAX_SINKS]; + unsigned sink_count; AVCodecContext *codec_ctx; AVFrame *frame; }; void -decoder_init(struct decoder *decoder, struct video_buffer *vb); - -bool -decoder_open(struct decoder *decoder, const AVCodec *codec); +decoder_init(struct decoder *decoder); void -decoder_close(struct decoder *decoder); +decoder_add_sink(struct decoder *decoder, struct sc_frame_sink *sink); #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index ad705c4ab7..e8715cbe0b 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -318,7 +318,7 @@ scrcpy(const struct scrcpy_options *options) { file_handler_initialized = true; } - decoder_init(&decoder, &video_buffer); + decoder_init(&decoder); dec = &decoder; } @@ -382,6 +382,8 @@ scrcpy(const struct scrcpy_options *options) { } screen_initialized = true; + decoder_add_sink(&decoder, &screen.frame_sink); + if (options->turn_screen_off) { struct control_msg msg; msg.type = CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE; From 8ede4b3f58744935edd99678f0f9c3ff7ca5d8e1 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 17/30] Move video_buffer to screen The video buffer is now an internal detail of the screen component. Since the screen is plugged to the decoder via the frame sink trait, the decoder does not access to the video buffer anymore. --- app/src/scrcpy.c | 15 +-------------- app/src/screen.c | 20 ++++++++++++++------ app/src/screen.h | 8 +++----- 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index e8715cbe0b..cab6352293 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -25,14 +25,12 @@ #include "server.h" #include "stream.h" #include "tiny_xpm.h" -#include "video_buffer.h" #include "util/log.h" #include "util/net.h" static struct server server; static struct screen screen; static struct fps_counter fps_counter; -static struct video_buffer video_buffer; static struct stream stream; static struct decoder decoder; static struct recorder recorder; @@ -247,7 +245,6 @@ scrcpy(const struct scrcpy_options *options) { bool server_started = false; bool fps_counter_initialized = false; - bool video_buffer_initialized = false; bool file_handler_initialized = false; bool recorder_initialized = false; bool stream_started = false; @@ -305,11 +302,6 @@ scrcpy(const struct scrcpy_options *options) { } fps_counter_initialized = true; - if (!video_buffer_init(&video_buffer)) { - goto end; - } - video_buffer_initialized = true; - if (options->control) { if (!file_handler_init(&file_handler, server.serial, options->push_target)) { @@ -376,8 +368,7 @@ scrcpy(const struct scrcpy_options *options) { .fullscreen = options->fullscreen, }; - if (!screen_init(&screen, &video_buffer, &fps_counter, - &screen_params)) { + if (!screen_init(&screen, &fps_counter, &screen_params)) { goto end; } screen_initialized = true; @@ -453,10 +444,6 @@ scrcpy(const struct scrcpy_options *options) { file_handler_destroy(&file_handler); } - if (video_buffer_initialized) { - video_buffer_destroy(&video_buffer); - } - if (fps_counter_initialized) { fps_counter_join(&fps_counter); fps_counter_destroy(&fps_counter); diff --git a/app/src/screen.c b/app/src/screen.c index 0aa0b832e5..c067ab7a6b 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -284,14 +284,12 @@ screen_frame_sink_close(struct sc_frame_sink *sink) { static bool screen_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { struct screen *screen = DOWNCAST(sink); - return video_buffer_push(screen->vb, frame); + return video_buffer_push(&screen->vb, frame); } bool -screen_init(struct screen *screen, struct video_buffer *vb, - struct fps_counter *fps_counter, +screen_init(struct screen *screen, struct fps_counter *fps_counter, const struct screen_params *params) { - screen->vb = vb; screen->fps_counter = fps_counter; screen->resize_pending = false; @@ -299,11 +297,17 @@ screen_init(struct screen *screen, struct video_buffer *vb, screen->fullscreen = false; screen->maximized = false; + bool ok = video_buffer_init(&screen->vb); + if (!ok) { + LOGE("Could not initialize video buffer"); + return false; + } + static const struct video_buffer_callbacks cbs = { .on_frame_available = on_frame_available, .on_frame_skipped = on_frame_skipped, }; - video_buffer_set_consumer_callbacks(vb, &cbs, screen); + video_buffer_set_consumer_callbacks(&screen->vb, &cbs, screen); screen->frame_size = params->frame_size; screen->rotation = params->rotation; @@ -349,6 +353,7 @@ screen_init(struct screen *screen, struct video_buffer *vb, if (!screen->renderer) { LOGC("Could not create renderer: %s", SDL_GetError()); SDL_DestroyWindow(screen->window); + video_buffer_destroy(&screen->vb); return false; } @@ -400,6 +405,7 @@ screen_init(struct screen *screen, struct video_buffer *vb, LOGC("Could not create texture: %s", SDL_GetError()); SDL_DestroyRenderer(screen->renderer); SDL_DestroyWindow(screen->window); + video_buffer_destroy(&screen->vb); return false; } @@ -409,6 +415,7 @@ screen_init(struct screen *screen, struct video_buffer *vb, SDL_DestroyTexture(screen->texture); SDL_DestroyRenderer(screen->renderer); SDL_DestroyWindow(screen->window); + video_buffer_destroy(&screen->vb); return false; } @@ -449,6 +456,7 @@ screen_destroy(struct screen *screen) { SDL_DestroyTexture(screen->texture); SDL_DestroyRenderer(screen->renderer); SDL_DestroyWindow(screen->window); + video_buffer_destroy(&screen->vb); } static void @@ -554,7 +562,7 @@ update_texture(struct screen *screen, const AVFrame *frame) { static bool screen_update_frame(struct screen *screen) { av_frame_unref(screen->frame); - video_buffer_consume(screen->vb, screen->frame); + video_buffer_consume(&screen->vb, screen->frame); AVFrame *frame = screen->frame; fps_counter_add_rendered_frame(screen->fps_counter); diff --git a/app/src/screen.h b/app/src/screen.h index d57b715280..3b4506e321 100644 --- a/app/src/screen.h +++ b/app/src/screen.h @@ -10,13 +10,12 @@ #include "coords.h" #include "opengl.h" #include "trait/frame_sink.h" - -struct video_buffer; +#include "video_buffer.h" struct screen { struct sc_frame_sink frame_sink; // frame sink trait - struct video_buffer *vb; + struct video_buffer vb; struct fps_counter *fps_counter; SDL_Window *window; @@ -63,8 +62,7 @@ struct screen_params { // initialize screen, create window, renderer and texture (window is hidden) bool -screen_init(struct screen *screen, struct video_buffer *vb, - struct fps_counter *fps_counter, +screen_init(struct screen *screen, struct fps_counter *fps_counter, const struct screen_params *params); // destroy window, renderer and texture (if any) From 7309a573dc2decf3db045d3db7a4729ffa876ad4 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 18/30] Remove video_buffer callbacks Now that screen is both the owner and the listener of the video buffer, execute the code directly without callbacks. --- app/src/screen.c | 49 ++++++++++++++++++------------------------ app/src/video_buffer.c | 31 +++++--------------------- app/src/video_buffer.h | 20 +---------------- 3 files changed, 27 insertions(+), 73 deletions(-) diff --git a/app/src/screen.c b/app/src/screen.c index c067ab7a6b..ddf81b08bc 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -193,27 +193,6 @@ screen_update_content_rect(struct screen *screen) { } } -static void -on_frame_available(struct video_buffer *vb, void *userdata) { - (void) vb; - (void) userdata; - - static SDL_Event new_frame_event = { - .type = EVENT_NEW_FRAME, - }; - - // Post the event on the UI thread - SDL_PushEvent(&new_frame_event); -} - -static void -on_frame_skipped(struct video_buffer *vb, void *userdata) { - (void) vb; - - struct screen *screen = userdata; - fps_counter_add_skipped_frame(screen->fps_counter); -} - static inline SDL_Texture * create_texture(struct screen *screen) { SDL_Renderer *renderer = screen->renderer; @@ -284,7 +263,27 @@ screen_frame_sink_close(struct sc_frame_sink *sink) { static bool screen_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { struct screen *screen = DOWNCAST(sink); - return video_buffer_push(&screen->vb, frame); + + bool previous_frame_skipped; + bool ok = video_buffer_push(&screen->vb, frame, &previous_frame_skipped); + if (!ok) { + return false; + } + + if (previous_frame_skipped) { + fps_counter_add_skipped_frame(screen->fps_counter); + // The EVENT_NEW_FRAME triggered for the previous frame will consume + // this new frame instead + } else { + static SDL_Event new_frame_event = { + .type = EVENT_NEW_FRAME, + }; + + // Post the event on the UI thread + SDL_PushEvent(&new_frame_event); + } + + return true; } bool @@ -303,12 +302,6 @@ screen_init(struct screen *screen, struct fps_counter *fps_counter, return false; } - static const struct video_buffer_callbacks cbs = { - .on_frame_available = on_frame_available, - .on_frame_skipped = on_frame_skipped, - }; - video_buffer_set_consumer_callbacks(&screen->vb, &cbs, screen); - screen->frame_size = params->frame_size; screen->rotation = params->rotation; if (screen->rotation) { diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index 18a180fa0b..7adf098bc6 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -29,10 +29,6 @@ video_buffer_init(struct video_buffer *vb) { // there is initially no frame, so consider it has already been consumed vb->pending_frame_consumed = true; - // The callbacks must be set by the consumer via - // video_buffer_set_consumer_callbacks() - vb->cbs = NULL; - return true; } @@ -50,21 +46,9 @@ swap_frames(AVFrame **lhs, AVFrame **rhs) { *rhs = tmp; } -void -video_buffer_set_consumer_callbacks(struct video_buffer *vb, - const struct video_buffer_callbacks *cbs, - void *cbs_userdata) { - assert(!vb->cbs); // must be set only once - assert(cbs); - assert(cbs->on_frame_available); - vb->cbs = cbs; - vb->cbs_userdata = cbs_userdata; -} - bool -video_buffer_push(struct video_buffer *vb, const AVFrame *frame) { - assert(vb->cbs); - +video_buffer_push(struct video_buffer *vb, const AVFrame *frame, + bool *previous_frame_skipped) { sc_mutex_lock(&vb->mutex); // Use a temporary frame to preserve pending_frame in case of error. @@ -80,18 +64,13 @@ video_buffer_push(struct video_buffer *vb, const AVFrame *frame) { swap_frames(&vb->pending_frame, &vb->tmp_frame); av_frame_unref(vb->tmp_frame); - bool skipped = !vb->pending_frame_consumed; + if (previous_frame_skipped) { + *previous_frame_skipped = !vb->pending_frame_consumed; + } vb->pending_frame_consumed = false; sc_mutex_unlock(&vb->mutex); - if (skipped) { - if (vb->cbs->on_frame_skipped) - vb->cbs->on_frame_skipped(vb, vb->cbs_userdata); - } else { - vb->cbs->on_frame_available(vb, vb->cbs_userdata); - } - return true; } diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h index cdecb25950..b9478f4c9e 100644 --- a/app/src/video_buffer.h +++ b/app/src/video_buffer.h @@ -33,19 +33,6 @@ struct video_buffer { sc_mutex mutex; bool pending_frame_consumed; - - const struct video_buffer_callbacks *cbs; - void *cbs_userdata; -}; - -struct video_buffer_callbacks { - // Called when a new frame can be consumed. - // This callback is mandatory (it must not be NULL). - void (*on_frame_available)(struct video_buffer *vb, void *userdata); - - // Called when a pending frame has been overwritten by the producer. - // This callback is optional (it may be NULL). - void (*on_frame_skipped)(struct video_buffer *vb, void *userdata); }; bool @@ -54,13 +41,8 @@ video_buffer_init(struct video_buffer *vb); void video_buffer_destroy(struct video_buffer *vb); -void -video_buffer_set_consumer_callbacks(struct video_buffer *vb, - const struct video_buffer_callbacks *cbs, - void *cbs_userdata); - bool -video_buffer_push(struct video_buffer *vb, const AVFrame *frame); +video_buffer_push(struct video_buffer *vb, const AVFrame *frame, bool *skipped); void video_buffer_consume(struct video_buffer *vb, AVFrame *dst); From c9a5611382c0b35482b29adfab45dd01f25e8320 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 11 Apr 2021 15:01:05 +0200 Subject: [PATCH 19/30] Assert screen closed on destroy The destruction order is important, but tricky, because the screen is open/close by the decoder, but destroyed by scrcpy.c on the main thread. Add assertions to guarantee that the screen is not destroyed before being closed. --- app/src/screen.c | 13 +++++++++++++ app/src/screen.h | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/app/src/screen.c b/app/src/screen.c index ddf81b08bc..1b3c5179e2 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -247,6 +247,9 @@ static bool screen_frame_sink_open(struct sc_frame_sink *sink) { struct screen *screen = DOWNCAST(sink); (void) screen; +#ifndef NDEBUG + screen->open = true; +#endif // nothing to do, the screen is already open on the main thread return true; @@ -256,6 +259,9 @@ static void screen_frame_sink_close(struct sc_frame_sink *sink) { struct screen *screen = DOWNCAST(sink); (void) screen; +#ifndef NDEBUG + screen->open = false; +#endif // nothing to do, the screen lifecycle is not managed by the frame producer } @@ -435,6 +441,10 @@ screen_init(struct screen *screen, struct fps_counter *fps_counter, screen->frame_sink.ops = &ops; +#ifndef NDEBUG + screen->open = false; +#endif + return true; } @@ -445,6 +455,9 @@ screen_show_window(struct screen *screen) { void screen_destroy(struct screen *screen) { +#ifndef NDEBUG + assert(!screen->open); +#endif av_frame_free(&screen->frame); SDL_DestroyTexture(screen->texture); SDL_DestroyRenderer(screen->renderer); diff --git a/app/src/screen.h b/app/src/screen.h index 3b4506e321..4a0bad09c0 100644 --- a/app/src/screen.h +++ b/app/src/screen.h @@ -15,6 +15,10 @@ struct screen { struct sc_frame_sink frame_sink; // frame sink trait +#ifndef NDEBUG + bool open; // track the open/close state to assert correct behavior +#endif + struct video_buffer vb; struct fps_counter *fps_counter; From 99c6a76eb5f7a45d594a5000e80c34eedb1a069a Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 13 Apr 2021 22:22:54 +0200 Subject: [PATCH 20/30] Hide the window immediately on close The screen may not be destroyed immediately on close to avoid undefined behavior, because it may still receive events from the decoder. But the visual window must still be closed immediately. --- app/src/scrcpy.c | 4 ++++ app/src/screen.c | 5 +++++ app/src/screen.h | 7 +++++++ 3 files changed, 16 insertions(+) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index cab6352293..4de6238989 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -398,6 +398,10 @@ scrcpy(const struct scrcpy_options *options) { ret = event_loop(options); LOGD("quit..."); + // Close the window immediately on closing, because screen_destroy() may + // only be called once the stream thread is joined (it may take time) + screen_hide_window(&screen); + end: // The stream is not stopped explicitly, because it will stop by itself on // end-of-stream diff --git a/app/src/screen.c b/app/src/screen.c index 1b3c5179e2..0598ccf4fa 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -453,6 +453,11 @@ screen_show_window(struct screen *screen) { SDL_ShowWindow(screen->window); } +void +screen_hide_window(struct screen *screen) { + SDL_HideWindow(screen->window); +} + void screen_destroy(struct screen *screen) { #ifndef NDEBUG diff --git a/app/src/screen.h b/app/src/screen.h index 4a0bad09c0..2921c701b1 100644 --- a/app/src/screen.h +++ b/app/src/screen.h @@ -73,6 +73,13 @@ screen_init(struct screen *screen, struct fps_counter *fps_counter, void screen_destroy(struct screen *screen); +// hide the window +// +// It is used to hide the window immediately on closing without waiting for +// screen_destroy() +void +screen_hide_window(struct screen *screen); + // render the texture to the renderer // // Set the update_content_rect flag if the window or content size may have From 49582e4a2edac54aa1e63d8578c5973272ec3f05 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 18 Apr 2021 11:25:58 +0200 Subject: [PATCH 21/30] Initialize recorder fields on open Only initialize ops and parameters copy from recorder_init(). It was inconsistent to initialize some fields from _init() and some others from _open(). --- app/src/recorder.c | 55 +++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index f0ec86dcb0..e96c4a5235 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -218,17 +218,40 @@ run_recorder(void *data) { static bool recorder_open(struct recorder *recorder, const AVCodec *input_codec) { + bool ok = sc_mutex_init(&recorder->mutex); + if (!ok) { + LOGC("Could not create mutex"); + return false; + } + + ok = sc_cond_init(&recorder->queue_cond); + if (!ok) { + LOGC("Could not create cond"); + sc_mutex_destroy(&recorder->mutex); + return false; + } + + queue_init(&recorder->queue); + recorder->stopped = false; + recorder->failed = false; + recorder->header_written = false; + recorder->previous = NULL; + const char *format_name = recorder_get_format_name(recorder->format); assert(format_name); const AVOutputFormat *format = find_muxer(format_name); if (!format) { LOGE("Could not find muxer"); + sc_cond_destroy(&recorder->queue_cond); + sc_mutex_destroy(&recorder->mutex); return false; } recorder->ctx = avformat_alloc_context(); if (!recorder->ctx) { LOGE("Could not allocate output context"); + sc_cond_destroy(&recorder->queue_cond); + sc_mutex_destroy(&recorder->mutex); return false; } @@ -244,6 +267,8 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { AVStream *ostream = avformat_new_stream(recorder->ctx, input_codec); if (!ostream) { avformat_free_context(recorder->ctx); + sc_cond_destroy(&recorder->queue_cond); + sc_mutex_destroy(&recorder->mutex); return false; } @@ -259,16 +284,20 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { LOGE("Failed to open output file: %s", recorder->filename); // ostream will be cleaned up during context cleaning avformat_free_context(recorder->ctx); + sc_cond_destroy(&recorder->queue_cond); + sc_mutex_destroy(&recorder->mutex); return false; } LOGD("Starting recorder thread"); - bool ok = sc_thread_create(&recorder->thread, run_recorder, "recorder", + ok = sc_thread_create(&recorder->thread, run_recorder, "recorder", recorder); if (!ok) { LOGC("Could not start recorder thread"); avio_close(recorder->ctx->pb); avformat_free_context(recorder->ctx); + sc_cond_destroy(&recorder->queue_cond); + sc_mutex_destroy(&recorder->mutex); return false; } @@ -288,6 +317,8 @@ recorder_close(struct recorder *recorder) { avio_close(recorder->ctx->pb); avformat_free_context(recorder->ctx); + sc_cond_destroy(&recorder->queue_cond); + sc_mutex_destroy(&recorder->mutex); } static bool @@ -344,28 +375,8 @@ recorder_init(struct recorder *recorder, return false; } - bool ok = sc_mutex_init(&recorder->mutex); - if (!ok) { - LOGC("Could not create mutex"); - free(recorder->filename); - return false; - } - - ok = sc_cond_init(&recorder->queue_cond); - if (!ok) { - LOGC("Could not create cond"); - sc_mutex_destroy(&recorder->mutex); - free(recorder->filename); - return false; - } - - queue_init(&recorder->queue); - recorder->stopped = false; - recorder->failed = false; recorder->format = format; recorder->declared_frame_size = declared_frame_size; - recorder->header_written = false; - recorder->previous = NULL; static const struct sc_packet_sink_ops ops = { .open = recorder_packet_sink_open, @@ -380,7 +391,5 @@ recorder_init(struct recorder *recorder, void recorder_destroy(struct recorder *recorder) { - sc_cond_destroy(&recorder->queue_cond); - sc_mutex_destroy(&recorder->mutex); free(recorder->filename); } From c8eb5cc6e310f48971c91e6a051deec8b84640b6 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 18 Apr 2021 11:32:21 +0200 Subject: [PATCH 22/30] Handle errors using gotos in recorder_open() There are many initializations in recorder_open(). Handle RAII-like deinitialization using gotos. --- app/src/recorder.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index e96c4a5235..479bdb3999 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -227,8 +227,7 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { ok = sc_cond_init(&recorder->queue_cond); if (!ok) { LOGC("Could not create cond"); - sc_mutex_destroy(&recorder->mutex); - return false; + goto error_mutex_destroy; } queue_init(&recorder->queue); @@ -242,17 +241,13 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { const AVOutputFormat *format = find_muxer(format_name); if (!format) { LOGE("Could not find muxer"); - sc_cond_destroy(&recorder->queue_cond); - sc_mutex_destroy(&recorder->mutex); - return false; + goto error_cond_destroy; } recorder->ctx = avformat_alloc_context(); if (!recorder->ctx) { LOGE("Could not allocate output context"); - sc_cond_destroy(&recorder->queue_cond); - sc_mutex_destroy(&recorder->mutex); - return false; + goto error_cond_destroy; } // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate() @@ -266,10 +261,7 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { AVStream *ostream = avformat_new_stream(recorder->ctx, input_codec); if (!ostream) { - avformat_free_context(recorder->ctx); - sc_cond_destroy(&recorder->queue_cond); - sc_mutex_destroy(&recorder->mutex); - return false; + goto error_avformat_free_context; } ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; @@ -283,10 +275,7 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { if (ret < 0) { LOGE("Failed to open output file: %s", recorder->filename); // ostream will be cleaned up during context cleaning - avformat_free_context(recorder->ctx); - sc_cond_destroy(&recorder->queue_cond); - sc_mutex_destroy(&recorder->mutex); - return false; + goto error_avformat_free_context; } LOGD("Starting recorder thread"); @@ -294,16 +283,23 @@ recorder_open(struct recorder *recorder, const AVCodec *input_codec) { recorder); if (!ok) { LOGC("Could not start recorder thread"); - avio_close(recorder->ctx->pb); - avformat_free_context(recorder->ctx); - sc_cond_destroy(&recorder->queue_cond); - sc_mutex_destroy(&recorder->mutex); - return false; + goto error_avio_close; } LOGI("Recording started to %s file: %s", format_name, recorder->filename); return true; + +error_avio_close: + avio_close(recorder->ctx->pb); +error_avformat_free_context: + avformat_free_context(recorder->ctx); +error_cond_destroy: + sc_cond_destroy(&recorder->queue_cond); +error_mutex_destroy: + sc_mutex_destroy(&recorder->mutex); + + return false; } static void From 066f4f5e88651f7100e8d361abcca22dddc4ce26 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 18 Apr 2021 17:13:58 +0200 Subject: [PATCH 23/30] Handle EAGAIN on send_packet in decoder EAGAIN was only handled on receive_frame. In practice, it should not be necessary, since one packet always contains one frame. But just in case. --- app/src/decoder.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index 34f2a15fd3..476158d9c6 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -99,8 +99,8 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) { return true; } - int ret; - if ((ret = avcodec_send_packet(decoder->codec_ctx, packet)) < 0) { + int ret = avcodec_send_packet(decoder->codec_ctx, packet); + if (ret < 0 && ret != AVERROR(EAGAIN)) { LOGE("Could not send video packet: %d", ret); return false; } From b2e67b5a60c73c734f51b59d0c713622aeae3173 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 18 Apr 2021 17:23:09 +0200 Subject: [PATCH 24/30] Fix recorder comment --- app/src/recorder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 479bdb3999..e33d1a3bbe 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -22,7 +22,7 @@ find_muxer(const char *name) { #else oformat = av_oformat_next(oformat); #endif - // until null or with name "mp4" + // until null or with having the requested name } while (oformat && strcmp(oformat->name, name)); return oformat; } From 9d3bf0947477f7522bd983a5ffe5b5a831704f9d Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 19 Apr 2021 09:22:53 +0200 Subject: [PATCH 25/30] Add strlist_contains() Add a function to know if a string list, using some separator, contains a specific string. --- app/src/util/str_util.c | 18 ++++++++++++++++++ app/src/util/str_util.h | 5 +++++ app/tests/test_strutil.c | 13 +++++++++++++ 3 files changed, 36 insertions(+) diff --git a/app/src/util/str_util.c b/app/src/util/str_util.c index 352d1d2f0e..287c08de27 100644 --- a/app/src/util/str_util.c +++ b/app/src/util/str_util.c @@ -140,6 +140,24 @@ parse_integer_with_suffix(const char *s, long *out) { return true; } +bool +strlist_contains(const char *list, char sep, const char *s) { + char *p; + do { + p = strchr(list, sep); + + size_t token_len = p ? (size_t) (p - list) : strlen(list); + if (!strncmp(list, s, token_len)) { + return true; + } + + if (p) { + list = p + 1; + } + } while (p); + return false; +} + size_t utf8_truncation_index(const char *utf8, size_t max_len) { size_t len = strlen(utf8); diff --git a/app/src/util/str_util.h b/app/src/util/str_util.h index 25bec4443b..c016a62590 100644 --- a/app/src/util/str_util.h +++ b/app/src/util/str_util.h @@ -43,6 +43,11 @@ parse_integers(const char *s, const char sep, size_t max_items, long *out); bool parse_integer_with_suffix(const char *s, long *out); +// search s in the list separated by sep +// for example, strlist_contains("a,bc,def", ',', "bc") returns true +bool +strlist_contains(const char *list, char sep, const char *s); + // return the index to truncate a UTF-8 string at a valid position size_t utf8_truncation_index(const char *utf8, size_t max_len); diff --git a/app/tests/test_strutil.c b/app/tests/test_strutil.c index ce0d5d3013..dfd996587d 100644 --- a/app/tests/test_strutil.c +++ b/app/tests/test_strutil.c @@ -287,6 +287,18 @@ static void test_parse_integer_with_suffix(void) { assert(!ok); } +static void test_strlist_contains(void) { + assert(strlist_contains("a,bc,def", ',', "bc")); + assert(!strlist_contains("a,bc,def", ',', "b")); + assert(strlist_contains("", ',', "")); + assert(strlist_contains("abc,", ',', "")); + assert(strlist_contains(",abc", ',', "")); + assert(strlist_contains("abc,,def", ',', "")); + assert(!strlist_contains("abc", ',', "")); + assert(strlist_contains(",,|x", '|', ",,")); + assert(strlist_contains("xyz", '\0', "xyz")); +} + int main(int argc, char *argv[]) { (void) argc; (void) argv; @@ -304,5 +316,6 @@ int main(int argc, char *argv[]) { test_parse_integer(); test_parse_integers(); test_parse_integer_with_suffix(); + test_strlist_contains(); return 0; } From 195073d628226b36a5e93b316458b14f141c0652 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 19 Apr 2021 09:28:28 +0200 Subject: [PATCH 26/30] Use strlist_contains() to find a muxer The AVOutputFormat name is a string list: it contains names (possibly only one) separated by '.' --- app/src/recorder.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index e33d1a3bbe..91390ff12a 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -4,6 +4,7 @@ #include #include "util/log.h" +#include "util/str_util.h" /** Downcast packet_sink to recorder */ #define DOWNCAST(SINK) container_of(SINK, struct recorder, packet_sink) @@ -22,8 +23,8 @@ find_muxer(const char *name) { #else oformat = av_oformat_next(oformat); #endif - // until null or with having the requested name - } while (oformat && strcmp(oformat->name, name)); + // until null or containing the requested name + } while (oformat && !strlist_contains(oformat->name, ',', name)); return oformat; } From a963d89a64750eb15e229a846ae315d4829044b7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 19 Apr 2021 18:42:20 +0200 Subject: [PATCH 27/30] Add --lock-video-orientation=initial Add a new mode to the --lock-video-orientation option, to lock the initial orientation of the device. This avoids to pass an explicit value (0, 1, 2 or 3) and think about which is the right one. --- README.md | 1 + app/scrcpy.1 | 4 +-- app/src/cli.c | 26 ++++++++++++++----- app/src/scrcpy.h | 14 ++++++++-- .../java/com/genymobile/scrcpy/Device.java | 3 +++ .../com/genymobile/scrcpy/ScreenInfo.java | 6 +++++ 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5f583b8116..106bc17fa7 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,7 @@ If `--max-size` is also specified, resizing is applied after cropping. To lock the orientation of the mirroring: ```bash +scrcpy --lock-video-orientation initial # initial (current) orientation scrcpy --lock-video-orientation 0 # natural orientation scrcpy --lock-video-orientation 1 # 90° counterclockwise scrcpy --lock-video-orientation 2 # 180° diff --git a/app/scrcpy.1 b/app/scrcpy.1 index ea9bce9ba6..89b7ee75ec 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -84,9 +84,9 @@ This is a workaround for some devices not behaving as expected when setting the .TP .BI "\-\-lock\-video\-orientation " value -Lock video orientation to \fIvalue\fR. Possible values are -1 (unlocked), 0, 1, 2 and 3. Natural device orientation is 0, and each increment adds a 90 degrees otation counterclockwise. +Lock video orientation to \fIvalue\fR. Possible values are "unlocked", "initial" (locked to the initial orientation), 0, 1, 2 and 3. Natural device orientation is 0, and each increment adds a 90 degrees otation counterclockwise. -Default is -1 (unlocked). +Default is "unlocked". .TP .BI "\-\-max\-fps " value diff --git a/app/src/cli.c b/app/src/cli.c index ec3c12941a..05e178e02b 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -81,10 +81,11 @@ scrcpy_print_usage(const char *arg0) { "\n" " --lock-video-orientation value\n" " Lock video orientation to value.\n" - " Possible values are -1 (unlocked), 0, 1, 2 and 3.\n" + " Possible values are \"unlocked\", \"initial\" (locked to the\n" + " initial orientation), 0, 1, 2 and 3.\n" " Natural device orientation is 0, and each increment adds a\n" " 90 degrees rotation counterclockwise.\n" - " Default is -1 (unlocked).\n" + " Default is \"unlocked\".\n" "\n" " --max-fps value\n" " Limit the frame rate of screen capture (officially supported\n" @@ -383,15 +384,27 @@ parse_max_fps(const char *s, uint16_t *max_fps) { } static bool -parse_lock_video_orientation(const char *s, int8_t *lock_video_orientation) { +parse_lock_video_orientation(const char *s, + enum sc_lock_video_orientation *lock_mode) { + if (!strcmp(s, "initial")) { + // Without argument, lock the initial orientation + *lock_mode = SC_LOCK_VIDEO_ORIENTATION_INITIAL; + return true; + } + + if (!strcmp(s, "unlocked")) { + *lock_mode = SC_LOCK_VIDEO_ORIENTATION_UNLOCKED; + return true; + } + long value; - bool ok = parse_integer_arg(s, &value, false, -1, 3, + bool ok = parse_integer_arg(s, &value, false, 0, 3, "lock video orientation"); if (!ok) { return false; } - *lock_video_orientation = (int8_t) value; + *lock_mode = (enum sc_lock_video_orientation) value; return true; } @@ -765,7 +778,8 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { } break; case OPT_LOCK_VIDEO_ORIENTATION: - if (!parse_lock_video_orientation(optarg, &opts->lock_video_orientation)) { + if (!parse_lock_video_orientation(optarg, + &opts->lock_video_orientation)) { return false; } break; diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index f91cb6b88f..7c2689ab9b 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -20,6 +20,16 @@ enum sc_record_format { SC_RECORD_FORMAT_MKV, }; +enum sc_lock_video_orientation { + SC_LOCK_VIDEO_ORIENTATION_UNLOCKED = -1, + // lock the current orientation when scrcpy starts + SC_LOCK_VIDEO_ORIENTATION_INITIAL = -2, + SC_LOCK_VIDEO_ORIENTATION_0 = 0, + SC_LOCK_VIDEO_ORIENTATION_1, + SC_LOCK_VIDEO_ORIENTATION_2, + SC_LOCK_VIDEO_ORIENTATION_3, +}; + #define SC_MAX_SHORTCUT_MODS 8 enum sc_shortcut_mod { @@ -59,7 +69,7 @@ struct scrcpy_options { uint16_t max_size; uint32_t bit_rate; uint16_t max_fps; - int8_t lock_video_orientation; + enum sc_lock_video_orientation lock_video_orientation; uint8_t rotation; int16_t window_x; // SC_WINDOW_POSITION_UNDEFINED for "auto" int16_t window_y; // SC_WINDOW_POSITION_UNDEFINED for "auto" @@ -106,7 +116,7 @@ struct scrcpy_options { .max_size = 0, \ .bit_rate = DEFAULT_BIT_RATE, \ .max_fps = 0, \ - .lock_video_orientation = -1, \ + .lock_video_orientation = SC_LOCK_VIDEO_ORIENTATION_UNLOCKED, \ .rotation = 0, \ .window_x = SC_WINDOW_POSITION_UNDEFINED, \ .window_y = SC_WINDOW_POSITION_UNDEFINED, \ diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.java index 624c9fa0d3..c38469893d 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Device.java +++ b/server/src/main/java/com/genymobile/scrcpy/Device.java @@ -25,6 +25,9 @@ public final class Device { public static final int POWER_MODE_OFF = SurfaceControl.POWER_MODE_OFF; public static final int POWER_MODE_NORMAL = SurfaceControl.POWER_MODE_NORMAL; + public static final int LOCK_VIDEO_ORIENTATION_UNLOCKED = -1; + public static final int LOCK_VIDEO_ORIENTATION_INITIAL = -2; + private static final ServiceManager SERVICE_MANAGER = new ServiceManager(); public interface RotationListener { diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java b/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java index 10acfb5061..c27322ef40 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java @@ -82,6 +82,12 @@ public ScreenInfo withDeviceRotation(int newDeviceRotation) { public static ScreenInfo computeScreenInfo(DisplayInfo displayInfo, Rect crop, int maxSize, int lockedVideoOrientation) { int rotation = displayInfo.getRotation(); + + if (lockedVideoOrientation == Device.LOCK_VIDEO_ORIENTATION_INITIAL) { + // The user requested to lock the video orientation to the current orientation + lockedVideoOrientation = rotation; + } + Size deviceSize = displayInfo.getSize(); Rect contentRect = new Rect(0, 0, deviceSize.getWidth(), deviceSize.getHeight()); if (crop != null) { From 860569641ff8d5de81922867397a2de886de78c3 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 19 Apr 2021 18:40:48 +0200 Subject: [PATCH 28/30] Make --lock-video-orientation argument optional If the option is set without argument, lock the initial device orientation (as if the value "initial" was passed). --- README.md | 2 +- app/scrcpy.1 | 4 +++- app/src/cli.c | 8 +++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 106bc17fa7..1ac5fd0948 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ If `--max-size` is also specified, resizing is applied after cropping. To lock the orientation of the mirroring: ```bash -scrcpy --lock-video-orientation initial # initial (current) orientation +scrcpy --lock-video-orientation # initial (current) orientation scrcpy --lock-video-orientation 0 # natural orientation scrcpy --lock-video-orientation 1 # 90° counterclockwise scrcpy --lock-video-orientation 2 # 180° diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 89b7ee75ec..51c618a8f4 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -83,11 +83,13 @@ Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+S This is a workaround for some devices not behaving as expected when setting the device clipboard programmatically. .TP -.BI "\-\-lock\-video\-orientation " value +.BI "\-\-lock\-video\-orientation " [value] Lock video orientation to \fIvalue\fR. Possible values are "unlocked", "initial" (locked to the initial orientation), 0, 1, 2 and 3. Natural device orientation is 0, and each increment adds a 90 degrees otation counterclockwise. Default is "unlocked". +Passing the option without argument is equivalent to passing "initial". + .TP .BI "\-\-max\-fps " value Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions). diff --git a/app/src/cli.c b/app/src/cli.c index 05e178e02b..6d5fd6b819 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -79,13 +79,15 @@ scrcpy_print_usage(const char *arg0) { " This is a workaround for some devices not behaving as\n" " expected when setting the device clipboard programmatically.\n" "\n" - " --lock-video-orientation value\n" + " --lock-video-orientation [value]\n" " Lock video orientation to value.\n" " Possible values are \"unlocked\", \"initial\" (locked to the\n" " initial orientation), 0, 1, 2 and 3.\n" " Natural device orientation is 0, and each increment adds a\n" " 90 degrees rotation counterclockwise.\n" " Default is \"unlocked\".\n" + " Passing the option without argument is equivalent to passing\n" + " \"initial\".\n" "\n" " --max-fps value\n" " Limit the frame rate of screen capture (officially supported\n" @@ -386,7 +388,7 @@ parse_max_fps(const char *s, uint16_t *max_fps) { static bool parse_lock_video_orientation(const char *s, enum sc_lock_video_orientation *lock_mode) { - if (!strcmp(s, "initial")) { + if (!s || !strcmp(s, "initial")) { // Without argument, lock the initial orientation *lock_mode = SC_LOCK_VIDEO_ORIENTATION_INITIAL; return true; @@ -693,7 +695,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { {"fullscreen", no_argument, NULL, 'f'}, {"help", no_argument, NULL, 'h'}, {"legacy-paste", no_argument, NULL, OPT_LEGACY_PASTE}, - {"lock-video-orientation", required_argument, NULL, + {"lock-video-orientation", optional_argument, NULL, OPT_LOCK_VIDEO_ORIENTATION}, {"max-fps", required_argument, NULL, OPT_MAX_FPS}, {"max-size", required_argument, NULL, 'm'}, From 0cd902bd07bbcdb94a50d06e1c0eb45f528c0068 Mon Sep 17 00:00:00 2001 From: Marco Martinelli <6640057+martinellimarco@users.noreply.github.com> Date: Sun, 4 Apr 2021 00:10:44 +0200 Subject: [PATCH 29/30] Add support for v4l2loopback It allows to send the video stream to /dev/videoN, so that it can be captured (like a webcam) by any V4L2-compatible tool. Refs #2232 Refs #2233 Co-authored-by: Romain Vimont --- app/meson.build | 12 ++ app/scrcpy.1 | 6 + app/src/cli.c | 31 ++++ app/src/decoder.h | 2 +- app/src/main.c | 14 ++ app/src/scrcpy.c | 35 ++++- app/src/scrcpy.h | 2 + app/src/v4l2_sink.c | 341 ++++++++++++++++++++++++++++++++++++++++++++ app/src/v4l2_sink.h | 39 +++++ 9 files changed, 480 insertions(+), 2 deletions(-) create mode 100644 app/src/v4l2_sink.c create mode 100644 app/src/v4l2_sink.h diff --git a/app/meson.build b/app/meson.build index c2a5974192..c1dbf48923 100644 --- a/app/meson.build +++ b/app/meson.build @@ -33,6 +33,11 @@ else src += [ 'src/sys/unix/process.c' ] endif +v4l2_support = host_machine.system() == 'linux' +if v4l2_support + src += [ 'src/v4l2_sink.c' ] +endif + check_functions = [ 'strdup' ] @@ -49,6 +54,10 @@ if not get_option('crossbuild_windows') dependency('sdl2'), ] + if v4l2_support + dependencies += dependency('libavdevice') + endif + else # cross-compile mingw32 build (from Linux to Windows) @@ -124,6 +133,9 @@ conf.set('SERVER_DEBUGGER', get_option('server_debugger')) # select the debugger method ('old' for Android < 9, 'new' for Android >= 9) conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == 'new') +# enable V4L2 support (linux only) +conf.set('HAVE_V4L2', v4l2_support) + configure_file(configuration: conf, output: 'config.h') src_dir = include_directories('src') diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 51c618a8f4..7ef69640c8 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -185,6 +185,12 @@ Enable "show touches" on start, restore the initial value on exit. It only shows physical touches (not clicks from scrcpy). +.TP +.BI "\-\-v4l2-sink " /dev/videoN +Output to v4l2loopback device. + +It requires to lock the video orientation (see --lock-video-orientation). + .TP .BI "\-V, \-\-verbosity " value Set the log level ("debug", "info", "warn" or "error"). diff --git a/app/src/cli.c b/app/src/cli.c index 6d5fd6b819..841b7c397c 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -176,6 +176,13 @@ scrcpy_print_usage(const char *arg0) { " on exit.\n" " It only shows physical touches (not clicks from scrcpy).\n" "\n" +#ifdef HAVE_V4L2 + " --v4l2-sink /dev/videoN\n" + " Output to v4l2loopback device.\n" + " It requires to lock the video orientation (see\n" + " --lock-video-orientation).\n" + "\n" +#endif " -V, --verbosity value\n" " Set the log level (debug, info, warn or error).\n" #ifndef NDEBUG @@ -676,6 +683,7 @@ guess_record_format(const char *filename) { #define OPT_LEGACY_PASTE 1024 #define OPT_ENCODER_NAME 1025 #define OPT_POWER_OFF_ON_CLOSE 1026 +#define OPT_V4L2_SINK 1027 bool scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { @@ -717,6 +725,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { {"show-touches", no_argument, NULL, 't'}, {"stay-awake", no_argument, NULL, 'w'}, {"turn-screen-off", no_argument, NULL, 'S'}, +#ifdef HAVE_V4L2 + {"v4l2-sink", required_argument, NULL, OPT_V4L2_SINK}, +#endif {"verbosity", required_argument, NULL, 'V'}, {"version", no_argument, NULL, 'v'}, {"window-title", required_argument, NULL, OPT_WINDOW_TITLE}, @@ -901,16 +912,36 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { case OPT_POWER_OFF_ON_CLOSE: opts->power_off_on_close = true; break; +#ifdef HAVE_V4L2 + case OPT_V4L2_SINK: + opts->v4l2_device = optarg; + break; +#endif default: // getopt prints the error message on stderr return false; } } +#ifdef HAVE_V4L2 + if (!opts->display && !opts->record_filename && !opts->v4l2_device) { + LOGE("-N/--no-display requires either screen recording (-r/--record)" + " or sink to v4l2loopback device (--v4l2-sink)"); + return false; + } + + if (opts->v4l2_device && opts->lock_video_orientation + == SC_LOCK_VIDEO_ORIENTATION_UNLOCKED) { + LOGI("Video orientation is locked for v4l2 sink. " + "See --lock-video-orientation."); + opts->lock_video_orientation = SC_LOCK_VIDEO_ORIENTATION_INITIAL; + } +#else if (!opts->display && !opts->record_filename) { LOGE("-N/--no-display requires screen recording (-r/--record)"); return false; } +#endif int index = optind; if (index < argc) { diff --git a/app/src/decoder.h b/app/src/decoder.h index bae97869c2..257f751a03 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -8,7 +8,7 @@ #include #include -#define DECODER_MAX_SINKS 1 +#define DECODER_MAX_SINKS 2 struct decoder { struct sc_packet_sink packet_sink; // packet sink trait diff --git a/app/src/main.c b/app/src/main.c index e1e44f688a..a468aed7f1 100644 --- a/app/src/main.c +++ b/app/src/main.c @@ -6,6 +6,9 @@ #include #include #include +#ifdef HAVE_V4L2 +# include +#endif #define SDL_MAIN_HANDLED // avoid link error on Linux Windows Subsystem #include @@ -28,6 +31,11 @@ print_version(void) { fprintf(stderr, " - libavutil %d.%d.%d\n", LIBAVUTIL_VERSION_MAJOR, LIBAVUTIL_VERSION_MINOR, LIBAVUTIL_VERSION_MICRO); +#ifdef HAVE_V4L2 + fprintf(stderr, " - libavdevice %d.%d.%d\n", LIBAVDEVICE_VERSION_MAJOR, + LIBAVDEVICE_VERSION_MINOR, + LIBAVDEVICE_VERSION_MICRO); +#endif } static SDL_LogPriority @@ -90,6 +98,12 @@ main(int argc, char *argv[]) { av_register_all(); #endif +#ifdef HAVE_V4L2 + if (args.opts.v4l2_device) { + avdevice_register_all(); + } +#endif + if (avformat_network_init()) { return 1; } diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 4de6238989..e847037e6d 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -27,6 +27,9 @@ #include "tiny_xpm.h" #include "util/log.h" #include "util/net.h" +#ifdef HAVE_V4L2 +# include "v4l2_sink.h" +#endif static struct server server; static struct screen screen; @@ -34,6 +37,9 @@ static struct fps_counter fps_counter; static struct stream stream; static struct decoder decoder; static struct recorder recorder; +#ifdef HAVE_V4L2 +static struct sc_v4l2_sink v4l2_sink; +#endif static struct controller controller; static struct file_handler file_handler; @@ -247,6 +253,9 @@ scrcpy(const struct scrcpy_options *options) { bool fps_counter_initialized = false; bool file_handler_initialized = false; bool recorder_initialized = false; +#ifdef HAVE_V4L2 + bool v4l2_sink_initialized = false; +#endif bool stream_started = false; bool controller_initialized = false; bool controller_started = false; @@ -295,7 +304,6 @@ scrcpy(const struct scrcpy_options *options) { goto end; } - struct decoder *dec = NULL; if (options->display) { if (!fps_counter_init(&fps_counter)) { goto end; @@ -309,7 +317,14 @@ scrcpy(const struct scrcpy_options *options) { } file_handler_initialized = true; } + } + struct decoder *dec = NULL; + bool needs_decoder = options->display; +#ifdef HAVE_V4L2 + needs_decoder |= !!options->v4l2_device; +#endif + if (needs_decoder) { decoder_init(&decoder); dec = &decoder; } @@ -386,6 +401,18 @@ scrcpy(const struct scrcpy_options *options) { } } +#ifdef HAVE_V4L2 + if (options->v4l2_device) { + if (!sc_v4l2_sink_init(&v4l2_sink, options->v4l2_device, frame_size)) { + goto end; + } + + decoder_add_sink(&decoder, &v4l2_sink.frame_sink); + + v4l2_sink_initialized = true; + } +#endif + // now we consumed the header values, the socket receives the video stream // start the stream if (!stream_start(&stream)) { @@ -426,6 +453,12 @@ scrcpy(const struct scrcpy_options *options) { stream_join(&stream); } +#ifdef HAVE_V4L2 + if (v4l2_sink_initialized) { + sc_v4l2_sink_destroy(&v4l2_sink); + } +#endif + // Destroy the screen only after the stream is guaranteed to be finished, // because otherwise the screen could receive new frames after destruction if (screen_initialized) { diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index 7c2689ab9b..405dc7f3b6 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -62,6 +62,7 @@ struct scrcpy_options { const char *render_driver; const char *codec_options; const char *encoder_name; + const char *v4l2_device; enum sc_log_level log_level; enum sc_record_format record_format; struct sc_port_range port_range; @@ -103,6 +104,7 @@ struct scrcpy_options { .render_driver = NULL, \ .codec_options = NULL, \ .encoder_name = NULL, \ + .v4l2_device = NULL, \ .log_level = SC_LOG_LEVEL_INFO, \ .record_format = SC_RECORD_FORMAT_AUTO, \ .port_range = { \ diff --git a/app/src/v4l2_sink.c b/app/src/v4l2_sink.c new file mode 100644 index 0000000000..b7d06bb6a6 --- /dev/null +++ b/app/src/v4l2_sink.c @@ -0,0 +1,341 @@ +#include "v4l2_sink.h" + +#include "util/log.h" +#include "util/str_util.h" + +/** Downcast frame_sink to sc_v4l2_sink */ +#define DOWNCAST(SINK) container_of(SINK, struct sc_v4l2_sink, frame_sink) + +static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us + +static const AVOutputFormat * +find_muxer(const char *name) { +#ifdef SCRCPY_LAVF_HAS_NEW_MUXER_ITERATOR_API + void *opaque = NULL; +#endif + const AVOutputFormat *oformat = NULL; + do { +#ifdef SCRCPY_LAVF_HAS_NEW_MUXER_ITERATOR_API + oformat = av_muxer_iterate(&opaque); +#else + oformat = av_oformat_next(oformat); +#endif + // until null or containing the requested name + } while (oformat && !strlist_contains(oformat->name, ',', name)); + return oformat; +} + +static bool +write_header(struct sc_v4l2_sink *vs, const AVPacket *packet) { + AVStream *ostream = vs->format_ctx->streams[0]; + + uint8_t *extradata = av_malloc(packet->size * sizeof(uint8_t)); + if (!extradata) { + LOGC("Could not allocate extradata"); + return false; + } + + // copy the first packet to the extra data + memcpy(extradata, packet->data, packet->size); + + ostream->codecpar->extradata = extradata; + ostream->codecpar->extradata_size = packet->size; + + int ret = avformat_write_header(vs->format_ctx, NULL); + if (ret < 0) { + LOGE("Failed to write header to %s", vs->device_name); + return false; + } + + return true; +} + +static void +rescale_packet(struct sc_v4l2_sink *vs, AVPacket *packet) { + AVStream *ostream = vs->format_ctx->streams[0]; + av_packet_rescale_ts(packet, SCRCPY_TIME_BASE, ostream->time_base); +} + +static bool +write_packet(struct sc_v4l2_sink *vs, AVPacket *packet) { + if (!vs->header_written) { + bool ok = write_header(vs, packet); + if (!ok) { + return false; + } + vs->header_written = true; + return true; + } + + rescale_packet(vs, packet); + + bool ok = av_write_frame(vs->format_ctx, packet) >= 0; + + // Failing to write the last frame is not very serious, no future frame may + // depend on it, so the resulting file will still be valid + (void) ok; + + return true; +} + +static bool +encode_and_write_frame(struct sc_v4l2_sink *vs, const AVFrame *frame) { + int ret = avcodec_send_frame(vs->encoder_ctx, frame); + if (ret < 0 && ret != AVERROR(EAGAIN)) { + LOGE("Could not send v4l2 video frame: %d", ret); + return false; + } + + AVPacket *packet = &vs->packet; + ret = avcodec_receive_packet(vs->encoder_ctx, packet); + if (ret == 0) { + // A packet was received + + bool ok = write_packet(vs, packet); + if (!ok) { + LOGW("Could not send packet to v4l2 sink"); + return false; + } + av_packet_unref(packet); + } else if (ret != AVERROR(EAGAIN)) { + LOGE("Could not receive v4l2 video packet: %d", ret); + return false; + } + + return true; +} + +static int +run_v4l2_sink(void *data) { + struct sc_v4l2_sink *vs = data; + + for (;;) { + sc_mutex_lock(&vs->mutex); + + while (!vs->stopped && vs->vb.pending_frame_consumed) { + sc_cond_wait(&vs->cond, &vs->mutex); + } + + if (vs->stopped) { + sc_mutex_unlock(&vs->mutex); + break; + } + + sc_mutex_unlock(&vs->mutex); + + video_buffer_consume(&vs->vb, vs->frame); + bool ok = encode_and_write_frame(vs, vs->frame); + if (!ok) { + LOGE("Could not send frame to v4l2 sink"); + break; + } + } + + LOGD("V4l2 thread ended"); + + return 0; +} + +static bool +sc_v4l2_sink_open(struct sc_v4l2_sink *vs) { + bool ok = video_buffer_init(&vs->vb); + if (!ok) { + return false; + } + + ok = sc_mutex_init(&vs->mutex); + if (!ok) { + LOGC("Could not create mutex"); + goto error_video_buffer_destroy; + } + + ok = sc_cond_init(&vs->cond); + if (!ok) { + LOGC("Could not create cond"); + goto error_mutex_destroy; + } + + // FIXME + const AVOutputFormat *format = find_muxer("video4linux2,v4l2"); + if (!format) { + LOGE("Could not find v4l2 muxer"); + goto error_cond_destroy; + } + + const AVCodec *encoder = avcodec_find_encoder(AV_CODEC_ID_RAWVIDEO); + if (!encoder) { + LOGE("Raw video encoder not found"); + return false; + } + + vs->format_ctx = avformat_alloc_context(); + if (!vs->format_ctx) { + LOGE("Could not allocate v4l2 output context"); + return false; + } + + // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate() + // returns (on purpose) a pointer-to-const, but AVFormatContext.oformat + // still expects a pointer-to-non-const (it has not be updated accordingly) + // + vs->format_ctx->oformat = (AVOutputFormat *) format; + vs->format_ctx->url = strdup(vs->device_name); + if (!vs->format_ctx->url) { + LOGE("Could not strdup v4l2 device name"); + goto error_avformat_free_context; + return false; + } + + AVStream *ostream = avformat_new_stream(vs->format_ctx, encoder); + if (!ostream) { + LOGE("Could not allocate new v4l2 stream"); + goto error_avformat_free_context; + return false; + } + + ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + ostream->codecpar->codec_id = encoder->id; + ostream->codecpar->format = AV_PIX_FMT_YUV420P; + ostream->codecpar->width = vs->frame_size.width; + ostream->codecpar->height = vs->frame_size.height; + + int ret = avio_open(&vs->format_ctx->pb, vs->device_name, AVIO_FLAG_WRITE); + if (ret < 0) { + LOGE("Failed to open output device: %s", vs->device_name); + // ostream will be cleaned up during context cleaning + goto error_avformat_free_context; + } + + vs->encoder_ctx = avcodec_alloc_context3(encoder); + if (!vs->encoder_ctx) { + LOGC("Could not allocate codec context for v4l2"); + goto error_avio_close; + } + + vs->encoder_ctx->width = vs->frame_size.width; + vs->encoder_ctx->height = vs->frame_size.height; + vs->encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P; + vs->encoder_ctx->time_base.num = 1; + vs->encoder_ctx->time_base.den = 1; + + if (avcodec_open2(vs->encoder_ctx, encoder, NULL) < 0) { + LOGE("Could not open codec for v4l2"); + goto error_avcodec_free_context; + } + + vs->frame = av_frame_alloc(); + if (!vs->frame) { + LOGE("Could not create v4l2 frame"); + goto error_avcodec_close; + } + + LOGD("Starting v4l2 thread"); + ok = sc_thread_create(&vs->thread, run_v4l2_sink, "v4l2", vs); + if (!ok) { + LOGC("Could not start v4l2 thread"); + goto error_av_frame_free; + } + + vs->header_written = false; + vs->stopped = false; + + LOGI("v4l2 sink started to device: %s", vs->device_name); + + return true; + +error_av_frame_free: + av_frame_free(&vs->frame); +error_avcodec_close: + avcodec_close(vs->encoder_ctx); +error_avcodec_free_context: + avcodec_free_context(&vs->encoder_ctx); +error_avio_close: + avio_close(vs->format_ctx->pb); +error_avformat_free_context: + avformat_free_context(vs->format_ctx); +error_cond_destroy: + sc_cond_destroy(&vs->cond); +error_mutex_destroy: + sc_mutex_destroy(&vs->mutex); +error_video_buffer_destroy: + video_buffer_destroy(&vs->vb); + + return false; +} + +static void +sc_v4l2_sink_close(struct sc_v4l2_sink *vs) { + sc_mutex_lock(&vs->mutex); + vs->stopped = true; + sc_cond_signal(&vs->cond); + sc_mutex_unlock(&vs->mutex); + + sc_thread_join(&vs->thread, NULL); + + av_frame_free(&vs->frame); + avcodec_close(vs->encoder_ctx); + avcodec_free_context(&vs->encoder_ctx); + avio_close(vs->format_ctx->pb); + avformat_free_context(vs->format_ctx); + sc_cond_destroy(&vs->cond); + sc_mutex_destroy(&vs->mutex); + video_buffer_destroy(&vs->vb); +} + +static bool +sc_v4l2_sink_push(struct sc_v4l2_sink *vs, const AVFrame *frame) { + bool ok = video_buffer_push(&vs->vb, frame, NULL); + if (!ok) { + return false; + } + + // signal possible change of vs->vb.pending_frame_consumed + sc_cond_signal(&vs->cond); + + return true; +} + +static bool +sc_v4l2_frame_sink_open(struct sc_frame_sink *sink) { + struct sc_v4l2_sink *vs = DOWNCAST(sink); + return sc_v4l2_sink_open(vs); +} + +static void +sc_v4l2_frame_sink_close(struct sc_frame_sink *sink) { + struct sc_v4l2_sink *vs = DOWNCAST(sink); + sc_v4l2_sink_close(vs); +} + +static bool +sc_v4l2_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { + struct sc_v4l2_sink *vs = DOWNCAST(sink); + return sc_v4l2_sink_push(vs, frame); +} + +bool +sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name, + struct size frame_size) { + vs->device_name = strdup(device_name); + if (!vs->device_name) { + LOGE("Could not strdup v4l2 device name"); + return false; + } + + vs->frame_size = frame_size; + + static const struct sc_frame_sink_ops ops = { + .open = sc_v4l2_frame_sink_open, + .close = sc_v4l2_frame_sink_close, + .push = sc_v4l2_frame_sink_push, + }; + + vs->frame_sink.ops = &ops; + + return true; +} + +void +sc_v4l2_sink_destroy(struct sc_v4l2_sink *vs) { + free(vs->device_name); +} diff --git a/app/src/v4l2_sink.h b/app/src/v4l2_sink.h new file mode 100644 index 0000000000..9d2ee1492e --- /dev/null +++ b/app/src/v4l2_sink.h @@ -0,0 +1,39 @@ +#ifndef SC_V4L2_SINK_H +#define SC_V4L2_SINK_H + +#include "common.h" + +#include "coords.h" +#include "trait/frame_sink.h" +#include "video_buffer.h" + +#include + +struct sc_v4l2_sink { + struct sc_frame_sink frame_sink; // frame sink trait + + struct video_buffer vb; + AVFormatContext *format_ctx; + AVCodecContext *encoder_ctx; + + char *device_name; + struct size frame_size; + + sc_thread thread; + sc_mutex mutex; + sc_cond cond; + bool stopped; + bool header_written; + + AVFrame *frame; + AVPacket packet; +}; + +bool +sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name, + struct size frame_size); + +void +sc_v4l2_sink_destroy(struct sc_v4l2_sink *vs); + +#endif From 59cd1f55ade8b179447762040e06d73a200547f2 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 19 Apr 2021 20:12:08 +0200 Subject: [PATCH 30/30] Document v4l2 sink in README --- README.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1ac5fd0948..531fbf2cfa 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,9 @@ error will give the available encoders: scrcpy --encoder _ ``` -### Recording +### Capture + +#### Recording It is possible to record the screen while mirroring: @@ -250,6 +252,58 @@ variation] does not impact the recorded file. [packet delay variation]: https://en.wikipedia.org/wiki/Packet_delay_variation +#### v4l2loopback + +On Linux, it is possible to send the video stream to a v4l2 loopback device, so +that the Android device can be opened like a webcam by any v4l2-capable tool. + +The module `v4l2loopback` must be installed: + +```bash +sudo apt install v4l2loopback-dkms +``` + +To create a v4l2 device: + +```bash +sudo modprobe v4l2loopback +``` + +This will create a new video device in `/dev/videoN`, where `N` is an integer +(more [options](https://github.com/umlaeute/v4l2loopback#options) are available +to create several devices or devices with specific IDs). + +To list the enabled devices: + +```bash +# requires v4l-utils package +v4l2-ctl --list-devices + +# simple but might be sufficient +ls /dev/video* +``` + +To start scrcpy using a v4l2 sink: + +```bash +scrcpy --v4l2-sink=/dev/videoN +scrcpy --v4l2-sink=/dev/videoN -N # --no-display to disable mirroring window +``` + +(replace `N` by the device ID, check with `ls /dev/video*`) + +Once enabled, you can open your video stream with a v4l2-capable tool: + +```bash +ffplay -i /dev/videoN +vlc v4l2:///dev/videoN # VLC might add some buffering delay +``` + +For example, you could capture the video within [OBS]. + +[OBS]: https://obsproject.com/ + + ### Connection #### Wireless