From a20615066d435cd3160f80196d25a2d7fb3e8ccc Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 27 Feb 2023 21:43:34 +0100 Subject: [PATCH 001/118] Simplify libusb prebuilt scripts In theory, include/ might be slightly different for win32 and win64 builds. Use each one separately to simplify. --- app/meson.build | 5 ++--- app/prebuilt-deps/prepare-libusb.sh | 7 +++---- cross_win32.txt | 3 +-- cross_win64.txt | 3 +-- release.mk | 4 ++-- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/app/meson.build b/app/meson.build index 5d779756df..a16a000b41 100644 --- a/app/meson.build +++ b/app/meson.build @@ -144,9 +144,8 @@ else ) prebuilt_libusb = meson.get_cross_property('prebuilt_libusb') - prebuilt_libusb_root = meson.get_cross_property('prebuilt_libusb_root') - libusb_bin_dir = meson.current_source_dir() + '/prebuilt-deps/data/' + prebuilt_libusb - libusb_include_dir = 'prebuilt-deps/data/' + prebuilt_libusb_root + '/include' + libusb_bin_dir = meson.current_source_dir() + '/prebuilt-deps/data/' + prebuilt_libusb + '/bin' + libusb_include_dir = 'prebuilt-deps/data/' + prebuilt_libusb + '/include' libusb = declare_dependency( dependencies: [ diff --git a/app/prebuilt-deps/prepare-libusb.sh b/app/prebuilt-deps/prepare-libusb.sh index a0c3721d84..47cf1df4ea 100755 --- a/app/prebuilt-deps/prepare-libusb.sh +++ b/app/prebuilt-deps/prepare-libusb.sh @@ -22,13 +22,12 @@ get_file "https://github.com/libusb/libusb/releases/download/v1.0.26/$FILENAME" mkdir "$DEP_DIR" cd "$DEP_DIR" -# include/ is the same in all folders of the archive 7z x "../$FILENAME" \ libusb-1.0.26-binaries/libusb-MinGW-Win32/bin/msys-usb-1.0.dll \ + libusb-1.0.26-binaries/libusb-MinGW-Win32/include/ \ libusb-1.0.26-binaries/libusb-MinGW-x64/bin/msys-usb-1.0.dll \ libusb-1.0.26-binaries/libusb-MinGW-x64/include/ -mv libusb-1.0.26-binaries/libusb-MinGW-Win32/bin MinGW-Win32 -mv libusb-1.0.26-binaries/libusb-MinGW-x64/bin MinGW-x64 -mv libusb-1.0.26-binaries/libusb-MinGW-x64/include . +mv libusb-1.0.26-binaries/libusb-MinGW-Win32 . +mv libusb-1.0.26-binaries/libusb-MinGW-x64 . rm -rf libusb-1.0.26-binaries diff --git a/cross_win32.txt b/cross_win32.txt index 3222694957..e50c0bc897 100644 --- a/cross_win32.txt +++ b/cross_win32.txt @@ -21,5 +21,4 @@ ffmpeg_avformat = 'avformat-58' ffmpeg_avutil = 'avutil-56' prebuilt_ffmpeg = 'ffmpeg-win32-4.3.1' prebuilt_sdl2 = 'SDL2-2.26.1/i686-w64-mingw32' -prebuilt_libusb_root = 'libusb-1.0.26' -prebuilt_libusb = 'libusb-1.0.26/MinGW-Win32' +prebuilt_libusb = 'libusb-1.0.26/libusb-MinGW-Win32' diff --git a/cross_win64.txt b/cross_win64.txt index 4dde4ab1ea..2dc876a6ed 100644 --- a/cross_win64.txt +++ b/cross_win64.txt @@ -21,5 +21,4 @@ ffmpeg_avformat = 'avformat-59' ffmpeg_avutil = 'avutil-57' prebuilt_ffmpeg = 'ffmpeg-win64-5.1.2' prebuilt_sdl2 = 'SDL2-2.26.1/x86_64-w64-mingw32' -prebuilt_libusb_root = 'libusb-1.0.26' -prebuilt_libusb = 'libusb-1.0.26/MinGW-x64' +prebuilt_libusb = 'libusb-1.0.26/libusb-MinGW-x64' diff --git a/release.mk b/release.mk index 06443e1ab7..67578022b2 100644 --- a/release.mk +++ b/release.mk @@ -109,7 +109,7 @@ dist-win32: build-server build-win32 cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinUsbApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/SDL2-2.26.1/i686-w64-mingw32/bin/SDL2.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/libusb-1.0.26/MinGW-Win32/msys-usb-1.0.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/libusb-1.0.26/libusb-MinGW-Win32/bin/msys-usb-1.0.dll "$(DIST)/$(WIN32_TARGET_DIR)/" dist-win64: build-server build-win64 mkdir -p "$(DIST)/$(WIN64_TARGET_DIR)" @@ -128,7 +128,7 @@ dist-win64: build-server build-win64 cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinUsbApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/SDL2-2.26.1/x86_64-w64-mingw32/bin/SDL2.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/libusb-1.0.26/MinGW-x64/msys-usb-1.0.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/libusb-1.0.26/libusb-MinGW-x64/bin/msys-usb-1.0.dll "$(DIST)/$(WIN64_TARGET_DIR)/" zip-win32: dist-win32 cd "$(DIST)"; \ From 0fc62bfcd63fdccba759c1421ca7e4b5a2278cb9 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 27 Feb 2023 21:43:48 +0100 Subject: [PATCH 002/118] Use minimal prebuilt FFmpeg for Windows On the scrcpy-deps repo, I built FFmpeg 5.1.2 binaries for Windows with only the features used by scrcpy. For comparison, here are the sizes of the dll for FFmpeg 5.1.2: - before: 89M - after: 4.7M It also allows to upgrade the old FFmpeg version (4.3.1) used for win32. Refs Refs --- app/meson.build | 11 ++---- app/prebuilt-deps/prepare-ffmpeg-win32.sh | 45 ----------------------- app/prebuilt-deps/prepare-ffmpeg-win64.sh | 36 ------------------ app/prebuilt-deps/prepare-ffmpeg.sh | 30 +++++++++++++++ cross_win32.txt | 5 +-- cross_win64.txt | 5 +-- release.mk | 36 ++++++++---------- 7 files changed, 50 insertions(+), 118 deletions(-) delete mode 100755 app/prebuilt-deps/prepare-ffmpeg-win32.sh delete mode 100755 app/prebuilt-deps/prepare-ffmpeg-win64.sh create mode 100755 app/prebuilt-deps/prepare-ffmpeg.sh diff --git a/app/meson.build b/app/meson.build index a16a000b41..f070db7204 100644 --- a/app/meson.build +++ b/app/meson.build @@ -129,16 +129,11 @@ else ffmpeg_bin_dir = meson.current_source_dir() + '/prebuilt-deps/data/' + prebuilt_ffmpeg + '/bin' ffmpeg_include_dir = 'prebuilt-deps/data/' + prebuilt_ffmpeg + '/include' - # ffmpeg versions are different for win32 and win64 builds - ffmpeg_avcodec = meson.get_cross_property('ffmpeg_avcodec') - ffmpeg_avformat = meson.get_cross_property('ffmpeg_avformat') - ffmpeg_avutil = meson.get_cross_property('ffmpeg_avutil') - ffmpeg = declare_dependency( dependencies: [ - cc.find_library(ffmpeg_avcodec, dirs: ffmpeg_bin_dir), - cc.find_library(ffmpeg_avformat, dirs: ffmpeg_bin_dir), - cc.find_library(ffmpeg_avutil, dirs: ffmpeg_bin_dir), + cc.find_library('avcodec-59', dirs: ffmpeg_bin_dir), + cc.find_library('avformat-59', dirs: ffmpeg_bin_dir), + cc.find_library('avutil-57', dirs: ffmpeg_bin_dir), ], include_directories: include_directories(ffmpeg_include_dir) ) diff --git a/app/prebuilt-deps/prepare-ffmpeg-win32.sh b/app/prebuilt-deps/prepare-ffmpeg-win32.sh deleted file mode 100755 index 2a6a38413e..0000000000 --- a/app/prebuilt-deps/prepare-ffmpeg-win32.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -e -DIR=$(dirname ${BASH_SOURCE[0]}) -cd "$DIR" -. common -mkdir -p "$PREBUILT_DATA_DIR" -cd "$PREBUILT_DATA_DIR" - -DEP_DIR=ffmpeg-win32-4.3.1 - -FILENAME_SHARED=ffmpeg-4.3.1-win32-shared.zip -SHA256SUM_SHARED=357af9901a456f4dcbacd107e83a934d344c9cb07ddad8aaf80612eeab7d26d2 - -FILENAME_DEV=ffmpeg-4.3.1-win32-dev.zip -SHA256SUM_DEV=230efb08e9bcf225bd474da29676c70e591fc94d8790a740ca801408fddcb78b - -if [[ -d "$DEP_DIR" ]] -then - echo "$DEP_DIR" found - exit 0 -fi - -get_file "https://github.com/Genymobile/scrcpy/releases/download/v1.16/$FILENAME_SHARED" \ - "$FILENAME_SHARED" "$SHA256SUM_SHARED" -get_file "https://github.com/Genymobile/scrcpy/releases/download/v1.16/$FILENAME_DEV" \ - "$FILENAME_DEV" "$SHA256SUM_DEV" - -mkdir "$DEP_DIR" -cd "$DEP_DIR" - -ZIP_PREFIX_SHARED=ffmpeg-4.3.1-win32-shared -unzip "../$FILENAME_SHARED" \ - "$ZIP_PREFIX_SHARED"/bin/avutil-56.dll \ - "$ZIP_PREFIX_SHARED"/bin/avcodec-58.dll \ - "$ZIP_PREFIX_SHARED"/bin/avformat-58.dll \ - "$ZIP_PREFIX_SHARED"/bin/swresample-3.dll \ - "$ZIP_PREFIX_SHARED"/bin/swscale-5.dll - -ZIP_PREFIX_DEV=ffmpeg-4.3.1-win32-dev -unzip "../$FILENAME_DEV" \ - "$ZIP_PREFIX_DEV/include/*" - -mv "$ZIP_PREFIX_SHARED"/* . -mv "$ZIP_PREFIX_DEV"/* . -rmdir "$ZIP_PREFIX_SHARED" "$ZIP_PREFIX_DEV" diff --git a/app/prebuilt-deps/prepare-ffmpeg-win64.sh b/app/prebuilt-deps/prepare-ffmpeg-win64.sh deleted file mode 100755 index f5d56e6f4d..0000000000 --- a/app/prebuilt-deps/prepare-ffmpeg-win64.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -e -DIR=$(dirname ${BASH_SOURCE[0]}) -cd "$DIR" -. common -mkdir -p "$PREBUILT_DATA_DIR" -cd "$PREBUILT_DATA_DIR" - -VERSION=5.1.2 -DEP_DIR=ffmpeg-win64-$VERSION - -FILENAME=ffmpeg-$VERSION-full_build-shared.7z -SHA256SUM=d9eb97b72d7cfdae4d0f7eaea59ccffb8c364d67d88018ea715d5e2e193f00e9 - -if [[ -d "$DEP_DIR" ]] -then - echo "$DEP_DIR" found - exit 0 -fi - -get_file "https://github.com/GyanD/codexffmpeg/releases/download/$VERSION/$FILENAME" \ - "$FILENAME" "$SHA256SUM" - -mkdir "$DEP_DIR" -cd "$DEP_DIR" - -ZIP_PREFIX=ffmpeg-$VERSION-full_build-shared -7z x "../$FILENAME" \ - "$ZIP_PREFIX"/bin/avutil-57.dll \ - "$ZIP_PREFIX"/bin/avcodec-59.dll \ - "$ZIP_PREFIX"/bin/avformat-59.dll \ - "$ZIP_PREFIX"/bin/swresample-4.dll \ - "$ZIP_PREFIX"/bin/swscale-6.dll \ - "$ZIP_PREFIX"/include -mv "$ZIP_PREFIX"/* . -rmdir "$ZIP_PREFIX" diff --git a/app/prebuilt-deps/prepare-ffmpeg.sh b/app/prebuilt-deps/prepare-ffmpeg.sh new file mode 100755 index 0000000000..dc8b1ca2a6 --- /dev/null +++ b/app/prebuilt-deps/prepare-ffmpeg.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -e +DIR=$(dirname ${BASH_SOURCE[0]}) +cd "$DIR" +. common +mkdir -p "$PREBUILT_DATA_DIR" +cd "$PREBUILT_DATA_DIR" + +VERSION=5.1.2-scrcpy +DEP_DIR="ffmpeg-$VERSION" + +FILENAME="$DEP_DIR".7z +SHA256SUM=93f32ffc29ddb3466d669f7078d3fd8030c3388bc8a18bcfeefb6428fc5ceef1 + +if [[ -d "$DEP_DIR" ]] +then + echo "$DEP_DIR" found + exit 0 +fi + +get_file "https://github.com/rom1v/scrcpy-deps/releases/download/$VERSION/$FILENAME" \ + "$FILENAME" "$SHA256SUM" + +mkdir "$DEP_DIR" +cd "$DEP_DIR" + +ZIP_PREFIX=ffmpeg +7z x "../$FILENAME" +mv "$ZIP_PREFIX"/* . +rmdir "$ZIP_PREFIX" diff --git a/cross_win32.txt b/cross_win32.txt index e50c0bc897..f89463d962 100644 --- a/cross_win32.txt +++ b/cross_win32.txt @@ -16,9 +16,6 @@ cpu = 'i686' endian = 'little' [properties] -ffmpeg_avcodec = 'avcodec-58' -ffmpeg_avformat = 'avformat-58' -ffmpeg_avutil = 'avutil-56' -prebuilt_ffmpeg = 'ffmpeg-win32-4.3.1' +prebuilt_ffmpeg = 'ffmpeg-5.1.2-scrcpy/win32' prebuilt_sdl2 = 'SDL2-2.26.1/i686-w64-mingw32' prebuilt_libusb = 'libusb-1.0.26/libusb-MinGW-Win32' diff --git a/cross_win64.txt b/cross_win64.txt index 2dc876a6ed..c30c46f2da 100644 --- a/cross_win64.txt +++ b/cross_win64.txt @@ -16,9 +16,6 @@ cpu = 'x86_64' endian = 'little' [properties] -ffmpeg_avcodec = 'avcodec-59' -ffmpeg_avformat = 'avformat-59' -ffmpeg_avutil = 'avutil-57' -prebuilt_ffmpeg = 'ffmpeg-win64-5.1.2' +prebuilt_ffmpeg = 'ffmpeg-5.1.2-scrcpy/win64' prebuilt_sdl2 = 'SDL2-2.26.1/x86_64-w64-mingw32' prebuilt_libusb = 'libusb-1.0.26/libusb-MinGW-x64' diff --git a/release.mk b/release.mk index 67578022b2..9c2820c769 100644 --- a/release.mk +++ b/release.mk @@ -11,7 +11,7 @@ .PHONY: default clean \ test \ build-server \ - prepare-deps-win32 prepare-deps-win64 \ + prepare-deps \ build-win32 build-win64 \ dist-win32 dist-win64 \ zip-win32 zip-win64 \ @@ -62,19 +62,13 @@ build-server: meson setup "$(SERVER_BUILD_DIR)" --buildtype release -Dcompile_app=false ) ninja -C "$(SERVER_BUILD_DIR)" -prepare-deps-win32: +prepare-deps: @app/prebuilt-deps/prepare-adb.sh @app/prebuilt-deps/prepare-sdl.sh - @app/prebuilt-deps/prepare-ffmpeg-win32.sh + @app/prebuilt-deps/prepare-ffmpeg.sh @app/prebuilt-deps/prepare-libusb.sh -prepare-deps-win64: - @app/prebuilt-deps/prepare-adb.sh - @app/prebuilt-deps/prepare-sdl.sh - @app/prebuilt-deps/prepare-ffmpeg-win64.sh - @app/prebuilt-deps/prepare-libusb.sh - -build-win32: prepare-deps-win32 +build-win32: prepare-deps [ -d "$(WIN32_BUILD_DIR)" ] || ( mkdir "$(WIN32_BUILD_DIR)" && \ meson setup "$(WIN32_BUILD_DIR)" \ --cross-file cross_win32.txt \ @@ -83,7 +77,7 @@ build-win32: prepare-deps-win32 -Dportable=true ) ninja -C "$(WIN32_BUILD_DIR)" -build-win64: prepare-deps-win64 +build-win64: prepare-deps [ -d "$(WIN64_BUILD_DIR)" ] || ( mkdir "$(WIN64_BUILD_DIR)" && \ meson setup "$(WIN64_BUILD_DIR)" \ --cross-file cross_win64.txt \ @@ -100,11 +94,11 @@ dist-win32: build-server build-win32 cp app/data/scrcpy-noconsole.vbs "$(DIST)/$(WIN32_TARGET_DIR)" cp app/data/icon.png "$(DIST)/$(WIN32_TARGET_DIR)" cp app/data/open_a_terminal_here.bat "$(DIST)/$(WIN32_TARGET_DIR)" - cp app/prebuilt-deps/data/ffmpeg-win32-4.3.1/bin/avutil-56.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-win32-4.3.1/bin/avcodec-58.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-win32-4.3.1/bin/avformat-58.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-win32-4.3.1/bin/swresample-3.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-win32-4.3.1/bin/swscale-5.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/avutil-57.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/avcodec-59.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/avformat-59.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/swresample-4.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/zlib1.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/adb.exe "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinUsbApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" @@ -119,11 +113,11 @@ dist-win64: build-server build-win64 cp app/data/scrcpy-noconsole.vbs "$(DIST)/$(WIN64_TARGET_DIR)" cp app/data/icon.png "$(DIST)/$(WIN64_TARGET_DIR)" cp app/data/open_a_terminal_here.bat "$(DIST)/$(WIN64_TARGET_DIR)" - cp app/prebuilt-deps/data/ffmpeg-win64-5.1.2/bin/avutil-57.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-win64-5.1.2/bin/avcodec-59.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-win64-5.1.2/bin/avformat-59.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-win64-5.1.2/bin/swresample-4.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-win64-5.1.2/bin/swscale-6.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/avutil-57.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/avcodec-59.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/avformat-59.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/swresample-4.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/zlib1.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/adb.exe "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinUsbApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" From 9d60d7880bdc467545c883ccfc8d2609927894cb Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 28 Feb 2023 11:56:42 +0100 Subject: [PATCH 003/118] Upgrade FFmpeg (6.0) for Windows Use the latest version (specifically built for scrcpy). Refs --- app/meson.build | 6 +++--- app/prebuilt-deps/prepare-ffmpeg.sh | 4 ++-- cross_win32.txt | 2 +- cross_win64.txt | 2 +- release.mk | 20 ++++++++++---------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/app/meson.build b/app/meson.build index f070db7204..b6a772a903 100644 --- a/app/meson.build +++ b/app/meson.build @@ -131,9 +131,9 @@ else ffmpeg = declare_dependency( dependencies: [ - cc.find_library('avcodec-59', dirs: ffmpeg_bin_dir), - cc.find_library('avformat-59', dirs: ffmpeg_bin_dir), - cc.find_library('avutil-57', dirs: ffmpeg_bin_dir), + cc.find_library('avcodec-60', dirs: ffmpeg_bin_dir), + cc.find_library('avformat-60', dirs: ffmpeg_bin_dir), + cc.find_library('avutil-58', dirs: ffmpeg_bin_dir), ], include_directories: include_directories(ffmpeg_include_dir) ) diff --git a/app/prebuilt-deps/prepare-ffmpeg.sh b/app/prebuilt-deps/prepare-ffmpeg.sh index dc8b1ca2a6..10e33903c0 100755 --- a/app/prebuilt-deps/prepare-ffmpeg.sh +++ b/app/prebuilt-deps/prepare-ffmpeg.sh @@ -6,11 +6,11 @@ cd "$DIR" mkdir -p "$PREBUILT_DATA_DIR" cd "$PREBUILT_DATA_DIR" -VERSION=5.1.2-scrcpy +VERSION=6.0-scrcpy DEP_DIR="ffmpeg-$VERSION" FILENAME="$DEP_DIR".7z -SHA256SUM=93f32ffc29ddb3466d669f7078d3fd8030c3388bc8a18bcfeefb6428fc5ceef1 +SHA256SUM=f3956295b4325a84aada05447ba3f314fbed96697811666d495de4de40d59f98 if [[ -d "$DEP_DIR" ]] then diff --git a/cross_win32.txt b/cross_win32.txt index f89463d962..deb70b77c6 100644 --- a/cross_win32.txt +++ b/cross_win32.txt @@ -16,6 +16,6 @@ cpu = 'i686' endian = 'little' [properties] -prebuilt_ffmpeg = 'ffmpeg-5.1.2-scrcpy/win32' +prebuilt_ffmpeg = 'ffmpeg-6.0-scrcpy/win32' prebuilt_sdl2 = 'SDL2-2.26.1/i686-w64-mingw32' prebuilt_libusb = 'libusb-1.0.26/libusb-MinGW-Win32' diff --git a/cross_win64.txt b/cross_win64.txt index c30c46f2da..3c4409dc7a 100644 --- a/cross_win64.txt +++ b/cross_win64.txt @@ -16,6 +16,6 @@ cpu = 'x86_64' endian = 'little' [properties] -prebuilt_ffmpeg = 'ffmpeg-5.1.2-scrcpy/win64' +prebuilt_ffmpeg = 'ffmpeg-6.0-scrcpy/win64' prebuilt_sdl2 = 'SDL2-2.26.1/x86_64-w64-mingw32' prebuilt_libusb = 'libusb-1.0.26/libusb-MinGW-x64' diff --git a/release.mk b/release.mk index 9c2820c769..f1084bd4f6 100644 --- a/release.mk +++ b/release.mk @@ -94,11 +94,11 @@ dist-win32: build-server build-win32 cp app/data/scrcpy-noconsole.vbs "$(DIST)/$(WIN32_TARGET_DIR)" cp app/data/icon.png "$(DIST)/$(WIN32_TARGET_DIR)" cp app/data/open_a_terminal_here.bat "$(DIST)/$(WIN32_TARGET_DIR)" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/avutil-57.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/avcodec-59.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/avformat-59.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/swresample-4.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win32/bin/zlib1.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/avutil-58.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/avcodec-60.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/avformat-60.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/swresample-4.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/zlib1.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/adb.exe "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinUsbApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" @@ -113,11 +113,11 @@ dist-win64: build-server build-win64 cp app/data/scrcpy-noconsole.vbs "$(DIST)/$(WIN64_TARGET_DIR)" cp app/data/icon.png "$(DIST)/$(WIN64_TARGET_DIR)" cp app/data/open_a_terminal_here.bat "$(DIST)/$(WIN64_TARGET_DIR)" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/avutil-57.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/avcodec-59.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/avformat-59.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/swresample-4.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-5.1.2-scrcpy/win64/bin/zlib1.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/avutil-58.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/avcodec-60.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/avformat-60.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/swresample-4.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/zlib1.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/adb.exe "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinUsbApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" From f30fd963a104649e7d388cc68f3fbc5d635baa4a Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 19:46:13 +0100 Subject: [PATCH 004/118] Upgrade FFmpeg custom builds for Windows Use a build which includes the pcm_s16le decoder, to support RAW audio. Refs --- app/prebuilt-deps/prepare-ffmpeg.sh | 4 ++-- cross_win32.txt | 2 +- cross_win64.txt | 2 +- release.mk | 20 ++++++++++---------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/app/prebuilt-deps/prepare-ffmpeg.sh b/app/prebuilt-deps/prepare-ffmpeg.sh index 10e33903c0..b156099a45 100755 --- a/app/prebuilt-deps/prepare-ffmpeg.sh +++ b/app/prebuilt-deps/prepare-ffmpeg.sh @@ -6,11 +6,11 @@ cd "$DIR" mkdir -p "$PREBUILT_DATA_DIR" cd "$PREBUILT_DATA_DIR" -VERSION=6.0-scrcpy +VERSION=6.0-scrcpy-2 DEP_DIR="ffmpeg-$VERSION" FILENAME="$DEP_DIR".7z -SHA256SUM=f3956295b4325a84aada05447ba3f314fbed96697811666d495de4de40d59f98 +SHA256SUM=98ef97f8607c97a5c4f9c5a0a991b78f105d002a3619145011d16ffb92501b14 if [[ -d "$DEP_DIR" ]] then diff --git a/cross_win32.txt b/cross_win32.txt index deb70b77c6..a02e798a6a 100644 --- a/cross_win32.txt +++ b/cross_win32.txt @@ -16,6 +16,6 @@ cpu = 'i686' endian = 'little' [properties] -prebuilt_ffmpeg = 'ffmpeg-6.0-scrcpy/win32' +prebuilt_ffmpeg = 'ffmpeg-6.0-scrcpy-2/win32' prebuilt_sdl2 = 'SDL2-2.26.1/i686-w64-mingw32' prebuilt_libusb = 'libusb-1.0.26/libusb-MinGW-Win32' diff --git a/cross_win64.txt b/cross_win64.txt index 3c4409dc7a..126de36e3c 100644 --- a/cross_win64.txt +++ b/cross_win64.txt @@ -16,6 +16,6 @@ cpu = 'x86_64' endian = 'little' [properties] -prebuilt_ffmpeg = 'ffmpeg-6.0-scrcpy/win64' +prebuilt_ffmpeg = 'ffmpeg-6.0-scrcpy-2/win64' prebuilt_sdl2 = 'SDL2-2.26.1/x86_64-w64-mingw32' prebuilt_libusb = 'libusb-1.0.26/libusb-MinGW-x64' diff --git a/release.mk b/release.mk index f1084bd4f6..75e5a9c0b1 100644 --- a/release.mk +++ b/release.mk @@ -94,11 +94,11 @@ dist-win32: build-server build-win32 cp app/data/scrcpy-noconsole.vbs "$(DIST)/$(WIN32_TARGET_DIR)" cp app/data/icon.png "$(DIST)/$(WIN32_TARGET_DIR)" cp app/data/open_a_terminal_here.bat "$(DIST)/$(WIN32_TARGET_DIR)" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/avutil-58.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/avcodec-60.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/avformat-60.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/swresample-4.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win32/bin/zlib1.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win32/bin/avutil-58.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win32/bin/avcodec-60.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win32/bin/avformat-60.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win32/bin/swresample-4.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win32/bin/zlib1.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/adb.exe "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinUsbApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" @@ -113,11 +113,11 @@ dist-win64: build-server build-win64 cp app/data/scrcpy-noconsole.vbs "$(DIST)/$(WIN64_TARGET_DIR)" cp app/data/icon.png "$(DIST)/$(WIN64_TARGET_DIR)" cp app/data/open_a_terminal_here.bat "$(DIST)/$(WIN64_TARGET_DIR)" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/avutil-58.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/avcodec-60.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/avformat-60.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/swresample-4.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy/win64/bin/zlib1.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win64/bin/avutil-58.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win64/bin/avcodec-60.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win64/bin/avformat-60.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win64/bin/swresample-4.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp app/prebuilt-deps/data/ffmpeg-6.0-scrcpy-2/win64/bin/zlib1.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/adb.exe "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp app/prebuilt-deps/data/platform-tools-33.0.3/AdbWinUsbApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" From 10e8295aea0163f80d88eab234be5b918a7571eb Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 23 Feb 2023 12:49:42 +0100 Subject: [PATCH 005/118] Move FFmpeg callback initialization Configure FFmpeg log redirection on start from a log helper. --- app/src/main.c | 2 ++ app/src/scrcpy.c | 39 --------------------------------------- app/src/util/log.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ app/src/util/log.h | 3 +++ 4 files changed, 49 insertions(+), 39 deletions(-) diff --git a/app/src/main.c b/app/src/main.c index 185f1d8fe5..cc3a85a743 100644 --- a/app/src/main.c +++ b/app/src/main.c @@ -75,6 +75,8 @@ main_scrcpy(int argc, char *argv[]) { return SCRCPY_EXIT_FAILURE; } + sc_log_configure(); + #ifdef HAVE_USB enum scrcpy_exit_code ret = args.opts.otg ? scrcpy_otg(&args.opts) : scrcpy(&args.opts); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 8932dd1d04..afd04d6da7 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -198,43 +198,6 @@ await_for_server(bool *connected) { return false; } -static SDL_LogPriority -sdl_priority_from_av_level(int level) { - switch (level) { - case AV_LOG_PANIC: - case AV_LOG_FATAL: - return SDL_LOG_PRIORITY_CRITICAL; - case AV_LOG_ERROR: - return SDL_LOG_PRIORITY_ERROR; - case AV_LOG_WARNING: - return SDL_LOG_PRIORITY_WARN; - case AV_LOG_INFO: - return SDL_LOG_PRIORITY_INFO; - } - // do not forward others, which are too verbose - return 0; -} - -static void -av_log_callback(void *avcl, int level, const char *fmt, va_list vl) { - (void) avcl; - SDL_LogPriority priority = sdl_priority_from_av_level(level); - if (priority == 0) { - return; - } - - size_t fmt_len = strlen(fmt); - char *local_fmt = malloc(fmt_len + 10); - if (!local_fmt) { - LOG_OOM(); - return; - } - memcpy(local_fmt, "[FFmpeg] ", 9); // do not write the final '\0' - memcpy(local_fmt + 9, fmt, fmt_len + 1); // include '\0' - SDL_LogMessageV(SDL_LOG_CATEGORY_VIDEO, priority, local_fmt, vl); - free(local_fmt); -} - static void sc_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, void *userdata) { (void) demuxer; @@ -426,8 +389,6 @@ scrcpy(struct scrcpy_options *options) { recorder_initialized = true; } - av_log_set_callback(av_log_callback); - static const struct sc_demuxer_callbacks demuxer_cbs = { .on_ended = sc_demuxer_on_ended, }; diff --git a/app/src/util/log.c b/app/src/util/log.c index 72cd287760..ef11d2d164 100644 --- a/app/src/util/log.c +++ b/app/src/util/log.c @@ -4,6 +4,7 @@ # include #endif #include +#include static SDL_LogPriority log_level_sc_to_sdl(enum sc_log_level level) { @@ -85,3 +86,46 @@ sc_log_windows_error(const char *prefix, int error) { return true; } #endif + +static SDL_LogPriority +sdl_priority_from_av_level(int level) { + switch (level) { + case AV_LOG_PANIC: + case AV_LOG_FATAL: + return SDL_LOG_PRIORITY_CRITICAL; + case AV_LOG_ERROR: + return SDL_LOG_PRIORITY_ERROR; + case AV_LOG_WARNING: + return SDL_LOG_PRIORITY_WARN; + case AV_LOG_INFO: + return SDL_LOG_PRIORITY_INFO; + } + // do not forward others, which are too verbose + return 0; +} + +static void +sc_av_log_callback(void *avcl, int level, const char *fmt, va_list vl) { + (void) avcl; + SDL_LogPriority priority = sdl_priority_from_av_level(level); + if (priority == 0) { + return; + } + + size_t fmt_len = strlen(fmt); + char *local_fmt = malloc(fmt_len + 10); + if (!local_fmt) { + LOG_OOM(); + return; + } + memcpy(local_fmt, "[FFmpeg] ", 9); // do not write the final '\0' + memcpy(local_fmt + 9, fmt, fmt_len + 1); // include '\0' + SDL_LogMessageV(SDL_LOG_CATEGORY_VIDEO, priority, local_fmt, vl); + free(local_fmt); +} + +void +sc_log_configure() { + // Redirect FFmpeg logs to SDL logs + av_log_set_callback(sc_av_log_callback); +} diff --git a/app/src/util/log.h b/app/src/util/log.h index 6bd8506c12..8e1b73a2c1 100644 --- a/app/src/util/log.h +++ b/app/src/util/log.h @@ -35,4 +35,7 @@ bool sc_log_windows_error(const char *prefix, int error); #endif +void +sc_log_configure(); + #endif From e30e692b3640e5e8db4fb5c31445194d1fad31b6 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 23 Feb 2023 13:25:04 +0100 Subject: [PATCH 006/118] Print FFmpeg logs FFmpeg logs are redirected to a specific SDL log category. Initialize the log level for this category to print them as expected. --- app/src/util/log.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/src/util/log.c b/app/src/util/log.c index ef11d2d164..25b1f26e8c 100644 --- a/app/src/util/log.c +++ b/app/src/util/log.c @@ -48,6 +48,7 @@ void sc_set_log_level(enum sc_log_level level) { SDL_LogPriority sdl_log = log_level_sc_to_sdl(level); SDL_LogSetPriority(SDL_LOG_CATEGORY_APPLICATION, sdl_log); + SDL_LogSetPriority(SDL_LOG_CATEGORY_CUSTOM, sdl_log); } enum sc_log_level @@ -120,7 +121,7 @@ sc_av_log_callback(void *avcl, int level, const char *fmt, va_list vl) { } memcpy(local_fmt, "[FFmpeg] ", 9); // do not write the final '\0' memcpy(local_fmt + 9, fmt, fmt_len + 1); // include '\0' - SDL_LogMessageV(SDL_LOG_CATEGORY_VIDEO, priority, local_fmt, vl); + SDL_LogMessageV(SDL_LOG_CATEGORY_CUSTOM, priority, local_fmt, vl); free(local_fmt); } From c78254fcd1c8d88fbc258d2a50ee925c282c5c64 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 22 Feb 2023 18:41:22 +0100 Subject: [PATCH 007/118] Split server stop() and join() For consistency with the other components, call stop() and join() separately. This allows to stop all components, then join them all. --- app/src/scrcpy.c | 4 ++++ app/src/server.c | 3 +++ app/src/server.h | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index afd04d6da7..e96fa187c4 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -698,6 +698,10 @@ scrcpy(struct scrcpy_options *options) { sc_file_pusher_destroy(&s->file_pusher); } + if (server_started) { + sc_server_join(&s->server); + } + sc_server_destroy(&s->server); return ret; diff --git a/app/src/server.c b/app/src/server.c index c916497298..413f02ee4a 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -909,7 +909,10 @@ sc_server_stop(struct sc_server *server) { sc_cond_signal(&server->cond_stopped); sc_intr_interrupt(&server->intr); sc_mutex_unlock(&server->mutex); +} +void +sc_server_join(struct sc_server *server) { sc_thread_join(&server->thread, NULL); } diff --git a/app/src/server.h b/app/src/server.h index d6b1401e0c..c05b1e5b8d 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -108,6 +108,10 @@ sc_server_start(struct sc_server *server); void sc_server_stop(struct sc_server *server); +// join the server thread +void +sc_server_join(struct sc_server *server); + // close and release sockets void sc_server_destroy(struct sc_server *server); From 9f8e96e895b8af51be2b1123a483c7d4d3decdf7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 22 Feb 2023 19:08:24 +0100 Subject: [PATCH 008/118] Fix --no-clipboard-autosync bash completion Fix typo. --- app/data/bash-completion/scrcpy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 0ddd8bcb2f..4590b6a803 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -24,7 +24,7 @@ _scrcpy() { -M --hid-mouse -m --max-size= --no-cleanup - --no-clipboard-on-error + --no-clipboard-autosync --no-downsize-on-error -n --no-control -N --no-display From b43938fa66bb854e45c97a76b4600edecdb0fb9b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 19:36:46 +0100 Subject: [PATCH 009/118] Do not print stacktraces when unnecessary User-friendly error messages are printed on specific configuration exceptions. In that case, do not print the stacktrace. Also handle the user-friendly error message directly where the error occurs, and print multiline messages in a single log call, to avoid confusing interleaving. --- .../scrcpy/ConfigurationException.java | 7 ++++ .../java/com/genymobile/scrcpy/Device.java | 18 +++++++++-- .../scrcpy/InvalidDisplayIdException.java | 21 ------------ .../scrcpy/InvalidEncoderException.java | 23 ------------- .../com/genymobile/scrcpy/ScreenEncoder.java | 20 +++++++++--- .../java/com/genymobile/scrcpy/Server.java | 32 ++++--------------- 6 files changed, 44 insertions(+), 77 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/ConfigurationException.java delete mode 100644 server/src/main/java/com/genymobile/scrcpy/InvalidDisplayIdException.java delete mode 100644 server/src/main/java/com/genymobile/scrcpy/InvalidEncoderException.java diff --git a/server/src/main/java/com/genymobile/scrcpy/ConfigurationException.java b/server/src/main/java/com/genymobile/scrcpy/ConfigurationException.java new file mode 100644 index 0000000000..76c8f52edc --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/ConfigurationException.java @@ -0,0 +1,7 @@ +package com.genymobile.scrcpy; + +public class ConfigurationException extends Exception { + public ConfigurationException(String message) { + super(message); + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.java index 30e64fd77c..c7f7c1f8b7 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Device.java +++ b/server/src/main/java/com/genymobile/scrcpy/Device.java @@ -61,12 +61,12 @@ public interface ClipboardListener { private final boolean supportsInputEvents; - public Device(Options options) { + public Device(Options options) throws ConfigurationException { displayId = options.getDisplayId(); DisplayInfo displayInfo = ServiceManager.getDisplayManager().getDisplayInfo(displayId); if (displayInfo == null) { - int[] displayIds = ServiceManager.getDisplayManager().getDisplayIds(); - throw new InvalidDisplayIdException(displayId, displayIds); + Ln.e(buildUnknownDisplayIdMessage(displayId)); + throw new ConfigurationException("Unknown display id: " + displayId); } int displayInfoFlags = displayInfo.getFlags(); @@ -130,6 +130,18 @@ public void dispatchPrimaryClipChanged() { } } + private static String buildUnknownDisplayIdMessage(int displayId) { + StringBuilder msg = new StringBuilder("Display ").append(displayId).append(" not found"); + int[] displayIds = ServiceManager.getDisplayManager().getDisplayIds(); + if (displayIds != null && displayIds.length > 0) { + msg.append("\nTry to use one of the available display ids:"); + for (int id : displayIds) { + msg.append("\n scrcpy --display=").append(id); + } + } + return msg.toString(); + } + public synchronized void setMaxSize(int newMaxSize) { maxSize = newMaxSize; screenInfo = ScreenInfo.computeScreenInfo(screenInfo.getReverseVideoRotation(), deviceSize, crop, newMaxSize, lockVideoOrientation); diff --git a/server/src/main/java/com/genymobile/scrcpy/InvalidDisplayIdException.java b/server/src/main/java/com/genymobile/scrcpy/InvalidDisplayIdException.java deleted file mode 100644 index 81e3b90377..0000000000 --- a/server/src/main/java/com/genymobile/scrcpy/InvalidDisplayIdException.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.genymobile.scrcpy; - -public class InvalidDisplayIdException extends RuntimeException { - - private final int displayId; - private final int[] availableDisplayIds; - - public InvalidDisplayIdException(int displayId, int[] availableDisplayIds) { - super("There is no display having id " + displayId); - this.displayId = displayId; - this.availableDisplayIds = availableDisplayIds; - } - - public int getDisplayId() { - return displayId; - } - - public int[] getAvailableDisplayIds() { - return availableDisplayIds; - } -} diff --git a/server/src/main/java/com/genymobile/scrcpy/InvalidEncoderException.java b/server/src/main/java/com/genymobile/scrcpy/InvalidEncoderException.java deleted file mode 100644 index b38e29b148..0000000000 --- a/server/src/main/java/com/genymobile/scrcpy/InvalidEncoderException.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.genymobile.scrcpy; - -import android.media.MediaCodecInfo; - -public class InvalidEncoderException extends RuntimeException { - - private final String name; - private final MediaCodecInfo[] availableEncoders; - - public InvalidEncoderException(String name, MediaCodecInfo[] availableEncoders) { - super("There is no encoder having name '" + name + "'"); - this.name = name; - this.availableEncoders = availableEncoders; - } - - public String getName() { - return name; - } - - public MediaCodecInfo[] getAvailableEncoders() { - return availableEncoders; - } -} diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index fed6f6c3b5..44ba75d185 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -63,7 +63,7 @@ public boolean consumeRotationChange() { return rotationChanged.getAndSet(false); } - public void streamScreen(Device device, Callbacks callbacks) throws IOException { + public void streamScreen(Device device, Callbacks callbacks) throws IOException, ConfigurationException { MediaCodec codec = createCodec(videoMimeType, encoderName); MediaFormat format = createFormat(videoMimeType, bitRate, maxFps, codecOptions); IBinder display = createDisplay(); @@ -207,14 +207,14 @@ private static MediaCodecInfo[] listEncoders(String videoMimeType) { return result.toArray(new MediaCodecInfo[result.size()]); } - private static MediaCodec createCodec(String videoMimeType, String encoderName) throws IOException { + private static MediaCodec createCodec(String videoMimeType, String encoderName) throws IOException, ConfigurationException { if (encoderName != null) { Ln.d("Creating encoder by name: '" + encoderName + "'"); try { return MediaCodec.createByCodecName(encoderName); } catch (IllegalArgumentException e) { - MediaCodecInfo[] encoders = listEncoders(videoMimeType); - throw new InvalidEncoderException(encoderName, encoders); + Ln.e(buildUnknownEncoderMessage(videoMimeType, encoderName)); + throw new ConfigurationException("Unknown encoder: " + encoderName); } } MediaCodec codec = MediaCodec.createEncoderByType(videoMimeType); @@ -222,6 +222,18 @@ private static MediaCodec createCodec(String videoMimeType, String encoderName) return codec; } + private static String buildUnknownEncoderMessage(String videoMimeType, String encoderName) { + StringBuilder msg = new StringBuilder("Encoder '").append(encoderName).append("' not found"); + MediaCodecInfo[] encoders = listEncoders(videoMimeType); + if (encoders != null && encoders.length > 0) { + msg.append("\nTry to use one of the available encoders:"); + for (MediaCodecInfo encoder : encoders) { + msg.append("\n scrcpy --encoder='").append(encoder.getName()).append("'"); + } + } + return msg.toString(); + } + private static void setCodecOption(MediaFormat format, CodecOption codecOption) { String key = codecOption.getKey(); Object value = codecOption.getValue(); diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 5a092061be..027050af26 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -1,7 +1,6 @@ package com.genymobile.scrcpy; import android.graphics.Rect; -import android.media.MediaCodecInfo; import android.os.BatteryManager; import android.os.Build; @@ -59,7 +58,7 @@ private static void initAndCleanUp(Options options) { } } - private static void scrcpy(Options options) throws IOException { + private static void scrcpy(Options options) throws IOException, ConfigurationException { Ln.i("Device: " + Build.MANUFACTURER + " " + Build.MODEL + " (Android " + Build.VERSION.RELEASE + ")"); final Device device = new Device(options); List codecOptions = options.getCodecOptions(); @@ -299,38 +298,19 @@ private static Rect parseCrop(String crop) { return new Rect(x, y, x + width, y + height); } - private static void suggestFix(Throwable e) { - if (e instanceof InvalidDisplayIdException) { - InvalidDisplayIdException idie = (InvalidDisplayIdException) e; - int[] displayIds = idie.getAvailableDisplayIds(); - if (displayIds != null && displayIds.length > 0) { - Ln.e("Try to use one of the available display ids:"); - for (int id : displayIds) { - Ln.e(" scrcpy --display=" + id); - } - } - } else if (e instanceof InvalidEncoderException) { - InvalidEncoderException iee = (InvalidEncoderException) e; - MediaCodecInfo[] encoders = iee.getAvailableEncoders(); - if (encoders != null && encoders.length > 0) { - Ln.e("Try to use one of the available encoders:"); - for (MediaCodecInfo encoder : encoders) { - Ln.e(" scrcpy --encoder='" + encoder.getName() + "'"); - } - } - } - } - public static void main(String... args) throws Exception { Thread.setDefaultUncaughtExceptionHandler((t, e) -> { Ln.e("Exception on thread " + t, e); - suggestFix(e); }); Options options = createOptions(args); Ln.initLogLevel(options.getLogLevel()); - scrcpy(options); + try { + scrcpy(options); + } catch (ConfigurationException e) { + // Do not print stack trace, a user-friendly error-message has already been logged + } } } From fa9976366895c1adbfdbf98fb624107e71d4b24e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 20:06:30 +0100 Subject: [PATCH 010/118] Fix --encoder documentation Mention that it depends on the codec provided by --codec (which is not necessarily H264 anymore). --- app/data/zsh-completion/_scrcpy | 2 +- app/scrcpy.1 | 2 +- app/src/cli.c | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index c57111cc55..961565e77a 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -18,7 +18,7 @@ arguments=( '--display=[Specify the display id to mirror]' '--display-buffer=[Add a buffering delay \(in milliseconds\) before displaying]' {-e,--select-tcpip}'[Use TCP/IP device]' - '--encoder=[Use a specific MediaCodec encoder \(must be a H.264 encoder\)]' + '--encoder=[Use a specific MediaCodec encoder]' '--force-adb-forward[Do not attempt to use \"adb reverse\" to connect to the device]' '--forward-all-clicks[Forward clicks to device]' {-f,--fullscreen}'[Start in fullscreen]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 8f028d7c7a..0f1147da90 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -82,7 +82,7 @@ Also see \fB\-d\fR (\fB\-\-select\-usb\fR). .TP .BI "\-\-encoder " name -Use a specific MediaCodec encoder (must be a H.264 encoder). +Use a specific MediaCodec encoder (depending on the codec provided by \fB\-\-codec\fR). .TP .B \-\-force\-adb\-forward diff --git a/app/src/cli.c b/app/src/cli.c index 1851bad6cc..ab46073257 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -175,7 +175,8 @@ static const struct sc_option options[] = { .longopt_id = OPT_ENCODER_NAME, .longopt = "encoder", .argdesc = "name", - .text = "Use a specific MediaCodec encoder (must be a H.264 encoder).", + .text = "Use a specific MediaCodec encoder (depending on the codec " + "provided by --codec).", }, { .longopt_id = OPT_FORCE_ADB_FORWARD, From 181fb555bb72ac5ba5460db6ec8efbf7366a12f3 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 17 Feb 2023 08:41:03 +0100 Subject: [PATCH 011/118] Change PTS origin type from uint64_t to int64_t It is initialized from AVPacket.pts, which is an int64_t. --- app/src/recorder.c | 6 ++---- app/src/recorder.h | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 455e1db174..d75f1b125b 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -11,8 +11,6 @@ /** Downcast packet_sink to recorder */ #define DOWNCAST(SINK) container_of(SINK, struct sc_recorder, packet_sink) -#define SC_PTS_ORIGIN_NONE UINT64_C(-1) - static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us static const AVOutputFormat * @@ -171,7 +169,7 @@ run_recorder(void *data) { sc_mutex_unlock(&recorder->mutex); - if (recorder->pts_origin == SC_PTS_ORIGIN_NONE + if (recorder->pts_origin == AV_NOPTS_VALUE && rec->packet->pts != AV_NOPTS_VALUE) { // First PTS received recorder->pts_origin = rec->packet->pts; @@ -257,7 +255,7 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) { recorder->failed = false; recorder->header_written = false; recorder->previous = NULL; - recorder->pts_origin = SC_PTS_ORIGIN_NONE; + recorder->pts_origin = AV_NOPTS_VALUE; const char *format_name = sc_recorder_get_format_name(recorder->format); assert(format_name); diff --git a/app/src/recorder.h b/app/src/recorder.h index a03c91d7e8..e6c66f9930 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -28,7 +28,7 @@ struct sc_recorder { struct sc_size declared_frame_size; bool header_written; - uint64_t pts_origin; + int64_t pts_origin; sc_thread thread; sc_mutex mutex; From b6744e788708e330ceef618d0961e1a2ce0a98e8 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 17 Feb 2023 08:46:11 +0100 Subject: [PATCH 012/118] Move pts_origin to a local variable It is only used from run_recorder(). --- app/src/recorder.c | 9 +++++---- app/src/recorder.h | 2 -- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index d75f1b125b..f71f632233 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -136,6 +136,8 @@ static int run_recorder(void *data) { struct sc_recorder *recorder = data; + int64_t pts_origin = AV_NOPTS_VALUE; + for (;;) { sc_mutex_lock(&recorder->mutex); @@ -169,15 +171,15 @@ run_recorder(void *data) { sc_mutex_unlock(&recorder->mutex); - if (recorder->pts_origin == AV_NOPTS_VALUE + if (pts_origin == AV_NOPTS_VALUE && rec->packet->pts != AV_NOPTS_VALUE) { // First PTS received - recorder->pts_origin = rec->packet->pts; + pts_origin = rec->packet->pts; } if (rec->packet->pts != AV_NOPTS_VALUE) { // Set PTS relatve to the origin - rec->packet->pts -= recorder->pts_origin; + rec->packet->pts -= pts_origin; rec->packet->dts = rec->packet->pts; } @@ -255,7 +257,6 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) { recorder->failed = false; recorder->header_written = false; recorder->previous = NULL; - recorder->pts_origin = AV_NOPTS_VALUE; const char *format_name = sc_recorder_get_format_name(recorder->format); assert(format_name); diff --git a/app/src/recorder.h b/app/src/recorder.h index e6c66f9930..373278e6a1 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -28,8 +28,6 @@ struct sc_recorder { struct sc_size declared_frame_size; bool header_written; - int64_t pts_origin; - sc_thread thread; sc_mutex mutex; sc_cond queue_cond; From db5751a76a9303e13adc470b45c0a203fc72980c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 17 Feb 2023 09:01:48 +0100 Subject: [PATCH 013/118] Move previous packet to a local variable It is only used from run_recorder(). --- app/src/recorder.c | 14 ++++++++------ app/src/recorder.h | 6 ------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index f71f632233..c52bbb4d1a 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -138,6 +138,10 @@ run_recorder(void *data) { int64_t pts_origin = AV_NOPTS_VALUE; + // We can write a packet only once we received the next one so that we can + // set its duration (next_pts - current_pts) + struct sc_record_packet *previous = NULL; + for (;;) { sc_mutex_lock(&recorder->mutex); @@ -150,7 +154,7 @@ run_recorder(void *data) { if (recorder->stopped && sc_queue_is_empty(&recorder->queue)) { sc_mutex_unlock(&recorder->mutex); - struct sc_record_packet *last = recorder->previous; + struct sc_record_packet *last = previous; if (last) { // assign an arbitrary duration to the last packet last->packet->duration = 100000; @@ -183,12 +187,9 @@ run_recorder(void *data) { rec->packet->dts = rec->packet->pts; } - // recorder->previous is only written from this thread, no need to lock - struct sc_record_packet *previous = recorder->previous; - recorder->previous = rec; - if (!previous) { // we just received the first packet + previous = rec; continue; } @@ -212,6 +213,8 @@ run_recorder(void *data) { sc_mutex_unlock(&recorder->mutex); break; } + + previous = rec; } if (!recorder->failed) { @@ -256,7 +259,6 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) { recorder->stopped = false; recorder->failed = false; recorder->header_written = false; - recorder->previous = NULL; const char *format_name = sc_recorder_get_format_name(recorder->format); assert(format_name); diff --git a/app/src/recorder.h b/app/src/recorder.h index 373278e6a1..05d3857eda 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -34,12 +34,6 @@ struct sc_recorder { bool stopped; // set on recorder_close() bool failed; // set on packet write failure struct sc_recorder_queue queue; - - // we can write a packet only once we received the next one so that we can - // set its duration (next_pts - current_pts) - // "previous" is only accessed from the recorder thread, so it does not - // need to be protected by the mutex - struct sc_record_packet *previous; }; bool From b1b33e3eaf00b94d6a7d7a3c625b8bbc8c7c8014 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 10 Feb 2023 18:10:24 +0100 Subject: [PATCH 014/118] Report recorder errors Stop scrcpy on recorder errors. It was previously indirectly stopped by the demuxer, which failed to push packets to a recorder in error. Report it directly instead: - it avoids to wait for the next demuxer call; - it will allow to open the target file from a separate thread and stop immediately on any I/O error. --- app/src/events.h | 1 + app/src/recorder.c | 13 ++++++++++--- app/src/recorder.h | 11 ++++++++++- app/src/scrcpy.c | 24 ++++++++++++++++++++---- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/app/src/events.h b/app/src/events.h index 7fa10761b0..0a45b652bb 100644 --- a/app/src/events.h +++ b/app/src/events.h @@ -4,3 +4,4 @@ #define SC_EVENT_SERVER_CONNECTED (SDL_USEREVENT + 3) #define SC_EVENT_USB_DEVICE_DISCONNECTED (SDL_USEREVENT + 4) #define SC_EVENT_DEMUXER_ERROR (SDL_USEREVENT + 5) +#define SC_EVENT_RECORDER_ERROR (SDL_USEREVENT + 6) diff --git a/app/src/recorder.c b/app/src/recorder.c index c52bbb4d1a..beca48aacf 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -240,6 +240,9 @@ run_recorder(void *data) { LOGD("Recorder thread ended"); + recorder->cbs->on_ended(recorder, !recorder->failed, + recorder->cbs_userdata); + return 0; } @@ -387,10 +390,10 @@ sc_recorder_packet_sink_push(struct sc_packet_sink *sink, } bool -sc_recorder_init(struct sc_recorder *recorder, - const char *filename, +sc_recorder_init(struct sc_recorder *recorder, const char *filename, enum sc_record_format format, - struct sc_size declared_frame_size) { + struct sc_size declared_frame_size, + const struct sc_recorder_callbacks *cbs, void *cbs_userdata) { recorder->filename = strdup(filename); if (!recorder->filename) { LOG_OOM(); @@ -400,6 +403,10 @@ sc_recorder_init(struct sc_recorder *recorder, recorder->format = format; recorder->declared_frame_size = declared_frame_size; + assert(cbs && cbs->on_ended); + recorder->cbs = cbs; + recorder->cbs_userdata = cbs_userdata; + static const struct sc_packet_sink_ops ops = { .open = sc_recorder_packet_sink_open, .close = sc_recorder_packet_sink_close, diff --git a/app/src/recorder.h b/app/src/recorder.h index 05d3857eda..de5827e3f8 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -34,12 +34,21 @@ struct sc_recorder { bool stopped; // set on recorder_close() bool failed; // set on packet write failure struct sc_recorder_queue queue; + + const struct sc_recorder_callbacks *cbs; + void *cbs_userdata; +}; + +struct sc_recorder_callbacks { + void (*on_ended)(struct sc_recorder *recorder, bool success, + void *userdata); }; bool sc_recorder_init(struct sc_recorder *recorder, const char *filename, enum sc_record_format format, - struct sc_size declared_frame_size); + struct sc_size declared_frame_size, + const struct sc_recorder_callbacks *cbs, void *cbs_userdata); void sc_recorder_destroy(struct sc_recorder *recorder); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index e96fa187c4..5a43a31337 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -161,6 +161,9 @@ event_loop(struct scrcpy *s) { case SC_EVENT_DEMUXER_ERROR: LOGE("Demuxer error"); return SCRCPY_EXIT_FAILURE; + case SC_EVENT_RECORDER_ERROR: + LOGE("Recorder error"); + return SCRCPY_EXIT_FAILURE; case SDL_QUIT: LOGD("User requested to quit"); return SCRCPY_EXIT_SUCCESS; @@ -198,6 +201,17 @@ await_for_server(bool *connected) { return false; } +static void +sc_recorder_on_ended(struct sc_recorder *recorder, bool success, + void *userdata) { + (void) recorder; + (void) userdata; + + if (!success) { + PUSH_EVENT(SC_EVENT_RECORDER_ERROR); + } +} + static void sc_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, void *userdata) { (void) demuxer; @@ -379,10 +393,12 @@ scrcpy(struct scrcpy_options *options) { struct sc_recorder *rec = NULL; if (options->record_filename) { - if (!sc_recorder_init(&s->recorder, - options->record_filename, - options->record_format, - info->frame_size)) { + static const struct sc_recorder_callbacks recorder_cbs = { + .on_ended = sc_recorder_on_ended, + }; + if (!sc_recorder_init(&s->recorder, options->record_filename, + options->record_format, info->frame_size, + &recorder_cbs, NULL)) { goto end; } rec = &s->recorder; From fb2913559144c1f3b062f08b7225216f0e31b3ee Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 14 Feb 2023 08:40:44 +0100 Subject: [PATCH 015/118] Initialize recorder fields from init() The recorder has two initialization phases: one to initialize the concrete recorder object, and one to open its packet_sink trait. Initialize mutex and condvar as part of the object initialization. If there were several packet_sink traits (spoiler: one for video, one for audio), then the mutex and condvar would still be initialized only once. --- app/src/recorder.c | 53 ++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index beca48aacf..2488a3b55a 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -248,33 +248,18 @@ run_recorder(void *data) { static bool sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) { - bool ok = sc_mutex_init(&recorder->mutex); - if (!ok) { - return false; - } - - ok = sc_cond_init(&recorder->queue_cond); - if (!ok) { - goto error_mutex_destroy; - } - - sc_queue_init(&recorder->queue); - recorder->stopped = false; - recorder->failed = false; - recorder->header_written = false; - const char *format_name = sc_recorder_get_format_name(recorder->format); assert(format_name); const AVOutputFormat *format = find_muxer(format_name); if (!format) { LOGE("Could not find muxer"); - goto error_cond_destroy; + return false; } recorder->ctx = avformat_alloc_context(); if (!recorder->ctx) { LOG_OOM(); - goto error_cond_destroy; + return false; } // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate() @@ -306,8 +291,8 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) { } LOGD("Starting recorder thread"); - ok = sc_thread_create(&recorder->thread, run_recorder, "scrcpy-recorder", - recorder); + bool ok = sc_thread_create(&recorder->thread, run_recorder, + "scrcpy-recorder", recorder); if (!ok) { LOGE("Could not start recorder thread"); goto error_avio_close; @@ -321,10 +306,6 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) { avio_close(recorder->ctx->pb); error_avformat_free_context: avformat_free_context(recorder->ctx); -error_cond_destroy: - sc_cond_destroy(&recorder->queue_cond); -error_mutex_destroy: - sc_mutex_destroy(&recorder->mutex); return false; } @@ -340,8 +321,6 @@ sc_recorder_close(struct sc_recorder *recorder) { avio_close(recorder->ctx->pb); avformat_free_context(recorder->ctx); - sc_cond_destroy(&recorder->queue_cond); - sc_mutex_destroy(&recorder->mutex); } static bool @@ -400,6 +379,21 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, return false; } + bool ok = sc_mutex_init(&recorder->mutex); + if (!ok) { + goto error_free_filename; + } + + ok = sc_cond_init(&recorder->queue_cond); + if (!ok) { + goto error_mutex_destroy; + } + + sc_queue_init(&recorder->queue); + recorder->stopped = false; + recorder->failed = false; + recorder->header_written = false; + recorder->format = format; recorder->declared_frame_size = declared_frame_size; @@ -416,9 +410,18 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, recorder->packet_sink.ops = &ops; return true; + +error_mutex_destroy: + sc_mutex_destroy(&recorder->mutex); +error_free_filename: + free(recorder->filename); + + return false; } void sc_recorder_destroy(struct sc_recorder *recorder) { + sc_cond_destroy(&recorder->queue_cond); + sc_mutex_destroy(&recorder->mutex); free(recorder->filename); } From 6b5dfef92357421fc033bf99a5ceeb006c322811 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 14 Feb 2023 09:37:36 +0100 Subject: [PATCH 016/118] Inline packet_sink impl in recorder Remove useless wrappers. --- app/src/recorder.c | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 2488a3b55a..998a2f7b82 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -247,7 +247,11 @@ run_recorder(void *data) { } static bool -sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) { +sc_recorder_packet_sink_open(struct sc_packet_sink *sink, + const AVCodec *codec) { + struct sc_recorder *recorder = DOWNCAST(sink); + assert(codec); + const char *format_name = sc_recorder_get_format_name(recorder->format); assert(format_name); const AVOutputFormat *format = find_muxer(format_name); @@ -271,13 +275,13 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) { av_dict_set(&recorder->ctx->metadata, "comment", "Recorded by scrcpy " SCRCPY_VERSION, 0); - AVStream *ostream = avformat_new_stream(recorder->ctx, input_codec); + AVStream *ostream = avformat_new_stream(recorder->ctx, codec); if (!ostream) { goto error_avformat_free_context; } ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; - ostream->codecpar->codec_id = input_codec->id; + ostream->codecpar->codec_id = codec->id; ostream->codecpar->format = AV_PIX_FMT_YUV420P; ostream->codecpar->width = recorder->declared_frame_size.width; ostream->codecpar->height = recorder->declared_frame_size.height; @@ -311,7 +315,9 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) { } static void -sc_recorder_close(struct sc_recorder *recorder) { +sc_recorder_packet_sink_close(struct sc_packet_sink *sink) { + struct sc_recorder *recorder = DOWNCAST(sink); + sc_mutex_lock(&recorder->mutex); recorder->stopped = true; sc_cond_signal(&recorder->queue_cond); @@ -324,7 +330,10 @@ sc_recorder_close(struct sc_recorder *recorder) { } static bool -sc_recorder_push(struct sc_recorder *recorder, const AVPacket *packet) { +sc_recorder_packet_sink_push(struct sc_packet_sink *sink, + const AVPacket *packet) { + struct sc_recorder *recorder = DOWNCAST(sink); + sc_mutex_lock(&recorder->mutex); assert(!recorder->stopped); @@ -348,26 +357,6 @@ sc_recorder_push(struct sc_recorder *recorder, const AVPacket *packet) { return true; } -static bool -sc_recorder_packet_sink_open(struct sc_packet_sink *sink, - const AVCodec *codec) { - struct sc_recorder *recorder = DOWNCAST(sink); - return sc_recorder_open(recorder, codec); -} - -static void -sc_recorder_packet_sink_close(struct sc_packet_sink *sink) { - struct sc_recorder *recorder = DOWNCAST(sink); - sc_recorder_close(recorder); -} - -static bool -sc_recorder_packet_sink_push(struct sc_packet_sink *sink, - const AVPacket *packet) { - struct sc_recorder *recorder = DOWNCAST(sink); - return sc_recorder_push(recorder, packet); -} - bool sc_recorder_init(struct sc_recorder *recorder, const char *filename, enum sc_record_format format, From a039124d5d21c3886aacb4463a747edd69da1a46 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 14 Feb 2023 09:25:50 +0100 Subject: [PATCH 017/118] Open recording file from the recorder thread The recorder opened the target file from the packet sink open() callback, called by the demuxer. Only then the recorder thread was started. One golden rule for the recorder is to never block the demuxer for I/O, because it would impact mirroring. This rule is respected on recording packets, but not for the initial recorder opening. Therefore, start the recorder thread from sc_recorder_init(), open the file immediately from the recorder thread, then make it wait for the stream to start (on packet sink open()). Now that the recorder can report errors directly (rather than making the demuxer call fail), it is possible to report file opening error even before the packet sink is open. --- app/src/recorder.c | 250 +++++++++++++++++++++++------------- app/src/recorder.h | 14 +- app/src/scrcpy.c | 4 + app/src/trait/packet_sink.h | 1 + 4 files changed, 178 insertions(+), 91 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 998a2f7b82..784ef7eeaf 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -132,10 +132,76 @@ sc_recorder_write(struct sc_recorder *recorder, AVPacket *packet) { return av_write_frame(recorder->ctx, packet) >= 0; } -static int -run_recorder(void *data) { - struct sc_recorder *recorder = data; +static bool +sc_recorder_open_output_file(struct sc_recorder *recorder) { + const char *format_name = sc_recorder_get_format_name(recorder->format); + assert(format_name); + const AVOutputFormat *format = find_muxer(format_name); + if (!format) { + LOGE("Could not find muxer"); + return false; + } + recorder->ctx = avformat_alloc_context(); + if (!recorder->ctx) { + LOG_OOM(); + return false; + } + + int ret = avio_open(&recorder->ctx->pb, recorder->filename, + AVIO_FLAG_WRITE); + if (ret < 0) { + LOGE("Failed to open output file: %s", recorder->filename); + avformat_free_context(recorder->ctx); + return false; + } + + // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate() + // returns (on purpose) a pointer-to-const, but AVFormatContext.oformat + // still expects a pointer-to-non-const (it has not be updated accordingly) + // + recorder->ctx->oformat = (AVOutputFormat *) format; + + av_dict_set(&recorder->ctx->metadata, "comment", + "Recorded by scrcpy " SCRCPY_VERSION, 0); + + LOGI("Recording started to %s file: %s", format_name, recorder->filename); + return true; +} + +static void +sc_recorder_close_output_file(struct sc_recorder *recorder) { + avio_close(recorder->ctx->pb); + avformat_free_context(recorder->ctx); +} + +static bool +sc_recorder_wait_video_stream(struct sc_recorder *recorder) { + sc_mutex_lock(&recorder->mutex); + while (!recorder->codec && !recorder->stopped) { + sc_cond_wait(&recorder->stream_cond, &recorder->mutex); + } + const AVCodec *codec = recorder->codec; + sc_mutex_unlock(&recorder->mutex); + + if (codec) { + AVStream *ostream = avformat_new_stream(recorder->ctx, codec); + if (!ostream) { + return false; + } + + ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + ostream->codecpar->codec_id = codec->id; + ostream->codecpar->format = AV_PIX_FMT_YUV420P; + ostream->codecpar->width = recorder->declared_frame_size.width; + ostream->codecpar->height = recorder->declared_frame_size.height; + } + + return true; +} + +static bool +sc_recorder_process_packets(struct sc_recorder *recorder) { int64_t pts_origin = AV_NOPTS_VALUE; // We can write a packet only once we received the next one so that we can @@ -206,42 +272,70 @@ run_recorder(void *data) { if (!ok) { LOGE("Could not record packet"); - sc_mutex_lock(&recorder->mutex); - recorder->failed = true; - // discard pending packets - sc_recorder_queue_clear(&recorder->queue); - sc_mutex_unlock(&recorder->mutex); - break; + return false; } previous = rec; } - if (!recorder->failed) { - if (recorder->header_written) { - int ret = av_write_trailer(recorder->ctx); - if (ret < 0) { - LOGE("Failed to write trailer to %s", recorder->filename); - recorder->failed = true; - } - } else { - // the recorded file is empty - recorder->failed = true; - } + if (!recorder->header_written) { + // the recorded file is empty + return false; } - if (recorder->failed) { - LOGE("Recording failed to %s", recorder->filename); - } else { + int ret = av_write_trailer(recorder->ctx); + if (ret < 0) { + LOGE("Failed to write trailer to %s", recorder->filename); + return false; + } + + return true; +} + +static bool +sc_recorder_record(struct sc_recorder *recorder) { + bool ok = sc_recorder_open_output_file(recorder); + if (!ok) { + return false; + } + + ok = sc_recorder_wait_video_stream(recorder); + if (!ok) { + sc_recorder_close_output_file(recorder); + return false; + } + + // If recorder->stopped, process any queued packet anyway + + ok = sc_recorder_process_packets(recorder); + sc_recorder_close_output_file(recorder); + return ok; +} + +static int +run_recorder(void *data) { + struct sc_recorder *recorder = data; + + bool success = sc_recorder_record(recorder); + + sc_mutex_lock(&recorder->mutex); + // Prevent the producer to push any new packet + recorder->stopped = true; + // Discard pending packets + sc_recorder_queue_clear(&recorder->queue); + sc_mutex_unlock(&recorder->mutex); + + if (success) { const char *format_name = sc_recorder_get_format_name(recorder->format); LOGI("Recording complete to %s file: %s", format_name, recorder->filename); + } else { + LOGE("Recording failed to %s", recorder->filename); } LOGD("Recorder thread ended"); - recorder->cbs->on_ended(recorder, !recorder->failed, - recorder->cbs_userdata); + recorder->cbs->on_ended(recorder, success, recorder->cbs_userdata); return 0; } @@ -252,66 +346,17 @@ sc_recorder_packet_sink_open(struct sc_packet_sink *sink, struct sc_recorder *recorder = DOWNCAST(sink); assert(codec); - const char *format_name = sc_recorder_get_format_name(recorder->format); - assert(format_name); - const AVOutputFormat *format = find_muxer(format_name); - if (!format) { - LOGE("Could not find muxer"); - return false; - } - - recorder->ctx = avformat_alloc_context(); - if (!recorder->ctx) { - LOG_OOM(); + sc_mutex_lock(&recorder->mutex); + if (recorder->stopped) { + sc_mutex_unlock(&recorder->mutex); return false; } - // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate() - // returns (on purpose) a pointer-to-const, but AVFormatContext.oformat - // still expects a pointer-to-non-const (it has not be updated accordingly) - // - recorder->ctx->oformat = (AVOutputFormat *) format; - - av_dict_set(&recorder->ctx->metadata, "comment", - "Recorded by scrcpy " SCRCPY_VERSION, 0); - - AVStream *ostream = avformat_new_stream(recorder->ctx, codec); - if (!ostream) { - goto error_avformat_free_context; - } - - ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; - ostream->codecpar->codec_id = codec->id; - ostream->codecpar->format = AV_PIX_FMT_YUV420P; - ostream->codecpar->width = recorder->declared_frame_size.width; - ostream->codecpar->height = recorder->declared_frame_size.height; - - int ret = avio_open(&recorder->ctx->pb, recorder->filename, - AVIO_FLAG_WRITE); - if (ret < 0) { - LOGE("Failed to open output file: %s", recorder->filename); - // ostream will be cleaned up during context cleaning - goto error_avformat_free_context; - } - - LOGD("Starting recorder thread"); - bool ok = sc_thread_create(&recorder->thread, run_recorder, - "scrcpy-recorder", recorder); - if (!ok) { - LOGE("Could not start recorder thread"); - goto error_avio_close; - } - - LOGI("Recording started to %s file: %s", format_name, recorder->filename); + recorder->codec = codec; + sc_cond_signal(&recorder->stream_cond); + sc_mutex_unlock(&recorder->mutex); return true; - -error_avio_close: - avio_close(recorder->ctx->pb); -error_avformat_free_context: - avformat_free_context(recorder->ctx); - - return false; } static void @@ -319,14 +364,10 @@ sc_recorder_packet_sink_close(struct sc_packet_sink *sink) { struct sc_recorder *recorder = DOWNCAST(sink); sc_mutex_lock(&recorder->mutex); + // EOS also stops the recorder recorder->stopped = true; sc_cond_signal(&recorder->queue_cond); sc_mutex_unlock(&recorder->mutex); - - sc_thread_join(&recorder->thread, NULL); - - avio_close(recorder->ctx->pb); - avformat_free_context(recorder->ctx); } static bool @@ -335,10 +376,9 @@ sc_recorder_packet_sink_push(struct sc_packet_sink *sink, struct sc_recorder *recorder = DOWNCAST(sink); sc_mutex_lock(&recorder->mutex); - assert(!recorder->stopped); - if (recorder->failed) { - // reject any new packet (this will stop the stream) + if (recorder->stopped) { + // reject any new packet sc_mutex_unlock(&recorder->mutex); return false; } @@ -378,11 +418,17 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, goto error_mutex_destroy; } + ok = sc_cond_init(&recorder->stream_cond); + if (!ok) { + goto error_queue_cond_destroy; + } + sc_queue_init(&recorder->queue); recorder->stopped = false; - recorder->failed = false; recorder->header_written = false; + recorder->codec = NULL; + recorder->format = format; recorder->declared_frame_size = declared_frame_size; @@ -398,8 +444,19 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, recorder->packet_sink.ops = &ops; + ok = sc_thread_create(&recorder->thread, run_recorder, "scrcpy-recorder", + recorder); + if (!ok) { + LOGE("Could not start recorder thread"); + goto error_stream_cond_destroy; + } + return true; +error_stream_cond_destroy: + sc_cond_destroy(&recorder->stream_cond); +error_queue_cond_destroy: + sc_cond_destroy(&recorder->queue_cond); error_mutex_destroy: sc_mutex_destroy(&recorder->mutex); error_free_filename: @@ -408,8 +465,23 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, return false; } +void +sc_recorder_stop(struct sc_recorder *recorder) { + sc_mutex_lock(&recorder->mutex); + recorder->stopped = true; + sc_cond_signal(&recorder->queue_cond); + sc_cond_signal(&recorder->stream_cond); + sc_mutex_unlock(&recorder->mutex); +} + +void +sc_recorder_join(struct sc_recorder *recorder) { + sc_thread_join(&recorder->thread, NULL); +} + void sc_recorder_destroy(struct sc_recorder *recorder) { + sc_cond_destroy(&recorder->stream_cond); sc_cond_destroy(&recorder->queue_cond); sc_mutex_destroy(&recorder->mutex); free(recorder->filename); diff --git a/app/src/recorder.h b/app/src/recorder.h index de5827e3f8..cab71678ec 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -31,10 +31,14 @@ struct sc_recorder { sc_thread thread; sc_mutex mutex; sc_cond queue_cond; - bool stopped; // set on recorder_close() - bool failed; // set on packet write failure + // set on sc_recorder_stop(), packet_sink close or recording failure + bool stopped; struct sc_recorder_queue queue; + // wake up the recorder thread once the codec in known + sc_cond stream_cond; + const AVCodec *codec; + const struct sc_recorder_callbacks *cbs; void *cbs_userdata; }; @@ -50,6 +54,12 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, struct sc_size declared_frame_size, const struct sc_recorder_callbacks *cbs, void *cbs_userdata); +void +sc_recorder_stop(struct sc_recorder *recorder); + +void +sc_recorder_join(struct sc_recorder *recorder); + void sc_recorder_destroy(struct sc_recorder *recorder); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 5a43a31337..90c6bd9ba7 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -660,6 +660,9 @@ scrcpy(struct scrcpy_options *options) { if (file_pusher_initialized) { sc_file_pusher_stop(&s->file_pusher); } + if (recorder_initialized) { + sc_recorder_stop(&s->recorder); + } if (screen_initialized) { sc_screen_interrupt(&s->screen); } @@ -706,6 +709,7 @@ scrcpy(struct scrcpy_options *options) { } if (recorder_initialized) { + sc_recorder_join(&s->recorder); sc_recorder_destroy(&s->recorder); } diff --git a/app/src/trait/packet_sink.h b/app/src/trait/packet_sink.h index 1fef765f61..9fc9fd246e 100644 --- a/app/src/trait/packet_sink.h +++ b/app/src/trait/packet_sink.h @@ -19,6 +19,7 @@ struct sc_packet_sink { }; struct sc_packet_sink_ops { + /* The codec instance is static, it is valid until the end of the program */ bool (*open)(struct sc_packet_sink *sink, const AVCodec *codec); void (*close)(struct sc_packet_sink *sink); bool (*push)(struct sc_packet_sink *sink, const AVPacket *packet); From 3c407773e9ddf9debd1de71ff0d12f571bd36718 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 23 Feb 2023 11:00:34 +0100 Subject: [PATCH 018/118] Add start() function for recorder For consistency with the other components, do not start the internal thread from an init() function. --- app/src/recorder.c | 21 ++++++++++++--------- app/src/recorder.h | 3 +++ app/src/scrcpy.c | 13 +++++++++++-- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 784ef7eeaf..aa3aea0e65 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -444,17 +444,8 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, recorder->packet_sink.ops = &ops; - ok = sc_thread_create(&recorder->thread, run_recorder, "scrcpy-recorder", - recorder); - if (!ok) { - LOGE("Could not start recorder thread"); - goto error_stream_cond_destroy; - } - return true; -error_stream_cond_destroy: - sc_cond_destroy(&recorder->stream_cond); error_queue_cond_destroy: sc_cond_destroy(&recorder->queue_cond); error_mutex_destroy: @@ -465,6 +456,18 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, return false; } +bool +sc_recorder_start(struct sc_recorder *recorder) { + bool ok = sc_thread_create(&recorder->thread, run_recorder, + "scrcpy-recorder", recorder); + if (!ok) { + LOGE("Could not start recorder thread"); + return false; + } + + return true; +} + void sc_recorder_stop(struct sc_recorder *recorder) { sc_mutex_lock(&recorder->mutex); diff --git a/app/src/recorder.h b/app/src/recorder.h index cab71678ec..e98d0ea292 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -54,6 +54,9 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, struct sc_size declared_frame_size, const struct sc_recorder_callbacks *cbs, void *cbs_userdata); +bool +sc_recorder_start(struct sc_recorder *recorder); + void sc_recorder_stop(struct sc_recorder *recorder); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 90c6bd9ba7..b636a03a0a 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -277,6 +277,7 @@ scrcpy(struct scrcpy_options *options) { bool server_started = false; bool file_pusher_initialized = false; bool recorder_initialized = false; + bool recorder_started = false; #ifdef HAVE_V4L2 bool v4l2_sink_initialized = false; #endif @@ -401,8 +402,14 @@ scrcpy(struct scrcpy_options *options) { &recorder_cbs, NULL)) { goto end; } - rec = &s->recorder; recorder_initialized = true; + + if (!sc_recorder_start(&s->recorder)) { + goto end; + } + recorder_started = true; + + rec = &s->recorder; } static const struct sc_demuxer_callbacks demuxer_cbs = { @@ -708,8 +715,10 @@ scrcpy(struct scrcpy_options *options) { sc_controller_destroy(&s->controller); } - if (recorder_initialized) { + if (recorder_started) { sc_recorder_join(&s->recorder); + } + if (recorder_initialized) { sc_recorder_destroy(&s->recorder); } From 4b246cd963ebe0cafa2fe2743e4f9b066dcec293 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 12:07:05 +0100 Subject: [PATCH 019/118] Move last packet recording Write the last packet at the end. --- app/src/recorder.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index aa3aea0e65..85f0fd2a73 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -220,19 +220,6 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { if (recorder->stopped && sc_queue_is_empty(&recorder->queue)) { sc_mutex_unlock(&recorder->mutex); - struct sc_record_packet *last = previous; - if (last) { - // assign an arbitrary duration to the last packet - last->packet->duration = 100000; - bool ok = sc_recorder_write(recorder, last->packet); - if (!ok) { - // failing to write the last frame is not very serious, no - // future frame may depend on it, so the resulting file - // will still be valid - LOGW("Could not record last packet"); - } - sc_record_packet_delete(last); - } break; } @@ -283,6 +270,21 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { return false; } + // Write the last packet + struct sc_record_packet *last = previous; + if (last) { + // assign an arbitrary duration to the last packet + last->packet->duration = 100000; + bool ok = sc_recorder_write(recorder, last->packet); + if (!ok) { + // failing to write the last frame is not very serious, no + // future frame may depend on it, so the resulting file + // will still be valid + LOGW("Could not record last packet"); + } + sc_record_packet_delete(last); + } + int ret = av_write_trailer(recorder->ctx); if (ret < 0) { LOGE("Failed to write trailer to %s", recorder->filename); From f9efe48aac25c37c4b974b2794fbe61a84d9e6db Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 12:07:30 +0100 Subject: [PATCH 020/118] Refactor recorder logic Process the initial config packet (necessary to write the header) separately. --- app/src/recorder.c | 123 ++++++++++++++++++++++++--------------------- app/src/recorder.h | 1 - 2 files changed, 67 insertions(+), 57 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 85f0fd2a73..7cc697787d 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -93,13 +93,7 @@ sc_recorder_write_header(struct sc_recorder *recorder, const AVPacket *packet) { ostream->codecpar->extradata = extradata; ostream->codecpar->extradata_size = packet->size; - int ret = avformat_write_header(recorder->ctx, NULL); - if (ret < 0) { - LOGE("Failed to write header to %s", recorder->filename); - return false; - } - - return true; + return avformat_write_header(recorder->ctx, NULL) >= 0; } static void @@ -110,24 +104,6 @@ sc_recorder_rescale_packet(struct sc_recorder *recorder, AVPacket *packet) { static bool sc_recorder_write(struct sc_recorder *recorder, AVPacket *packet) { - if (!recorder->header_written) { - if (packet->pts != AV_NOPTS_VALUE) { - LOGE("The first packet is not a config packet"); - return false; - } - bool ok = sc_recorder_write_header(recorder, packet); - if (!ok) { - return false; - } - recorder->header_written = true; - return true; - } - - if (packet->pts == AV_NOPTS_VALUE) { - // ignore config packets - return true; - } - sc_recorder_rescale_packet(recorder, packet); return av_write_frame(recorder->ctx, packet) >= 0; } @@ -200,6 +176,40 @@ sc_recorder_wait_video_stream(struct sc_recorder *recorder) { return true; } +static bool +sc_recorder_process_header(struct sc_recorder *recorder) { + sc_mutex_lock(&recorder->mutex); + + while (!recorder->stopped && sc_queue_is_empty(&recorder->queue)) { + sc_cond_wait(&recorder->queue_cond, &recorder->mutex); + } + + if (recorder->stopped && sc_queue_is_empty(&recorder->queue)) { + sc_mutex_unlock(&recorder->mutex); + return false; + } + + struct sc_record_packet *rec; + sc_queue_take(&recorder->queue, next, &rec); + + sc_mutex_unlock(&recorder->mutex); + + if (rec->packet->pts != AV_NOPTS_VALUE) { + LOGE("The first packet is not a config packet"); + sc_record_packet_delete(rec); + return false; + } + + bool ok = sc_recorder_write_header(recorder, rec->packet); + sc_record_packet_delete(rec); + if (!ok) { + LOGE("Failed to write header to %s", recorder->filename); + return false; + } + + return true; +} + static bool sc_recorder_process_packets(struct sc_recorder *recorder) { int64_t pts_origin = AV_NOPTS_VALUE; @@ -208,6 +218,11 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { // set its duration (next_pts - current_pts) struct sc_record_packet *previous = NULL; + bool header_written = sc_recorder_process_header(recorder); + if (!header_written) { + return false; + } + for (;;) { sc_mutex_lock(&recorder->mutex); @@ -228,46 +243,43 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { sc_mutex_unlock(&recorder->mutex); - if (pts_origin == AV_NOPTS_VALUE - && rec->packet->pts != AV_NOPTS_VALUE) { - // First PTS received - pts_origin = rec->packet->pts; - } + if (rec->packet->pts == AV_NOPTS_VALUE) { + // Ignore further config packets (e.g. on device orientation + // change). The next non-config packet will have the config packet + // data prepended. + sc_record_packet_delete(rec); + } else { + assert(rec->packet->pts != AV_NOPTS_VALUE); + + if (!previous) { + // This is the first non-config packet + assert(pts_origin == AV_NOPTS_VALUE); + pts_origin = rec->packet->pts; + rec->packet->pts = 0; + rec->packet->dts = 0; + previous = rec; + continue; + } + + assert(previous); + assert(pts_origin != AV_NOPTS_VALUE); - if (rec->packet->pts != AV_NOPTS_VALUE) { - // Set PTS relatve to the origin rec->packet->pts -= pts_origin; rec->packet->dts = rec->packet->pts; - } - if (!previous) { - // we just received the first packet - previous = rec; - continue; - } - - // config packets have no PTS, we must ignore them - if (rec->packet->pts != AV_NOPTS_VALUE - && previous->packet->pts != AV_NOPTS_VALUE) { // we now know the duration of the previous packet previous->packet->duration = rec->packet->pts - previous->packet->pts; - } - bool ok = sc_recorder_write(recorder, previous->packet); - sc_record_packet_delete(previous); - if (!ok) { - LOGE("Could not record packet"); + bool ok = sc_recorder_write(recorder, previous->packet); + sc_record_packet_delete(previous); + if (!ok) { + LOGE("Could not record packet"); + return false; + } - return false; + previous = rec; } - - previous = rec; - } - - if (!recorder->header_written) { - // the recorded file is empty - return false; } // Write the last packet @@ -427,7 +439,6 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, sc_queue_init(&recorder->queue); recorder->stopped = false; - recorder->header_written = false; recorder->codec = NULL; diff --git a/app/src/recorder.h b/app/src/recorder.h index e98d0ea292..287030bc6f 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -26,7 +26,6 @@ struct sc_recorder { enum sc_record_format format; AVFormatContext *ctx; struct sc_size declared_frame_size; - bool header_written; sc_thread thread; sc_mutex mutex; From ef6a3b97a73f3aa85cb2226b21f4cc1b4372fb12 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 23 Feb 2023 12:36:59 +0100 Subject: [PATCH 021/118] Reorder initialization Initialize components in the pipeline order: demuxer first, decoder and recorder second. --- app/src/scrcpy.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index b636a03a0a..6866512576 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -382,17 +382,20 @@ scrcpy(struct scrcpy_options *options) { file_pusher_initialized = true; } - struct sc_decoder *dec = NULL; + static const struct sc_demuxer_callbacks demuxer_cbs = { + .on_ended = sc_demuxer_on_ended, + }; + sc_demuxer_init(&s->demuxer, s->server.video_socket, &demuxer_cbs, NULL); + bool needs_decoder = options->display; #ifdef HAVE_V4L2 needs_decoder |= !!options->v4l2_device; #endif if (needs_decoder) { sc_decoder_init(&s->decoder); - dec = &s->decoder; + sc_demuxer_add_sink(&s->demuxer, &s->decoder.packet_sink); } - struct sc_recorder *rec = NULL; if (options->record_filename) { static const struct sc_recorder_callbacks recorder_cbs = { .on_ended = sc_recorder_on_ended, @@ -409,20 +412,7 @@ scrcpy(struct scrcpy_options *options) { } recorder_started = true; - rec = &s->recorder; - } - - static const struct sc_demuxer_callbacks demuxer_cbs = { - .on_ended = sc_demuxer_on_ended, - }; - sc_demuxer_init(&s->demuxer, s->server.video_socket, &demuxer_cbs, NULL); - - if (dec) { - sc_demuxer_add_sink(&s->demuxer, &dec->packet_sink); - } - - if (rec) { - sc_demuxer_add_sink(&s->demuxer, &rec->packet_sink); + sc_demuxer_add_sink(&s->demuxer, &s->recorder.packet_sink); } struct sc_controller *controller = NULL; From 5b2ec662220d87aa95d3794e7748a00c1163bcd6 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Feb 2023 16:53:44 +0100 Subject: [PATCH 022/118] Simplify error handling on socket creation On any error, all previously opened sockets must be closed. Handle these errors in a single catch-block. Currently, there are only 2 sockets, but this will simplify even more with more sockets. Note: this commit is better displayed with --ignore-space-change (-b). --- .../genymobile/scrcpy/DesktopConnection.java | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java index 1f8f46e45a..3cb36a09f0 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java +++ b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java @@ -58,34 +58,34 @@ private static String getSocketName(int scid) { public static DesktopConnection open(int scid, boolean tunnelForward, boolean control, boolean sendDummyByte) throws IOException { String socketName = getSocketName(scid); - LocalSocket videoSocket; + LocalSocket videoSocket = null; LocalSocket controlSocket = null; - if (tunnelForward) { - try (LocalServerSocket localServerSocket = new LocalServerSocket(socketName)) { - videoSocket = localServerSocket.accept(); - if (sendDummyByte) { - // send one byte so the client may read() to detect a connection error - videoSocket.getOutputStream().write(0); - } - if (control) { - try { + try { + if (tunnelForward) { + try (LocalServerSocket localServerSocket = new LocalServerSocket(socketName)) { + videoSocket = localServerSocket.accept(); + if (sendDummyByte) { + // send one byte so the client may read() to detect a connection error + videoSocket.getOutputStream().write(0); + } + if (control) { controlSocket = localServerSocket.accept(); - } catch (IOException | RuntimeException e) { - videoSocket.close(); - throw e; } } - } - } else { - videoSocket = connect(socketName); - if (control) { - try { + } else { + videoSocket = connect(socketName); + if (control) { controlSocket = connect(socketName); - } catch (IOException | RuntimeException e) { - videoSocket.close(); - throw e; } } + } catch (IOException | RuntimeException e) { + if (videoSocket != null) { + videoSocket.close(); + } + if (controlSocket != null) { + controlSocket.close(); + } + throw e; } return new DesktopConnection(videoSocket, controlSocket); From ae29e5b56200035a81c7321c1617fe05928fc5b0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 6 Feb 2023 11:44:18 +0100 Subject: [PATCH 023/118] Use VideoStreamer directly from ScreenEncoder The Callbacks interface notifies new packets. But in addition, the screen encoder will need to write headers on start. We could add a function onStart(), but for simplicity, just remove the interface, which brings no value, and call the streamer directly. Refs 87972e2022686b1176bfaf0c678e703856c2b027 --- .../java/com/genymobile/scrcpy/ScreenEncoder.java | 12 ++++-------- .../java/com/genymobile/scrcpy/VideoStreamer.java | 5 ++--- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index 44ba75d185..3ca2e80cce 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -21,10 +21,6 @@ public class ScreenEncoder implements Device.RotationListener { - public interface Callbacks { - void onPacket(ByteBuffer codecBuffer, MediaCodec.BufferInfo bufferInfo) throws IOException; - } - private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms private static final String KEY_MAX_FPS_TO_ENCODER = "max-fps-to-encoder"; @@ -63,7 +59,7 @@ public boolean consumeRotationChange() { return rotationChanged.getAndSet(false); } - public void streamScreen(Device device, Callbacks callbacks) throws IOException, ConfigurationException { + public void streamScreen(Device device, VideoStreamer streamer) throws IOException, ConfigurationException { MediaCodec codec = createCodec(videoMimeType, encoderName); MediaFormat format = createFormat(videoMimeType, bitRate, maxFps, codecOptions); IBinder display = createDisplay(); @@ -92,7 +88,7 @@ public void streamScreen(Device device, Callbacks callbacks) throws IOException, codec.start(); - alive = encode(codec, callbacks); + alive = encode(codec, streamer); // do not call stop() on exception, it would trigger an IllegalStateException codec.stop(); } catch (IllegalStateException | IllegalArgumentException e) { @@ -161,7 +157,7 @@ private static int chooseMaxSizeFallback(Size failedSize) { return 0; } - private boolean encode(MediaCodec codec, Callbacks callbacks) throws IOException { + private boolean encode(MediaCodec codec, VideoStreamer streamer) throws IOException { boolean eof = false; MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); @@ -184,7 +180,7 @@ private boolean encode(MediaCodec codec, Callbacks callbacks) throws IOException consecutiveErrors = 0; } - callbacks.onPacket(codecBuffer, bufferInfo); + streamer.writePacket(codecBuffer, bufferInfo); } } finally { if (outputBufferId >= 0) { diff --git a/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java b/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java index 943c641ddc..cbde141e8c 100644 --- a/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java +++ b/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java @@ -6,7 +6,7 @@ import java.io.IOException; import java.nio.ByteBuffer; -public final class VideoStreamer implements ScreenEncoder.Callbacks { +public final class VideoStreamer { private static final long PACKET_FLAG_CONFIG = 1L << 63; private static final long PACKET_FLAG_KEY_FRAME = 1L << 62; @@ -28,8 +28,7 @@ public void writeHeader(int codecId) throws IOException { IO.writeFully(fd, buffer); } - @Override - public void onPacket(ByteBuffer codecBuffer, MediaCodec.BufferInfo bufferInfo) throws IOException { + public void writePacket(ByteBuffer codecBuffer, MediaCodec.BufferInfo bufferInfo) throws IOException { if (sendFrameMeta) { writeFrameMeta(fd, bufferInfo, codecBuffer.remaining()); } From 51628201b77c7f7c990e26471aa88c5ef930061c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 6 Feb 2023 11:57:37 +0100 Subject: [PATCH 024/118] Write streamer header from ScreenEncoder The screen encoder is responsible for writing data to the video streamer. --- .../com/genymobile/scrcpy/ScreenEncoder.java | 3 +++ .../java/com/genymobile/scrcpy/Server.java | 5 +---- .../com/genymobile/scrcpy/VideoStreamer.java | 18 ++++++++++++------ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index 3ca2e80cce..c86bc9a5eb 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -64,6 +64,9 @@ public void streamScreen(Device device, VideoStreamer streamer) throws IOExcepti MediaFormat format = createFormat(videoMimeType, bitRate, maxFps, codecOptions); IBinder display = createDisplay(); device.setRotationListener(this); + + streamer.writeHeader(); + boolean alive; try { do { diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 027050af26..d75e83102d 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -104,10 +104,7 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc try { // synchronous - VideoStreamer videoStreamer = new VideoStreamer(connection.getVideoFd(), options.getSendFrameMeta()); - if (options.getSendCodecId()) { - videoStreamer.writeHeader(codec.getId()); - } + VideoStreamer videoStreamer = new VideoStreamer(connection.getVideoFd(), codec, options.getSendCodecId(), options.getSendFrameMeta()); screenEncoder.streamScreen(device, videoStreamer); } catch (IOException e) { // Broken pipe is expected on close, because the socket is closed by the client diff --git a/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java b/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java index cbde141e8c..5858d7d84b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java +++ b/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java @@ -12,20 +12,26 @@ public final class VideoStreamer { private static final long PACKET_FLAG_KEY_FRAME = 1L << 62; private final FileDescriptor fd; + private final VideoCodec codec; + private final boolean sendCodecId; private final boolean sendFrameMeta; private final ByteBuffer headerBuffer = ByteBuffer.allocate(12); - public VideoStreamer(FileDescriptor fd, boolean sendFrameMeta) { + public VideoStreamer(FileDescriptor fd, VideoCodec codec, boolean sendCodecId, boolean sendFrameMeta) { this.fd = fd; + this.codec = codec; + this.sendCodecId = sendCodecId; this.sendFrameMeta = sendFrameMeta; } - public void writeHeader(int codecId) throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(4); - buffer.putInt(codecId); - buffer.flip(); - IO.writeFully(fd, buffer); + public void writeHeader() throws IOException { + if (sendCodecId) { + ByteBuffer buffer = ByteBuffer.allocate(4); + buffer.putInt(codec.getId()); + buffer.flip(); + IO.writeFully(fd, buffer); + } } public void writePacket(ByteBuffer codecBuffer, MediaCodec.BufferInfo bufferInfo) throws IOException { From ae9b08b90540d0f357255b97e5b2d3d51b9b0d4e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 6 Feb 2023 14:09:20 +0100 Subject: [PATCH 025/118] Move screen encoder initialization This prepares further refactors. --- server/src/main/java/com/genymobile/scrcpy/Server.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index d75e83102d..ed1dc1bb3c 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -91,8 +91,6 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc Size videoSize = device.getScreenInfo().getVideoSize(); connection.sendDeviceMeta(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight()); } - ScreenEncoder screenEncoder = new ScreenEncoder(codec.getMimeType(), options.getBitRate(), options.getMaxFps(), codecOptions, - options.getEncoderName(), options.getDownsizeOnError()); if (control) { controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn()); @@ -102,9 +100,11 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc device.setClipboardListener(text -> controllerRef.getSender().pushClipboardText(text)); } + VideoStreamer videoStreamer = new VideoStreamer(connection.getVideoFd(), codec, options.getSendCodecId(), options.getSendFrameMeta()); + ScreenEncoder screenEncoder = new ScreenEncoder(codec.getMimeType(), options.getBitRate(), options.getMaxFps(), codecOptions, + options.getEncoderName(), options.getDownsizeOnError()); try { // synchronous - VideoStreamer videoStreamer = new VideoStreamer(connection.getVideoFd(), codec, options.getSendCodecId(), options.getSendFrameMeta()); screenEncoder.streamScreen(device, videoStreamer); } catch (IOException e) { // Broken pipe is expected on close, because the socket is closed by the client From c2267979917215415befb67115bce5cbca1e5006 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 6 Feb 2023 13:46:19 +0100 Subject: [PATCH 026/118] Pass all args to ScreenEncoder constructor There is no good reason to pass some of them in the constructor and some others as parameters of the streamScreen() method. --- .../java/com/genymobile/scrcpy/ScreenEncoder.java | 12 ++++++++---- .../src/main/java/com/genymobile/scrcpy/Server.java | 4 ++-- .../java/com/genymobile/scrcpy/VideoStreamer.java | 4 ++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index c86bc9a5eb..6f47c7f0d7 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -31,7 +31,8 @@ public class ScreenEncoder implements Device.RotationListener { private final AtomicBoolean rotationChanged = new AtomicBoolean(); - private final String videoMimeType; + private final Device device; + private final VideoStreamer streamer; private final String encoderName; private final List codecOptions; private final int bitRate; @@ -41,8 +42,10 @@ public class ScreenEncoder implements Device.RotationListener { private boolean firstFrameSent; private int consecutiveErrors; - public ScreenEncoder(String videoMimeType, int bitRate, int maxFps, List codecOptions, String encoderName, boolean downsizeOnError) { - this.videoMimeType = videoMimeType; + public ScreenEncoder(Device device, VideoStreamer streamer, int bitRate, int maxFps, List codecOptions, String encoderName, + boolean downsizeOnError) { + this.device = device; + this.streamer = streamer; this.bitRate = bitRate; this.maxFps = maxFps; this.codecOptions = codecOptions; @@ -59,7 +62,8 @@ public boolean consumeRotationChange() { return rotationChanged.getAndSet(false); } - public void streamScreen(Device device, VideoStreamer streamer) throws IOException, ConfigurationException { + public void streamScreen() throws IOException, ConfigurationException { + String videoMimeType = streamer.getCodec().getMimeType(); MediaCodec codec = createCodec(videoMimeType, encoderName); MediaFormat format = createFormat(videoMimeType, bitRate, maxFps, codecOptions); IBinder display = createDisplay(); diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index ed1dc1bb3c..85234c4ee7 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -101,11 +101,11 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc } VideoStreamer videoStreamer = new VideoStreamer(connection.getVideoFd(), codec, options.getSendCodecId(), options.getSendFrameMeta()); - ScreenEncoder screenEncoder = new ScreenEncoder(codec.getMimeType(), options.getBitRate(), options.getMaxFps(), codecOptions, + ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getBitRate(), options.getMaxFps(), codecOptions, options.getEncoderName(), options.getDownsizeOnError()); try { // synchronous - screenEncoder.streamScreen(device, videoStreamer); + screenEncoder.streamScreen(); } catch (IOException e) { // Broken pipe is expected on close, because the socket is closed by the client if (!IO.isBrokenPipe(e)) { diff --git a/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java b/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java index 5858d7d84b..24f480827f 100644 --- a/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java +++ b/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java @@ -25,6 +25,10 @@ public VideoStreamer(FileDescriptor fd, VideoCodec codec, boolean sendCodecId, b this.sendFrameMeta = sendFrameMeta; } + public VideoCodec getCodec() { + return codec; + } + public void writeHeader() throws IOException { if (sendCodecId) { ByteBuffer buffer = ByteBuffer.allocate(4); From 10ce0f376a89579408337b084be3e05ebcd18d92 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 6 Feb 2023 14:52:09 +0100 Subject: [PATCH 027/118] Make streamer independent of codec type Rename VideoStreamer to Streamer, and extract a Codec interface which will also support audio codecs. PR #3757 --- .../main/java/com/genymobile/scrcpy/Codec.java | 16 ++++++++++++++++ .../com/genymobile/scrcpy/ScreenEncoder.java | 6 +++--- .../main/java/com/genymobile/scrcpy/Server.java | 6 +++--- .../scrcpy/{VideoStreamer.java => Streamer.java} | 8 ++++---- .../java/com/genymobile/scrcpy/VideoCodec.java | 14 +++++++++++++- 5 files changed, 39 insertions(+), 11 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/Codec.java rename server/src/main/java/com/genymobile/scrcpy/{VideoStreamer.java => Streamer.java} (89%) diff --git a/server/src/main/java/com/genymobile/scrcpy/Codec.java b/server/src/main/java/com/genymobile/scrcpy/Codec.java new file mode 100644 index 0000000000..50e8acada2 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/Codec.java @@ -0,0 +1,16 @@ +package com.genymobile.scrcpy; + +public interface Codec { + + enum Type { + VIDEO, + } + + Type getType(); + + int getId(); + + String getName(); + + String getMimeType(); +} diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index 6f47c7f0d7..a79ce0795a 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -32,7 +32,7 @@ public class ScreenEncoder implements Device.RotationListener { private final AtomicBoolean rotationChanged = new AtomicBoolean(); private final Device device; - private final VideoStreamer streamer; + private final Streamer streamer; private final String encoderName; private final List codecOptions; private final int bitRate; @@ -42,7 +42,7 @@ public class ScreenEncoder implements Device.RotationListener { private boolean firstFrameSent; private int consecutiveErrors; - public ScreenEncoder(Device device, VideoStreamer streamer, int bitRate, int maxFps, List codecOptions, String encoderName, + public ScreenEncoder(Device device, Streamer streamer, int bitRate, int maxFps, List codecOptions, String encoderName, boolean downsizeOnError) { this.device = device; this.streamer = streamer; @@ -164,7 +164,7 @@ private static int chooseMaxSizeFallback(Size failedSize) { return 0; } - private boolean encode(MediaCodec codec, VideoStreamer streamer) throws IOException { + private boolean encode(MediaCodec codec, Streamer streamer) throws IOException { boolean eof = false; MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 85234c4ee7..d570a9fb50 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -100,9 +100,9 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc device.setClipboardListener(text -> controllerRef.getSender().pushClipboardText(text)); } - VideoStreamer videoStreamer = new VideoStreamer(connection.getVideoFd(), codec, options.getSendCodecId(), options.getSendFrameMeta()); - ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getBitRate(), options.getMaxFps(), codecOptions, - options.getEncoderName(), options.getDownsizeOnError()); + Streamer videoStreamer = new Streamer(connection.getVideoFd(), codec, options.getSendCodecId(), options.getSendFrameMeta()); + ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getBitRate(), options.getMaxFps(), + codecOptions, options.getEncoderName(), options.getDownsizeOnError()); try { // synchronous screenEncoder.streamScreen(); diff --git a/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java b/server/src/main/java/com/genymobile/scrcpy/Streamer.java similarity index 89% rename from server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java rename to server/src/main/java/com/genymobile/scrcpy/Streamer.java index 24f480827f..d6bf6780c1 100644 --- a/server/src/main/java/com/genymobile/scrcpy/VideoStreamer.java +++ b/server/src/main/java/com/genymobile/scrcpy/Streamer.java @@ -6,26 +6,26 @@ import java.io.IOException; import java.nio.ByteBuffer; -public final class VideoStreamer { +public final class Streamer { private static final long PACKET_FLAG_CONFIG = 1L << 63; private static final long PACKET_FLAG_KEY_FRAME = 1L << 62; private final FileDescriptor fd; - private final VideoCodec codec; + private final Codec codec; private final boolean sendCodecId; private final boolean sendFrameMeta; private final ByteBuffer headerBuffer = ByteBuffer.allocate(12); - public VideoStreamer(FileDescriptor fd, VideoCodec codec, boolean sendCodecId, boolean sendFrameMeta) { + public Streamer(FileDescriptor fd, Codec codec, boolean sendCodecId, boolean sendFrameMeta) { this.fd = fd; this.codec = codec; this.sendCodecId = sendCodecId; this.sendFrameMeta = sendFrameMeta; } - public VideoCodec getCodec() { + public Codec getCodec() { return codec; } diff --git a/server/src/main/java/com/genymobile/scrcpy/VideoCodec.java b/server/src/main/java/com/genymobile/scrcpy/VideoCodec.java index e19b27f017..43531f1e0b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/VideoCodec.java +++ b/server/src/main/java/com/genymobile/scrcpy/VideoCodec.java @@ -3,7 +3,7 @@ import android.annotation.SuppressLint; import android.media.MediaFormat; -public enum VideoCodec { +public enum VideoCodec implements Codec { H264(0x68_32_36_34, "h264", MediaFormat.MIMETYPE_VIDEO_AVC), H265(0x68_32_36_35, "h265", MediaFormat.MIMETYPE_VIDEO_HEVC), @SuppressLint("InlinedApi") // introduced in API 21 @@ -19,10 +19,22 @@ public enum VideoCodec { this.mimeType = mimeType; } + @Override + public Type getType() { + return Type.VIDEO; + } + + @Override public int getId() { return id; } + @Override + public String getName() { + return name; + } + + @Override public String getMimeType() { return mimeType; } From 90d88a6927901345afbe81550c7a489304a61e63 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 19:51:50 +0100 Subject: [PATCH 028/118] Rename "codec" variable to "mediaCodec" This will allow to use "codec" for the Codec type. PR #3757 --- .../com/genymobile/scrcpy/ScreenEncoder.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index a79ce0795a..c697cfa2e0 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -64,7 +64,7 @@ public boolean consumeRotationChange() { public void streamScreen() throws IOException, ConfigurationException { String videoMimeType = streamer.getCodec().getMimeType(); - MediaCodec codec = createCodec(videoMimeType, encoderName); + MediaCodec mediaCodec = createMediaCodec(videoMimeType, encoderName); MediaFormat format = createFormat(videoMimeType, bitRate, maxFps, codecOptions); IBinder display = createDisplay(); device.setRotationListener(this); @@ -84,8 +84,8 @@ public void streamScreen() throws IOException, ConfigurationException { Surface surface = null; try { - codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - surface = codec.createInputSurface(); + mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + surface = mediaCodec.createInputSurface(); // does not include the locked video orientation Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect(); @@ -93,11 +93,11 @@ public void streamScreen() throws IOException, ConfigurationException { int layerStack = device.getLayerStack(); setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack); - codec.start(); + mediaCodec.start(); - alive = encode(codec, streamer); + alive = encode(mediaCodec, streamer); // do not call stop() on exception, it would trigger an IllegalStateException - codec.stop(); + mediaCodec.stop(); } catch (IllegalStateException | IllegalArgumentException e) { Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage()); if (!prepareRetry(device, screenInfo)) { @@ -106,14 +106,14 @@ public void streamScreen() throws IOException, ConfigurationException { Ln.i("Retrying..."); alive = true; } finally { - codec.reset(); + mediaCodec.reset(); if (surface != null) { surface.release(); } } } while (alive); } finally { - codec.release(); + mediaCodec.release(); device.setRotationListener(null); SurfaceControl.destroyDisplay(display); } @@ -210,7 +210,7 @@ private static MediaCodecInfo[] listEncoders(String videoMimeType) { return result.toArray(new MediaCodecInfo[result.size()]); } - private static MediaCodec createCodec(String videoMimeType, String encoderName) throws IOException, ConfigurationException { + private static MediaCodec createMediaCodec(String videoMimeType, String encoderName) throws IOException, ConfigurationException { if (encoderName != null) { Ln.d("Creating encoder by name: '" + encoderName + "'"); try { @@ -220,9 +220,9 @@ private static MediaCodec createCodec(String videoMimeType, String encoderName) throw new ConfigurationException("Unknown encoder: " + encoderName); } } - MediaCodec codec = MediaCodec.createEncoderByType(videoMimeType); - Ln.d("Using encoder: '" + codec.getName() + "'"); - return codec; + MediaCodec mediaCodec = MediaCodec.createEncoderByType(videoMimeType); + Ln.d("Using encoder: '" + mediaCodec.getName() + "'"); + return mediaCodec; } private static String buildUnknownEncoderMessage(String videoMimeType, String encoderName) { From 10ef8da95d9f5d66b3755b88b4d1fda032163260 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 19:58:05 +0100 Subject: [PATCH 029/118] Improve error message for unknown encoder The provided encoder name depends on the selected codec. Improve the error message and the suggestions. PR #3757 --- .../com/genymobile/scrcpy/ScreenEncoder.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index c697cfa2e0..fdd23bf393 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -63,9 +63,9 @@ public boolean consumeRotationChange() { } public void streamScreen() throws IOException, ConfigurationException { - String videoMimeType = streamer.getCodec().getMimeType(); - MediaCodec mediaCodec = createMediaCodec(videoMimeType, encoderName); - MediaFormat format = createFormat(videoMimeType, bitRate, maxFps, codecOptions); + Codec codec = streamer.getCodec(); + MediaCodec mediaCodec = createMediaCodec(codec, encoderName); + MediaFormat format = createFormat(codec.getMimeType(), bitRate, maxFps, codecOptions); IBinder display = createDisplay(); device.setRotationListener(this); @@ -210,28 +210,28 @@ private static MediaCodecInfo[] listEncoders(String videoMimeType) { return result.toArray(new MediaCodecInfo[result.size()]); } - private static MediaCodec createMediaCodec(String videoMimeType, String encoderName) throws IOException, ConfigurationException { + private static MediaCodec createMediaCodec(Codec codec, String encoderName) throws IOException, ConfigurationException { if (encoderName != null) { Ln.d("Creating encoder by name: '" + encoderName + "'"); try { return MediaCodec.createByCodecName(encoderName); } catch (IllegalArgumentException e) { - Ln.e(buildUnknownEncoderMessage(videoMimeType, encoderName)); + Ln.e(buildUnknownEncoderMessage(codec, encoderName)); throw new ConfigurationException("Unknown encoder: " + encoderName); } } - MediaCodec mediaCodec = MediaCodec.createEncoderByType(videoMimeType); + MediaCodec mediaCodec = MediaCodec.createEncoderByType(codec.getMimeType()); Ln.d("Using encoder: '" + mediaCodec.getName() + "'"); return mediaCodec; } - private static String buildUnknownEncoderMessage(String videoMimeType, String encoderName) { - StringBuilder msg = new StringBuilder("Encoder '").append(encoderName).append("' not found"); - MediaCodecInfo[] encoders = listEncoders(videoMimeType); + private static String buildUnknownEncoderMessage(Codec codec, String encoderName) { + StringBuilder msg = new StringBuilder("Encoder '").append(encoderName).append("' for ").append(codec.getName()).append(" not found"); + MediaCodecInfo[] encoders = listEncoders(codec.getMimeType()); if (encoders != null && encoders.length > 0) { msg.append("\nTry to use one of the available encoders:"); for (MediaCodecInfo encoder : encoders) { - msg.append("\n scrcpy --encoder='").append(encoder.getName()).append("'"); + msg.append("\n scrcpy --codec=").append(codec.getName()).append(" --encoder='").append(encoder.getName()).append("'"); } } return msg.toString(); From a5541b3476fe32306de7cb02472cf9077dd45498 Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Fri, 27 Jan 2023 20:13:37 +0800 Subject: [PATCH 030/118] Add a fake Android Context Since scrcpy-server is not an Android application (it's a java executable), it has no Context. Some features will require a Context instance to get the package name and the UID. Add a FakeContext for this purpose. PR #3757 Co-authored-by: Romain Vimont Signed-off-by: Romain Vimont --- .../com/genymobile/scrcpy/FakeContext.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 server/src/main/java/com/genymobile/scrcpy/FakeContext.java diff --git a/server/src/main/java/com/genymobile/scrcpy/FakeContext.java b/server/src/main/java/com/genymobile/scrcpy/FakeContext.java new file mode 100644 index 0000000000..ddd6177bc8 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/FakeContext.java @@ -0,0 +1,40 @@ +package com.genymobile.scrcpy; + +import android.annotation.TargetApi; +import android.content.AttributionSource; +import android.content.ContextWrapper; +import android.os.Build; +import android.os.Process; + +public final class FakeContext extends ContextWrapper { + + public static final String PACKAGE_NAME = "com.android.shell"; + + private static final FakeContext INSTANCE = new FakeContext(); + + public static FakeContext get() { + return INSTANCE; + } + + private FakeContext() { + super(null); + } + + @Override + public String getPackageName() { + return PACKAGE_NAME; + } + + @Override + public String getOpPackageName() { + return PACKAGE_NAME; + } + + @TargetApi(Build.VERSION_CODES.S) + @Override + public AttributionSource getAttributionSource() { + AttributionSource.Builder builder = new AttributionSource.Builder(Process.SHELL_UID); + builder.setPackageName(PACKAGE_NAME); + return builder.build(); + } +} From 9a815ceba8eccc53c724ef0eb79c6ab277446ad9 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 31 Jan 2023 22:31:15 +0100 Subject: [PATCH 031/118] Use AttributionSource from FakeContext FakeContext already provides an AttributeSource instance. PR #3757 Co-authored-by: Simon Chan <1330321+yume-chan@users.noreply.github.com> --- .../scrcpy/wrappers/ContentProvider.java | 55 ++++++++----------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java index 47eae64d1e..854d413627 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java @@ -1,9 +1,12 @@ package com.genymobile.scrcpy.wrappers; +import com.genymobile.scrcpy.FakeContext; import com.genymobile.scrcpy.Ln; import com.genymobile.scrcpy.SettingsException; import android.annotation.SuppressLint; +import android.content.AttributionSource; +import android.os.Build; import android.os.Bundle; import android.os.IBinder; @@ -51,11 +54,10 @@ public class ContentProvider implements Closeable { @SuppressLint("PrivateApi") private Method getCallMethod() throws NoSuchMethodException { if (callMethod == null) { - try { - Class attributionSourceClass = Class.forName("android.content.AttributionSource"); - callMethod = provider.getClass().getMethod("call", attributionSourceClass, String.class, String.class, String.class, Bundle.class); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + callMethod = provider.getClass().getMethod("call", AttributionSource.class, String.class, String.class, String.class, Bundle.class); callMethodVersion = 0; - } catch (NoSuchMethodException | ClassNotFoundException e0) { + } else { // old versions try { callMethod = provider.getClass() @@ -75,40 +77,29 @@ private Method getCallMethod() throws NoSuchMethodException { return callMethod; } - @SuppressLint("PrivateApi") - private Object getAttributionSource() - throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { - if (attributionSource == null) { - Class cl = Class.forName("android.content.AttributionSource$Builder"); - Object builder = cl.getConstructor(int.class).newInstance(ServiceManager.USER_ID); - cl.getDeclaredMethod("setPackageName", String.class).invoke(builder, ServiceManager.PACKAGE_NAME); - attributionSource = cl.getDeclaredMethod("build").invoke(builder); - } - - return attributionSource; - } - private Bundle call(String callMethod, String arg, Bundle extras) - throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException { + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { try { Method method = getCallMethod(); Object[] args; - switch (callMethodVersion) { - case 0: - args = new Object[]{getAttributionSource(), "settings", callMethod, arg, extras}; - break; - case 1: - args = new Object[]{ServiceManager.PACKAGE_NAME, null, "settings", callMethod, arg, extras}; - break; - case 2: - args = new Object[]{ServiceManager.PACKAGE_NAME, "settings", callMethod, arg, extras}; - break; - default: - args = new Object[]{ServiceManager.PACKAGE_NAME, callMethod, arg, extras}; - break; + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && callMethodVersion == 0) { + args = new Object[]{FakeContext.get().getAttributionSource(), "settings", callMethod, arg, extras}; + } else { + switch (callMethodVersion) { + case 1: + args = new Object[]{ServiceManager.PACKAGE_NAME, null, "settings", callMethod, arg, extras}; + break; + case 2: + args = new Object[]{ServiceManager.PACKAGE_NAME, "settings", callMethod, arg, extras}; + break; + default: + args = new Object[]{ServiceManager.PACKAGE_NAME, callMethod, arg, extras}; + break; + } } return (Bundle) method.invoke(provider, args); - } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException | ClassNotFoundException | InstantiationException e) { + } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { Ln.e("Could not invoke method", e); throw e; } From 820189b6a6b570388187f166b8def499d73ced8a Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 31 Jan 2023 22:32:33 +0100 Subject: [PATCH 032/118] Use PACKAGE_NAME from FakeContext Remove duplicated constant. PR #3757 --- .../scrcpy/wrappers/ClipboardManager.java | 19 ++++++++++--------- .../scrcpy/wrappers/ContentProvider.java | 6 +++--- .../scrcpy/wrappers/ServiceManager.java | 1 - 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java index f43a76bc1c..e0af7923ac 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java @@ -1,5 +1,6 @@ package com.genymobile.scrcpy.wrappers; +import com.genymobile.scrcpy.FakeContext; import com.genymobile.scrcpy.Ln; import android.content.ClipData; @@ -58,22 +59,22 @@ private Method getSetPrimaryClipMethod() throws NoSuchMethodException { private static ClipData getPrimaryClip(Method method, boolean alternativeMethod, IInterface manager) throws InvocationTargetException, IllegalAccessException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { - return (ClipData) method.invoke(manager, ServiceManager.PACKAGE_NAME); + return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME); } if (alternativeMethod) { - return (ClipData) method.invoke(manager, ServiceManager.PACKAGE_NAME, null, ServiceManager.USER_ID); + return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, ServiceManager.USER_ID); } - return (ClipData) method.invoke(manager, ServiceManager.PACKAGE_NAME, ServiceManager.USER_ID); + return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, ServiceManager.USER_ID); } private static void setPrimaryClip(Method method, boolean alternativeMethod, IInterface manager, ClipData clipData) throws InvocationTargetException, IllegalAccessException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { - method.invoke(manager, clipData, ServiceManager.PACKAGE_NAME); + method.invoke(manager, clipData, FakeContext.PACKAGE_NAME); } else if (alternativeMethod) { - method.invoke(manager, clipData, ServiceManager.PACKAGE_NAME, null, ServiceManager.USER_ID); + method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, null, ServiceManager.USER_ID); } else { - method.invoke(manager, clipData, ServiceManager.PACKAGE_NAME, ServiceManager.USER_ID); + method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, ServiceManager.USER_ID); } } @@ -106,11 +107,11 @@ public boolean setText(CharSequence text) { private static void addPrimaryClipChangedListener(Method method, boolean alternativeMethod, IInterface manager, IOnPrimaryClipChangedListener listener) throws InvocationTargetException, IllegalAccessException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { - method.invoke(manager, listener, ServiceManager.PACKAGE_NAME); + method.invoke(manager, listener, FakeContext.PACKAGE_NAME); } else if (alternativeMethod) { - method.invoke(manager, listener, ServiceManager.PACKAGE_NAME, null, ServiceManager.USER_ID); + method.invoke(manager, listener, FakeContext.PACKAGE_NAME, null, ServiceManager.USER_ID); } else { - method.invoke(manager, listener, ServiceManager.PACKAGE_NAME, ServiceManager.USER_ID); + method.invoke(manager, listener, FakeContext.PACKAGE_NAME, ServiceManager.USER_ID); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java index 854d413627..2bdbe1752e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java @@ -88,13 +88,13 @@ private Bundle call(String callMethod, String arg, Bundle extras) } else { switch (callMethodVersion) { case 1: - args = new Object[]{ServiceManager.PACKAGE_NAME, null, "settings", callMethod, arg, extras}; + args = new Object[]{FakeContext.PACKAGE_NAME, null, "settings", callMethod, arg, extras}; break; case 2: - args = new Object[]{ServiceManager.PACKAGE_NAME, "settings", callMethod, arg, extras}; + args = new Object[]{FakeContext.PACKAGE_NAME, "settings", callMethod, arg, extras}; break; default: - args = new Object[]{ServiceManager.PACKAGE_NAME, callMethod, arg, extras}; + args = new Object[]{FakeContext.PACKAGE_NAME, callMethod, arg, extras}; break; } } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java index cb6863b6d6..ff4b5e23a5 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java @@ -10,7 +10,6 @@ @SuppressLint("PrivateApi,DiscouragedPrivateApi") public final class ServiceManager { - public static final String PACKAGE_NAME = "com.android.shell"; public static final int USER_ID = 0; private static final Method GET_SERVICE_METHOD; From 42285ae86906447708def6783d3edee433624406 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 26 Feb 2023 22:29:58 +0100 Subject: [PATCH 033/118] Use ROOT_UID from FakeContext Remove USER_ID from ServiceManager, and replace it by a constant in FakeContext. This is the same as android.os.Process.ROOT_UID, but this constant has been introduced in API 29. PR #3757 --- .../main/java/com/genymobile/scrcpy/FakeContext.java | 1 + .../genymobile/scrcpy/wrappers/ActivityManager.java | 5 +++-- .../genymobile/scrcpy/wrappers/ClipboardManager.java | 12 ++++++------ .../genymobile/scrcpy/wrappers/ContentProvider.java | 4 ++-- .../genymobile/scrcpy/wrappers/ServiceManager.java | 2 -- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/FakeContext.java b/server/src/main/java/com/genymobile/scrcpy/FakeContext.java index ddd6177bc8..844d6bd820 100644 --- a/server/src/main/java/com/genymobile/scrcpy/FakeContext.java +++ b/server/src/main/java/com/genymobile/scrcpy/FakeContext.java @@ -9,6 +9,7 @@ public final class FakeContext extends ContextWrapper { public static final String PACKAGE_NAME = "com.android.shell"; + public static final int ROOT_UID = 0; // Like android.os.Process.ROOT_UID, but before API 29 private static final FakeContext INSTANCE = new FakeContext(); diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java index 93ed452875..76aab5f1bb 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java @@ -1,5 +1,6 @@ package com.genymobile.scrcpy.wrappers; +import com.genymobile.scrcpy.FakeContext; import com.genymobile.scrcpy.Ln; import android.os.Binder; @@ -48,10 +49,10 @@ private ContentProvider getContentProviderExternal(String name, IBinder token) { Object[] args; if (getContentProviderExternalMethodNewVersion) { // new version - args = new Object[]{name, ServiceManager.USER_ID, token, null}; + args = new Object[]{name, FakeContext.ROOT_UID, token, null}; } else { // old version - args = new Object[]{name, ServiceManager.USER_ID, token}; + args = new Object[]{name, FakeContext.ROOT_UID, token}; } // ContentProviderHolder providerHolder = getContentProviderExternal(...); Object providerHolder = method.invoke(manager, args); diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java index e0af7923ac..0c1777ecef 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ClipboardManager.java @@ -62,9 +62,9 @@ private static ClipData getPrimaryClip(Method method, boolean alternativeMethod, return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME); } if (alternativeMethod) { - return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, ServiceManager.USER_ID); + return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID); } - return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, ServiceManager.USER_ID); + return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, FakeContext.ROOT_UID); } private static void setPrimaryClip(Method method, boolean alternativeMethod, IInterface manager, ClipData clipData) @@ -72,9 +72,9 @@ private static void setPrimaryClip(Method method, boolean alternativeMethod, IIn if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { method.invoke(manager, clipData, FakeContext.PACKAGE_NAME); } else if (alternativeMethod) { - method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, null, ServiceManager.USER_ID); + method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID); } else { - method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, ServiceManager.USER_ID); + method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, FakeContext.ROOT_UID); } } @@ -109,9 +109,9 @@ private static void addPrimaryClipChangedListener(Method method, boolean alterna if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { method.invoke(manager, listener, FakeContext.PACKAGE_NAME); } else if (alternativeMethod) { - method.invoke(manager, listener, FakeContext.PACKAGE_NAME, null, ServiceManager.USER_ID); + method.invoke(manager, listener, FakeContext.PACKAGE_NAME, null, FakeContext.ROOT_UID); } else { - method.invoke(manager, listener, FakeContext.PACKAGE_NAME, ServiceManager.USER_ID); + method.invoke(manager, listener, FakeContext.PACKAGE_NAME, FakeContext.ROOT_UID); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java index 2bdbe1752e..4917f5eb75 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ContentProvider.java @@ -138,7 +138,7 @@ private static String getPutMethod(String table) { public String getValue(String table, String key) throws SettingsException { String method = getGetMethod(table); Bundle arg = new Bundle(); - arg.putInt(CALL_METHOD_USER_KEY, ServiceManager.USER_ID); + arg.putInt(CALL_METHOD_USER_KEY, FakeContext.ROOT_UID); try { Bundle bundle = call(method, key, arg); if (bundle == null) { @@ -154,7 +154,7 @@ public String getValue(String table, String key) throws SettingsException { public void putValue(String table, String key, String value) throws SettingsException { String method = getPutMethod(table); Bundle arg = new Bundle(); - arg.putInt(CALL_METHOD_USER_KEY, ServiceManager.USER_ID); + arg.putInt(CALL_METHOD_USER_KEY, FakeContext.ROOT_UID); arg.putString(NAME_VALUE_TABLE_VALUE, value); try { call(method, key, arg); diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java index ff4b5e23a5..ee2f0fa978 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ServiceManager.java @@ -10,8 +10,6 @@ @SuppressLint("PrivateApi,DiscouragedPrivateApi") public final class ServiceManager { - public static final int USER_ID = 0; - private static final Method GET_SERVICE_METHOD; static { try { From 84ba6435bb39c1bee194fd658a7aeb913f4ca0d9 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 31 Jan 2023 22:33:30 +0100 Subject: [PATCH 034/118] Use shell package name for workarounds For consistency. PR #3757 --- server/src/main/java/com/genymobile/scrcpy/Workarounds.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Workarounds.java b/server/src/main/java/com/genymobile/scrcpy/Workarounds.java index 0f473bc1b7..e2345e11f4 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Workarounds.java +++ b/server/src/main/java/com/genymobile/scrcpy/Workarounds.java @@ -50,7 +50,7 @@ public static void fillAppInfo() { Object appBindData = appBindDataConstructor.newInstance(); ApplicationInfo applicationInfo = new ApplicationInfo(); - applicationInfo.packageName = "com.genymobile.scrcpy"; + applicationInfo.packageName = FakeContext.PACKAGE_NAME; // appBindData.appInfo = applicationInfo; Field appInfoField = appBindDataClass.getDeclaredField("appInfo"); From 8487ddba644b5d2aad762bea586654f9a7891b5e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 31 Jan 2023 22:48:50 +0100 Subject: [PATCH 035/118] Use FakeContext for Application instance This will expose the correct package name and UID to the application context. PR #3757 --- .../java/com/genymobile/scrcpy/Workarounds.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Workarounds.java b/server/src/main/java/com/genymobile/scrcpy/Workarounds.java index e2345e11f4..64cc127232 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Workarounds.java +++ b/server/src/main/java/com/genymobile/scrcpy/Workarounds.java @@ -2,14 +2,12 @@ import android.annotation.SuppressLint; import android.app.Application; -import android.app.Instrumentation; -import android.content.Context; +import android.content.ContextWrapper; import android.content.pm.ApplicationInfo; import android.os.Looper; import java.lang.reflect.Constructor; import java.lang.reflect.Field; -import java.lang.reflect.Method; public final class Workarounds { private Workarounds() { @@ -62,11 +60,10 @@ public static void fillAppInfo() { mBoundApplicationField.setAccessible(true); mBoundApplicationField.set(activityThread, appBindData); - // Context ctx = activityThread.getSystemContext(); - Method getSystemContextMethod = activityThreadClass.getDeclaredMethod("getSystemContext"); - Context ctx = (Context) getSystemContextMethod.invoke(activityThread); - - Application app = Instrumentation.newApplication(Application.class, ctx); + Application app = Application.class.newInstance(); + Field baseField = ContextWrapper.class.getDeclaredField("mBase"); + baseField.setAccessible(true); + baseField.set(app, FakeContext.get()); // activityThread.mInitialApplication = app; Field mInitialApplicationField = activityThreadClass.getDeclaredField("mInitialApplication"); From 3cf03e4a4bea7b93766f6e5fb66d8e8878a6a906 Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Fri, 3 Feb 2023 16:27:34 +0100 Subject: [PATCH 036/118] Add --no-audio option Audio will be enabled by default (when supported). Add an option to disable it. PR #3757 Co-authored-by: Romain Vimont Signed-off-by: Romain Vimont --- app/data/bash-completion/scrcpy | 1 + app/data/zsh-completion/_scrcpy | 1 + app/src/cli.c | 9 +++++++++ app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 1 + app/src/server.c | 3 +++ app/src/server.h | 1 + server/src/main/java/com/genymobile/scrcpy/Options.java | 9 +++++++++ server/src/main/java/com/genymobile/scrcpy/Server.java | 4 ++++ 10 files changed, 31 insertions(+) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 4590b6a803..22dc4ceef3 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -23,6 +23,7 @@ _scrcpy() { --max-fps= -M --hid-mouse -m --max-size= + --no-audio --no-cleanup --no-clipboard-autosync --no-downsize-on-error diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 961565e77a..17e1de9f64 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -29,6 +29,7 @@ arguments=( '--max-fps=[Limit the frame rate of screen capture]' {-M,--hid-mouse}'[Simulate a physical mouse by using HID over AOAv2]' {-m,--max-size=}'[Limit both the width and height of the video to value]' + '--no-audio[Disable audio forwarding]' '--no-cleanup[Disable device cleanup actions on exit]' '--no-clipboard-autosync[Disable automatic clipboard synchronization]' '--no-downsize-on-error[Disable lowering definition on MediaCodec error]' diff --git a/app/src/cli.c b/app/src/cli.c index ab46073257..c1ee1e6dea 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -59,6 +59,7 @@ enum { OPT_PRINT_FPS, OPT_NO_POWER_ON, OPT_CODEC, + OPT_NO_AUDIO, }; struct sc_option { @@ -267,6 +268,11 @@ static const struct sc_option options[] = { "is preserved.\n" "Default is 0 (unlimited).", }, + { + .longopt_id = OPT_NO_AUDIO, + .longopt = "no-audio", + .text = "Disable audio forwarding.", + }, { .longopt_id = OPT_NO_CLEANUP, .longopt = "no-cleanup", @@ -1630,6 +1636,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_NO_DOWNSIZE_ON_ERROR: opts->downsize_on_error = false; break; + case OPT_NO_AUDIO: + opts->audio = false; + break; case OPT_NO_CLEANUP: opts->cleanup = false; break; diff --git a/app/src/options.c b/app/src/options.c index a75e584e05..0854067f0e 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -67,4 +67,5 @@ const struct scrcpy_options scrcpy_options_default = { .cleanup = true, .start_fps_counter = false, .power_on = true, + .audio = true, }; diff --git a/app/src/options.h b/app/src/options.h index b9d237e0b2..7bf3001106 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -147,6 +147,7 @@ struct scrcpy_options { bool cleanup; bool start_fps_counter; bool power_on; + bool audio; }; extern const struct scrcpy_options scrcpy_options_default; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 6866512576..6d63a7a150 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -312,6 +312,7 @@ scrcpy(struct scrcpy_options *options) { .lock_video_orientation = options->lock_video_orientation, .control = options->control, .display_id = options->display_id, + .audio = options->audio, .show_touches = options->show_touches, .stay_awake = options->stay_awake, .codec_options = options->codec_options, diff --git a/app/src/server.c b/app/src/server.c index 413f02ee4a..0ff0d3c8f4 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -217,6 +217,9 @@ execute_server(struct sc_server *server, ADD_PARAM("log_level=%s", log_level_to_server_string(params->log_level)); ADD_PARAM("bit_rate=%" PRIu32, params->bit_rate); + if (!params->audio) { + ADD_PARAM("audio=false"); + } if (params->codec != SC_CODEC_H264) { ADD_PARAM("codec=%s", sc_server_get_codec_name(params->codec)); } diff --git a/app/src/server.h b/app/src/server.h index c05b1e5b8d..95e24b4128 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -38,6 +38,7 @@ struct sc_server_params { int8_t lock_video_orientation; bool control; uint32_t display_id; + bool audio; bool show_touches; bool stay_awake; bool force_adb_forward; diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 5c59ec8eaf..0778997454 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -8,6 +8,7 @@ public class Options { private Ln.Level logLevel = Ln.Level.DEBUG; private int scid = -1; // 31-bit non-negative value, or -1 + private boolean audio = true; private int maxSize; private VideoCodec codec = VideoCodec.H264; private int bitRate = 8000000; @@ -49,6 +50,14 @@ public void setScid(int scid) { this.scid = scid; } + public boolean getAudio() { + return audio; + } + + public void setAudio(boolean audio) { + this.audio = audio; + } + public int getMaxSize() { return maxSize; } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index d570a9fb50..2aeeb79ebd 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -169,6 +169,10 @@ private static Options createOptions(String... args) { Ln.Level level = Ln.Level.valueOf(value.toUpperCase(Locale.ENGLISH)); options.setLogLevel(level); break; + case "audio": + boolean audio = Boolean.parseBoolean(value); + options.setAudio(audio); + break; case "codec": VideoCodec codec = VideoCodec.findByName(value); if (codec == null) { From e841241a8ea135772a339687b9e7c45779c7864c Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Fri, 3 Feb 2023 16:50:42 +0100 Subject: [PATCH 037/118] Add a new socket for audio stream When audio is enabled, open a new socket to send the audio stream from the device to the client. PR #3757 Co-authored-by: Romain Vimont Signed-off-by: Romain Vimont --- app/src/server.c | 39 +++++++++++++++++++ app/src/server.h | 1 + .../genymobile/scrcpy/DesktopConnection.java | 30 ++++++++++++-- .../java/com/genymobile/scrcpy/Server.java | 3 +- 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/app/src/server.c b/app/src/server.c index 0ff0d3c8f4..88e3421f87 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -391,6 +391,7 @@ sc_server_init(struct sc_server *server, const struct sc_server_params *params, server->stopped = false; server->video_socket = SC_SOCKET_NONE; + server->audio_socket = SC_SOCKET_NONE; server->control_socket = SC_SOCKET_NONE; sc_adb_tunnel_init(&server->tunnel); @@ -434,9 +435,11 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { const char *serial = server->serial; assert(serial); + bool audio = server->params.audio; bool control = server->params.control; sc_socket video_socket = SC_SOCKET_NONE; + sc_socket audio_socket = SC_SOCKET_NONE; sc_socket control_socket = SC_SOCKET_NONE; if (!tunnel->forward) { video_socket = net_accept_intr(&server->intr, tunnel->server_socket); @@ -444,6 +447,14 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { goto fail; } + if (audio) { + audio_socket = + net_accept_intr(&server->intr, tunnel->server_socket); + if (audio_socket == SC_SOCKET_NONE) { + goto fail; + } + } + if (control) { control_socket = net_accept_intr(&server->intr, tunnel->server_socket); @@ -470,6 +481,18 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { goto fail; } + if (audio) { + audio_socket = net_socket(); + if (audio_socket == SC_SOCKET_NONE) { + goto fail; + } + bool ok = net_connect_intr(&server->intr, audio_socket, tunnel_host, + tunnel_port); + if (!ok) { + goto fail; + } + } + if (control) { // we know that the device is listening, we don't need several // attempts @@ -496,9 +519,11 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { } assert(video_socket != SC_SOCKET_NONE); + assert(!audio || audio_socket != SC_SOCKET_NONE); assert(!control || control_socket != SC_SOCKET_NONE); server->video_socket = video_socket; + server->audio_socket = audio_socket; server->control_socket = control_socket; return true; @@ -510,6 +535,12 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) { } } + if (audio_socket != SC_SOCKET_NONE) { + if (!net_close(audio_socket)) { + LOGW("Could not close audio socket"); + } + } + if (control_socket != SC_SOCKET_NONE) { if (!net_close(control_socket)) { LOGW("Could not close control socket"); @@ -860,6 +891,11 @@ run_server(void *data) { assert(server->video_socket != SC_SOCKET_NONE); net_interrupt(server->video_socket); + if (server->audio_socket != SC_SOCKET_NONE) { + // There is no audio_socket if --no-audio is set + net_interrupt(server->audio_socket); + } + if (server->control_socket != SC_SOCKET_NONE) { // There is no control_socket if --no-control is set net_interrupt(server->control_socket); @@ -924,6 +960,9 @@ sc_server_destroy(struct sc_server *server) { if (server->video_socket != SC_SOCKET_NONE) { net_close(server->video_socket); } + if (server->audio_socket != SC_SOCKET_NONE) { + net_close(server->audio_socket); + } if (server->control_socket != SC_SOCKET_NONE) { net_close(server->control_socket); } diff --git a/app/src/server.h b/app/src/server.h index 95e24b4128..3005ebd263 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -70,6 +70,7 @@ struct sc_server { struct sc_adb_tunnel tunnel; sc_socket video_socket; + sc_socket audio_socket; sc_socket control_socket; const struct sc_server_callbacks *cbs; diff --git a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java index 3cb36a09f0..3e743621c9 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java +++ b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java @@ -20,6 +20,9 @@ public final class DesktopConnection implements Closeable { private final LocalSocket videoSocket; private final FileDescriptor videoFd; + private final LocalSocket audioSocket; + private final FileDescriptor audioFd; + private final LocalSocket controlSocket; private final InputStream controlInputStream; private final OutputStream controlOutputStream; @@ -27,9 +30,10 @@ public final class DesktopConnection implements Closeable { private final ControlMessageReader reader = new ControlMessageReader(); private final DeviceMessageWriter writer = new DeviceMessageWriter(); - private DesktopConnection(LocalSocket videoSocket, LocalSocket controlSocket) throws IOException { + private DesktopConnection(LocalSocket videoSocket, LocalSocket audioSocket, LocalSocket controlSocket) throws IOException { this.videoSocket = videoSocket; this.controlSocket = controlSocket; + this.audioSocket = audioSocket; if (controlSocket != null) { controlInputStream = controlSocket.getInputStream(); controlOutputStream = controlSocket.getOutputStream(); @@ -38,6 +42,7 @@ private DesktopConnection(LocalSocket videoSocket, LocalSocket controlSocket) th controlOutputStream = null; } videoFd = videoSocket.getFileDescriptor(); + audioFd = audioSocket != null ? audioSocket.getFileDescriptor() : null; } private static LocalSocket connect(String abstractName) throws IOException { @@ -55,10 +60,11 @@ private static String getSocketName(int scid) { return SOCKET_NAME_PREFIX + String.format("_%08x", scid); } - public static DesktopConnection open(int scid, boolean tunnelForward, boolean control, boolean sendDummyByte) throws IOException { + public static DesktopConnection open(int scid, boolean tunnelForward, boolean audio, boolean control, boolean sendDummyByte) throws IOException { String socketName = getSocketName(scid); LocalSocket videoSocket = null; + LocalSocket audioSocket = null; LocalSocket controlSocket = null; try { if (tunnelForward) { @@ -68,12 +74,18 @@ public static DesktopConnection open(int scid, boolean tunnelForward, boolean co // send one byte so the client may read() to detect a connection error videoSocket.getOutputStream().write(0); } + if (audio) { + audioSocket = localServerSocket.accept(); + } if (control) { controlSocket = localServerSocket.accept(); } } } else { videoSocket = connect(socketName); + if (audio) { + audioSocket = connect(socketName); + } if (control) { controlSocket = connect(socketName); } @@ -82,19 +94,27 @@ public static DesktopConnection open(int scid, boolean tunnelForward, boolean co if (videoSocket != null) { videoSocket.close(); } + if (audioSocket != null) { + audioSocket.close(); + } if (controlSocket != null) { controlSocket.close(); } throw e; } - return new DesktopConnection(videoSocket, controlSocket); + return new DesktopConnection(videoSocket, audioSocket, controlSocket); } public void close() throws IOException { videoSocket.shutdownInput(); videoSocket.shutdownOutput(); videoSocket.close(); + if (audioSocket != null) { + audioSocket.shutdownInput(); + audioSocket.shutdownOutput(); + audioSocket.close(); + } if (controlSocket != null) { controlSocket.shutdownInput(); controlSocket.shutdownOutput(); @@ -121,6 +141,10 @@ public FileDescriptor getVideoFd() { return videoFd; } + public FileDescriptor getAudioFd() { + return audioFd; + } + public ControlMessage receiveControlMessage() throws IOException { ControlMessage msg = reader.next(); while (msg == null) { diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 2aeeb79ebd..55b38c6daf 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -68,6 +68,7 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc int scid = options.getScid(); boolean tunnelForward = options.isTunnelForward(); boolean control = options.getControl(); + boolean audio = options.getAudio(); boolean sendDummyByte = options.getSendDummyByte(); Workarounds.prepareMainLooper(); @@ -85,7 +86,7 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc Controller controller = null; - try (DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, control, sendDummyByte)) { + try (DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, audio, control, sendDummyByte)) { VideoCodec codec = options.getCodec(); if (options.getSendDeviceMeta()) { Size videoSize = device.getScreenInfo().getVideoSize(); From 11d32616a91f90ec49515a8f72f386bb9322766b Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Fri, 27 Jan 2023 20:13:37 +0800 Subject: [PATCH 038/118] Capture device audio Create an AudioRecorder to capture the audio source REMOTE_SUBMIX. For now, the captured packets are just logged into the console. PR #3757 Co-authored-by: Romain Vimont Signed-off-by: Romain Vimont --- .../com/genymobile/scrcpy/AudioEncoder.java | 80 +++++++++++++++++++ .../java/com/genymobile/scrcpy/Server.java | 38 ++++++--- 2 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java new file mode 100644 index 0000000000..45e217fcad --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -0,0 +1,80 @@ +package com.genymobile.scrcpy; + +import android.annotation.SuppressLint; +import android.annotation.TargetApi; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.MediaRecorder; +import android.os.Build; + +public final class AudioEncoder { + + private static final int SAMPLE_RATE = 48000; + private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO; + private static final int CHANNELS = 2; + private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT; + private static final int BYTES_PER_SAMPLE = 2; + + private static final int READ_MS = 5; // milliseconds + private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000; + + private Thread thread; + + private static AudioFormat createAudioFormat() { + AudioFormat.Builder builder = new AudioFormat.Builder(); + builder.setEncoding(FORMAT); + builder.setSampleRate(SAMPLE_RATE); + builder.setChannelMask(CHANNEL_CONFIG); + return builder.build(); + } + + @TargetApi(Build.VERSION_CODES.M) + @SuppressLint({"WrongConstant", "MissingPermission"}) + private static AudioRecord createAudioRecord() { + AudioRecord.Builder builder = new AudioRecord.Builder(); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + // On older APIs, Workarounds.fillAppInfo() must be called beforehand + builder.setContext(FakeContext.get()); + } + builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX); + builder.setAudioFormat(createAudioFormat()); + int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT); + // This buffer size does not impact latency + builder.setBufferSizeInBytes(8 * minBufferSize); + return builder.build(); + } + + public void start() { + AudioRecord recorder = createAudioRecord(); + + thread = new Thread(() -> { + recorder.startRecording(); + try { + byte[] buf = new byte[READ_SIZE]; + while (!Thread.currentThread().isInterrupted()) { + int r = recorder.read(buf, 0, READ_SIZE); + if (r > 0) { + Ln.i("Audio captured: " + r + " bytes"); + } else { + Ln.e("Audio capture error: " + r); + } + } + } finally { + recorder.stop(); + } + }); + thread.start(); + } + + public void stop() { + if (thread != null) { + thread.interrupt(); + } + } + + public void join() throws InterruptedException { + if (thread != null) { + thread.join(); + } + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 55b38c6daf..c32e4612ed 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -72,19 +72,28 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc boolean sendDummyByte = options.getSendDummyByte(); Workarounds.prepareMainLooper(); - if (Build.BRAND.equalsIgnoreCase("meizu")) { - // Workarounds must be applied for Meizu phones: - // - - // - - // - - // - // But only apply when strictly necessary, since workarounds can cause other issues: - // - - // - + + // Workarounds must be applied for Meizu phones: + // - + // - + // - + // + // But only apply when strictly necessary, since workarounds can cause other issues: + // - + // - + boolean mustFillAppInfo = Build.BRAND.equalsIgnoreCase("meizu"); + + // Before Android 11, audio is not supported. + // Since Android 12, we can properly set a context on the AudioRecord. + // Only on Android 11 we must fill app info for the AudioRecord to work. + mustFillAppInfo |= audio && Build.VERSION.SDK_INT == Build.VERSION_CODES.R; + + if (mustFillAppInfo) { Workarounds.fillAppInfo(); } Controller controller = null; + AudioEncoder audioEncoder = null; try (DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, audio, control, sendDummyByte)) { VideoCodec codec = options.getCodec(); @@ -101,6 +110,11 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc device.setClipboardListener(text -> controllerRef.getSender().pushClipboardText(text)); } + if (audio) { + audioEncoder = new AudioEncoder(); + audioEncoder.start(); + } + Streamer videoStreamer = new Streamer(connection.getVideoFd(), codec, options.getSendCodecId(), options.getSendFrameMeta()); ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getBitRate(), options.getMaxFps(), codecOptions, options.getEncoderName(), options.getDownsizeOnError()); @@ -116,12 +130,18 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc } finally { Ln.d("Screen streaming stopped"); initThread.interrupt(); + if (audioEncoder != null) { + audioEncoder.stop(); + } if (controller != null) { controller.stop(); } try { initThread.join(); + if (audioEncoder != null) { + audioEncoder.join(); + } if (controller != null) { controller.join(); } From 464a35b05e7896ce5173a1fa3e27f00e0899f19e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 14:07:32 +0100 Subject: [PATCH 039/118] Make streamer more generic Expose a method to write a packet from raw metadata (without BufferInfo). --- .../java/com/genymobile/scrcpy/Streamer.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Streamer.java b/server/src/main/java/com/genymobile/scrcpy/Streamer.java index d6bf6780c1..ae43710400 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Streamer.java +++ b/server/src/main/java/com/genymobile/scrcpy/Streamer.java @@ -38,23 +38,30 @@ public void writeHeader() throws IOException { } } - public void writePacket(ByteBuffer codecBuffer, MediaCodec.BufferInfo bufferInfo) throws IOException { + public void writePacket(ByteBuffer buffer, long pts, boolean config, boolean keyFrame) throws IOException { if (sendFrameMeta) { - writeFrameMeta(fd, bufferInfo, codecBuffer.remaining()); + writeFrameMeta(fd, buffer.remaining(), pts, config, keyFrame); } - IO.writeFully(fd, codecBuffer); + IO.writeFully(fd, buffer); + } + + public void writePacket(ByteBuffer codecBuffer, MediaCodec.BufferInfo bufferInfo) throws IOException { + long pts = bufferInfo.presentationTimeUs; + boolean config = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; + boolean keyFrame = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0; + writePacket(codecBuffer, pts, config, keyFrame); } - private void writeFrameMeta(FileDescriptor fd, MediaCodec.BufferInfo bufferInfo, int packetSize) throws IOException { + private void writeFrameMeta(FileDescriptor fd, int packetSize, long pts, boolean config, boolean keyFrame) throws IOException { headerBuffer.clear(); long ptsAndFlags; - if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + if (config) { ptsAndFlags = PACKET_FLAG_CONFIG; // non-media data packet } else { - ptsAndFlags = bufferInfo.presentationTimeUs; - if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) { + ptsAndFlags = pts; + if (keyFrame) { ptsAndFlags |= PACKET_FLAG_KEY_FRAME; } } From 5eed2c52c2f38192544d909ee639a3f02720fe56 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 7 Feb 2023 23:08:57 +0100 Subject: [PATCH 040/118] Encode recorded audio on the device For now, the encoded packets are just logged into the console. PR #3757 --- .../com/genymobile/scrcpy/AudioEncoder.java | 274 +++++++++++++++++- 1 file changed, 260 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 45e217fcad..7b57112667 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -4,21 +4,63 @@ import android.annotation.TargetApi; import android.media.AudioFormat; import android.media.AudioRecord; +import android.media.AudioTimestamp; +import android.media.MediaCodec; +import android.media.MediaFormat; import android.media.MediaRecorder; import android.os.Build; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Looper; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; public final class AudioEncoder { + private static class InputTask { + private final int index; + + InputTask(int index) { + this.index = index; + } + } + + private static class OutputTask { + private final int index; + private final MediaCodec.BufferInfo bufferInfo; + + OutputTask(int index, MediaCodec.BufferInfo bufferInfo) { + this.index = index; + this.bufferInfo = bufferInfo; + } + } + + private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS; private static final int SAMPLE_RATE = 48000; private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO; private static final int CHANNELS = 2; private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT; private static final int BYTES_PER_SAMPLE = 2; + private static final int BIT_RATE = 128000; private static final int READ_MS = 5; // milliseconds private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000; + // Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4). + // So many pending tasks would lead to an unacceptable delay anyway. + private final BlockingQueue inputTasks = new ArrayBlockingQueue<>(64); + private final BlockingQueue outputTasks = new ArrayBlockingQueue<>(64); + private Thread thread; + private HandlerThread mediaCodecThread; + + private Thread inputThread; + private Thread outputThread; + + private boolean ended; private static AudioFormat createAudioFormat() { AudioFormat.Builder builder = new AudioFormat.Builder(); @@ -44,23 +86,80 @@ private static AudioRecord createAudioRecord() { return builder.build(); } - public void start() { - AudioRecord recorder = createAudioRecord(); + private static MediaFormat createFormat() { + MediaFormat format = new MediaFormat(); + format.setString(MediaFormat.KEY_MIME, MIMETYPE); + format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE); + format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS); + format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE); + return format; + } + + @TargetApi(Build.VERSION_CODES.N) + private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException { + final AudioTimestamp timestamp = new AudioTimestamp(); + long previousPts = 0; + long nextPts = 0; + + while (!Thread.currentThread().isInterrupted()) { + InputTask task = inputTasks.take(); + ByteBuffer buffer = mediaCodec.getInputBuffer(task.index); + int r = recorder.read(buffer, READ_SIZE); + if (r < 0) { + throw new IOException("Could not read audio: " + r); + } + + long pts; + + int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC); + if (ret == AudioRecord.SUCCESS) { + pts = timestamp.nanoTime / 1000; + } else { + if (nextPts == 0) { + Ln.w("Could not get any audio timestamp"); + } + // compute from previous timestamp and packet size + pts = nextPts; + } + + long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE); + nextPts = pts + durationUs; + if (previousPts != 0 && pts < previousPts) { + // Audio PTS may come from two sources: + // - recorder.getTimestamp() if the call works; + // - an estimation from the previous PTS and the packet size as a fallback. + // + // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it. + pts = previousPts + 1; + } + + previousPts = pts; + + mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0); + } + } + + private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException { + while (!Thread.currentThread().isInterrupted()) { + OutputTask task = outputTasks.take(); + ByteBuffer buffer = mediaCodec.getOutputBuffer(task.index); + try { + Ln.i("Audio packet [pts=" + task.bufferInfo.presentationTimeUs + "] " + buffer.remaining() + " bytes"); + } finally { + mediaCodec.releaseOutputBuffer(task.index, false); + } + } + } + + public void start() { thread = new Thread(() -> { - recorder.startRecording(); try { - byte[] buf = new byte[READ_SIZE]; - while (!Thread.currentThread().isInterrupted()) { - int r = recorder.read(buf, 0, READ_SIZE); - if (r > 0) { - Ln.i("Audio captured: " + r + " bytes"); - } else { - Ln.e("Audio capture error: " + r); - } - } + encode(); + } catch (IOException e) { + Ln.e("Audio encoding error", e); } finally { - recorder.stop(); + Ln.d("Audio encoder stopped"); } }); thread.start(); @@ -68,7 +167,8 @@ public void start() { public void stop() { if (thread != null) { - thread.interrupt(); + // Just wake up the blocking wait from the thread, so that it properly releases all its resources and terminates + end(); } } @@ -77,4 +177,150 @@ public void join() throws InterruptedException { thread.join(); } } + + private synchronized void end() { + ended = true; + notify(); + } + + private synchronized void waitEnded() { + try { + while (!ended) { + wait(); + } + } catch (InterruptedException e) { + // ignore + } + } + + @TargetApi(Build.VERSION_CODES.M) + public void encode() throws IOException { + MediaCodec mediaCodec = null; + AudioRecord recorder = null; + + boolean mediaCodecStarted = false; + boolean recorderStarted = false; + try { + mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException + + mediaCodecThread = new HandlerThread("AudioEncoder"); + mediaCodecThread.start(); + + MediaFormat format = createFormat(); + mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper())); + mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + + recorder = createAudioRecord(); + recorder.startRecording(); + recorderStarted = true; + + final MediaCodec mediaCodecRef = mediaCodec; + final AudioRecord recorderRef = recorder; + inputThread = new Thread(() -> { + try { + inputThread(mediaCodecRef, recorderRef); + } catch (IOException | InterruptedException e) { + Ln.e("Audio capture error", e); + } finally { + end(); + } + }); + + outputThread = new Thread(() -> { + try { + outputThread(mediaCodecRef); + } catch (InterruptedException e) { + // this is expected on close + } catch (IOException e) { + // Broken pipe is expected on close, because the socket is closed by the client + if (!IO.isBrokenPipe(e)) { + Ln.e("Audio encoding error", e); + } + } finally { + end(); + } + }); + + mediaCodec.start(); + mediaCodecStarted = true; + inputThread.start(); + outputThread.start(); + + waitEnded(); + } finally { + // Cleanup everything (either at the end or on error at any step of the initialization) + if (mediaCodecThread != null) { + Looper looper = mediaCodecThread.getLooper(); + if (looper != null) { + looper.quitSafely(); + } + } + if (inputThread != null) { + inputThread.interrupt(); + } + if (outputThread != null) { + outputThread.interrupt(); + } + + try { + if (mediaCodecThread != null) { + mediaCodecThread.join(); + } + if (inputThread != null) { + inputThread.join(); + } + if (outputThread != null) { + outputThread.join(); + } + } catch (InterruptedException e) { + // Should never happen + throw new AssertionError(e); + } + + if (mediaCodec != null) { + if (mediaCodecStarted) { + mediaCodec.stop(); + } + mediaCodec.release(); + } + if (recorder != null) { + if (recorderStarted) { + recorder.stop(); + } + recorder.release(); + } + } + } + + private class EncoderCallback extends MediaCodec.Callback { + @TargetApi(Build.VERSION_CODES.N) + @Override + public void onInputBufferAvailable(MediaCodec codec, int index) { + try { + inputTasks.put(new InputTask(index)); + } catch (InterruptedException e) { + end(); + } + } + + @Override + public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) { + try { + outputTasks.put(new OutputTask(index, bufferInfo)); + } catch (InterruptedException e) { + end(); + } + } + + @Override + public void onError(MediaCodec codec, MediaCodec.CodecException e) { + Ln.e("MediaCodec error", e); + end(); + } + + @Override + public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) { + // ignore + } + } } From 7cf5cf5875b217a5d2e141214d1dc344bdde4e73 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 9 Feb 2023 21:37:16 +0100 Subject: [PATCH 041/118] Use a streamer to send the audio stream Send each encoded audio packet using a streamer. PR #3757 --- .../com/genymobile/scrcpy/AudioCodec.java | 46 +++++++++++++++++++ .../com/genymobile/scrcpy/AudioEncoder.java | 10 +++- .../java/com/genymobile/scrcpy/Codec.java | 1 + .../java/com/genymobile/scrcpy/Server.java | 3 +- 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/AudioCodec.java diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java b/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java new file mode 100644 index 0000000000..4d9e320158 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java @@ -0,0 +1,46 @@ +package com.genymobile.scrcpy; + +import android.media.MediaFormat; + +public enum AudioCodec implements Codec { + OPUS(0x6f_70_75_73, "opus", MediaFormat.MIMETYPE_AUDIO_OPUS); + + private final int id; // 4-byte ASCII representation of the name + private final String name; + private final String mimeType; + + AudioCodec(int id, String name, String mimeType) { + this.id = id; + this.name = name; + this.mimeType = mimeType; + } + + @Override + public Type getType() { + return Type.AUDIO; + } + + @Override + public int getId() { + return id; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getMimeType() { + return mimeType; + } + + public static AudioCodec findByName(String name) { + for (AudioCodec codec : values()) { + if (codec.name.equals(name)) { + return codec; + } + } + return null; + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 7b57112667..74481f99e1 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -49,6 +49,8 @@ private static class OutputTask { private static final int READ_MS = 5; // milliseconds private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000; + private final Streamer streamer; + // Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4). // So many pending tasks would lead to an unacceptable delay anyway. private final BlockingQueue inputTasks = new ArrayBlockingQueue<>(64); @@ -62,6 +64,10 @@ private static class OutputTask { private boolean ended; + public AudioEncoder(Streamer streamer) { + this.streamer = streamer; + } + private static AudioFormat createAudioFormat() { AudioFormat.Builder builder = new AudioFormat.Builder(); builder.setEncoding(FORMAT); @@ -141,11 +147,13 @@ private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOE } private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException { + streamer.writeHeader(); + while (!Thread.currentThread().isInterrupted()) { OutputTask task = outputTasks.take(); ByteBuffer buffer = mediaCodec.getOutputBuffer(task.index); try { - Ln.i("Audio packet [pts=" + task.bufferInfo.presentationTimeUs + "] " + buffer.remaining() + " bytes"); + streamer.writePacket(buffer, task.bufferInfo); } finally { mediaCodec.releaseOutputBuffer(task.index, false); } diff --git a/server/src/main/java/com/genymobile/scrcpy/Codec.java b/server/src/main/java/com/genymobile/scrcpy/Codec.java index 50e8acada2..7e905af376 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Codec.java +++ b/server/src/main/java/com/genymobile/scrcpy/Codec.java @@ -4,6 +4,7 @@ public interface Codec { enum Type { VIDEO, + AUDIO, } Type getType(); diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index c32e4612ed..eb0c13846d 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -111,7 +111,8 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc } if (audio) { - audioEncoder = new AudioEncoder(); + Streamer audioStreamer = new Streamer(connection.getAudioFd(), AudioCodec.OPUS, options.getSendCodecId(), options.getSendFrameMeta()); + audioEncoder = new AudioEncoder(audioStreamer); audioEncoder.start(); } From 15556d1f3be9aa0cf98c43205b516e622e9c7856 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 17:35:54 +0100 Subject: [PATCH 042/118] Extract OPUS extradata For OPUS codec, FFmpeg expects the raw extradata, but MediaCodec wraps it in some structure. Fix the config packet to send only the raw extradata. PR #3757 --- .../java/com/genymobile/scrcpy/Streamer.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/server/src/main/java/com/genymobile/scrcpy/Streamer.java b/server/src/main/java/com/genymobile/scrcpy/Streamer.java index ae43710400..77d9eefa9a 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Streamer.java +++ b/server/src/main/java/com/genymobile/scrcpy/Streamer.java @@ -11,6 +11,8 @@ public final class Streamer { private static final long PACKET_FLAG_CONFIG = 1L << 63; private static final long PACKET_FLAG_KEY_FRAME = 1L << 62; + private static final long AOPUSHDR = 0x5244485355504F41L; // "AOPUSHDR" in ASCII (little-endian) + private final FileDescriptor fd; private final Codec codec; private final boolean sendCodecId; @@ -39,6 +41,10 @@ public void writeHeader() throws IOException { } public void writePacket(ByteBuffer buffer, long pts, boolean config, boolean keyFrame) throws IOException { + if (config && codec == AudioCodec.OPUS) { + fixOpusConfigPacket(buffer); + } + if (sendFrameMeta) { writeFrameMeta(fd, buffer.remaining(), pts, config, keyFrame); } @@ -71,4 +77,44 @@ private void writeFrameMeta(FileDescriptor fd, int packetSize, long pts, boolean headerBuffer.flip(); IO.writeFully(fd, headerBuffer); } + + private static void fixOpusConfigPacket(ByteBuffer buffer) throws IOException { + // Here is an example of the config packet received for an OPUS stream: + // + // 00000000 41 4f 50 55 53 48 44 52 13 00 00 00 00 00 00 00 |AOPUSHDR........| + // -------------- BELOW IS THE PART WE MUST PUT AS EXTRADATA ------------------- + // 00000010 4f 70 75 73 48 65 61 64 01 01 38 01 80 bb 00 00 |OpusHead..8.....| + // 00000020 00 00 00 |... | + // ------------------------------------------------------------------------------ + // 00000020 41 4f 50 55 53 44 4c 59 08 00 00 00 00 | AOPUSDLY.....| + // 00000030 00 00 00 a0 2e 63 00 00 00 00 00 41 4f 50 55 53 |.....c.....AOPUS| + // 00000040 50 52 4c 08 00 00 00 00 00 00 00 00 b4 c4 04 00 |PRL.............| + // 00000050 00 00 00 |...| + // + // Each "section" is prefixed by a 64-bit ID and a 64-bit length. + // + // + + if (buffer.remaining() < 16) { + throw new IOException("Not enough data in OPUS config packet"); + } + + long id = buffer.getLong(); + if (id != AOPUSHDR) { + throw new IOException("OPUS header not found"); + } + + long sizeLong = buffer.getLong(); + if (sizeLong < 0 || sizeLong >= 0x7FFFFFFF) { + throw new IOException("Invalid block size in OPUS header: " + sizeLong); + } + + int size = (int) sizeLong; + if (buffer.remaining() < size) { + throw new IOException("Not enough data in OPUS header (invalid size: " + size + ")"); + } + + // Set the buffer to point to the OPUS header slice + buffer.limit(buffer.position() + size); + } } From e9876788c9901d43d75031b090c81c0e604ed994 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 6 Feb 2023 10:08:01 +0100 Subject: [PATCH 043/118] Rename demuxer to video_demuxer There will be another demuxer instance for audio. PR #3757 --- app/src/scrcpy.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 6d63a7a150..e4e1355c46 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -40,7 +40,7 @@ struct scrcpy { struct sc_server server; struct sc_screen screen; - struct sc_demuxer demuxer; + struct sc_demuxer video_demuxer; struct sc_decoder decoder; struct sc_recorder recorder; #ifdef HAVE_V4L2 @@ -213,7 +213,8 @@ sc_recorder_on_ended(struct sc_recorder *recorder, bool success, } static void -sc_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, void *userdata) { +sc_video_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, + void *userdata) { (void) demuxer; (void) userdata; @@ -281,7 +282,7 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_V4L2 bool v4l2_sink_initialized = false; #endif - bool demuxer_started = false; + bool video_demuxer_started = false; #ifdef HAVE_USB bool aoa_hid_initialized = false; bool hid_keyboard_initialized = false; @@ -383,10 +384,11 @@ scrcpy(struct scrcpy_options *options) { file_pusher_initialized = true; } - static const struct sc_demuxer_callbacks demuxer_cbs = { - .on_ended = sc_demuxer_on_ended, + static const struct sc_demuxer_callbacks video_demuxer_cbs = { + .on_ended = sc_video_demuxer_on_ended, }; - sc_demuxer_init(&s->demuxer, s->server.video_socket, &demuxer_cbs, NULL); + sc_demuxer_init(&s->video_demuxer, s->server.video_socket, + &video_demuxer_cbs, NULL); bool needs_decoder = options->display; #ifdef HAVE_V4L2 @@ -394,7 +396,7 @@ scrcpy(struct scrcpy_options *options) { #endif if (needs_decoder) { sc_decoder_init(&s->decoder); - sc_demuxer_add_sink(&s->demuxer, &s->decoder.packet_sink); + sc_demuxer_add_sink(&s->video_demuxer, &s->decoder.packet_sink); } if (options->record_filename) { @@ -413,7 +415,7 @@ scrcpy(struct scrcpy_options *options) { } recorder_started = true; - sc_demuxer_add_sink(&s->demuxer, &s->recorder.packet_sink); + sc_demuxer_add_sink(&s->video_demuxer, &s->recorder.packet_sink); } struct sc_controller *controller = NULL; @@ -621,17 +623,17 @@ scrcpy(struct scrcpy_options *options) { #endif // now we consumed the header values, the socket receives the video stream - // start the demuxer - if (!sc_demuxer_start(&s->demuxer)) { + // start the video demuxer + if (!sc_demuxer_start(&s->video_demuxer)) { goto end; } - demuxer_started = true; + video_demuxer_started = true; ret = event_loop(s); LOGD("quit..."); // Close the window immediately on closing, because screen_destroy() may - // only be called once the demuxer thread is joined (it may take time) + // only be called once the video demuxer thread is joined (it may take time) sc_screen_hide_window(&s->screen); end: @@ -672,8 +674,8 @@ scrcpy(struct scrcpy_options *options) { // now that the sockets are shutdown, the demuxer and controller are // interrupted, we can join them - if (demuxer_started) { - sc_demuxer_join(&s->demuxer); + if (video_demuxer_started) { + sc_demuxer_join(&s->video_demuxer); } #ifdef HAVE_V4L2 @@ -692,8 +694,9 @@ scrcpy(struct scrcpy_options *options) { } #endif - // Destroy the screen only after the demuxer is guaranteed to be finished, - // because otherwise the screen could receive new frames after destruction + // Destroy the screen only after the video demuxer is guaranteed to be + // finished, because otherwise the screen could receive new frames after + // destruction if (screen_initialized) { sc_screen_join(&s->screen); sc_screen_destroy(&s->screen); From d499f890e7fe019ac3c60bec7ceb61d39aaca97b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 00:13:54 +0100 Subject: [PATCH 044/118] Give a name to demuxer instances This will be useful in logs. PR #3757 --- app/src/demuxer.c | 13 +++++++------ app/src/demuxer.h | 5 ++++- app/src/scrcpy.c | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/app/src/demuxer.c b/app/src/demuxer.c index c83d6bfa30..e5f0762823 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -122,7 +122,7 @@ static bool sc_demuxer_push_packet(struct sc_demuxer *demuxer, AVPacket *packet) { bool ok = push_packet_to_sinks(demuxer, packet); if (!ok) { - LOGE("Could not process packet"); + LOGE("Demuxer '%s': could not process packet", demuxer->name); return false; } @@ -177,7 +177,7 @@ run_demuxer(void *data) { const AVCodec *codec = avcodec_find_decoder(codec_id); if (!codec) { - LOGE("Decoder not found"); + LOGE("Demuxer '%s': decoder not found", demuxer->name); goto end; } @@ -217,7 +217,7 @@ run_demuxer(void *data) { } } - LOGD("End of frames"); + LOGD("Demuxer '%s': end of frames", demuxer->name); sc_packet_merger_destroy(&merger); @@ -231,8 +231,9 @@ run_demuxer(void *data) { } void -sc_demuxer_init(struct sc_demuxer *demuxer, sc_socket socket, +sc_demuxer_init(struct sc_demuxer *demuxer, const char *name, sc_socket socket, const struct sc_demuxer_callbacks *cbs, void *cbs_userdata) { + demuxer->name = name; // statically allocated demuxer->socket = socket; demuxer->sink_count = 0; @@ -252,12 +253,12 @@ sc_demuxer_add_sink(struct sc_demuxer *demuxer, struct sc_packet_sink *sink) { bool sc_demuxer_start(struct sc_demuxer *demuxer) { - LOGD("Starting demuxer thread"); + LOGD("Demuxer '%s': starting thread", demuxer->name); bool ok = sc_thread_create(&demuxer->thread, run_demuxer, "scrcpy-demuxer", demuxer); if (!ok) { - LOGE("Could not start demuxer thread"); + LOGE("Demuxer '%s': could not start thread", demuxer->name); return false; } return true; diff --git a/app/src/demuxer.h b/app/src/demuxer.h index e403fe3588..73166b41ef 100644 --- a/app/src/demuxer.h +++ b/app/src/demuxer.h @@ -15,6 +15,8 @@ #define SC_DEMUXER_MAX_SINKS 2 struct sc_demuxer { + const char *name; // must be statically allocated (e.g. a string literal) + sc_socket socket; sc_thread thread; @@ -29,8 +31,9 @@ struct sc_demuxer_callbacks { void (*on_ended)(struct sc_demuxer *demuxer, bool eos, void *userdata); }; +// The name must be statically allocated (e.g. a string literal) void -sc_demuxer_init(struct sc_demuxer *demuxer, sc_socket socket, +sc_demuxer_init(struct sc_demuxer *demuxer, const char *name, sc_socket socket, const struct sc_demuxer_callbacks *cbs, void *cbs_userdata); void diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index e4e1355c46..820f332841 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -387,7 +387,7 @@ scrcpy(struct scrcpy_options *options) { static const struct sc_demuxer_callbacks video_demuxer_cbs = { .on_ended = sc_video_demuxer_on_ended, }; - sc_demuxer_init(&s->video_demuxer, s->server.video_socket, + sc_demuxer_init(&s->video_demuxer, "video", s->server.video_socket, &video_demuxer_cbs, NULL); bool needs_decoder = options->display; From f60b5767f4aec15d07e8053e25d5a68961618411 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 13:49:53 +0100 Subject: [PATCH 045/118] Force --no-audio if no display and no recording The client does not use the audio stream if there is no display and no recording (i.e. only V4L2), so disable audio so that the device does not attempt to capture it. PR #3757 --- app/src/cli.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/src/cli.c b/app/src/cli.c index c1ee1e6dea..4f023da031 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -1740,6 +1740,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } #endif + if (opts->audio && !opts->display && !opts->record_filename) { + LOGI("No display and no recording: audio disabled"); + opts->audio = false; + } + if ((opts->tunnel_host || opts->tunnel_port) && !opts->force_adb_forward) { LOGI("Tunnel host/port is set, " "--force-adb-forward automatically enabled."); From de430bc4aa3b63e8a602032a42613d9672f7c181 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 6 Feb 2023 10:33:47 +0100 Subject: [PATCH 046/118] Add an audio demuxer Add a demuxer which will read the stream from the audio socket. PR #3757 --- app/src/demuxer.c | 5 +++++ app/src/scrcpy.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/app/src/demuxer.c b/app/src/demuxer.c index e5f0762823..c5898060f5 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -23,6 +23,7 @@ sc_demuxer_to_avcodec_id(uint32_t codec_id) { #define SC_CODEC_ID_H264 UINT32_C(0x68323634) // "h264" in ASCII #define SC_CODEC_ID_H265 UINT32_C(0x68323635) // "h265" in ASCII #define SC_CODEC_ID_AV1 UINT32_C(0x00617631) // "av1" in ASCII +#define SC_CODEC_ID_OPUS UINT32_C(0x6f707573) // "opus" in ASCII switch (codec_id) { case SC_CODEC_ID_H264: return AV_CODEC_ID_H264; @@ -30,6 +31,8 @@ sc_demuxer_to_avcodec_id(uint32_t codec_id) { return AV_CODEC_ID_HEVC; case SC_CODEC_ID_AV1: return AV_CODEC_ID_AV1; + case SC_CODEC_ID_OPUS: + return AV_CODEC_ID_OPUS; default: LOGE("Unknown codec id 0x%08" PRIx32, codec_id); return AV_CODEC_ID_NONE; @@ -233,6 +236,8 @@ run_demuxer(void *data) { void sc_demuxer_init(struct sc_demuxer *demuxer, const char *name, sc_socket socket, const struct sc_demuxer_callbacks *cbs, void *cbs_userdata) { + assert(socket != SC_SOCKET_NONE); + demuxer->name = name; // statically allocated demuxer->socket = socket; demuxer->sink_count = 0; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 820f332841..0c84612398 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -41,6 +41,7 @@ struct scrcpy { struct sc_server server; struct sc_screen screen; struct sc_demuxer video_demuxer; + struct sc_demuxer audio_demuxer; struct sc_decoder decoder; struct sc_recorder recorder; #ifdef HAVE_V4L2 @@ -225,6 +226,16 @@ sc_video_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, } } +static void +sc_audio_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, + void *userdata) { + (void) demuxer; + (void) eos; + (void) userdata; + + // Contrary to the video demuxer, keep mirroring if only the audio fails +} + static void sc_server_on_connection_failed(struct sc_server *server, void *userdata) { (void) server; @@ -283,6 +294,7 @@ scrcpy(struct scrcpy_options *options) { bool v4l2_sink_initialized = false; #endif bool video_demuxer_started = false; + bool audio_demuxer_started = false; #ifdef HAVE_USB bool aoa_hid_initialized = false; bool hid_keyboard_initialized = false; @@ -390,6 +402,14 @@ scrcpy(struct scrcpy_options *options) { sc_demuxer_init(&s->video_demuxer, "video", s->server.video_socket, &video_demuxer_cbs, NULL); + if (options->audio) { + static const struct sc_demuxer_callbacks audio_demuxer_cbs = { + .on_ended = sc_audio_demuxer_on_ended, + }; + sc_demuxer_init(&s->audio_demuxer, "audio", s->server.audio_socket, + &audio_demuxer_cbs, NULL); + } + bool needs_decoder = options->display; #ifdef HAVE_V4L2 needs_decoder |= !!options->v4l2_device; @@ -629,6 +649,13 @@ scrcpy(struct scrcpy_options *options) { } video_demuxer_started = true; + if (options->audio) { + if (!sc_demuxer_start(&s->audio_demuxer)) { + goto end; + } + audio_demuxer_started = true; + } + ret = event_loop(s); LOGD("quit..."); @@ -678,6 +705,10 @@ scrcpy(struct scrcpy_options *options) { sc_demuxer_join(&s->video_demuxer); } + if (audio_demuxer_started) { + sc_demuxer_join(&s->audio_demuxer); + } + #ifdef HAVE_V4L2 if (v4l2_sink_initialized) { sc_v4l2_sink_destroy(&s->v4l2_sink); From 609b098a97322b73de8ec6cd375926ac1f6f8307 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 17:37:58 +0100 Subject: [PATCH 047/118] Do not merge config audio packets For video streams (at least H.264 and H.265), the config packet containing SPS/PPS must be prepended to the next packet (the following keyframe). For audio streams (at least OPUS), they must not be merged. PR #3757 --- app/src/demuxer.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/app/src/demuxer.c b/app/src/demuxer.c index c5898060f5..968e6db093 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -188,8 +188,15 @@ run_demuxer(void *data) { goto end; } + // Config packets must be merged with the next non-config packet only for + // video streams + bool must_merge_config_packet = codec->type == AVMEDIA_TYPE_VIDEO; + struct sc_packet_merger merger; - sc_packet_merger_init(&merger); + + if (must_merge_config_packet) { + sc_packet_merger_init(&merger); + } AVPacket *packet = av_packet_alloc(); if (!packet) { @@ -205,11 +212,13 @@ run_demuxer(void *data) { break; } - // Prepend any config packet to the next media packet - ok = sc_packet_merger_merge(&merger, packet); - if (!ok) { - av_packet_unref(packet); - break; + if (must_merge_config_packet) { + // Prepend any config packet to the next media packet + ok = sc_packet_merger_merge(&merger, packet); + if (!ok) { + av_packet_unref(packet); + break; + } } ok = sc_demuxer_push_packet(demuxer, packet); @@ -222,7 +231,9 @@ run_demuxer(void *data) { LOGD("Demuxer '%s': end of frames", demuxer->name); - sc_packet_merger_destroy(&merger); + if (must_merge_config_packet) { + sc_packet_merger_destroy(&merger); + } av_packet_free(&packet); finally_close_sinks: From 3d29f6ef0684aa9ef7a47690aa82326b6f830293 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 15 Feb 2023 10:06:10 +0100 Subject: [PATCH 048/118] Rename video-specific variables in recorder This paves the way to add audio-specific variables. PR #3757 --- app/src/recorder.c | 58 ++++++++++++++++++++++++---------------------- app/src/recorder.h | 8 +++---- app/src/scrcpy.c | 2 +- 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 7cc697787d..868176812a 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -8,8 +8,9 @@ #include "util/log.h" #include "util/str.h" -/** Downcast packet_sink to recorder */ -#define DOWNCAST(SINK) container_of(SINK, struct sc_recorder, packet_sink) +/** Downcast video packet_sink to recorder */ +#define DOWNCAST_VIDEO(SINK) \ + container_of(SINK, struct sc_recorder, video_packet_sink) static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us @@ -154,10 +155,10 @@ sc_recorder_close_output_file(struct sc_recorder *recorder) { static bool sc_recorder_wait_video_stream(struct sc_recorder *recorder) { sc_mutex_lock(&recorder->mutex); - while (!recorder->codec && !recorder->stopped) { + while (!recorder->video_codec && !recorder->stopped) { sc_cond_wait(&recorder->stream_cond, &recorder->mutex); } - const AVCodec *codec = recorder->codec; + const AVCodec *codec = recorder->video_codec; sc_mutex_unlock(&recorder->mutex); if (codec) { @@ -180,17 +181,17 @@ static bool sc_recorder_process_header(struct sc_recorder *recorder) { sc_mutex_lock(&recorder->mutex); - while (!recorder->stopped && sc_queue_is_empty(&recorder->queue)) { + while (!recorder->stopped && sc_queue_is_empty(&recorder->video_queue)) { sc_cond_wait(&recorder->queue_cond, &recorder->mutex); } - if (recorder->stopped && sc_queue_is_empty(&recorder->queue)) { + if (recorder->stopped && sc_queue_is_empty(&recorder->video_queue)) { sc_mutex_unlock(&recorder->mutex); return false; } struct sc_record_packet *rec; - sc_queue_take(&recorder->queue, next, &rec); + sc_queue_take(&recorder->video_queue, next, &rec); sc_mutex_unlock(&recorder->mutex); @@ -226,20 +227,21 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { for (;;) { sc_mutex_lock(&recorder->mutex); - while (!recorder->stopped && sc_queue_is_empty(&recorder->queue)) { + while (!recorder->stopped + && sc_queue_is_empty(&recorder->video_queue)) { sc_cond_wait(&recorder->queue_cond, &recorder->mutex); } // if stopped is set, continue to process the remaining events (to // finish the recording) before actually stopping - if (recorder->stopped && sc_queue_is_empty(&recorder->queue)) { + if (recorder->stopped && sc_queue_is_empty(&recorder->video_queue)) { sc_mutex_unlock(&recorder->mutex); break; } struct sc_record_packet *rec; - sc_queue_take(&recorder->queue, next, &rec); + sc_queue_take(&recorder->video_queue, next, &rec); sc_mutex_unlock(&recorder->mutex); @@ -336,7 +338,7 @@ run_recorder(void *data) { // Prevent the producer to push any new packet recorder->stopped = true; // Discard pending packets - sc_recorder_queue_clear(&recorder->queue); + sc_recorder_queue_clear(&recorder->video_queue); sc_mutex_unlock(&recorder->mutex); if (success) { @@ -355,9 +357,9 @@ run_recorder(void *data) { } static bool -sc_recorder_packet_sink_open(struct sc_packet_sink *sink, - const AVCodec *codec) { - struct sc_recorder *recorder = DOWNCAST(sink); +sc_recorder_video_packet_sink_open(struct sc_packet_sink *sink, + const AVCodec *codec) { + struct sc_recorder *recorder = DOWNCAST_VIDEO(sink); assert(codec); sc_mutex_lock(&recorder->mutex); @@ -366,7 +368,7 @@ sc_recorder_packet_sink_open(struct sc_packet_sink *sink, return false; } - recorder->codec = codec; + recorder->video_codec = codec; sc_cond_signal(&recorder->stream_cond); sc_mutex_unlock(&recorder->mutex); @@ -374,8 +376,8 @@ sc_recorder_packet_sink_open(struct sc_packet_sink *sink, } static void -sc_recorder_packet_sink_close(struct sc_packet_sink *sink) { - struct sc_recorder *recorder = DOWNCAST(sink); +sc_recorder_video_packet_sink_close(struct sc_packet_sink *sink) { + struct sc_recorder *recorder = DOWNCAST_VIDEO(sink); sc_mutex_lock(&recorder->mutex); // EOS also stops the recorder @@ -385,9 +387,9 @@ sc_recorder_packet_sink_close(struct sc_packet_sink *sink) { } static bool -sc_recorder_packet_sink_push(struct sc_packet_sink *sink, - const AVPacket *packet) { - struct sc_recorder *recorder = DOWNCAST(sink); +sc_recorder_video_packet_sink_push(struct sc_packet_sink *sink, + const AVPacket *packet) { + struct sc_recorder *recorder = DOWNCAST_VIDEO(sink); sc_mutex_lock(&recorder->mutex); @@ -404,7 +406,7 @@ sc_recorder_packet_sink_push(struct sc_packet_sink *sink, return false; } - sc_queue_push(&recorder->queue, next, rec); + sc_queue_push(&recorder->video_queue, next, rec); sc_cond_signal(&recorder->queue_cond); sc_mutex_unlock(&recorder->mutex); @@ -437,10 +439,10 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, goto error_queue_cond_destroy; } - sc_queue_init(&recorder->queue); + sc_queue_init(&recorder->video_queue); recorder->stopped = false; - recorder->codec = NULL; + recorder->video_codec = NULL; recorder->format = format; recorder->declared_frame_size = declared_frame_size; @@ -449,13 +451,13 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, recorder->cbs = cbs; recorder->cbs_userdata = cbs_userdata; - static const struct sc_packet_sink_ops ops = { - .open = sc_recorder_packet_sink_open, - .close = sc_recorder_packet_sink_close, - .push = sc_recorder_packet_sink_push, + static const struct sc_packet_sink_ops video_ops = { + .open = sc_recorder_video_packet_sink_open, + .close = sc_recorder_video_packet_sink_close, + .push = sc_recorder_video_packet_sink_push, }; - recorder->packet_sink.ops = &ops; + recorder->video_packet_sink.ops = &video_ops; return true; diff --git a/app/src/recorder.h b/app/src/recorder.h index 287030bc6f..b7a10e5e29 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -20,7 +20,7 @@ struct sc_record_packet { struct sc_recorder_queue SC_QUEUE(struct sc_record_packet); struct sc_recorder { - struct sc_packet_sink packet_sink; // packet sink trait + struct sc_packet_sink video_packet_sink; // packet sink trait char *filename; enum sc_record_format format; @@ -32,11 +32,11 @@ struct sc_recorder { sc_cond queue_cond; // set on sc_recorder_stop(), packet_sink close or recording failure bool stopped; - struct sc_recorder_queue queue; + struct sc_recorder_queue video_queue; - // wake up the recorder thread once the codec in known + // wake up the recorder thread once the video codec is known sc_cond stream_cond; - const AVCodec *codec; + const AVCodec *video_codec; const struct sc_recorder_callbacks *cbs; void *cbs_userdata; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 0c84612398..437ae650b3 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -435,7 +435,7 @@ scrcpy(struct scrcpy_options *options) { } recorder_started = true; - sc_demuxer_add_sink(&s->video_demuxer, &s->recorder.packet_sink); + sc_demuxer_add_sink(&s->video_demuxer, &s->recorder.video_packet_sink); } struct sc_controller *controller = NULL; From 7de062221465fef6c6be5998b3884bdc833c8526 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 18:02:43 +0100 Subject: [PATCH 049/118] Add audio recording support Make the recorder accept two input sources (video and audio), and mux them into a single file. PR #3757 --- app/src/recorder.c | 393 ++++++++++++++++++++++++++++++++++++--------- app/src/recorder.h | 14 +- app/src/scrcpy.c | 8 +- 3 files changed, 334 insertions(+), 81 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 868176812a..93c3321f0f 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -8,9 +8,11 @@ #include "util/log.h" #include "util/str.h" -/** Downcast video packet_sink to recorder */ +/** Downcast packet sinks to recorder */ #define DOWNCAST_VIDEO(SINK) \ container_of(SINK, struct sc_recorder, video_packet_sink) +#define DOWNCAST_AUDIO(SINK) \ + container_of(SINK, struct sc_recorder, audio_packet_sink) static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us @@ -79,9 +81,7 @@ sc_recorder_get_format_name(enum sc_record_format format) { } static bool -sc_recorder_write_header(struct sc_recorder *recorder, const AVPacket *packet) { - AVStream *ostream = recorder->ctx->streams[0]; - +sc_recorder_set_extradata(AVStream *ostream, const AVPacket *packet) { uint8_t *extradata = av_malloc(packet->size * sizeof(uint8_t)); if (!extradata) { LOG_OOM(); @@ -93,20 +93,32 @@ sc_recorder_write_header(struct sc_recorder *recorder, const AVPacket *packet) { ostream->codecpar->extradata = extradata; ostream->codecpar->extradata_size = packet->size; - - return avformat_write_header(recorder->ctx, NULL) >= 0; + return true; } -static void -sc_recorder_rescale_packet(struct sc_recorder *recorder, AVPacket *packet) { - AVStream *ostream = recorder->ctx->streams[0]; - av_packet_rescale_ts(packet, SCRCPY_TIME_BASE, ostream->time_base); +static inline void +sc_recorder_rescale_packet(AVStream *stream, AVPacket *packet) { + av_packet_rescale_ts(packet, SCRCPY_TIME_BASE, stream->time_base); } static bool -sc_recorder_write(struct sc_recorder *recorder, AVPacket *packet) { - sc_recorder_rescale_packet(recorder, packet); - return av_write_frame(recorder->ctx, packet) >= 0; +sc_recorder_write_stream(struct sc_recorder *recorder, int stream_index, + AVPacket *packet) { + AVStream *stream = recorder->ctx->streams[stream_index]; + sc_recorder_rescale_packet(stream, packet); + return av_interleaved_write_frame(recorder->ctx, packet) >= 0; +} + +static inline bool +sc_recorder_write_video(struct sc_recorder *recorder, AVPacket *packet) { + return sc_recorder_write_stream(recorder, recorder->video_stream_index, + packet); +} + +static inline bool +sc_recorder_write_audio(struct sc_recorder *recorder, AVPacket *packet) { + return sc_recorder_write_stream(recorder, recorder->audio_stream_index, + packet); } static bool @@ -162,134 +174,270 @@ sc_recorder_wait_video_stream(struct sc_recorder *recorder) { sc_mutex_unlock(&recorder->mutex); if (codec) { - AVStream *ostream = avformat_new_stream(recorder->ctx, codec); - if (!ostream) { + AVStream *stream = avformat_new_stream(recorder->ctx, codec); + if (!stream) { return false; } - ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; - ostream->codecpar->codec_id = codec->id; - ostream->codecpar->format = AV_PIX_FMT_YUV420P; - ostream->codecpar->width = recorder->declared_frame_size.width; - ostream->codecpar->height = recorder->declared_frame_size.height; + stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + stream->codecpar->codec_id = codec->id; + stream->codecpar->format = AV_PIX_FMT_YUV420P; + stream->codecpar->width = recorder->declared_frame_size.width; + stream->codecpar->height = recorder->declared_frame_size.height; + + recorder->video_stream_index = stream->index; } return true; } +static bool +sc_recorder_wait_audio_stream(struct sc_recorder *recorder) { + sc_mutex_lock(&recorder->mutex); + while (!recorder->audio_codec && !recorder->stopped) { + sc_cond_wait(&recorder->stream_cond, &recorder->mutex); + } + const AVCodec *codec = recorder->audio_codec; + sc_mutex_unlock(&recorder->mutex); + + if (codec) { + AVStream *stream = avformat_new_stream(recorder->ctx, codec); + if (!stream) { + return false; + } + + stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; + stream->codecpar->codec_id = codec->id; + stream->codecpar->ch_layout.nb_channels = 2; + stream->codecpar->sample_rate = 48000; + + recorder->audio_stream_index = stream->index; + } + + return true; +} + +static inline bool +sc_recorder_has_empty_queues(struct sc_recorder *recorder) { + if (sc_queue_is_empty(&recorder->video_queue)) { + // The video queue is empty + return true; + } + + if (recorder->audio && sc_queue_is_empty(&recorder->audio_queue)) { + // The audio queue is empty (when audio is enabled) + return true; + } + + // No queue is empty + return false; +} + static bool sc_recorder_process_header(struct sc_recorder *recorder) { sc_mutex_lock(&recorder->mutex); - while (!recorder->stopped && sc_queue_is_empty(&recorder->video_queue)) { + while (!recorder->stopped && sc_recorder_has_empty_queues(recorder)) { sc_cond_wait(&recorder->queue_cond, &recorder->mutex); } - if (recorder->stopped && sc_queue_is_empty(&recorder->video_queue)) { + if (sc_recorder_has_empty_queues(recorder)) { + assert(recorder->stopped); sc_mutex_unlock(&recorder->mutex); return false; } - struct sc_record_packet *rec; - sc_queue_take(&recorder->video_queue, next, &rec); + struct sc_record_packet *video_pkt; + sc_queue_take(&recorder->video_queue, next, &video_pkt); + + struct sc_record_packet *audio_pkt; + if (recorder->audio) { + sc_queue_take(&recorder->audio_queue, next, &audio_pkt); + } sc_mutex_unlock(&recorder->mutex); - if (rec->packet->pts != AV_NOPTS_VALUE) { - LOGE("The first packet is not a config packet"); - sc_record_packet_delete(rec); - return false; + int ret = false; + + if (video_pkt->packet->pts != AV_NOPTS_VALUE) { + LOGE("The first video packet is not a config packet"); + goto end; } - bool ok = sc_recorder_write_header(recorder, rec->packet); - sc_record_packet_delete(rec); + assert(recorder->video_stream_index >= 0); + AVStream *video_stream = + recorder->ctx->streams[recorder->video_stream_index]; + bool ok = sc_recorder_set_extradata(video_stream, video_pkt->packet); + if (!ok) { + goto end; + } + + if (recorder->audio) { + if (audio_pkt->packet->pts != AV_NOPTS_VALUE) { + LOGE("The first audio packet is not a config packet"); + goto end; + } + + assert(recorder->audio_stream_index >= 0); + AVStream *audio_stream = + recorder->ctx->streams[recorder->audio_stream_index]; + ok = sc_recorder_set_extradata(audio_stream, audio_pkt->packet); + if (!ok) { + goto end; + } + } + + ok = avformat_write_header(recorder->ctx, NULL) >= 0; if (!ok) { LOGE("Failed to write header to %s", recorder->filename); - return false; + goto end; } - return true; + ret = true; + +end: + sc_record_packet_delete(video_pkt); + if (recorder->audio) { + sc_record_packet_delete(audio_pkt); + } + + return ret; } static bool sc_recorder_process_packets(struct sc_recorder *recorder) { int64_t pts_origin = AV_NOPTS_VALUE; - // We can write a packet only once we received the next one so that we can - // set its duration (next_pts - current_pts) - struct sc_record_packet *previous = NULL; - bool header_written = sc_recorder_process_header(recorder); if (!header_written) { return false; } + struct sc_record_packet *video_pkt = NULL; + struct sc_record_packet *audio_pkt = NULL; + + // We can write a video packet only once we received the next one so that + // we can set its duration (next_pts - current_pts) + struct sc_record_packet *video_pkt_previous = NULL; + + bool error = false; + for (;;) { sc_mutex_lock(&recorder->mutex); - while (!recorder->stopped - && sc_queue_is_empty(&recorder->video_queue)) { + while (!recorder->stopped) { + if (!video_pkt && !sc_queue_is_empty(&recorder->video_queue)) { + // A new packet may be assigned to video_pkt and be processed + break; + } + if (recorder->audio && !audio_pkt + && !sc_queue_is_empty(&recorder->audio_queue)) { + // A new packet may be assigned to audio_pkt and be processed + break; + } sc_cond_wait(&recorder->queue_cond, &recorder->mutex); } - // if stopped is set, continue to process the remaining events (to - // finish the recording) before actually stopping + // If stopped is set, continue to process the remaining events (to + // finish the recording) before actually stopping. + + // If there is no audio, then the audio_queue will remain empty forever + // and audio_pkt will always be NULL. + assert(recorder->audio + || (!audio_pkt && sc_queue_is_empty(&recorder->audio_queue))); + + if (!video_pkt && !sc_queue_is_empty(&recorder->video_queue)) { + sc_queue_take(&recorder->video_queue, next, &video_pkt); + } + + if (!audio_pkt && !sc_queue_is_empty(&recorder->audio_queue)) { + sc_queue_take(&recorder->audio_queue, next, &audio_pkt); + } - if (recorder->stopped && sc_queue_is_empty(&recorder->video_queue)) { + if (recorder->stopped && !video_pkt && !audio_pkt) { + assert(sc_queue_is_empty(&recorder->video_queue)); + assert(sc_queue_is_empty(&recorder->audio_queue)); sc_mutex_unlock(&recorder->mutex); break; } - struct sc_record_packet *rec; - sc_queue_take(&recorder->video_queue, next, &rec); + assert(video_pkt || audio_pkt); // at least one sc_mutex_unlock(&recorder->mutex); - if (rec->packet->pts == AV_NOPTS_VALUE) { - // Ignore further config packets (e.g. on device orientation - // change). The next non-config packet will have the config packet - // data prepended. - sc_record_packet_delete(rec); - } else { - assert(rec->packet->pts != AV_NOPTS_VALUE); - - if (!previous) { - // This is the first non-config packet - assert(pts_origin == AV_NOPTS_VALUE); - pts_origin = rec->packet->pts; - rec->packet->pts = 0; - rec->packet->dts = 0; - previous = rec; + // Ignore further config packets (e.g. on device orientation + // change). The next non-config packet will have the config packet + // data prepended. + if (video_pkt && video_pkt->packet->pts == AV_NOPTS_VALUE) { + sc_record_packet_delete(video_pkt); + video_pkt = NULL; + } + + if (audio_pkt && audio_pkt->packet->pts == AV_NOPTS_VALUE) { + sc_record_packet_delete(audio_pkt); + audio_pkt= NULL; + } + + if (pts_origin == AV_NOPTS_VALUE) { + if (!recorder->audio) { + assert(video_pkt); + pts_origin = video_pkt->packet->pts; + } else if (video_pkt && audio_pkt) { + pts_origin = + MIN(video_pkt->packet->pts, audio_pkt->packet->pts); + } else { + // We need both video and audio packets to initialize pts_origin continue; } + } - assert(previous); - assert(pts_origin != AV_NOPTS_VALUE); + assert(pts_origin != AV_NOPTS_VALUE); + + if (video_pkt) { + video_pkt->packet->pts -= pts_origin; + video_pkt->packet->dts = video_pkt->packet->pts; + + if (video_pkt_previous) { + // we now know the duration of the previous packet + video_pkt_previous->packet->duration = + video_pkt->packet->pts - video_pkt_previous->packet->pts; + + bool ok = sc_recorder_write_video(recorder, + video_pkt_previous->packet); + sc_record_packet_delete(video_pkt_previous); + if (!ok) { + LOGE("Could not record video packet"); + error = true; + goto end; + } + } - rec->packet->pts -= pts_origin; - rec->packet->dts = rec->packet->pts; + video_pkt_previous = video_pkt; + video_pkt = NULL; + } - // we now know the duration of the previous packet - previous->packet->duration = - rec->packet->pts - previous->packet->pts; + if (audio_pkt) { + audio_pkt->packet->pts -= pts_origin; + audio_pkt->packet->dts = audio_pkt->packet->pts; - bool ok = sc_recorder_write(recorder, previous->packet); - sc_record_packet_delete(previous); + bool ok = sc_recorder_write_audio(recorder, audio_pkt->packet); if (!ok) { - LOGE("Could not record packet"); - return false; + LOGE("Could not record audio packet"); + error = true; + goto end; } - previous = rec; + sc_record_packet_delete(audio_pkt); + audio_pkt = NULL; } } - // Write the last packet - struct sc_record_packet *last = previous; + // Write the last video packet + struct sc_record_packet *last = video_pkt_previous; if (last) { // assign an arbitrary duration to the last packet last->packet->duration = 100000; - bool ok = sc_recorder_write(recorder, last->packet); + bool ok = sc_recorder_write_video(recorder, last->packet); if (!ok) { // failing to write the last frame is not very serious, no // future frame may depend on it, so the resulting file @@ -302,10 +450,18 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { int ret = av_write_trailer(recorder->ctx); if (ret < 0) { LOGE("Failed to write trailer to %s", recorder->filename); - return false; + error = false; } - return true; +end: + if (video_pkt) { + sc_record_packet_delete(video_pkt); + } + if (audio_pkt) { + sc_record_packet_delete(audio_pkt); + } + + return !error; } static bool @@ -321,6 +477,14 @@ sc_recorder_record(struct sc_recorder *recorder) { return false; } + if (recorder->audio) { + ok = sc_recorder_wait_audio_stream(recorder); + if (!ok) { + sc_recorder_close_output_file(recorder); + return false; + } + } + // If recorder->stopped, process any queued packet anyway ok = sc_recorder_process_packets(recorder); @@ -339,6 +503,7 @@ run_recorder(void *data) { recorder->stopped = true; // Discard pending packets sc_recorder_queue_clear(&recorder->video_queue); + sc_recorder_queue_clear(&recorder->audio_queue); sc_mutex_unlock(&recorder->mutex); if (success) { @@ -406,6 +571,8 @@ sc_recorder_video_packet_sink_push(struct sc_packet_sink *sink, return false; } + rec->packet->stream_index = recorder->video_stream_index; + sc_queue_push(&recorder->video_queue, next, rec); sc_cond_signal(&recorder->queue_cond); @@ -413,9 +580,66 @@ sc_recorder_video_packet_sink_push(struct sc_packet_sink *sink, return true; } +static bool +sc_recorder_audio_packet_sink_open(struct sc_packet_sink *sink, + const AVCodec *codec) { + struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); + assert(recorder->audio); + assert(codec); + + sc_mutex_lock(&recorder->mutex); + recorder->audio_codec = codec; + sc_cond_signal(&recorder->stream_cond); + sc_mutex_unlock(&recorder->mutex); + + return true; +} + +static void +sc_recorder_audio_packet_sink_close(struct sc_packet_sink *sink) { + struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); + assert(recorder->audio); + + sc_mutex_lock(&recorder->mutex); + // EOS also stops the recorder + recorder->stopped = true; + sc_cond_signal(&recorder->queue_cond); + sc_mutex_unlock(&recorder->mutex); +} + +static bool +sc_recorder_audio_packet_sink_push(struct sc_packet_sink *sink, + const AVPacket *packet) { + struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); + assert(recorder->audio); + + sc_mutex_lock(&recorder->mutex); + + if (recorder->stopped) { + // reject any new packet + sc_mutex_unlock(&recorder->mutex); + return false; + } + + struct sc_record_packet *rec = sc_record_packet_new(packet); + if (!rec) { + LOG_OOM(); + sc_mutex_unlock(&recorder->mutex); + return false; + } + + rec->packet->stream_index = recorder->audio_stream_index; + + sc_queue_push(&recorder->audio_queue, next, rec); + sc_cond_signal(&recorder->queue_cond); + + sc_mutex_unlock(&recorder->mutex); + return true; +} + bool sc_recorder_init(struct sc_recorder *recorder, const char *filename, - enum sc_record_format format, + enum sc_record_format format, bool audio, struct sc_size declared_frame_size, const struct sc_recorder_callbacks *cbs, void *cbs_userdata) { recorder->filename = strdup(filename); @@ -439,10 +663,17 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, goto error_queue_cond_destroy; } + recorder->audio = audio; + sc_queue_init(&recorder->video_queue); + sc_queue_init(&recorder->audio_queue); recorder->stopped = false; recorder->video_codec = NULL; + recorder->audio_codec = NULL; + + recorder->video_stream_index = -1; + recorder->audio_stream_index = -1; recorder->format = format; recorder->declared_frame_size = declared_frame_size; @@ -459,6 +690,16 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, recorder->video_packet_sink.ops = &video_ops; + if (audio) { + static const struct sc_packet_sink_ops audio_ops = { + .open = sc_recorder_audio_packet_sink_open, + .close = sc_recorder_audio_packet_sink_close, + .push = sc_recorder_audio_packet_sink_push, + }; + + recorder->audio_packet_sink.ops = &audio_ops; + } + return true; error_queue_cond_destroy: diff --git a/app/src/recorder.h b/app/src/recorder.h index b7a10e5e29..ed880de0f0 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -20,7 +20,10 @@ struct sc_record_packet { struct sc_recorder_queue SC_QUEUE(struct sc_record_packet); struct sc_recorder { - struct sc_packet_sink video_packet_sink; // packet sink trait + struct sc_packet_sink video_packet_sink; + struct sc_packet_sink audio_packet_sink; + + bool audio; char *filename; enum sc_record_format format; @@ -33,10 +36,15 @@ struct sc_recorder { // set on sc_recorder_stop(), packet_sink close or recording failure bool stopped; struct sc_recorder_queue video_queue; + struct sc_recorder_queue audio_queue; - // wake up the recorder thread once the video codec is known + // wake up the recorder thread once the video or audio codec is known sc_cond stream_cond; const AVCodec *video_codec; + const AVCodec *audio_codec; + + int video_stream_index; + int audio_stream_index; const struct sc_recorder_callbacks *cbs; void *cbs_userdata; @@ -49,7 +57,7 @@ struct sc_recorder_callbacks { bool sc_recorder_init(struct sc_recorder *recorder, const char *filename, - enum sc_record_format format, + enum sc_record_format format, bool audio, struct sc_size declared_frame_size, const struct sc_recorder_callbacks *cbs, void *cbs_userdata); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 437ae650b3..6750f6a1c9 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -424,8 +424,8 @@ scrcpy(struct scrcpy_options *options) { .on_ended = sc_recorder_on_ended, }; if (!sc_recorder_init(&s->recorder, options->record_filename, - options->record_format, info->frame_size, - &recorder_cbs, NULL)) { + options->record_format, options->audio, + info->frame_size, &recorder_cbs, NULL)) { goto end; } recorder_initialized = true; @@ -436,6 +436,10 @@ scrcpy(struct scrcpy_options *options) { recorder_started = true; sc_demuxer_add_sink(&s->video_demuxer, &s->recorder.video_packet_sink); + if (options->audio) { + sc_demuxer_add_sink(&s->audio_demuxer, + &s->recorder.audio_packet_sink); + } } struct sc_controller *controller = NULL; From 13a3395a332eca55c884edc05a5052457b519f60 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 18:09:18 +0100 Subject: [PATCH 050/118] Disable audio on initialization error By default, audio is enabled (--no-audio must be explicitly passed to disable it). However, some devices may not support audio capture (typically devices below Android 11, or Android 11 when the shell application is not foreground on start). In that case, make the server notify the client to dynamically disable audio forwarding so that it does not wait indefinitely for an audio stream. Also disable audio on unknown codec or missing decoder on the client-side, for the same reasons. PR #3757 --- app/src/demuxer.c | 28 +++++++++++++-- app/src/recorder.c | 34 ++++++++++++++++++- app/src/recorder.h | 11 ++++++ app/src/trait/packet_sink.h | 10 ++++++ .../com/genymobile/scrcpy/AudioEncoder.java | 4 +++ .../java/com/genymobile/scrcpy/Streamer.java | 6 ++++ 6 files changed, 90 insertions(+), 3 deletions(-) diff --git a/app/src/demuxer.c b/app/src/demuxer.c index 968e6db093..482f2e0484 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -158,6 +158,16 @@ sc_demuxer_open_sinks(struct sc_demuxer *demuxer, const AVCodec *codec) { return true; } +static void +sc_demuxer_disable_sinks(struct sc_demuxer *demuxer) { + for (unsigned i = 0; i < demuxer->sink_count; ++i) { + struct sc_packet_sink *sink = demuxer->sinks[i]; + if (sink->ops->disable) { + sink->ops->disable(sink); + } + } +} + static int run_demuxer(void *data) { struct sc_demuxer *demuxer = data; @@ -168,19 +178,33 @@ run_demuxer(void *data) { uint32_t raw_codec_id; bool ok = sc_demuxer_recv_codec_id(demuxer, &raw_codec_id); if (!ok) { + LOGE("Demuxer '%s': stream disabled due to connection error", + demuxer->name); + eos = true; + goto end; + } + + if (raw_codec_id == 0) { + LOGW("Demuxer '%s': stream explicitly disabled by the device", + demuxer->name); + sc_demuxer_disable_sinks(demuxer); eos = true; goto end; } enum AVCodecID codec_id = sc_demuxer_to_avcodec_id(raw_codec_id); if (codec_id == AV_CODEC_ID_NONE) { - // Error already logged + LOGE("Demuxer '%s': stream disabled due to unsupported codec", + demuxer->name); + sc_demuxer_disable_sinks(demuxer); goto end; } const AVCodec *codec = avcodec_find_decoder(codec_id); if (!codec) { - LOGE("Demuxer '%s': decoder not found", demuxer->name); + LOGE("Demuxer '%s': stream disabled due to missing decoder", + demuxer->name); + sc_demuxer_disable_sinks(demuxer); goto end; } diff --git a/app/src/recorder.c b/app/src/recorder.c index 93c3321f0f..c694f022f9 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -194,9 +194,17 @@ sc_recorder_wait_video_stream(struct sc_recorder *recorder) { static bool sc_recorder_wait_audio_stream(struct sc_recorder *recorder) { sc_mutex_lock(&recorder->mutex); - while (!recorder->audio_codec && !recorder->stopped) { + while (!recorder->audio_codec && !recorder->audio_disabled + && !recorder->stopped) { sc_cond_wait(&recorder->stream_cond, &recorder->mutex); } + + if (recorder->audio_disabled) { + // Reset audio flag. From there, the recorder thread may access this + // flag without any mutex. + recorder->audio = false; + } + const AVCodec *codec = recorder->audio_codec; sc_mutex_unlock(&recorder->mutex); @@ -585,6 +593,8 @@ sc_recorder_audio_packet_sink_open(struct sc_packet_sink *sink, const AVCodec *codec) { struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); assert(recorder->audio); + // only written from this thread, no need to lock + assert(!recorder->audio_disabled); assert(codec); sc_mutex_lock(&recorder->mutex); @@ -599,6 +609,8 @@ static void sc_recorder_audio_packet_sink_close(struct sc_packet_sink *sink) { struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); assert(recorder->audio); + // only written from this thread, no need to lock + assert(!recorder->audio_disabled); sc_mutex_lock(&recorder->mutex); // EOS also stops the recorder @@ -612,6 +624,8 @@ sc_recorder_audio_packet_sink_push(struct sc_packet_sink *sink, const AVPacket *packet) { struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); assert(recorder->audio); + // only written from this thread, no need to lock + assert(!recorder->audio_disabled); sc_mutex_lock(&recorder->mutex); @@ -637,6 +651,22 @@ sc_recorder_audio_packet_sink_push(struct sc_packet_sink *sink, return true; } +static void +sc_recorder_audio_packet_sink_disable(struct sc_packet_sink *sink) { + struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); + assert(recorder->audio); + // only written from this thread, no need to lock + assert(!recorder->audio_disabled); + assert(!recorder->audio_codec); + + LOGW("Audio stream recording disabled"); + + sc_mutex_lock(&recorder->mutex); + recorder->audio_disabled = true; + sc_cond_signal(&recorder->stream_cond); + sc_mutex_unlock(&recorder->mutex); +} + bool sc_recorder_init(struct sc_recorder *recorder, const char *filename, enum sc_record_format format, bool audio, @@ -671,6 +701,7 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, recorder->video_codec = NULL; recorder->audio_codec = NULL; + recorder->audio_disabled = false; recorder->video_stream_index = -1; recorder->audio_stream_index = -1; @@ -695,6 +726,7 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, .open = sc_recorder_audio_packet_sink_open, .close = sc_recorder_audio_packet_sink_close, .push = sc_recorder_audio_packet_sink_push, + .disable = sc_recorder_audio_packet_sink_disable, }; recorder->audio_packet_sink.ops = &audio_ops; diff --git a/app/src/recorder.h b/app/src/recorder.h index ed880de0f0..6fe72401df 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -23,6 +23,14 @@ struct sc_recorder { struct sc_packet_sink video_packet_sink; struct sc_packet_sink audio_packet_sink; + /* The audio flag is unprotected: + * - it is initialized from sc_recorder_init() from the main thread; + * - it may be reset once from the recorder thread if the audio is + * disabled dynamically. + * + * Therefore, once the recorder thread is started, only the recorder thread + * may access it without data races. + */ bool audio; char *filename; @@ -42,6 +50,9 @@ struct sc_recorder { sc_cond stream_cond; const AVCodec *video_codec; const AVCodec *audio_codec; + // Instead of providing an audio_codec, the demuxer may notify that the + // stream is disabled if the device could not capture audio + bool audio_disabled; int video_stream_index; int audio_stream_index; diff --git a/app/src/trait/packet_sink.h b/app/src/trait/packet_sink.h index 9fc9fd246e..099c8c521d 100644 --- a/app/src/trait/packet_sink.h +++ b/app/src/trait/packet_sink.h @@ -23,6 +23,16 @@ struct sc_packet_sink_ops { bool (*open)(struct sc_packet_sink *sink, const AVCodec *codec); void (*close)(struct sc_packet_sink *sink); bool (*push)(struct sc_packet_sink *sink, const AVPacket *packet); + + /*/ + * Called when the input stream has been disabled at runtime. + * + * If it is called, then open(), close() and push() will never be called. + * + * It is useful to notify the recorder that the requested audio stream has + * finally been disabled because the device could not capture it. + */ + void (*disable)(struct sc_packet_sink *sink); }; #endif diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 74481f99e1..8bc25e8e9e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -255,6 +255,10 @@ public void encode() throws IOException { outputThread.start(); waitEnded(); + } catch (Throwable e) { + // Notify the client that the audio could not be captured + streamer.writeDisableStream(); + throw e; } finally { // Cleanup everything (either at the end or on error at any step of the initialization) if (mediaCodecThread != null) { diff --git a/server/src/main/java/com/genymobile/scrcpy/Streamer.java b/server/src/main/java/com/genymobile/scrcpy/Streamer.java index 77d9eefa9a..7cc065ebdf 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Streamer.java +++ b/server/src/main/java/com/genymobile/scrcpy/Streamer.java @@ -40,6 +40,12 @@ public void writeHeader() throws IOException { } } + public void writeDisableStream() throws IOException { + // Writing 0 (32-bit) as codec-id means that the device disables the stream (because it could not capture) + byte[] zeros = new byte[4]; + IO.writeFully(fd, zeros, 0, zeros.length); + } + public void writePacket(ByteBuffer buffer, long pts, boolean config, boolean keyFrame) throws IOException { if (config && codec == AudioCodec.OPUS) { fixOpusConfigPacket(buffer); From 80c0780b7779acc27a563ee5acdd3ac645108763 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 15:24:08 +0100 Subject: [PATCH 051/118] Disable audio before Android 11 The permission "android.permission.RECORD_AUDIO" has been added for shell in Android 11. Moreover, on lower versions, it may make the server segfault on the device (happened on a Nexus 5 with Android 6.0.1). Refs PR #3757 --- .../src/main/java/com/genymobile/scrcpy/AudioEncoder.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 8bc25e8e9e..d06898d672 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -203,6 +203,12 @@ private synchronized void waitEnded() { @TargetApi(Build.VERSION_CODES.M) public void encode() throws IOException { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { + Ln.w("Audio disabled: it is not supported before Android 11"); + streamer.writeDisableStream(); + return; + } + MediaCodec mediaCodec = null; AudioRecord recorder = null; From 17d5301c0fd6f93c5b9d4dd325e4303739d93ffb Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 00:55:36 +0100 Subject: [PATCH 052/118] Record at least video packets on stop If the recorder is stopped while it has not received any audio packet yet, make sure the video stream is correctly recorded. PR #3757 --- app/src/recorder.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index c694f022f9..9fc15dacf8 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -249,8 +249,10 @@ sc_recorder_process_header(struct sc_recorder *recorder) { sc_cond_wait(&recorder->queue_cond, &recorder->mutex); } - if (sc_recorder_has_empty_queues(recorder)) { + if (sc_queue_is_empty(&recorder->video_queue)) { assert(recorder->stopped); + // Don't process anything if there are not at least video packets (when + // the recorder is stopped) sc_mutex_unlock(&recorder->mutex); return false; } @@ -258,8 +260,9 @@ sc_recorder_process_header(struct sc_recorder *recorder) { struct sc_record_packet *video_pkt; sc_queue_take(&recorder->video_queue, next, &video_pkt); - struct sc_record_packet *audio_pkt; - if (recorder->audio) { + struct sc_record_packet *audio_pkt = NULL; + if (!sc_queue_is_empty(&recorder->audio_queue)) { + assert(recorder->audio); sc_queue_take(&recorder->audio_queue, next, &audio_pkt); } @@ -280,7 +283,7 @@ sc_recorder_process_header(struct sc_recorder *recorder) { goto end; } - if (recorder->audio) { + if (audio_pkt) { if (audio_pkt->packet->pts != AV_NOPTS_VALUE) { LOGE("The first audio packet is not a config packet"); goto end; @@ -305,7 +308,7 @@ sc_recorder_process_header(struct sc_recorder *recorder) { end: sc_record_packet_delete(video_pkt); - if (recorder->audio) { + if (audio_pkt) { sc_record_packet_delete(audio_pkt); } @@ -393,6 +396,16 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { } else if (video_pkt && audio_pkt) { pts_origin = MIN(video_pkt->packet->pts, audio_pkt->packet->pts); + } else if (recorder->stopped) { + if (video_pkt) { + // The recorder is stopped without audio, record the video + // packets + pts_origin = video_pkt->packet->pts; + } else { + // Fail if there is no video + error = true; + goto end; + } } else { // We need both video and audio packets to initialize pts_origin continue; From a1802dab763d5031bc26576e45ab1d388a8904fc Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 18:21:14 +0100 Subject: [PATCH 053/118] Remove default bit-rate on client side If no bit-rate is passed, let the server use the default value (8Mbps). This avoids to define a default value on both sides, and to pass the default bit-rate as an argument when starting the server. PR #3757 --- app/meson.build | 4 ---- app/scrcpy.1 | 2 +- app/src/cli.c | 2 +- app/src/options.c | 2 +- app/src/server.c | 4 +++- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/app/meson.build b/app/meson.build index b6a772a903..2ea3b317a1 100644 --- a/app/meson.build +++ b/app/meson.build @@ -195,10 +195,6 @@ conf.set('PORTABLE', get_option('portable')) conf.set('DEFAULT_LOCAL_PORT_RANGE_FIRST', '27183') conf.set('DEFAULT_LOCAL_PORT_RANGE_LAST', '27199') -# the default video bitrate, in bits/second -# overridden by option --bit-rate -conf.set('DEFAULT_BIT_RATE', '8000000') # 8Mbps - # run a server debugger and wait for a client to be attached conf.set('SERVER_DEBUGGER', get_option('server_debugger')) diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 0f1147da90..186d8ad53c 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -23,7 +23,7 @@ Make scrcpy window always on top (above other windows). .BI "\-b, \-\-bit\-rate " value Encode the video at the given bit\-rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). -Default is 8000000. +Default is 8M (8000000). .TP .BI "\-\-codec " name diff --git a/app/src/cli.c b/app/src/cli.c index 4f023da031..d7a0a7ae43 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -107,7 +107,7 @@ static const struct sc_option options[] = { .argdesc = "value", .text = "Encode the video at the given bit-rate, expressed in bits/s. " "Unit suffixes are supported: 'K' (x1000) and 'M' (x1000000).\n" - "Default is " STR(DEFAULT_BIT_RATE) ".", + "Default is 8M (8000000).", }, { .longopt_id = OPT_CODEC, diff --git a/app/src/options.c b/app/src/options.c index 0854067f0e..64ec5b3b05 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -28,7 +28,7 @@ const struct scrcpy_options scrcpy_options_default = { .count = 2, }, .max_size = 0, - .bit_rate = DEFAULT_BIT_RATE, + .bit_rate = 0, .max_fps = 0, .lock_video_orientation = SC_LOCK_VIDEO_ORIENTATION_UNLOCKED, .rotation = 0, diff --git a/app/src/server.c b/app/src/server.c index 88e3421f87..6bf0eb6e47 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -215,8 +215,10 @@ execute_server(struct sc_server *server, ADD_PARAM("scid=%08x", params->scid); ADD_PARAM("log_level=%s", log_level_to_server_string(params->log_level)); - ADD_PARAM("bit_rate=%" PRIu32, params->bit_rate); + if (params->bit_rate) { + ADD_PARAM("bit_rate=%" PRIu32, params->bit_rate); + } if (!params->audio) { ADD_PARAM("audio=false"); } From cee40ca047f02c68e59372ae7305fce6d6ead8f0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 20 Feb 2023 21:19:36 +0100 Subject: [PATCH 054/118] Rename --codec to --video-codec This prepares the introduction of --audio-codec. PR #3757 --- README.md | 10 +++---- app/data/bash-completion/scrcpy | 4 +-- app/data/zsh-completion/_scrcpy | 2 +- app/scrcpy.1 | 14 +++++----- app/src/cli.c | 27 ++++++++++++++----- app/src/options.c | 2 +- app/src/options.h | 2 +- app/src/scrcpy.c | 2 +- app/src/server.c | 5 ++-- app/src/server.h | 2 +- .../java/com/genymobile/scrcpy/Options.java | 10 +++---- .../com/genymobile/scrcpy/ScreenEncoder.java | 2 +- .../java/com/genymobile/scrcpy/Server.java | 16 +++++------ 13 files changed, 56 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index a19aec0f59..4b898d2699 100644 --- a/README.md +++ b/README.md @@ -258,9 +258,9 @@ The video codec can be selected. The possible values are `h264` (default), `h265` and `av1`: ```bash -scrcpy --codec=h264 # default -scrcpy --codec=h265 -scrcpy --codec=av1 +scrcpy --video-codec=h264 # default +scrcpy --video-codec=h265 +scrcpy --video-codec=av1 ``` @@ -277,8 +277,8 @@ To list the available encoders, you can pass an invalid encoder name; the error will give the available encoders: ```bash -scrcpy --encoder=_ # for the default codec -scrcpy --codec=h265 --encoder=_ # for a specific codec +scrcpy --encoder=_ # for the default codec +scrcpy --video-codec=h265 --encoder=_ # for a specific codec ``` ### Capture diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 22dc4ceef3..d92cf0094a 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -3,7 +3,6 @@ _scrcpy() { local opts=" --always-on-top -b --bit-rate= - --codec= --codec-options= --crop= -d --select-usb @@ -55,6 +54,7 @@ _scrcpy() { --v4l2-sink= -V --verbosity= -v --version + --video-codec= -w --stay-awake --window-borderless --window-title= @@ -66,7 +66,7 @@ _scrcpy() { _init_completion -s || return case "$prev" in - --codec) + --video-codec) COMPREPLY=($(compgen -W 'h264 h265 av1' -- "$cur")) return ;; diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 17e1de9f64..b9c94e1e7b 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -10,7 +10,6 @@ local arguments arguments=( '--always-on-top[Make scrcpy window always on top \(above other windows\)]' {-b,--bit-rate=}'[Encode the video at the given bit-rate]' - '--codec=[Select the video codec]:codec:(h264 h265 av1)' '--codec-options=[Set a list of comma-separated key\:type=value options for the device encoder]' '--crop=[\[width\:height\:x\:y\] Crop the device screen on the server]' {-d,--select-usb}'[Use USB device]' @@ -60,6 +59,7 @@ arguments=( '--v4l2-sink=[\[\/dev\/videoN\] Output to v4l2loopback device]' {-V,--verbosity=}'[Set the log level]:verbosity:(verbose debug info warn error)' {-v,--version}'[Print the version of scrcpy]' + '--video-codec=[Select the video codec]:codec:(h264 h265 av1)' {-w,--stay-awake}'[Keep the device on while scrcpy is running, when the device is plugged in]' '--window-borderless[Disable window decorations \(display borderless window\)]' '--window-title=[Set a custom window title]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 186d8ad53c..32bb84647b 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -25,12 +25,6 @@ Encode the video at the given bit\-rate, expressed in bits/s. Unit suffixes are Default is 8M (8000000). -.TP -.BI "\-\-codec " name -Select a video codec (h264, h265 or av1). - -Default is h264. - .TP .BI "\-\-codec\-options " key\fR[:\fItype\fR]=\fIvalue\fR[,...] Set a list of comma-separated key:type=value options for the device encoder. @@ -82,7 +76,7 @@ Also see \fB\-d\fR (\fB\-\-select\-usb\fR). .TP .BI "\-\-encoder " name -Use a specific MediaCodec encoder (depending on the codec provided by \fB\-\-codec\fR). +Use a specific MediaCodec encoder (depending on the codec provided by \fB\-\-video\-codec\fR). .TP .B \-\-force\-adb\-forward @@ -329,6 +323,12 @@ Default is "info" for release builds, "debug" for debug builds. .B \-v, \-\-version Print the version of scrcpy. +.TP +.BI "\-\-video\-codec " name +Select a video codec (h264, h265 or av1). + +Default is h264. + .TP .B \-w, \-\-stay-awake Keep the device on while scrcpy is running, when the device is plugged in. diff --git a/app/src/cli.c b/app/src/cli.c index d7a0a7ae43..9163ba6021 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -59,6 +59,7 @@ enum { OPT_PRINT_FPS, OPT_NO_POWER_ON, OPT_CODEC, + OPT_VIDEO_CODEC, OPT_NO_AUDIO, }; @@ -110,11 +111,13 @@ static const struct sc_option options[] = { "Default is 8M (8000000).", }, { + // Not really deprecated (--codec has never been released), but without + // declaring an explicit --codec option, getopt_long() partial matching + // behavior would consider --codec to be equivalent to --codec-options, + // which would be confusing. .longopt_id = OPT_CODEC, .longopt = "codec", - .argdesc = "name", - .text = "Select a video codec (h264, h265 or av1).\n" - "Default is h264.", + .argdesc = "value", }, { .longopt_id = OPT_CODEC_OPTIONS, @@ -177,7 +180,7 @@ static const struct sc_option options[] = { .longopt = "encoder", .argdesc = "name", .text = "Use a specific MediaCodec encoder (depending on the codec " - "provided by --codec).", + "provided by --video-codec).", }, { .longopt_id = OPT_FORCE_ADB_FORWARD, @@ -519,6 +522,13 @@ static const struct sc_option options[] = { .longopt = "version", .text = "Print the version of scrcpy.", }, + { + .longopt_id = OPT_VIDEO_CODEC, + .longopt = "video-codec", + .argdesc = "name", + .text = "Select a video codec (h264, h265 or av1).\n" + "Default is h264.", + }, { .shortopt = 'w', .longopt = "stay-awake", @@ -1395,7 +1405,7 @@ guess_record_format(const char *filename) { } static bool -parse_codec(const char *optarg, enum sc_codec *codec) { +parse_video_codec(const char *optarg, enum sc_codec *codec) { if (!strcmp(optarg, "h264")) { *codec = SC_CODEC_H264; return true; @@ -1408,7 +1418,7 @@ parse_codec(const char *optarg, enum sc_codec *codec) { *codec = SC_CODEC_AV1; return true; } - LOGE("Unsupported codec: %s (expected h264, h265 or av1)", optarg); + LOGE("Unsupported video codec: %s (expected h264, h265 or av1)", optarg); return false; } @@ -1649,7 +1659,10 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], opts->start_fps_counter = true; break; case OPT_CODEC: - if (!parse_codec(optarg, &opts->codec)) { + LOGW("--codec is deprecated, use --video-codec instead."); + // fall through + case OPT_VIDEO_CODEC: + if (!parse_video_codec(optarg, &opts->video_codec)) { return false; } break; diff --git a/app/src/options.c b/app/src/options.c index 64ec5b3b05..0368ffcc9d 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -13,7 +13,7 @@ const struct scrcpy_options scrcpy_options_default = { .v4l2_device = NULL, #endif .log_level = SC_LOG_LEVEL_INFO, - .codec = SC_CODEC_H264, + .video_codec = SC_CODEC_H264, .record_format = SC_RECORD_FORMAT_AUTO, .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, .mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT, diff --git a/app/src/options.h b/app/src/options.h index 7bf3001106..f6ba324b2c 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -99,7 +99,7 @@ struct scrcpy_options { const char *v4l2_device; #endif enum sc_log_level log_level; - enum sc_codec codec; + enum sc_codec video_codec; enum sc_record_format record_format; enum sc_keyboard_input_mode keyboard_input_mode; enum sc_mouse_input_mode mouse_input_mode; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 6750f6a1c9..35b999a890 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -314,7 +314,7 @@ scrcpy(struct scrcpy_options *options) { .select_usb = options->select_usb, .select_tcpip = options->select_tcpip, .log_level = options->log_level, - .codec = options->codec, + .video_codec = options->video_codec, .crop = options->crop, .port_range = options->port_range, .tunnel_host = options->tunnel_host, diff --git a/app/src/server.c b/app/src/server.c index 6bf0eb6e47..20433ea0ed 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -222,8 +222,9 @@ execute_server(struct sc_server *server, if (!params->audio) { ADD_PARAM("audio=false"); } - if (params->codec != SC_CODEC_H264) { - ADD_PARAM("codec=%s", sc_server_get_codec_name(params->codec)); + if (params->video_codec != SC_CODEC_H264) { + ADD_PARAM("video_codec=%s", + sc_server_get_codec_name(params->video_codec)); } if (params->max_size) { ADD_PARAM("max_size=%" PRIu16, params->max_size); diff --git a/app/src/server.h b/app/src/server.h index 3005ebd263..8914349f56 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -25,7 +25,7 @@ struct sc_server_params { uint32_t scid; const char *req_serial; enum sc_log_level log_level; - enum sc_codec codec; + enum sc_codec video_codec; const char *crop; const char *codec_options; const char *encoder_name; diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 0778997454..73a303d8f8 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -10,7 +10,7 @@ public class Options { private int scid = -1; // 31-bit non-negative value, or -1 private boolean audio = true; private int maxSize; - private VideoCodec codec = VideoCodec.H264; + private VideoCodec videoCodec = VideoCodec.H264; private int bitRate = 8000000; private int maxFps; private int lockVideoOrientation = -1; @@ -66,12 +66,12 @@ public void setMaxSize(int maxSize) { this.maxSize = maxSize; } - public VideoCodec getCodec() { - return codec; + public VideoCodec getVideoCodec() { + return videoCodec; } - public void setCodec(VideoCodec codec) { - this.codec = codec; + public void setVideoCodec(VideoCodec videoCodec) { + this.videoCodec = videoCodec; } public int getBitRate() { diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index fdd23bf393..30e988f099 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -231,7 +231,7 @@ private static String buildUnknownEncoderMessage(Codec codec, String encoderName if (encoders != null && encoders.length > 0) { msg.append("\nTry to use one of the available encoders:"); for (MediaCodecInfo encoder : encoders) { - msg.append("\n scrcpy --codec=").append(codec.getName()).append(" --encoder='").append(encoder.getName()).append("'"); + msg.append("\n scrcpy --video-codec=").append(codec.getName()).append(" --encoder='").append(encoder.getName()).append("'"); } } return msg.toString(); diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index eb0c13846d..a926f443ab 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -96,7 +96,6 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc AudioEncoder audioEncoder = null; try (DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, audio, control, sendDummyByte)) { - VideoCodec codec = options.getCodec(); if (options.getSendDeviceMeta()) { Size videoSize = device.getScreenInfo().getVideoSize(); connection.sendDeviceMeta(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight()); @@ -116,9 +115,10 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc audioEncoder.start(); } - Streamer videoStreamer = new Streamer(connection.getVideoFd(), codec, options.getSendCodecId(), options.getSendFrameMeta()); - ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getBitRate(), options.getMaxFps(), - codecOptions, options.getEncoderName(), options.getDownsizeOnError()); + Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecId(), + options.getSendFrameMeta()); + ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getBitRate(), options.getMaxFps(), codecOptions, + options.getEncoderName(), options.getDownsizeOnError()); try { // synchronous screenEncoder.streamScreen(); @@ -195,12 +195,12 @@ private static Options createOptions(String... args) { boolean audio = Boolean.parseBoolean(value); options.setAudio(audio); break; - case "codec": - VideoCodec codec = VideoCodec.findByName(value); - if (codec == null) { + case "video_codec": + VideoCodec videoCodec = VideoCodec.findByName(value); + if (videoCodec == null) { throw new IllegalArgumentException("Video codec " + value + " not supported"); } - options.setCodec(codec); + options.setVideoCodec(videoCodec); break; case "max_size": int maxSize = Integer.parseInt(value) & ~7; // multiple of 8 From 9087e85c3f076c54b81a8895a9936bbb49a41812 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 21 Feb 2023 19:56:44 +0100 Subject: [PATCH 055/118] Rename --bit-rate to --video-bit-rate This prepares the introduction of --audio-bit-rate. PR #3757 --- README.md | 4 ++-- app/data/bash-completion/scrcpy | 4 ++-- app/data/zsh-completion/_scrcpy | 2 +- app/scrcpy.1 | 2 +- app/src/cli.c | 14 ++++++++++++-- app/src/options.c | 2 +- app/src/options.h | 2 +- app/src/scrcpy.c | 2 +- app/src/server.c | 4 ++-- app/src/server.h | 2 +- app/tests/test_cli.c | 4 ++-- .../main/java/com/genymobile/scrcpy/Options.java | 10 +++++----- .../java/com/genymobile/scrcpy/ScreenEncoder.java | 8 ++++---- .../main/java/com/genymobile/scrcpy/Server.java | 8 ++++---- 14 files changed, 39 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 4b898d2699..81371b929a 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ preserved. That way, a device in 1920×1080 will be mirrored at 1024×576. The default bit-rate is 8 Mbps. To change the video bitrate (e.g. to 2 Mbps): ```bash -scrcpy --bit-rate=2M +scrcpy --video-bit-rate=2M scrcpy -b 2M # short version ``` @@ -444,7 +444,7 @@ none found, try running `adb disconnect`, and then run those two commands again. It may be useful to decrease the bit-rate and the resolution: ```bash -scrcpy --bit-rate=2M --max-size=800 +scrcpy --video-bit-rate=2M --max-size=800 scrcpy -b2M -m800 # short version ``` diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index d92cf0094a..1fe79765a4 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -2,7 +2,7 @@ _scrcpy() { local cur prev words cword local opts=" --always-on-top - -b --bit-rate= + -b --video-bit-rate= --codec-options= --crop= -d --select-usb @@ -104,7 +104,7 @@ _scrcpy() { COMPREPLY=($(compgen -W "$("${ADB:-adb}" devices | awk '$2 == "device" {print $1}')" -- ${cur})) return ;; - -b|--bit-rate \ + -b|--video-bit-rate \ |--codec-options \ |--crop \ |--display \ diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index b9c94e1e7b..ac3ec0234b 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -9,7 +9,7 @@ local arguments arguments=( '--always-on-top[Make scrcpy window always on top \(above other windows\)]' - {-b,--bit-rate=}'[Encode the video at the given bit-rate]' + {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]' '--codec-options=[Set a list of comma-separated key\:type=value options for the device encoder]' '--crop=[\[width\:height\:x\:y\] Crop the device screen on the server]' {-d,--select-usb}'[Use USB device]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 32bb84647b..41ce28a4d7 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -20,7 +20,7 @@ provides display and control of Android devices connected on USB (or over TCP/IP Make scrcpy window always on top (above other windows). .TP -.BI "\-b, \-\-bit\-rate " value +.BI "\-b, \-\-video\-bit\-rate " value Encode the video at the given bit\-rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). Default is 8M (8000000). diff --git a/app/src/cli.c b/app/src/cli.c index 9163ba6021..57e85aa437 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -19,6 +19,7 @@ enum { OPT_RENDER_EXPIRED_FRAMES = 1000, + OPT_BIT_RATE, OPT_WINDOW_TITLE, OPT_PUSH_TARGET, OPT_ALWAYS_ON_TOP, @@ -104,12 +105,18 @@ static const struct sc_option options[] = { }, { .shortopt = 'b', - .longopt = "bit-rate", + .longopt = "video-bit-rate", .argdesc = "value", .text = "Encode the video at the given bit-rate, expressed in bits/s. " "Unit suffixes are supported: 'K' (x1000) and 'M' (x1000000).\n" "Default is 8M (8000000).", }, + { + // deprecated + .longopt_id = OPT_BIT_RATE, + .longopt = "bit-rate", + .argdesc = "value", + }, { // Not really deprecated (--codec has never been released), but without // declaring an explicit --codec option, getopt_long() partial matching @@ -1432,8 +1439,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], int c; while ((c = getopt_long(argc, argv, optstring, longopts, NULL)) != -1) { switch (c) { + case OPT_BIT_RATE: + LOGW("--bit-rate is deprecated, use --video-bit-rate instead."); + // fall through case 'b': - if (!parse_bit_rate(optarg, &opts->bit_rate)) { + if (!parse_bit_rate(optarg, &opts->video_bit_rate)) { return false; } break; diff --git a/app/src/options.c b/app/src/options.c index 0368ffcc9d..a087f50728 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -28,7 +28,7 @@ const struct scrcpy_options scrcpy_options_default = { .count = 2, }, .max_size = 0, - .bit_rate = 0, + .video_bit_rate = 0, .max_fps = 0, .lock_video_orientation = SC_LOCK_VIDEO_ORIENTATION_UNLOCKED, .rotation = 0, diff --git a/app/src/options.h b/app/src/options.h index f6ba324b2c..d22078e440 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -108,7 +108,7 @@ struct scrcpy_options { uint16_t tunnel_port; struct sc_shortcut_mods shortcut_mods; uint16_t max_size; - uint32_t bit_rate; + uint32_t video_bit_rate; uint16_t max_fps; enum sc_lock_video_orientation lock_video_orientation; uint8_t rotation; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 35b999a890..b09de541d3 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -320,7 +320,7 @@ scrcpy(struct scrcpy_options *options) { .tunnel_host = options->tunnel_host, .tunnel_port = options->tunnel_port, .max_size = options->max_size, - .bit_rate = options->bit_rate, + .video_bit_rate = options->video_bit_rate, .max_fps = options->max_fps, .lock_video_orientation = options->lock_video_orientation, .control = options->control, diff --git a/app/src/server.c b/app/src/server.c index 20433ea0ed..7c8fba8dac 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -216,8 +216,8 @@ execute_server(struct sc_server *server, ADD_PARAM("scid=%08x", params->scid); ADD_PARAM("log_level=%s", log_level_to_server_string(params->log_level)); - if (params->bit_rate) { - ADD_PARAM("bit_rate=%" PRIu32, params->bit_rate); + if (params->video_bit_rate) { + ADD_PARAM("video_bit_rate=%" PRIu32, params->video_bit_rate); } if (!params->audio) { ADD_PARAM("audio=false"); diff --git a/app/src/server.h b/app/src/server.h index 8914349f56..d30192885a 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -33,7 +33,7 @@ struct sc_server_params { uint32_t tunnel_host; uint16_t tunnel_port; uint16_t max_size; - uint32_t bit_rate; + uint32_t video_bit_rate; uint16_t max_fps; int8_t lock_video_orientation; bool control; diff --git a/app/tests/test_cli.c b/app/tests/test_cli.c index 5ea54b7fc5..3e9a248a31 100644 --- a/app/tests/test_cli.c +++ b/app/tests/test_cli.c @@ -46,7 +46,7 @@ static void test_options(void) { char *argv[] = { "scrcpy", "--always-on-top", - "--bit-rate", "5M", + "--video-bit-rate", "5M", "--crop", "100:200:300:400", "--fullscreen", "--max-fps", "30", @@ -75,7 +75,7 @@ static void test_options(void) { const struct scrcpy_options *opts = &args.opts; assert(opts->always_on_top); - assert(opts->bit_rate == 5000000); + assert(opts->video_bit_rate == 5000000); assert(!strcmp(opts->crop, "100:200:300:400")); assert(opts->fullscreen); assert(opts->max_fps == 30); diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 73a303d8f8..53257cf382 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -11,7 +11,7 @@ public class Options { private boolean audio = true; private int maxSize; private VideoCodec videoCodec = VideoCodec.H264; - private int bitRate = 8000000; + private int videoBitRate = 8000000; private int maxFps; private int lockVideoOrientation = -1; private boolean tunnelForward; @@ -74,12 +74,12 @@ public void setVideoCodec(VideoCodec videoCodec) { this.videoCodec = videoCodec; } - public int getBitRate() { - return bitRate; + public int getVideoBitRate() { + return videoBitRate; } - public void setBitRate(int bitRate) { - this.bitRate = bitRate; + public void setVideoBitRate(int videoBitRate) { + this.videoBitRate = videoBitRate; } public int getMaxFps() { diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index 30e988f099..d646995b4c 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -35,18 +35,18 @@ public class ScreenEncoder implements Device.RotationListener { private final Streamer streamer; private final String encoderName; private final List codecOptions; - private final int bitRate; + private final int videoBitRate; private final int maxFps; private final boolean downsizeOnError; private boolean firstFrameSent; private int consecutiveErrors; - public ScreenEncoder(Device device, Streamer streamer, int bitRate, int maxFps, List codecOptions, String encoderName, + public ScreenEncoder(Device device, Streamer streamer, int videoBitRate, int maxFps, List codecOptions, String encoderName, boolean downsizeOnError) { this.device = device; this.streamer = streamer; - this.bitRate = bitRate; + this.videoBitRate = videoBitRate; this.maxFps = maxFps; this.codecOptions = codecOptions; this.encoderName = encoderName; @@ -65,7 +65,7 @@ public boolean consumeRotationChange() { public void streamScreen() throws IOException, ConfigurationException { Codec codec = streamer.getCodec(); MediaCodec mediaCodec = createMediaCodec(codec, encoderName); - MediaFormat format = createFormat(codec.getMimeType(), bitRate, maxFps, codecOptions); + MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions); IBinder display = createDisplay(); device.setRotationListener(this); diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index a926f443ab..f764804cb3 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -117,7 +117,7 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecId(), options.getSendFrameMeta()); - ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getBitRate(), options.getMaxFps(), codecOptions, + ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(), codecOptions, options.getEncoderName(), options.getDownsizeOnError()); try { // synchronous @@ -206,9 +206,9 @@ private static Options createOptions(String... args) { int maxSize = Integer.parseInt(value) & ~7; // multiple of 8 options.setMaxSize(maxSize); break; - case "bit_rate": - int bitRate = Integer.parseInt(value); - options.setBitRate(bitRate); + case "video_bit_rate": + int videoBitRate = Integer.parseInt(value); + options.setVideoBitRate(videoBitRate); break; case "max_fps": int maxFps = Integer.parseInt(value); From 31555fa5309b80f6e65a19dc3aa8932fe9265b8d Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 21 Feb 2023 21:46:34 +0100 Subject: [PATCH 056/118] Rename --codec-options to --video-codec-options This prepares the introduction of --audio-codec-options. PR #3757 --- app/data/bash-completion/scrcpy | 2 +- app/data/zsh-completion/_scrcpy | 2 +- app/scrcpy.1 | 20 +++++++------- app/src/cli.c | 27 +++++++++++++------ app/src/options.c | 2 +- app/src/options.h | 2 +- app/src/scrcpy.c | 2 +- app/src/server.c | 8 +++--- app/src/server.h | 2 +- .../java/com/genymobile/scrcpy/Options.java | 10 +++---- .../java/com/genymobile/scrcpy/Server.java | 11 ++++---- 11 files changed, 49 insertions(+), 39 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 1fe79765a4..167f736fe0 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -3,7 +3,6 @@ _scrcpy() { local opts=" --always-on-top -b --video-bit-rate= - --codec-options= --crop= -d --select-usb --disable-screensaver @@ -55,6 +54,7 @@ _scrcpy() { -V --verbosity= -v --version --video-codec= + --video-codec-options= -w --stay-awake --window-borderless --window-title= diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index ac3ec0234b..29bec42dce 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -10,7 +10,6 @@ local arguments arguments=( '--always-on-top[Make scrcpy window always on top \(above other windows\)]' {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]' - '--codec-options=[Set a list of comma-separated key\:type=value options for the device encoder]' '--crop=[\[width\:height\:x\:y\] Crop the device screen on the server]' {-d,--select-usb}'[Use USB device]' '--disable-screensaver[Disable screensaver while scrcpy is running]' @@ -60,6 +59,7 @@ arguments=( {-V,--verbosity=}'[Set the log level]:verbosity:(verbose debug info warn error)' {-v,--version}'[Print the version of scrcpy]' '--video-codec=[Select the video codec]:codec:(h264 h265 av1)' + '--video-codec-options=[Set a list of comma-separated key\:type=value options for the device video encoder]' {-w,--stay-awake}'[Keep the device on while scrcpy is running, when the device is plugged in]' '--window-borderless[Disable window decorations \(display borderless window\)]' '--window-title=[Set a custom window title]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 41ce28a4d7..49d05a1498 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -25,16 +25,6 @@ Encode the video at the given bit\-rate, expressed in bits/s. Unit suffixes are Default is 8M (8000000). -.TP -.BI "\-\-codec\-options " key\fR[:\fItype\fR]=\fIvalue\fR[,...] -Set a list of comma-separated key:type=value options for the device encoder. - -The possible values for 'type' are 'int' (default), 'long', 'float' and 'string'. - -The list of possible codec options is available in the Android documentation -.UR https://d.android.com/reference/android/media/MediaFormat -.UE . - .TP .BI "\-\-crop " width\fR:\fIheight\fR:\fIx\fR:\fIy Crop the device screen on the server. @@ -329,6 +319,16 @@ Select a video codec (h264, h265 or av1). Default is h264. +.TP +.BI "\-\-video\-codec\-options " key\fR[:\fItype\fR]=\fIvalue\fR[,...] +Set a list of comma-separated key:type=value options for the device video encoder. + +The possible values for 'type' are 'int' (default), 'long', 'float' and 'string'. + +The list of possible codec options is available in the Android documentation +.UR https://d.android.com/reference/android/media/MediaFormat +.UE . + .TP .B \-w, \-\-stay-awake Keep the device on while scrcpy is running, when the device is plugged in. diff --git a/app/src/cli.c b/app/src/cli.c index 57e85aa437..cb1af46e3b 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -38,6 +38,7 @@ enum { OPT_RENDER_DRIVER, OPT_NO_MIPMAPS, OPT_CODEC_OPTIONS, + OPT_VIDEO_CODEC_OPTIONS, OPT_FORCE_ADB_FORWARD, OPT_DISABLE_SCREENSAVER, OPT_SHORTCUT_MOD, @@ -127,16 +128,10 @@ static const struct sc_option options[] = { .argdesc = "value", }, { + // deprecated .longopt_id = OPT_CODEC_OPTIONS, .longopt = "codec-options", .argdesc = "key[:type]=value[,...]", - .text = "Set a list of comma-separated key:type=value options for the " - "device encoder.\n" - "The possible values for 'type' are 'int' (default), 'long', " - "'float' and 'string'.\n" - "The list of possible codec options is available in the " - "Android documentation: " - "", }, { .longopt_id = OPT_CROP, @@ -536,6 +531,18 @@ static const struct sc_option options[] = { .text = "Select a video codec (h264, h265 or av1).\n" "Default is h264.", }, + { + .longopt_id = OPT_VIDEO_CODEC_OPTIONS, + .longopt = "video-codec-options", + .argdesc = "key[:type]=value[,...]", + .text = "Set a list of comma-separated key:type=value options for the " + "device video encoder.\n" + "The possible values for 'type' are 'int' (default), 'long', " + "'float' and 'string'.\n" + "The list of possible codec options is available in the " + "Android documentation: " + "", + }, { .shortopt = 'w', .longopt = "stay-awake", @@ -1616,7 +1623,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], opts->forward_key_repeat = false; break; case OPT_CODEC_OPTIONS: - opts->codec_options = optarg; + LOGW("--codec-options is deprecated, use --video-codec-options " + "instead."); + // fall through + case OPT_VIDEO_CODEC_OPTIONS: + opts->video_codec_options = optarg; break; case OPT_ENCODER_NAME: opts->encoder_name = optarg; diff --git a/app/src/options.c b/app/src/options.c index a087f50728..0547da1b93 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -7,7 +7,7 @@ const struct scrcpy_options scrcpy_options_default = { .window_title = NULL, .push_target = NULL, .render_driver = NULL, - .codec_options = NULL, + .video_codec_options = NULL, .encoder_name = NULL, #ifdef HAVE_V4L2 .v4l2_device = NULL, diff --git a/app/src/options.h b/app/src/options.h index d22078e440..bde7968720 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -93,7 +93,7 @@ struct scrcpy_options { const char *window_title; const char *push_target; const char *render_driver; - const char *codec_options; + const char *video_codec_options; const char *encoder_name; #ifdef HAVE_V4L2 const char *v4l2_device; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index b09de541d3..2bb5794c15 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -328,7 +328,7 @@ scrcpy(struct scrcpy_options *options) { .audio = options->audio, .show_touches = options->show_touches, .stay_awake = options->stay_awake, - .codec_options = options->codec_options, + .video_codec_options = options->video_codec_options, .encoder_name = options->encoder_name, .force_adb_forward = options->force_adb_forward, .power_off_on_close = options->power_off_on_close, diff --git a/app/src/server.c b/app/src/server.c index 7c8fba8dac..eb91b2b2c9 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -71,7 +71,7 @@ sc_server_params_destroy(struct sc_server_params *params) { // The server stores a copy of the params provided by the user free((char *) params->req_serial); free((char *) params->crop); - free((char *) params->codec_options); + free((char *) params->video_codec_options); free((char *) params->encoder_name); free((char *) params->tcpip_dst); } @@ -95,7 +95,7 @@ sc_server_params_copy(struct sc_server_params *dst, COPY(req_serial); COPY(crop); - COPY(codec_options); + COPY(video_codec_options); COPY(encoder_name); COPY(tcpip_dst); #undef COPY @@ -255,8 +255,8 @@ execute_server(struct sc_server *server, if (params->stay_awake) { ADD_PARAM("stay_awake=true"); } - if (params->codec_options) { - ADD_PARAM("codec_options=%s", params->codec_options); + if (params->video_codec_options) { + ADD_PARAM("video_codec_options=%s", params->video_codec_options); } if (params->encoder_name) { ADD_PARAM("encoder_name=%s", params->encoder_name); diff --git a/app/src/server.h b/app/src/server.h index d30192885a..352d2caed6 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -27,7 +27,7 @@ struct sc_server_params { enum sc_log_level log_level; enum sc_codec video_codec; const char *crop; - const char *codec_options; + const char *video_codec_options; const char *encoder_name; struct sc_port_range port_range; uint32_t tunnel_host; diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 53257cf382..9cfb787105 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -20,7 +20,7 @@ public class Options { private int displayId; private boolean showTouches; private boolean stayAwake; - private List codecOptions; + private List videoCodecOptions; private String encoderName; private boolean powerOffScreenOnClose; private boolean clipboardAutosync = true; @@ -146,12 +146,12 @@ public void setStayAwake(boolean stayAwake) { this.stayAwake = stayAwake; } - public List getCodecOptions() { - return codecOptions; + public List getVideoCodecOptions() { + return videoCodecOptions; } - public void setCodecOptions(List codecOptions) { - this.codecOptions = codecOptions; + public void setVideoCodecOptions(List videoCodecOptions) { + this.videoCodecOptions = videoCodecOptions; } public String getEncoderName() { diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index f764804cb3..4ff938b4ca 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -61,7 +61,6 @@ private static void initAndCleanUp(Options options) { private static void scrcpy(Options options) throws IOException, ConfigurationException { Ln.i("Device: " + Build.MANUFACTURER + " " + Build.MODEL + " (Android " + Build.VERSION.RELEASE + ")"); final Device device = new Device(options); - List codecOptions = options.getCodecOptions(); Thread initThread = startInitThread(options); @@ -117,8 +116,8 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecId(), options.getSendFrameMeta()); - ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(), codecOptions, - options.getEncoderName(), options.getDownsizeOnError()); + ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(), + options.getVideoCodecOptions(), options.getEncoderName(), options.getDownsizeOnError()); try { // synchronous screenEncoder.streamScreen(); @@ -242,9 +241,9 @@ private static Options createOptions(String... args) { boolean stayAwake = Boolean.parseBoolean(value); options.setStayAwake(stayAwake); break; - case "codec_options": - List codecOptions = CodecOption.parse(value); - options.setCodecOptions(codecOptions); + case "video_codec_options": + List videoCodecOptions = CodecOption.parse(value); + options.setVideoCodecOptions(videoCodecOptions); break; case "encoder_name": if (!value.isEmpty()) { From e694619d53d1f38ed02b269be1649c7a67a11916 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 22 Feb 2023 22:44:01 +0100 Subject: [PATCH 057/118] Rename --encoder to --video-encoder This prepares the introduction of --audio-encoder. PR #3757 --- README.md | 6 ++--- app/data/bash-completion/scrcpy | 2 +- app/data/zsh-completion/_scrcpy | 2 +- app/scrcpy.1 | 8 +++---- app/src/cli.c | 22 ++++++++++++++----- app/src/options.c | 2 +- app/src/options.h | 2 +- app/src/scrcpy.c | 2 +- app/src/server.c | 8 +++---- app/src/server.h | 2 +- .../java/com/genymobile/scrcpy/Options.java | 10 ++++----- .../java/com/genymobile/scrcpy/Server.java | 6 ++--- 12 files changed, 41 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 81371b929a..8eabafa985 100644 --- a/README.md +++ b/README.md @@ -270,15 +270,15 @@ Some devices have more than one encoder for a specific codec, and some of them may cause issues or crash. It is possible to select a different encoder: ```bash -scrcpy --encoder=OMX.qcom.video.encoder.avc +scrcpy --video-encoder=OMX.qcom.video.encoder.avc ``` To list the available encoders, you can pass an invalid encoder name; the error will give the available encoders: ```bash -scrcpy --encoder=_ # for the default codec -scrcpy --video-codec=h265 --encoder=_ # for a specific codec +scrcpy --video-encoder=_ # for the default codec +scrcpy --video-codec=h265 --video-encoder=_ # for a specific codec ``` ### Capture diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 167f736fe0..450bd32d31 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -9,7 +9,6 @@ _scrcpy() { --display= --display-buffer= -e --select-tcpip - --encoder= --force-adb-forward --forward-all-clicks -f --fullscreen @@ -55,6 +54,7 @@ _scrcpy() { -v --version --video-codec= --video-codec-options= + --video-encoder= -w --stay-awake --window-borderless --window-title= diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 29bec42dce..86d9ffbf6f 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -16,7 +16,6 @@ arguments=( '--display=[Specify the display id to mirror]' '--display-buffer=[Add a buffering delay \(in milliseconds\) before displaying]' {-e,--select-tcpip}'[Use TCP/IP device]' - '--encoder=[Use a specific MediaCodec encoder]' '--force-adb-forward[Do not attempt to use \"adb reverse\" to connect to the device]' '--forward-all-clicks[Forward clicks to device]' {-f,--fullscreen}'[Start in fullscreen]' @@ -60,6 +59,7 @@ arguments=( {-v,--version}'[Print the version of scrcpy]' '--video-codec=[Select the video codec]:codec:(h264 h265 av1)' '--video-codec-options=[Set a list of comma-separated key\:type=value options for the device video encoder]' + '--video-encoder=[Use a specific MediaCodec video encoder]' {-w,--stay-awake}'[Keep the device on while scrcpy is running, when the device is plugged in]' '--window-borderless[Disable window decorations \(display borderless window\)]' '--window-title=[Set a custom window title]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 49d05a1498..34bb750ee9 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -64,10 +64,6 @@ Use TCP/IP device (if there is exactly one, like adb -e). Also see \fB\-d\fR (\fB\-\-select\-usb\fR). -.TP -.BI "\-\-encoder " name -Use a specific MediaCodec encoder (depending on the codec provided by \fB\-\-video\-codec\fR). - .TP .B \-\-force\-adb\-forward Do not attempt to use "adb reverse" to connect to the device. @@ -329,6 +325,10 @@ The list of possible codec options is available in the Android documentation .UR https://d.android.com/reference/android/media/MediaFormat .UE . +.TP +.BI "\-\-video\-encoder " name +Use a specific MediaCodec video encoder (depending on the codec provided by \fB\-\-video\-codec\fR). + .TP .B \-w, \-\-stay-awake Keep the device on while scrcpy is running, when the device is plugged in. diff --git a/app/src/cli.c b/app/src/cli.c index cb1af46e3b..685c2f3da9 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -45,7 +45,8 @@ enum { OPT_NO_KEY_REPEAT, OPT_FORWARD_ALL_CLICKS, OPT_LEGACY_PASTE, - OPT_ENCODER_NAME, + OPT_ENCODER, + OPT_VIDEO_ENCODER, OPT_POWER_OFF_ON_CLOSE, OPT_V4L2_SINK, OPT_DISPLAY_BUFFER, @@ -178,11 +179,10 @@ static const struct sc_option options[] = { "Also see -d (--select-usb).", }, { - .longopt_id = OPT_ENCODER_NAME, + // deprecated + .longopt_id = OPT_ENCODER, .longopt = "encoder", .argdesc = "name", - .text = "Use a specific MediaCodec encoder (depending on the codec " - "provided by --video-codec).", }, { .longopt_id = OPT_FORCE_ADB_FORWARD, @@ -543,6 +543,13 @@ static const struct sc_option options[] = { "Android documentation: " "", }, + { + .longopt_id = OPT_VIDEO_ENCODER, + .longopt = "video-encoder", + .argdesc = "name", + .text = "Use a specific MediaCodec video encoder (depending on the " + "codec provided by --video-codec).", + }, { .shortopt = 'w', .longopt = "stay-awake", @@ -1629,8 +1636,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_VIDEO_CODEC_OPTIONS: opts->video_codec_options = optarg; break; - case OPT_ENCODER_NAME: - opts->encoder_name = optarg; + case OPT_ENCODER: + LOGW("--encoder is deprecated, use --video-encoder instead."); + // fall through + case OPT_VIDEO_ENCODER: + opts->video_encoder = optarg; break; case OPT_FORCE_ADB_FORWARD: opts->force_adb_forward = true; diff --git a/app/src/options.c b/app/src/options.c index 0547da1b93..fa025dea44 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -8,7 +8,7 @@ const struct scrcpy_options scrcpy_options_default = { .push_target = NULL, .render_driver = NULL, .video_codec_options = NULL, - .encoder_name = NULL, + .video_encoder = NULL, #ifdef HAVE_V4L2 .v4l2_device = NULL, #endif diff --git a/app/src/options.h b/app/src/options.h index bde7968720..3c602b7e18 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -94,7 +94,7 @@ struct scrcpy_options { const char *push_target; const char *render_driver; const char *video_codec_options; - const char *encoder_name; + const char *video_encoder; #ifdef HAVE_V4L2 const char *v4l2_device; #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 2bb5794c15..776f5d132c 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -329,7 +329,7 @@ scrcpy(struct scrcpy_options *options) { .show_touches = options->show_touches, .stay_awake = options->stay_awake, .video_codec_options = options->video_codec_options, - .encoder_name = options->encoder_name, + .video_encoder = options->video_encoder, .force_adb_forward = options->force_adb_forward, .power_off_on_close = options->power_off_on_close, .clipboard_autosync = options->clipboard_autosync, diff --git a/app/src/server.c b/app/src/server.c index eb91b2b2c9..583c338e3a 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -72,7 +72,7 @@ sc_server_params_destroy(struct sc_server_params *params) { free((char *) params->req_serial); free((char *) params->crop); free((char *) params->video_codec_options); - free((char *) params->encoder_name); + free((char *) params->video_encoder); free((char *) params->tcpip_dst); } @@ -96,7 +96,7 @@ sc_server_params_copy(struct sc_server_params *dst, COPY(req_serial); COPY(crop); COPY(video_codec_options); - COPY(encoder_name); + COPY(video_encoder); COPY(tcpip_dst); #undef COPY @@ -258,8 +258,8 @@ execute_server(struct sc_server *server, if (params->video_codec_options) { ADD_PARAM("video_codec_options=%s", params->video_codec_options); } - if (params->encoder_name) { - ADD_PARAM("encoder_name=%s", params->encoder_name); + if (params->video_encoder) { + ADD_PARAM("video_encoder=%s", params->video_encoder); } if (params->power_off_on_close) { ADD_PARAM("power_off_on_close=true"); diff --git a/app/src/server.h b/app/src/server.h index 352d2caed6..97c9aea2e4 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -28,7 +28,7 @@ struct sc_server_params { enum sc_codec video_codec; const char *crop; const char *video_codec_options; - const char *encoder_name; + const char *video_encoder; struct sc_port_range port_range; uint32_t tunnel_host; uint16_t tunnel_port; diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 9cfb787105..c518bf073f 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -21,7 +21,7 @@ public class Options { private boolean showTouches; private boolean stayAwake; private List videoCodecOptions; - private String encoderName; + private String videoEncoder; private boolean powerOffScreenOnClose; private boolean clipboardAutosync = true; private boolean downsizeOnError = true; @@ -154,12 +154,12 @@ public void setVideoCodecOptions(List videoCodecOptions) { this.videoCodecOptions = videoCodecOptions; } - public String getEncoderName() { - return encoderName; + public String getVideoEncoder() { + return videoEncoder; } - public void setEncoderName(String encoderName) { - this.encoderName = encoderName; + public void setVideoEncoder(String videoEncoder) { + this.videoEncoder = videoEncoder; } public void setPowerOffScreenOnClose(boolean powerOffScreenOnClose) { diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 4ff938b4ca..b809e90f96 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -117,7 +117,7 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecId(), options.getSendFrameMeta()); ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(), - options.getVideoCodecOptions(), options.getEncoderName(), options.getDownsizeOnError()); + options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError()); try { // synchronous screenEncoder.streamScreen(); @@ -245,9 +245,9 @@ private static Options createOptions(String... args) { List videoCodecOptions = CodecOption.parse(value); options.setVideoCodecOptions(videoCodecOptions); break; - case "encoder_name": + case "video_encoder": if (!value.isEmpty()) { - options.setEncoderName(value); + options.setVideoEncoder(value); } break; case "power_off_on_close": From 8e640dc90f66e9b7f6b22feeb7e5bc9d2bb3e6f0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 02:27:39 +0100 Subject: [PATCH 058/118] Disable MethodLength checkstyle on createOptions() This method will grow as needed to initialize options. PR #3757 --- server/src/main/java/com/genymobile/scrcpy/Server.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index b809e90f96..800b2fb66e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -157,6 +157,7 @@ private static Thread startInitThread(final Options options) { return thread; } + @SuppressWarnings("MethodLength") private static Options createOptions(String... args) { if (args.length < 1) { throw new IllegalArgumentException("Missing client version"); From 0870b8c8be18261b0930721dfa8e9f5454bd3081 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 18:32:43 +0100 Subject: [PATCH 059/118] Add --audio-bit-rate Add an option to configure the audio bit-rate. PR #3757 --- app/data/bash-completion/scrcpy | 1 + app/data/zsh-completion/_scrcpy | 1 + app/scrcpy.1 | 6 ++++++ app/src/cli.c | 14 ++++++++++++++ app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 1 + app/src/server.c | 2 ++ app/src/server.h | 1 + .../java/com/genymobile/scrcpy/AudioEncoder.java | 11 ++++++----- .../main/java/com/genymobile/scrcpy/Options.java | 9 +++++++++ .../main/java/com/genymobile/scrcpy/Server.java | 6 +++++- 12 files changed, 48 insertions(+), 6 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 450bd32d31..02ade8d047 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -2,6 +2,7 @@ _scrcpy() { local cur prev words cword local opts=" --always-on-top + --audio-bit-rate= -b --video-bit-rate= --crop= -d --select-usb diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 86d9ffbf6f..28d017e3c7 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -9,6 +9,7 @@ local arguments arguments=( '--always-on-top[Make scrcpy window always on top \(above other windows\)]' + '--audio-bit-rate=[Encode the audio at the given bit-rate]' {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]' '--crop=[\[width\:height\:x\:y\] Crop the device screen on the server]' {-d,--select-usb}'[Use USB device]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 34bb750ee9..7c11f6e592 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -19,6 +19,12 @@ provides display and control of Android devices connected on USB (or over TCP/IP .B \-\-always\-on\-top Make scrcpy window always on top (above other windows). +.TP +.BI "\-\-audio\-bit\-rate " value +Encode the audio at the given bit\-rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). + +Default is 128K (128000). + .TP .BI "\-b, \-\-video\-bit\-rate " value Encode the video at the given bit\-rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). diff --git a/app/src/cli.c b/app/src/cli.c index 685c2f3da9..7187b8783e 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -64,6 +64,7 @@ enum { OPT_CODEC, OPT_VIDEO_CODEC, OPT_NO_AUDIO, + OPT_AUDIO_BIT_RATE, }; struct sc_option { @@ -105,6 +106,14 @@ static const struct sc_option options[] = { .longopt = "always-on-top", .text = "Make scrcpy window always on top (above other windows).", }, + { + .longopt_id = OPT_AUDIO_BIT_RATE, + .longopt = "audio-bit-rate", + .argdesc = "value", + .text = "Encode the audio at the given bit-rate, expressed in bits/s. " + "Unit suffixes are supported: 'K' (x1000) and 'M' (x1000000).\n" + "Default is 128K (128000).", + }, { .shortopt = 'b', .longopt = "video-bit-rate", @@ -1461,6 +1470,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } break; + case OPT_AUDIO_BIT_RATE: + if (!parse_bit_rate(optarg, &opts->audio_bit_rate)) { + return false; + } + break; case OPT_CROP: opts->crop = optarg; break; diff --git a/app/src/options.c b/app/src/options.c index fa025dea44..70d26a6fba 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -29,6 +29,7 @@ const struct scrcpy_options scrcpy_options_default = { }, .max_size = 0, .video_bit_rate = 0, + .audio_bit_rate = 0, .max_fps = 0, .lock_video_orientation = SC_LOCK_VIDEO_ORIENTATION_UNLOCKED, .rotation = 0, diff --git a/app/src/options.h b/app/src/options.h index 3c602b7e18..92a536534c 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -109,6 +109,7 @@ struct scrcpy_options { struct sc_shortcut_mods shortcut_mods; uint16_t max_size; uint32_t video_bit_rate; + uint32_t audio_bit_rate; uint16_t max_fps; enum sc_lock_video_orientation lock_video_orientation; uint8_t rotation; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 776f5d132c..478f0e878d 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -321,6 +321,7 @@ scrcpy(struct scrcpy_options *options) { .tunnel_port = options->tunnel_port, .max_size = options->max_size, .video_bit_rate = options->video_bit_rate, + .audio_bit_rate = options->audio_bit_rate, .max_fps = options->max_fps, .lock_video_orientation = options->lock_video_orientation, .control = options->control, diff --git a/app/src/server.c b/app/src/server.c index 583c338e3a..fa8d83001b 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -221,6 +221,8 @@ execute_server(struct sc_server *server, } if (!params->audio) { ADD_PARAM("audio=false"); + } else if (params->audio_bit_rate) { + ADD_PARAM("audio_bit_rate=%" PRIu32, params->audio_bit_rate); } if (params->video_codec != SC_CODEC_H264) { ADD_PARAM("video_codec=%s", diff --git a/app/src/server.h b/app/src/server.h index 97c9aea2e4..805bdaf279 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -34,6 +34,7 @@ struct sc_server_params { uint16_t tunnel_port; uint16_t max_size; uint32_t video_bit_rate; + uint32_t audio_bit_rate; uint16_t max_fps; int8_t lock_video_orientation; bool control; diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index d06898d672..5704f76896 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -44,12 +44,12 @@ private static class OutputTask { private static final int CHANNELS = 2; private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT; private static final int BYTES_PER_SAMPLE = 2; - private static final int BIT_RATE = 128000; private static final int READ_MS = 5; // milliseconds private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000; private final Streamer streamer; + private final int bitRate; // Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4). // So many pending tasks would lead to an unacceptable delay anyway. @@ -64,8 +64,9 @@ private static class OutputTask { private boolean ended; - public AudioEncoder(Streamer streamer) { + public AudioEncoder(Streamer streamer, int bitRate) { this.streamer = streamer; + this.bitRate = bitRate; } private static AudioFormat createAudioFormat() { @@ -92,10 +93,10 @@ private static AudioRecord createAudioRecord() { return builder.build(); } - private static MediaFormat createFormat() { + private static MediaFormat createFormat(int bitRate) { MediaFormat format = new MediaFormat(); format.setString(MediaFormat.KEY_MIME, MIMETYPE); - format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE); + format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS); format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE); return format; @@ -220,7 +221,7 @@ public void encode() throws IOException { mediaCodecThread = new HandlerThread("AudioEncoder"); mediaCodecThread.start(); - MediaFormat format = createFormat(); + MediaFormat format = createFormat(bitRate); mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper())); mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index c518bf073f..44bc73ecc7 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -12,6 +12,7 @@ public class Options { private int maxSize; private VideoCodec videoCodec = VideoCodec.H264; private int videoBitRate = 8000000; + private int audioBitRate = 128000; private int maxFps; private int lockVideoOrientation = -1; private boolean tunnelForward; @@ -82,6 +83,14 @@ public void setVideoBitRate(int videoBitRate) { this.videoBitRate = videoBitRate; } + public int getAudioBitRate() { + return audioBitRate; + } + + public void setAudioBitRate(int audioBitRate) { + this.audioBitRate = audioBitRate; + } + public int getMaxFps() { return maxFps; } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 800b2fb66e..c10e3209bc 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -110,7 +110,7 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc if (audio) { Streamer audioStreamer = new Streamer(connection.getAudioFd(), AudioCodec.OPUS, options.getSendCodecId(), options.getSendFrameMeta()); - audioEncoder = new AudioEncoder(audioStreamer); + audioEncoder = new AudioEncoder(audioStreamer, options.getAudioBitRate()); audioEncoder.start(); } @@ -210,6 +210,10 @@ private static Options createOptions(String... args) { int videoBitRate = Integer.parseInt(value); options.setVideoBitRate(videoBitRate); break; + case "audio_bit_rate": + int audioBitRate = Integer.parseInt(value); + options.setAudioBitRate(audioBitRate); + break; case "max_fps": int maxFps = Integer.parseInt(value); options.setMaxFps(maxFps); From 839b842aa71a848510d9407dacc7742c61a17fea Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 19:05:43 +0100 Subject: [PATCH 060/118] Add --audio-codec Introduce the selection mechanism. Alternative codecs will be added later. PR #3757 --- app/data/bash-completion/scrcpy | 5 ++++ app/data/zsh-completion/_scrcpy | 1 + app/scrcpy.1 | 6 +++++ app/src/cli.c | 23 +++++++++++++++++++ app/src/options.c | 1 + app/src/options.h | 2 ++ app/src/scrcpy.c | 1 + app/src/server.c | 6 +++++ app/src/server.h | 1 + .../com/genymobile/scrcpy/AudioEncoder.java | 10 ++++---- .../java/com/genymobile/scrcpy/Options.java | 9 ++++++++ .../java/com/genymobile/scrcpy/Server.java | 10 +++++++- 12 files changed, 69 insertions(+), 6 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 02ade8d047..5a50f6c5a3 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -3,6 +3,7 @@ _scrcpy() { local opts=" --always-on-top --audio-bit-rate= + --audio-codec= -b --video-bit-rate= --crop= -d --select-usb @@ -71,6 +72,10 @@ _scrcpy() { COMPREPLY=($(compgen -W 'h264 h265 av1' -- "$cur")) return ;; + --audio-codec) + COMPREPLY=($(compgen -W 'opus' -- "$cur")) + return + ;; --lock-video-orientation) COMPREPLY=($(compgen -W 'unlocked initial 0 1 2 3' -- "$cur")) return diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 28d017e3c7..4f7ad5ef07 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -10,6 +10,7 @@ local arguments arguments=( '--always-on-top[Make scrcpy window always on top \(above other windows\)]' '--audio-bit-rate=[Encode the audio at the given bit-rate]' + '--audio-codec=[Select the audio codec]:codec:(opus)' {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]' '--crop=[\[width\:height\:x\:y\] Crop the device screen on the server]' {-d,--select-usb}'[Use USB device]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 7c11f6e592..89533a1fd8 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -25,6 +25,12 @@ Encode the audio at the given bit\-rate, expressed in bits/s. Unit suffixes are Default is 128K (128000). +.TP +.BI "\-\-audio\-codec " name +Select an audio codec (opus). + +Default is opus. + .TP .BI "\-b, \-\-video\-bit\-rate " value Encode the video at the given bit\-rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). diff --git a/app/src/cli.c b/app/src/cli.c index 7187b8783e..5f28164e99 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -65,6 +65,7 @@ enum { OPT_VIDEO_CODEC, OPT_NO_AUDIO, OPT_AUDIO_BIT_RATE, + OPT_AUDIO_CODEC, }; struct sc_option { @@ -114,6 +115,13 @@ static const struct sc_option options[] = { "Unit suffixes are supported: 'K' (x1000) and 'M' (x1000000).\n" "Default is 128K (128000).", }, + { + .longopt_id = OPT_AUDIO_CODEC, + .longopt = "audio-codec", + .argdesc = "name", + .text = "Select an audio codec (opus).\n" + "Default is opus.", + }, { .shortopt = 'b', .longopt = "video-bit-rate", @@ -1452,6 +1460,16 @@ parse_video_codec(const char *optarg, enum sc_codec *codec) { return false; } +static bool +parse_audio_codec(const char *optarg, enum sc_codec *codec) { + if (!strcmp(optarg, "opus")) { + *codec = SC_CODEC_OPUS; + return true; + } + LOGE("Unsupported audio codec: %s (expected opus)", optarg); + return false; +} + static bool parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], const char *optstring, const struct option *longopts) { @@ -1711,6 +1729,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } break; + case OPT_AUDIO_CODEC: + if (!parse_audio_codec(optarg, &opts->audio_codec)) { + return false; + } + break; case OPT_OTG: #ifdef HAVE_USB opts->otg = true; diff --git a/app/src/options.c b/app/src/options.c index 70d26a6fba..72f34e43ac 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -14,6 +14,7 @@ const struct scrcpy_options scrcpy_options_default = { #endif .log_level = SC_LOG_LEVEL_INFO, .video_codec = SC_CODEC_H264, + .audio_codec = SC_CODEC_OPUS, .record_format = SC_RECORD_FORMAT_AUTO, .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, .mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT, diff --git a/app/src/options.h b/app/src/options.h index 92a536534c..c698e6e3cc 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -27,6 +27,7 @@ enum sc_codec { SC_CODEC_H264, SC_CODEC_H265, SC_CODEC_AV1, + SC_CODEC_OPUS, }; enum sc_lock_video_orientation { @@ -100,6 +101,7 @@ struct scrcpy_options { #endif enum sc_log_level log_level; enum sc_codec video_codec; + enum sc_codec audio_codec; enum sc_record_format record_format; enum sc_keyboard_input_mode keyboard_input_mode; enum sc_mouse_input_mode mouse_input_mode; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 478f0e878d..8b96477c86 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -315,6 +315,7 @@ scrcpy(struct scrcpy_options *options) { .select_tcpip = options->select_tcpip, .log_level = options->log_level, .video_codec = options->video_codec, + .audio_codec = options->audio_codec, .crop = options->crop, .port_range = options->port_range, .tunnel_host = options->tunnel_host, diff --git a/app/src/server.c b/app/src/server.c index fa8d83001b..a797f01db0 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -165,6 +165,8 @@ sc_server_get_codec_name(enum sc_codec codec) { return "h265"; case SC_CODEC_AV1: return "av1"; + case SC_CODEC_OPUS: + return "opus"; default: return NULL; } @@ -228,6 +230,10 @@ execute_server(struct sc_server *server, ADD_PARAM("video_codec=%s", sc_server_get_codec_name(params->video_codec)); } + if (params->audio_codec != SC_CODEC_OPUS) { + ADD_PARAM("audio_codec=%s", + sc_server_get_codec_name(params->audio_codec)); + } if (params->max_size) { ADD_PARAM("max_size=%" PRIu16, params->max_size); } diff --git a/app/src/server.h b/app/src/server.h index 805bdaf279..55a86605f5 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -26,6 +26,7 @@ struct sc_server_params { const char *req_serial; enum sc_log_level log_level; enum sc_codec video_codec; + enum sc_codec audio_codec; const char *crop; const char *video_codec_options; const char *video_encoder; diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 5704f76896..710e5f7dcd 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -38,7 +38,6 @@ private static class OutputTask { } } - private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS; private static final int SAMPLE_RATE = 48000; private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO; private static final int CHANNELS = 2; @@ -93,9 +92,9 @@ private static AudioRecord createAudioRecord() { return builder.build(); } - private static MediaFormat createFormat(int bitRate) { + private static MediaFormat createFormat(String mimeType, int bitRate) { MediaFormat format = new MediaFormat(); - format.setString(MediaFormat.KEY_MIME, MIMETYPE); + format.setString(MediaFormat.KEY_MIME, mimeType); format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS); format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE); @@ -216,12 +215,13 @@ public void encode() throws IOException { boolean mediaCodecStarted = false; boolean recorderStarted = false; try { - mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException + String mimeType = streamer.getCodec().getMimeType(); + mediaCodec = MediaCodec.createEncoderByType(mimeType); // may throw IOException mediaCodecThread = new HandlerThread("AudioEncoder"); mediaCodecThread.start(); - MediaFormat format = createFormat(bitRate); + MediaFormat format = createFormat(mimeType, bitRate); mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper())); mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 44bc73ecc7..bdeab85183 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -11,6 +11,7 @@ public class Options { private boolean audio = true; private int maxSize; private VideoCodec videoCodec = VideoCodec.H264; + private AudioCodec audioCodec = AudioCodec.OPUS; private int videoBitRate = 8000000; private int audioBitRate = 128000; private int maxFps; @@ -75,6 +76,14 @@ public void setVideoCodec(VideoCodec videoCodec) { this.videoCodec = videoCodec; } + public AudioCodec getAudioCodec() { + return audioCodec; + } + + public void setAudioCodec(AudioCodec audioCodec) { + this.audioCodec = audioCodec; + } + public int getVideoBitRate() { return videoBitRate; } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index c10e3209bc..4c15bd39c2 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -109,7 +109,8 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc } if (audio) { - Streamer audioStreamer = new Streamer(connection.getAudioFd(), AudioCodec.OPUS, options.getSendCodecId(), options.getSendFrameMeta()); + Streamer audioStreamer = new Streamer(connection.getAudioFd(), options.getAudioCodec(), options.getSendCodecId(), + options.getSendFrameMeta()); audioEncoder = new AudioEncoder(audioStreamer, options.getAudioBitRate()); audioEncoder.start(); } @@ -202,6 +203,13 @@ private static Options createOptions(String... args) { } options.setVideoCodec(videoCodec); break; + case "audio_codec": + AudioCodec audioCodec = AudioCodec.findByName(value); + if (audioCodec == null) { + throw new IllegalArgumentException("Audio codec " + value + " not supported"); + } + options.setAudioCodec(audioCodec); + break; case "max_size": int maxSize = Integer.parseInt(value) & ~7; // multiple of 8 options.setMaxSize(maxSize); From 4601735e51b14e85b67fdc557868f022653b4230 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 18 Feb 2023 19:30:36 +0100 Subject: [PATCH 061/118] Add support for AAC audio codec Add option --audio-codec=aac. PR #3757 --- app/data/bash-completion/scrcpy | 2 +- app/data/zsh-completion/_scrcpy | 2 +- app/scrcpy.1 | 2 +- app/src/cli.c | 8 ++++++-- app/src/demuxer.c | 3 +++ app/src/options.h | 1 + app/src/server.c | 2 ++ .../src/main/java/com/genymobile/scrcpy/AudioCodec.java | 3 ++- 8 files changed, 17 insertions(+), 6 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 5a50f6c5a3..f303ff66e3 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -73,7 +73,7 @@ _scrcpy() { return ;; --audio-codec) - COMPREPLY=($(compgen -W 'opus' -- "$cur")) + COMPREPLY=($(compgen -W 'opus aac' -- "$cur")) return ;; --lock-video-orientation) diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 4f7ad5ef07..a0d83a0522 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -10,7 +10,7 @@ local arguments arguments=( '--always-on-top[Make scrcpy window always on top \(above other windows\)]' '--audio-bit-rate=[Encode the audio at the given bit-rate]' - '--audio-codec=[Select the audio codec]:codec:(opus)' + '--audio-codec=[Select the audio codec]:codec:(opus aac)' {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]' '--crop=[\[width\:height\:x\:y\] Crop the device screen on the server]' {-d,--select-usb}'[Use USB device]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 89533a1fd8..3ccbb11105 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -27,7 +27,7 @@ Default is 128K (128000). .TP .BI "\-\-audio\-codec " name -Select an audio codec (opus). +Select an audio codec (opus or aac). Default is opus. diff --git a/app/src/cli.c b/app/src/cli.c index 5f28164e99..afd060b8d7 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -119,7 +119,7 @@ static const struct sc_option options[] = { .longopt_id = OPT_AUDIO_CODEC, .longopt = "audio-codec", .argdesc = "name", - .text = "Select an audio codec (opus).\n" + .text = "Select an audio codec (opus or aac).\n" "Default is opus.", }, { @@ -1466,7 +1466,11 @@ parse_audio_codec(const char *optarg, enum sc_codec *codec) { *codec = SC_CODEC_OPUS; return true; } - LOGE("Unsupported audio codec: %s (expected opus)", optarg); + if (!strcmp(optarg, "aac")) { + *codec = SC_CODEC_AAC; + return true; + } + LOGE("Unsupported audio codec: %s (expected opus or aac)", optarg); return false; } diff --git a/app/src/demuxer.c b/app/src/demuxer.c index 482f2e0484..64bf30a34f 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -24,6 +24,7 @@ sc_demuxer_to_avcodec_id(uint32_t codec_id) { #define SC_CODEC_ID_H265 UINT32_C(0x68323635) // "h265" in ASCII #define SC_CODEC_ID_AV1 UINT32_C(0x00617631) // "av1" in ASCII #define SC_CODEC_ID_OPUS UINT32_C(0x6f707573) // "opus" in ASCII +#define SC_CODEC_ID_AAC UINT32_C(0x00616163) // "aac in ASCII" switch (codec_id) { case SC_CODEC_ID_H264: return AV_CODEC_ID_H264; @@ -33,6 +34,8 @@ sc_demuxer_to_avcodec_id(uint32_t codec_id) { return AV_CODEC_ID_AV1; case SC_CODEC_ID_OPUS: return AV_CODEC_ID_OPUS; + case SC_CODEC_ID_AAC: + return AV_CODEC_ID_AAC; default: LOGE("Unknown codec id 0x%08" PRIx32, codec_id); return AV_CODEC_ID_NONE; diff --git a/app/src/options.h b/app/src/options.h index c698e6e3cc..3efa2dd60c 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -28,6 +28,7 @@ enum sc_codec { SC_CODEC_H265, SC_CODEC_AV1, SC_CODEC_OPUS, + SC_CODEC_AAC, }; enum sc_lock_video_orientation { diff --git a/app/src/server.c b/app/src/server.c index a797f01db0..36146d86a7 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -167,6 +167,8 @@ sc_server_get_codec_name(enum sc_codec codec) { return "av1"; case SC_CODEC_OPUS: return "opus"; + case SC_CODEC_AAC: + return "aac"; default: return NULL; } diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java b/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java index 4d9e320158..dc000e98dc 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java @@ -3,7 +3,8 @@ import android.media.MediaFormat; public enum AudioCodec implements Codec { - OPUS(0x6f_70_75_73, "opus", MediaFormat.MIMETYPE_AUDIO_OPUS); + OPUS(0x6f_70_75_73, "opus", MediaFormat.MIMETYPE_AUDIO_OPUS), + AAC(0x00_61_61_63, "aac", MediaFormat.MIMETYPE_AUDIO_AAC); private final int id; // 4-byte ASCII representation of the name private final String name; From 58cf8e540199ca5a1a25bb0e9bdac56f8de17a14 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 20:03:04 +0100 Subject: [PATCH 062/118] Extract application of codec options This will allow to reuse the same code for audio codec options. PR #3757 --- .../com/genymobile/scrcpy/CodecUtils.java | 28 +++++++++++++++++++ .../com/genymobile/scrcpy/ScreenEncoder.java | 22 +++------------ 2 files changed, 32 insertions(+), 18 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/CodecUtils.java diff --git a/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java b/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java new file mode 100644 index 0000000000..2a808c596f --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java @@ -0,0 +1,28 @@ +package com.genymobile.scrcpy; + +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; +import android.media.MediaFormat; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public final class CodecUtils { + + private CodecUtils() { + // not instantiable + } + + public static void setCodecOption(MediaFormat format, String key, Object value) { + if (value instanceof Integer) { + format.setInteger(key, (Integer) value); + } else if (value instanceof Long) { + format.setLong(key, (Long) value); + } else if (value instanceof Float) { + format.setFloat(key, (Float) value); + } else if (value instanceof String) { + format.setString(key, (String) value); + } + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index d646995b4c..1c3ccf726f 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -237,23 +237,6 @@ private static String buildUnknownEncoderMessage(Codec codec, String encoderName return msg.toString(); } - private static void setCodecOption(MediaFormat format, CodecOption codecOption) { - String key = codecOption.getKey(); - Object value = codecOption.getValue(); - - if (value instanceof Integer) { - format.setInteger(key, (Integer) value); - } else if (value instanceof Long) { - format.setLong(key, (Long) value); - } else if (value instanceof Float) { - format.setFloat(key, (Float) value); - } else if (value instanceof String) { - format.setString(key, (String) value); - } - - Ln.d("Codec option set: " + key + " (" + value.getClass().getSimpleName() + ") = " + value); - } - private static MediaFormat createFormat(String videoMimeType, int bitRate, int maxFps, List codecOptions) { MediaFormat format = new MediaFormat(); format.setString(MediaFormat.KEY_MIME, videoMimeType); @@ -273,7 +256,10 @@ private static MediaFormat createFormat(String videoMimeType, int bitRate, int m if (codecOptions != null) { for (CodecOption option : codecOptions) { - setCodecOption(format, option); + String key = option.getKey(); + Object value = option.getValue(); + CodecUtils.setCodecOption(format, key, value); + Ln.d("Video codec option set: " + key + " (" + value.getClass().getSimpleName() + ") = " + value); } } From b03c864c70c4388ae788c494cfffeed2c00aabb8 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 22 Feb 2023 22:48:23 +0100 Subject: [PATCH 063/118] Add --audio-codec-options Similar to --video-codec-options, but for audio. PR #3757 --- app/data/bash-completion/scrcpy | 1 + app/data/zsh-completion/_scrcpy | 1 + app/scrcpy.1 | 10 ++++++++++ app/src/cli.c | 16 ++++++++++++++++ app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 1 + app/src/server.c | 5 +++++ app/src/server.h | 1 + .../com/genymobile/scrcpy/AudioEncoder.java | 19 ++++++++++++++++--- .../java/com/genymobile/scrcpy/Options.java | 10 ++++++++++ .../java/com/genymobile/scrcpy/Server.java | 6 +++++- 12 files changed, 68 insertions(+), 4 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index f303ff66e3..da245acc82 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -4,6 +4,7 @@ _scrcpy() { --always-on-top --audio-bit-rate= --audio-codec= + --audio-codec-options= -b --video-bit-rate= --crop= -d --select-usb diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index a0d83a0522..aa7928c62b 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -11,6 +11,7 @@ arguments=( '--always-on-top[Make scrcpy window always on top \(above other windows\)]' '--audio-bit-rate=[Encode the audio at the given bit-rate]' '--audio-codec=[Select the audio codec]:codec:(opus aac)' + '--audio-codec-options=[Set a list of comma-separated key\:type=value options for the device audio encoder]' {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]' '--crop=[\[width\:height\:x\:y\] Crop the device screen on the server]' {-d,--select-usb}'[Use USB device]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 3ccbb11105..fd7746c465 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -31,6 +31,16 @@ Select an audio codec (opus or aac). Default is opus. +.TP +.BI "\-\-audio\-codec\-options " key\fR[:\fItype\fR]=\fIvalue\fR[,...] +Set a list of comma-separated key:type=value options for the device audio encoder. + +The possible values for 'type' are 'int' (default), 'long', 'float' and 'string'. + +The list of possible codec options is available in the Android documentation +.UR https://d.android.com/reference/android/media/MediaFormat +.UE . + .TP .BI "\-b, \-\-video\-bit\-rate " value Encode the video at the given bit\-rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). diff --git a/app/src/cli.c b/app/src/cli.c index afd060b8d7..9f61e6cbb5 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -66,6 +66,7 @@ enum { OPT_NO_AUDIO, OPT_AUDIO_BIT_RATE, OPT_AUDIO_CODEC, + OPT_AUDIO_CODEC_OPTIONS, }; struct sc_option { @@ -122,6 +123,18 @@ static const struct sc_option options[] = { .text = "Select an audio codec (opus or aac).\n" "Default is opus.", }, + { + .longopt_id = OPT_AUDIO_CODEC_OPTIONS, + .longopt = "audio-codec-options", + .argdesc = "key[:type]=value[,...]", + .text = "Set a list of comma-separated key:type=value options for the " + "device audio encoder.\n" + "The possible values for 'type' are 'int' (default), 'long', " + "'float' and 'string'.\n" + "The list of possible codec options is available in the " + "Android documentation: " + "", + }, { .shortopt = 'b', .longopt = "video-bit-rate", @@ -1672,6 +1685,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_VIDEO_CODEC_OPTIONS: opts->video_codec_options = optarg; break; + case OPT_AUDIO_CODEC_OPTIONS: + opts->audio_codec_options = optarg; + break; case OPT_ENCODER: LOGW("--encoder is deprecated, use --video-encoder instead."); // fall through diff --git a/app/src/options.c b/app/src/options.c index 72f34e43ac..a9be5dfa5d 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -8,6 +8,7 @@ const struct scrcpy_options scrcpy_options_default = { .push_target = NULL, .render_driver = NULL, .video_codec_options = NULL, + .audio_codec_options = NULL, .video_encoder = NULL, #ifdef HAVE_V4L2 .v4l2_device = NULL, diff --git a/app/src/options.h b/app/src/options.h index 3efa2dd60c..bbb52eb7a2 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -96,6 +96,7 @@ struct scrcpy_options { const char *push_target; const char *render_driver; const char *video_codec_options; + const char *audio_codec_options; const char *video_encoder; #ifdef HAVE_V4L2 const char *v4l2_device; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 8b96477c86..a43c26876b 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -331,6 +331,7 @@ scrcpy(struct scrcpy_options *options) { .show_touches = options->show_touches, .stay_awake = options->stay_awake, .video_codec_options = options->video_codec_options, + .audio_codec_options = options->audio_codec_options, .video_encoder = options->video_encoder, .force_adb_forward = options->force_adb_forward, .power_off_on_close = options->power_off_on_close, diff --git a/app/src/server.c b/app/src/server.c index 36146d86a7..95e4670d23 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -72,6 +72,7 @@ sc_server_params_destroy(struct sc_server_params *params) { free((char *) params->req_serial); free((char *) params->crop); free((char *) params->video_codec_options); + free((char *) params->audio_codec_options); free((char *) params->video_encoder); free((char *) params->tcpip_dst); } @@ -96,6 +97,7 @@ sc_server_params_copy(struct sc_server_params *dst, COPY(req_serial); COPY(crop); COPY(video_codec_options); + COPY(audio_codec_options); COPY(video_encoder); COPY(tcpip_dst); #undef COPY @@ -268,6 +270,9 @@ execute_server(struct sc_server *server, if (params->video_codec_options) { ADD_PARAM("video_codec_options=%s", params->video_codec_options); } + if (params->audio_codec_options) { + ADD_PARAM("audio_codec_options=%s", params->audio_codec_options); + } if (params->video_encoder) { ADD_PARAM("video_encoder=%s", params->video_encoder); } diff --git a/app/src/server.h b/app/src/server.h index 55a86605f5..d96f997e3c 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -29,6 +29,7 @@ struct sc_server_params { enum sc_codec audio_codec; const char *crop; const char *video_codec_options; + const char *audio_codec_options; const char *video_encoder; struct sc_port_range port_range; uint32_t tunnel_host; diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 710e5f7dcd..56ff207fba 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -49,6 +50,7 @@ private static class OutputTask { private final Streamer streamer; private final int bitRate; + private final List codecOptions; // Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4). // So many pending tasks would lead to an unacceptable delay anyway. @@ -63,9 +65,10 @@ private static class OutputTask { private boolean ended; - public AudioEncoder(Streamer streamer, int bitRate) { + public AudioEncoder(Streamer streamer, int bitRate, List codecOptions) { this.streamer = streamer; this.bitRate = bitRate; + this.codecOptions = codecOptions; } private static AudioFormat createAudioFormat() { @@ -92,12 +95,22 @@ private static AudioRecord createAudioRecord() { return builder.build(); } - private static MediaFormat createFormat(String mimeType, int bitRate) { + private static MediaFormat createFormat(String mimeType, int bitRate, List codecOptions) { MediaFormat format = new MediaFormat(); format.setString(MediaFormat.KEY_MIME, mimeType); format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS); format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE); + + if (codecOptions != null) { + for (CodecOption option : codecOptions) { + String key = option.getKey(); + Object value = option.getValue(); + CodecUtils.setCodecOption(format, key, value); + Ln.d("Audio codec option set: " + key + " (" + value.getClass().getSimpleName() + ") = " + value); + } + } + return format; } @@ -221,7 +234,7 @@ public void encode() throws IOException { mediaCodecThread = new HandlerThread("AudioEncoder"); mediaCodecThread.start(); - MediaFormat format = createFormat(mimeType, bitRate); + MediaFormat format = createFormat(mimeType, bitRate, codecOptions); mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper())); mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index bdeab85183..4cb21e287f 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -23,6 +23,8 @@ public class Options { private boolean showTouches; private boolean stayAwake; private List videoCodecOptions; + private List audioCodecOptions; + private String videoEncoder; private boolean powerOffScreenOnClose; private boolean clipboardAutosync = true; @@ -172,6 +174,14 @@ public void setVideoCodecOptions(List videoCodecOptions) { this.videoCodecOptions = videoCodecOptions; } + public List getAudioCodecOptions() { + return audioCodecOptions; + } + + public void setAudioCodecOptions(List audioCodecOptions) { + this.audioCodecOptions = audioCodecOptions; + } + public String getVideoEncoder() { return videoEncoder; } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 4c15bd39c2..f4e36bffc9 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -111,7 +111,7 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc if (audio) { Streamer audioStreamer = new Streamer(connection.getAudioFd(), options.getAudioCodec(), options.getSendCodecId(), options.getSendFrameMeta()); - audioEncoder = new AudioEncoder(audioStreamer, options.getAudioBitRate()); + audioEncoder = new AudioEncoder(audioStreamer, options.getAudioBitRate(), options.getAudioCodecOptions()); audioEncoder.start(); } @@ -258,6 +258,10 @@ private static Options createOptions(String... args) { List videoCodecOptions = CodecOption.parse(value); options.setVideoCodecOptions(videoCodecOptions); break; + case "audio_codec_options": + List audioCodecOptions = CodecOption.parse(value); + options.setAudioCodecOptions(audioCodecOptions); + break; case "video_encoder": if (!value.isEmpty()) { options.setVideoEncoder(value); From 6f332a2bc73853884c894ee2400ad9c92aabc34c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 20:03:04 +0100 Subject: [PATCH 064/118] Extract unknown encoder error message This will allow to reuse the same code for audio encoder selection. PR #3757 --- .../com/genymobile/scrcpy/CodecUtils.java | 25 +++++++++++++++++ .../com/genymobile/scrcpy/ScreenEncoder.java | 28 +------------------ 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java b/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java index 2a808c596f..96887c145d 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java +++ b/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java @@ -25,4 +25,29 @@ public static void setCodecOption(MediaFormat format, String key, Object value) format.setString(key, (String) value); } } + + public static String buildUnknownEncoderMessage(Codec codec, String encoderName) { + StringBuilder msg = new StringBuilder("Encoder '").append(encoderName).append("' for ").append(codec.getName()).append(" not found"); + MediaCodecInfo[] encoders = listEncoders(codec.getMimeType()); + if (encoders != null && encoders.length > 0) { + msg.append("\nTry to use one of the available encoders:"); + String codecOption = codec.getType() == Codec.Type.VIDEO ? "video-codec" : "audio-codec"; + for (MediaCodecInfo encoder : encoders) { + msg.append("\n scrcpy --").append(codecOption).append("=").append(codec.getName()); + msg.append(" --encoder='").append(encoder.getName()).append("'"); + } + } + return msg.toString(); + } + + private static MediaCodecInfo[] listEncoders(String mimeType) { + List result = new ArrayList<>(); + MediaCodecList list = new MediaCodecList(MediaCodecList.REGULAR_CODECS); + for (MediaCodecInfo codecInfo : list.getCodecInfos()) { + if (codecInfo.isEncoder() && Arrays.asList(codecInfo.getSupportedTypes()).contains(mimeType)) { + result.add(codecInfo); + } + } + return result.toArray(new MediaCodecInfo[result.size()]); + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index 1c3ccf726f..77cd1de437 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -5,7 +5,6 @@ import android.graphics.Rect; import android.media.MediaCodec; import android.media.MediaCodecInfo; -import android.media.MediaCodecList; import android.media.MediaFormat; import android.os.Build; import android.os.IBinder; @@ -14,8 +13,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -199,24 +196,13 @@ private boolean encode(MediaCodec codec, Streamer streamer) throws IOException { return !eof; } - private static MediaCodecInfo[] listEncoders(String videoMimeType) { - List result = new ArrayList<>(); - MediaCodecList list = new MediaCodecList(MediaCodecList.REGULAR_CODECS); - for (MediaCodecInfo codecInfo : list.getCodecInfos()) { - if (codecInfo.isEncoder() && Arrays.asList(codecInfo.getSupportedTypes()).contains(videoMimeType)) { - result.add(codecInfo); - } - } - return result.toArray(new MediaCodecInfo[result.size()]); - } - private static MediaCodec createMediaCodec(Codec codec, String encoderName) throws IOException, ConfigurationException { if (encoderName != null) { Ln.d("Creating encoder by name: '" + encoderName + "'"); try { return MediaCodec.createByCodecName(encoderName); } catch (IllegalArgumentException e) { - Ln.e(buildUnknownEncoderMessage(codec, encoderName)); + Ln.e(CodecUtils.buildUnknownEncoderMessage(codec, encoderName)); throw new ConfigurationException("Unknown encoder: " + encoderName); } } @@ -225,18 +211,6 @@ private static MediaCodec createMediaCodec(Codec codec, String encoderName) thro return mediaCodec; } - private static String buildUnknownEncoderMessage(Codec codec, String encoderName) { - StringBuilder msg = new StringBuilder("Encoder '").append(encoderName).append("' for ").append(codec.getName()).append(" not found"); - MediaCodecInfo[] encoders = listEncoders(codec.getMimeType()); - if (encoders != null && encoders.length > 0) { - msg.append("\nTry to use one of the available encoders:"); - for (MediaCodecInfo encoder : encoders) { - msg.append("\n scrcpy --video-codec=").append(codec.getName()).append(" --encoder='").append(encoder.getName()).append("'"); - } - } - return msg.toString(); - } - private static MediaFormat createFormat(String videoMimeType, int bitRate, int maxFps, List codecOptions) { MediaFormat format = new MediaFormat(); format.setString(MediaFormat.KEY_MIME, videoMimeType); From f9960e959fa7b46cc1ed6b15115fe2958724766d Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 19 Feb 2023 20:20:29 +0100 Subject: [PATCH 065/118] Add --audio-encoder Similar to --video-encoder, but for audio. PR #3757 --- app/data/bash-completion/scrcpy | 1 + app/data/zsh-completion/_scrcpy | 1 + app/scrcpy.1 | 4 +++ app/src/cli.c | 11 +++++++ app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 1 + app/src/server.c | 5 ++++ app/src/server.h | 1 + .../com/genymobile/scrcpy/AudioEncoder.java | 29 +++++++++++++++---- .../java/com/genymobile/scrcpy/Options.java | 9 ++++++ .../java/com/genymobile/scrcpy/Server.java | 6 +++- 12 files changed, 64 insertions(+), 6 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index da245acc82..c860707f3b 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -5,6 +5,7 @@ _scrcpy() { --audio-bit-rate= --audio-codec= --audio-codec-options= + --audio-encoder= -b --video-bit-rate= --crop= -d --select-usb diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index aa7928c62b..b122587f2e 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -12,6 +12,7 @@ arguments=( '--audio-bit-rate=[Encode the audio at the given bit-rate]' '--audio-codec=[Select the audio codec]:codec:(opus aac)' '--audio-codec-options=[Set a list of comma-separated key\:type=value options for the device audio encoder]' + '--audio-encoder=[Use a specific MediaCodec audio encoder]' {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]' '--crop=[\[width\:height\:x\:y\] Crop the device screen on the server]' {-d,--select-usb}'[Use USB device]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index fd7746c465..ef17465a5a 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -41,6 +41,10 @@ The list of possible codec options is available in the Android documentation .UR https://d.android.com/reference/android/media/MediaFormat .UE . +.TP +.BI "\-\-audio\-encoder " name +Use a specific MediaCodec audio encoder (depending on the codec provided by \fB\-\-audio\-codec\fR). + .TP .BI "\-b, \-\-video\-bit\-rate " value Encode the video at the given bit\-rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). diff --git a/app/src/cli.c b/app/src/cli.c index 9f61e6cbb5..68629fd257 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -67,6 +67,7 @@ enum { OPT_AUDIO_BIT_RATE, OPT_AUDIO_CODEC, OPT_AUDIO_CODEC_OPTIONS, + OPT_AUDIO_ENCODER, }; struct sc_option { @@ -135,6 +136,13 @@ static const struct sc_option options[] = { "Android documentation: " "", }, + { + .longopt_id = OPT_AUDIO_ENCODER, + .longopt = "audio-encoder", + .argdesc = "name", + .text = "Use a specific MediaCodec audio encoder (depending on the " + "codec provided by --audio-codec).", + }, { .shortopt = 'b', .longopt = "video-bit-rate", @@ -1694,6 +1702,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_VIDEO_ENCODER: opts->video_encoder = optarg; break; + case OPT_AUDIO_ENCODER: + opts->audio_encoder = optarg; + break; case OPT_FORCE_ADB_FORWARD: opts->force_adb_forward = true; break; diff --git a/app/src/options.c b/app/src/options.c index a9be5dfa5d..40f84fdd89 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -10,6 +10,7 @@ const struct scrcpy_options scrcpy_options_default = { .video_codec_options = NULL, .audio_codec_options = NULL, .video_encoder = NULL, + .audio_encoder = NULL, #ifdef HAVE_V4L2 .v4l2_device = NULL, #endif diff --git a/app/src/options.h b/app/src/options.h index bbb52eb7a2..804fba9354 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -98,6 +98,7 @@ struct scrcpy_options { const char *video_codec_options; const char *audio_codec_options; const char *video_encoder; + const char *audio_encoder; #ifdef HAVE_V4L2 const char *v4l2_device; #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index a43c26876b..6bfed2951d 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -333,6 +333,7 @@ scrcpy(struct scrcpy_options *options) { .video_codec_options = options->video_codec_options, .audio_codec_options = options->audio_codec_options, .video_encoder = options->video_encoder, + .audio_encoder = options->audio_encoder, .force_adb_forward = options->force_adb_forward, .power_off_on_close = options->power_off_on_close, .clipboard_autosync = options->clipboard_autosync, diff --git a/app/src/server.c b/app/src/server.c index 95e4670d23..b50003c96a 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -74,6 +74,7 @@ sc_server_params_destroy(struct sc_server_params *params) { free((char *) params->video_codec_options); free((char *) params->audio_codec_options); free((char *) params->video_encoder); + free((char *) params->audio_encoder); free((char *) params->tcpip_dst); } @@ -99,6 +100,7 @@ sc_server_params_copy(struct sc_server_params *dst, COPY(video_codec_options); COPY(audio_codec_options); COPY(video_encoder); + COPY(audio_encoder); COPY(tcpip_dst); #undef COPY @@ -276,6 +278,9 @@ execute_server(struct sc_server *server, if (params->video_encoder) { ADD_PARAM("video_encoder=%s", params->video_encoder); } + if (params->audio_encoder) { + ADD_PARAM("audio_encoder=%s", params->audio_encoder); + } if (params->power_off_on_close) { ADD_PARAM("power_off_on_close=true"); } diff --git a/app/src/server.h b/app/src/server.h index d96f997e3c..c20508e03a 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -31,6 +31,7 @@ struct sc_server_params { const char *video_codec_options; const char *audio_codec_options; const char *video_encoder; + const char *audio_encoder; struct sc_port_range port_range; uint32_t tunnel_host; uint16_t tunnel_port; diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 56ff207fba..a70a475b37 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -51,6 +51,7 @@ private static class OutputTask { private final Streamer streamer; private final int bitRate; private final List codecOptions; + private final String encoderName; // Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4). // So many pending tasks would lead to an unacceptable delay anyway. @@ -65,10 +66,11 @@ private static class OutputTask { private boolean ended; - public AudioEncoder(Streamer streamer, int bitRate, List codecOptions) { + public AudioEncoder(Streamer streamer, int bitRate, List codecOptions, String encoderName) { this.streamer = streamer; this.bitRate = bitRate; this.codecOptions = codecOptions; + this.encoderName = encoderName; } private static AudioFormat createAudioFormat() { @@ -177,6 +179,8 @@ public void start() { thread = new Thread(() -> { try { encode(); + } catch (ConfigurationException e) { + // Do not print stack trace, a user-friendly error-message has already been logged } catch (IOException e) { Ln.e("Audio encoding error", e); } finally { @@ -215,7 +219,7 @@ private synchronized void waitEnded() { } @TargetApi(Build.VERSION_CODES.M) - public void encode() throws IOException { + public void encode() throws IOException, ConfigurationException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { Ln.w("Audio disabled: it is not supported before Android 11"); streamer.writeDisableStream(); @@ -228,13 +232,13 @@ public void encode() throws IOException { boolean mediaCodecStarted = false; boolean recorderStarted = false; try { - String mimeType = streamer.getCodec().getMimeType(); - mediaCodec = MediaCodec.createEncoderByType(mimeType); // may throw IOException + Codec codec = streamer.getCodec(); + mediaCodec = createMediaCodec(codec, encoderName); mediaCodecThread = new HandlerThread("AudioEncoder"); mediaCodecThread.start(); - MediaFormat format = createFormat(mimeType, bitRate, codecOptions); + MediaFormat format = createFormat(codec.getMimeType(), bitRate, codecOptions); mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper())); mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); @@ -324,6 +328,21 @@ public void encode() throws IOException { } } + private static MediaCodec createMediaCodec(Codec codec, String encoderName) throws IOException, ConfigurationException { + if (encoderName != null) { + Ln.d("Creating audio encoder by name: '" + encoderName + "'"); + try { + return MediaCodec.createByCodecName(encoderName); + } catch (IllegalArgumentException e) { + Ln.e(CodecUtils.buildUnknownEncoderMessage(codec, encoderName)); + throw new ConfigurationException("Unknown encoder: " + encoderName); + } + } + MediaCodec mediaCodec = MediaCodec.createEncoderByType(codec.getMimeType()); + Ln.d("Using audio encoder: '" + mediaCodec.getName() + "'"); + return mediaCodec; + } + private class EncoderCallback extends MediaCodec.Callback { @TargetApi(Build.VERSION_CODES.N) @Override diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 4cb21e287f..8683802209 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -26,6 +26,7 @@ public class Options { private List audioCodecOptions; private String videoEncoder; + private String audioEncoder; private boolean powerOffScreenOnClose; private boolean clipboardAutosync = true; private boolean downsizeOnError = true; @@ -190,6 +191,14 @@ public void setVideoEncoder(String videoEncoder) { this.videoEncoder = videoEncoder; } + public String getAudioEncoder() { + return audioEncoder; + } + + public void setAudioEncoder(String audioEncoder) { + this.audioEncoder = audioEncoder; + } + public void setPowerOffScreenOnClose(boolean powerOffScreenOnClose) { this.powerOffScreenOnClose = powerOffScreenOnClose; } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index f4e36bffc9..f30d65f69e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -111,7 +111,7 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc if (audio) { Streamer audioStreamer = new Streamer(connection.getAudioFd(), options.getAudioCodec(), options.getSendCodecId(), options.getSendFrameMeta()); - audioEncoder = new AudioEncoder(audioStreamer, options.getAudioBitRate(), options.getAudioCodecOptions()); + audioEncoder = new AudioEncoder(audioStreamer, options.getAudioBitRate(), options.getAudioCodecOptions(), options.getAudioEncoder()); audioEncoder.start(); } @@ -267,6 +267,10 @@ private static Options createOptions(String... args) { options.setVideoEncoder(value); } break; + case "audio_encoder": + if (!value.isEmpty()) { + options.setAudioEncoder(value); + } case "power_off_on_close": boolean powerOffScreenOnClose = Boolean.parseBoolean(value); options.setPowerOffScreenOnClose(powerOffScreenOnClose); From b7e5284adf1074d6d851a0a7d164461325ccaa58 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 22 Feb 2023 23:12:21 +0100 Subject: [PATCH 066/118] Move await_for_server() logs Print the logs on the caller side. This will allow to call the function in another context without printing the logs. PR #3757 --- app/src/scrcpy.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 6bfed2951d..81affe47b2 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -183,14 +183,11 @@ await_for_server(bool *connected) { while (SDL_WaitEvent(&event)) { switch (event.type) { case SDL_QUIT: - LOGD("User requested to quit"); *connected = false; return true; case SC_EVENT_SERVER_CONNECTION_FAILED: - LOGE("Server connection failed"); return false; case SC_EVENT_SERVER_CONNECTED: - LOGD("Server connected"); *connected = true; return true; default: @@ -374,15 +371,19 @@ scrcpy(struct scrcpy_options *options) { // Await for server without blocking Ctrl+C handling bool connected; if (!await_for_server(&connected)) { + LOGE("Server connection failed"); goto end; } if (!connected) { // This is not an error, user requested to quit + LOGD("User requested to quit"); ret = SCRCPY_EXIT_SUCCESS; goto end; } + LOGD("Server connected"); + // It is necessarily initialized here, since the device is connected struct sc_server_info *info = &s->server.info; From 9196dc156376760eb637024d5c0448ceeaa2a8ff Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 22 Feb 2023 23:15:15 +0100 Subject: [PATCH 067/118] Add --list-encoders Add an option to list the device encoders properly. PR #3757 --- README.md | 6 +- app/data/bash-completion/scrcpy | 1 + app/data/zsh-completion/_scrcpy | 1 + app/scrcpy.1 | 8 ++ app/src/cli.c | 15 +++- app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 15 +++- app/src/server.c | 27 +++++-- app/src/server.h | 1 + .../com/genymobile/scrcpy/AudioEncoder.java | 2 +- .../java/com/genymobile/scrcpy/CleanUp.java | 2 +- .../com/genymobile/scrcpy/CodecUtils.java | 79 ++++++++++++++++--- .../java/com/genymobile/scrcpy/Options.java | 10 +++ .../com/genymobile/scrcpy/ScreenEncoder.java | 2 +- .../java/com/genymobile/scrcpy/Server.java | 15 ++++ 16 files changed, 157 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 8eabafa985..5575fc4d8a 100644 --- a/README.md +++ b/README.md @@ -273,12 +273,10 @@ may cause issues or crash. It is possible to select a different encoder: scrcpy --video-encoder=OMX.qcom.video.encoder.avc ``` -To list the available encoders, you can pass an invalid encoder name; the -error will give the available encoders: +To list the available encoders: ```bash -scrcpy --video-encoder=_ # for the default codec -scrcpy --video-codec=h265 --video-encoder=_ # for a specific codec +scrcpy --list-encoders ``` ### Capture diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index c860707f3b..7069501964 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -19,6 +19,7 @@ _scrcpy() { -K --hid-keyboard -h --help --legacy-paste + --list-encoders --lock-video-orientation --lock-video-orientation= --max-fps= diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index b122587f2e..268aa62647 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -26,6 +26,7 @@ arguments=( {-K,--hid-keyboard}'[Simulate a physical keyboard by using HID over AOAv2]' {-h,--help}'[Print the help]' '--legacy-paste[Inject computer clipboard text as a sequence of key events on Ctrl+v]' + '--list-encoders[List video and audio encoders available on the device]' '--lock-video-orientation=[Lock video orientation]:orientation:(unlocked initial 0 1 2 3)' '--max-fps=[Limit the frame rate of screen capture]' {-M,--hid-mouse}'[Simulate a physical mouse by using HID over AOAv2]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index ef17465a5a..add263c2ef 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -45,6 +45,8 @@ The list of possible codec options is available in the Android documentation .BI "\-\-audio\-encoder " name Use a specific MediaCodec audio encoder (depending on the codec provided by \fB\-\-audio\-codec\fR). +The available encoders can be listed by \-\-list\-encoders. + .TP .BI "\-b, \-\-video\-bit\-rate " value Encode the video at the given bit\-rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). @@ -128,6 +130,10 @@ Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+S This is a workaround for some devices not behaving as expected when setting the device clipboard programmatically. +.TP +.B \-\-list\-encoders +List video and audio encoders available on the device. + .TP \fB\-\-lock\-video\-orientation\fR[=\fIvalue\fR] Lock video orientation to \fIvalue\fR. Possible values are "unlocked", "initial" (locked to the initial orientation), 0, 1, 2 and 3. Natural device orientation is 0, and each increment adds a 90 degrees rotation counterclockwise. @@ -355,6 +361,8 @@ The list of possible codec options is available in the Android documentation .BI "\-\-video\-encoder " name Use a specific MediaCodec video encoder (depending on the codec provided by \fB\-\-video\-codec\fR). +The available encoders can be listed by \-\-list\-encoders. + .TP .B \-w, \-\-stay-awake Keep the device on while scrcpy is running, when the device is plugged in. diff --git a/app/src/cli.c b/app/src/cli.c index 68629fd257..edb694a618 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -68,6 +68,7 @@ enum { OPT_AUDIO_CODEC, OPT_AUDIO_CODEC_OPTIONS, OPT_AUDIO_ENCODER, + OPT_LIST_ENCODERS, }; struct sc_option { @@ -141,7 +142,8 @@ static const struct sc_option options[] = { .longopt = "audio-encoder", .argdesc = "name", .text = "Use a specific MediaCodec audio encoder (depending on the " - "codec provided by --audio-codec).", + "codec provided by --audio-codec).\n" + "The available encoders can be listed by --list-encoders.", }, { .shortopt = 'b', @@ -270,6 +272,11 @@ static const struct sc_option options[] = { "This is a workaround for some devices not behaving as " "expected when setting the device clipboard programmatically.", }, + { + .longopt_id = OPT_LIST_ENCODERS, + .longopt = "list-encoders", + .text = "List video and audio encoders available on the device.", + }, { .longopt_id = OPT_LOCK_VIDEO_ORIENTATION, .longopt = "lock-video-orientation", @@ -586,7 +593,8 @@ static const struct sc_option options[] = { .longopt = "video-encoder", .argdesc = "name", .text = "Use a specific MediaCodec video encoder (depending on the " - "codec provided by --video-codec).", + "codec provided by --video-codec).\n" + "The available encoders can be listed by --list-encoders.", }, { .shortopt = 'w', @@ -1792,6 +1800,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], LOGE("V4L2 (--v4l2-buffer) is only available on Linux."); return false; #endif + case OPT_LIST_ENCODERS: + opts->list_encoders = true; + break; default: // getopt prints the error message on stderr return false; diff --git a/app/src/options.c b/app/src/options.c index 40f84fdd89..1839df6e61 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -72,4 +72,5 @@ const struct scrcpy_options scrcpy_options_default = { .start_fps_counter = false, .power_on = true, .audio = true, + .list_encoders = false, }; diff --git a/app/src/options.h b/app/src/options.h index 804fba9354..568b8155f1 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -154,6 +154,7 @@ struct scrcpy_options { bool start_fps_counter; bool power_on; bool audio; + bool list_encoders; }; extern const struct scrcpy_options scrcpy_options_default; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 81affe47b2..6d0fac9e1f 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -183,12 +183,16 @@ await_for_server(bool *connected) { while (SDL_WaitEvent(&event)) { switch (event.type) { case SDL_QUIT: - *connected = false; + if (connected) { + *connected = false; + } return true; case SC_EVENT_SERVER_CONNECTION_FAILED: return false; case SC_EVENT_SERVER_CONNECTED: - *connected = true; + if (connected) { + *connected = true; + } return true; default: break; @@ -339,6 +343,7 @@ scrcpy(struct scrcpy_options *options) { .tcpip_dst = options->tcpip_dst, .cleanup = options->cleanup, .power_on = options->power_on, + .list_encoders = options->list_encoders, }; static const struct sc_server_callbacks cbs = { @@ -356,6 +361,12 @@ scrcpy(struct scrcpy_options *options) { server_started = true; + if (options->list_encoders) { + bool ok = await_for_server(NULL); + ret = ok ? SCRCPY_EXIT_SUCCESS : SCRCPY_EXIT_FAILURE; + goto end; + } + if (options->display) { sdl_set_hints(options->render_driver); } diff --git a/app/src/server.c b/app/src/server.c index b50003c96a..077614a894 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -300,6 +300,9 @@ execute_server(struct sc_server *server, // By default, power_on is true ADD_PARAM("power_on=false"); } + if (params->list_encoders) { + ADD_PARAM("list_encoders=true"); + } #undef ADD_PARAM @@ -848,6 +851,25 @@ run_server(void *data) { assert(serial); LOGD("Device serial: %s", serial); + ok = push_server(&server->intr, serial); + if (!ok) { + goto error_connection_failed; + } + + // If --list-encoders is passed, then the server just prints the encoders + // then exits. + if (params->list_encoders) { + sc_pid pid = execute_server(server, params); + if (pid == SC_PROCESS_NONE) { + goto error_connection_failed; + } + sc_process_wait(pid, NULL); // ignore exit code + sc_process_close(pid); + // Wake up await_for_server() + server->cbs->on_connected(server, server->cbs_userdata); + return 0; + } + int r = asprintf(&server->device_socket_name, SC_SOCKET_NAME_PREFIX "%08x", params->scid); if (r == -1) { @@ -857,11 +879,6 @@ run_server(void *data) { assert(r == sizeof(SC_SOCKET_NAME_PREFIX) - 1 + 8); assert(server->device_socket_name); - ok = push_server(&server->intr, serial); - if (!ok) { - goto error_connection_failed; - } - ok = sc_adb_tunnel_open(&server->tunnel, &server->intr, serial, server->device_socket_name, params->port_range, params->force_adb_forward); diff --git a/app/src/server.h b/app/src/server.h index c20508e03a..ada04baade 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -55,6 +55,7 @@ struct sc_server_params { bool select_tcpip; bool cleanup; bool power_on; + bool list_encoders; }; struct sc_server { diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index a70a475b37..540d8306bd 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -334,7 +334,7 @@ private static MediaCodec createMediaCodec(Codec codec, String encoderName) thro try { return MediaCodec.createByCodecName(encoderName); } catch (IllegalArgumentException e) { - Ln.e(CodecUtils.buildUnknownEncoderMessage(codec, encoderName)); + Ln.e("Encoder '" + encoderName + "' for " + codec.getName() + " not found\n" + CodecUtils.buildAudioEncoderListMessage()); throw new ConfigurationException("Unknown encoder: " + encoderName); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/CleanUp.java b/server/src/main/java/com/genymobile/scrcpy/CleanUp.java index 831dc994ad..0bcd1a5470 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CleanUp.java +++ b/server/src/main/java/com/genymobile/scrcpy/CleanUp.java @@ -139,7 +139,7 @@ private static void startProcess(Config config) throws IOException { builder.start(); } - private static void unlinkSelf() { + public static void unlinkSelf() { try { new File(SERVER_PATH).delete(); } catch (Exception e) { diff --git a/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java b/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java index 96887c145d..aca54d20a0 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java +++ b/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java @@ -10,6 +10,24 @@ public final class CodecUtils { + public static final class DeviceEncoder { + private final Codec codec; + private final MediaCodecInfo info; + + DeviceEncoder(Codec codec, MediaCodecInfo info) { + this.codec = codec; + this.info = info; + } + + public Codec getCodec() { + return codec; + } + + public MediaCodecInfo getInfo() { + return info; + } + } + private CodecUtils() { // not instantiable } @@ -26,28 +44,63 @@ public static void setCodecOption(MediaFormat format, String key, Object value) } } - public static String buildUnknownEncoderMessage(Codec codec, String encoderName) { - StringBuilder msg = new StringBuilder("Encoder '").append(encoderName).append("' for ").append(codec.getName()).append(" not found"); - MediaCodecInfo[] encoders = listEncoders(codec.getMimeType()); - if (encoders != null && encoders.length > 0) { - msg.append("\nTry to use one of the available encoders:"); - String codecOption = codec.getType() == Codec.Type.VIDEO ? "video-codec" : "audio-codec"; - for (MediaCodecInfo encoder : encoders) { - msg.append("\n scrcpy --").append(codecOption).append("=").append(codec.getName()); - msg.append(" --encoder='").append(encoder.getName()).append("'"); + public static String buildVideoEncoderListMessage() { + StringBuilder builder = new StringBuilder("List of video encoders:"); + List videoEncoders = CodecUtils.listVideoEncoders(); + if (videoEncoders.isEmpty()) { + builder.append("\n (none)"); + } else { + for (CodecUtils.DeviceEncoder encoder : videoEncoders) { + builder.append("\n --video-codec=").append(encoder.getCodec().getName()); + builder.append(" --video-encoder='").append(encoder.getInfo().getName()).append("'"); + } + } + return builder.toString(); + } + + public static String buildAudioEncoderListMessage() { + StringBuilder builder = new StringBuilder("List of audio encoders:"); + List audioEncoders = CodecUtils.listAudioEncoders(); + if (audioEncoders.isEmpty()) { + builder.append("\n (none)"); + } else { + for (CodecUtils.DeviceEncoder encoder : audioEncoders) { + builder.append("\n --audio-codec=").append(encoder.getCodec().getName()); + builder.append(" --audio-encoder='").append(encoder.getInfo().getName()).append("'"); } } - return msg.toString(); + return builder.toString(); } - private static MediaCodecInfo[] listEncoders(String mimeType) { + private static MediaCodecInfo[] getEncoders(MediaCodecList codecs, String mimeType) { List result = new ArrayList<>(); - MediaCodecList list = new MediaCodecList(MediaCodecList.REGULAR_CODECS); - for (MediaCodecInfo codecInfo : list.getCodecInfos()) { + for (MediaCodecInfo codecInfo : codecs.getCodecInfos()) { if (codecInfo.isEncoder() && Arrays.asList(codecInfo.getSupportedTypes()).contains(mimeType)) { result.add(codecInfo); } } return result.toArray(new MediaCodecInfo[result.size()]); } + + public static List listVideoEncoders() { + List encoders = new ArrayList<>(); + MediaCodecList codecs = new MediaCodecList(MediaCodecList.REGULAR_CODECS); + for (VideoCodec codec : VideoCodec.values()) { + for (MediaCodecInfo info : getEncoders(codecs, codec.getMimeType())) { + encoders.add(new DeviceEncoder(codec, info)); + } + } + return encoders; + } + + public static List listAudioEncoders() { + List encoders = new ArrayList<>(); + MediaCodecList codecs = new MediaCodecList(MediaCodecList.REGULAR_CODECS); + for (AudioCodec codec : AudioCodec.values()) { + for (MediaCodecInfo info : getEncoders(codecs, codec.getMimeType())) { + encoders.add(new DeviceEncoder(codec, info)); + } + } + return encoders; + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 8683802209..8cac5e2c56 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -33,6 +33,8 @@ public class Options { private boolean cleanup = true; private boolean powerOn = true; + private boolean listEncoders; + // Options not used by the scrcpy client, but useful to use scrcpy-server directly private boolean sendDeviceMeta = true; // send device name and size private boolean sendFrameMeta = true; // send PTS so that the client may record properly @@ -239,6 +241,14 @@ public void setPowerOn(boolean powerOn) { this.powerOn = powerOn; } + public boolean getListEncoders() { + return listEncoders; + } + + public void setListEncoders(boolean listEncoders) { + this.listEncoders = listEncoders; + } + public boolean getSendDeviceMeta() { return sendDeviceMeta; } diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index 77cd1de437..668a4ed0ce 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -202,7 +202,7 @@ private static MediaCodec createMediaCodec(Codec codec, String encoderName) thro try { return MediaCodec.createByCodecName(encoderName); } catch (IllegalArgumentException e) { - Ln.e(CodecUtils.buildUnknownEncoderMessage(codec, encoderName)); + Ln.e("Encoder '" + encoderName + "' for " + codec.getName() + " not found\n" + CodecUtils.buildVideoEncoderListMessage()); throw new ConfigurationException("Unknown encoder: " + encoderName); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index f30d65f69e..adfbef2af5 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -291,6 +291,10 @@ private static Options createOptions(String... args) { boolean powerOn = Boolean.parseBoolean(value); options.setPowerOn(powerOn); break; + case "list_encoders": + boolean listEncoders = Boolean.parseBoolean(value); + options.setListEncoders(listEncoders); + break; case "send_device_meta": boolean sendDeviceMeta = Boolean.parseBoolean(value); options.setSendDeviceMeta(sendDeviceMeta); @@ -350,6 +354,17 @@ public static void main(String... args) throws Exception { Ln.initLogLevel(options.getLogLevel()); + if (options.getListEncoders()) { + if (options.getCleanup()) { + CleanUp.unlinkSelf(); + } + + Ln.i(CodecUtils.buildVideoEncoderListMessage()); + Ln.i(CodecUtils.buildAudioEncoderListMessage()); + // Just print the available encoders, do not mirror + return; + } + try { scrcpy(options); } catch (ConfigurationException e) { From 50d56a9a2bb75398c562e71b131de5c3ed43e95e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 23 Feb 2023 21:27:11 +0100 Subject: [PATCH 068/118] Quit on audio configuration failure When audio capture fails on the device, scrcpy continues mirroring the video stream. This allows to enable audio by default only when supported. However, if an audio configuration occurs (for example the user explicitly selected an unknown audio encoder), this must be treated as an error and scrcpy must exit. PR #3757 --- app/src/demuxer.c | 6 ++++++ app/src/scrcpy.c | 13 +++++++++++-- .../java/com/genymobile/scrcpy/AudioEncoder.java | 8 ++++++-- .../main/java/com/genymobile/scrcpy/Streamer.java | 13 +++++++++---- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/app/src/demuxer.c b/app/src/demuxer.c index 64bf30a34f..d80a5dda8e 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -195,6 +195,12 @@ run_demuxer(void *data) { goto end; } + if (raw_codec_id == 1) { + LOGE("Demuxer '%s': stream configuration error on the device", + demuxer->name); + goto end; + } + enum AVCodecID codec_id = sc_demuxer_to_avcodec_id(raw_codec_id); if (codec_id == AV_CODEC_ID_NONE) { LOGE("Demuxer '%s': stream disabled due to unsupported codec", diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 6d0fac9e1f..5739c3b29f 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -231,10 +231,19 @@ static void sc_audio_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, void *userdata) { (void) demuxer; - (void) eos; (void) userdata; - // Contrary to the video demuxer, keep mirroring if only the audio fails + // Contrary to the video demuxer, keep mirroring if only the audio fails. + // 'eos' is true on end-of-stream, including when audio capture is not + // possible on the device (so that scrcpy continue to mirror video without + // failing). + // However, if an audio configuration failure occurs (for example the user + // explicitly selected an unknown audio encoder), 'eos' is false and scrcpy + // must exit. + + if (!eos) { + PUSH_EVENT(SC_EVENT_DEMUXER_ERROR); + } } static void diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 540d8306bd..66950004be 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -222,7 +222,7 @@ private synchronized void waitEnded() { public void encode() throws IOException, ConfigurationException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { Ln.w("Audio disabled: it is not supported before Android 11"); - streamer.writeDisableStream(); + streamer.writeDisableStream(false); return; } @@ -279,9 +279,13 @@ public void encode() throws IOException, ConfigurationException { outputThread.start(); waitEnded(); + } catch (ConfigurationException e) { + // Notify the error to make scrcpy exit + streamer.writeDisableStream(true); + throw e; } catch (Throwable e) { // Notify the client that the audio could not be captured - streamer.writeDisableStream(); + streamer.writeDisableStream(false); throw e; } finally { // Cleanup everything (either at the end or on error at any step of the initialization) diff --git a/server/src/main/java/com/genymobile/scrcpy/Streamer.java b/server/src/main/java/com/genymobile/scrcpy/Streamer.java index 7cc065ebdf..9bfe7e91ff 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Streamer.java +++ b/server/src/main/java/com/genymobile/scrcpy/Streamer.java @@ -40,10 +40,15 @@ public void writeHeader() throws IOException { } } - public void writeDisableStream() throws IOException { - // Writing 0 (32-bit) as codec-id means that the device disables the stream (because it could not capture) - byte[] zeros = new byte[4]; - IO.writeFully(fd, zeros, 0, zeros.length); + public void writeDisableStream(boolean error) throws IOException { + // Writing a specific code as codec-id means that the device disables the stream + // code 0: it explicitly disables the stream (because it could not capture audio), scrcpy should continue mirroring video only + // code 1: a configuration error occurred, scrcpy must be stopped + byte[] code = new byte[4]; + if (error) { + code[3] = 1; + } + IO.writeFully(fd, code, 0, code.length); } public void writePacket(ByteBuffer buffer, long pts, boolean config, boolean keyFrame) throws IOException { From 2596ca02f0d9b53ddf322873ad141197216f00f4 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 23 Feb 2023 23:09:25 +0100 Subject: [PATCH 069/118] Move log message helpers to LogUtils This class will also contain other log helpers. --- .../com/genymobile/scrcpy/AudioEncoder.java | 2 +- .../com/genymobile/scrcpy/CodecUtils.java | 28 -------------- .../java/com/genymobile/scrcpy/LogUtils.java | 38 +++++++++++++++++++ .../com/genymobile/scrcpy/ScreenEncoder.java | 2 +- .../java/com/genymobile/scrcpy/Server.java | 4 +- 5 files changed, 42 insertions(+), 32 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/LogUtils.java diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 66950004be..1ce4107f56 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -338,7 +338,7 @@ private static MediaCodec createMediaCodec(Codec codec, String encoderName) thro try { return MediaCodec.createByCodecName(encoderName); } catch (IllegalArgumentException e) { - Ln.e("Encoder '" + encoderName + "' for " + codec.getName() + " not found\n" + CodecUtils.buildAudioEncoderListMessage()); + Ln.e("Encoder '" + encoderName + "' for " + codec.getName() + " not found\n" + LogUtils.buildAudioEncoderListMessage()); throw new ConfigurationException("Unknown encoder: " + encoderName); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java b/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java index aca54d20a0..afb6f904a3 100644 --- a/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java +++ b/server/src/main/java/com/genymobile/scrcpy/CodecUtils.java @@ -44,34 +44,6 @@ public static void setCodecOption(MediaFormat format, String key, Object value) } } - public static String buildVideoEncoderListMessage() { - StringBuilder builder = new StringBuilder("List of video encoders:"); - List videoEncoders = CodecUtils.listVideoEncoders(); - if (videoEncoders.isEmpty()) { - builder.append("\n (none)"); - } else { - for (CodecUtils.DeviceEncoder encoder : videoEncoders) { - builder.append("\n --video-codec=").append(encoder.getCodec().getName()); - builder.append(" --video-encoder='").append(encoder.getInfo().getName()).append("'"); - } - } - return builder.toString(); - } - - public static String buildAudioEncoderListMessage() { - StringBuilder builder = new StringBuilder("List of audio encoders:"); - List audioEncoders = CodecUtils.listAudioEncoders(); - if (audioEncoders.isEmpty()) { - builder.append("\n (none)"); - } else { - for (CodecUtils.DeviceEncoder encoder : audioEncoders) { - builder.append("\n --audio-codec=").append(encoder.getCodec().getName()); - builder.append(" --audio-encoder='").append(encoder.getInfo().getName()).append("'"); - } - } - return builder.toString(); - } - private static MediaCodecInfo[] getEncoders(MediaCodecList codecs, String mimeType) { List result = new ArrayList<>(); for (MediaCodecInfo codecInfo : codecs.getCodecInfos()) { diff --git a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java new file mode 100644 index 0000000000..e74b7e9707 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java @@ -0,0 +1,38 @@ +package com.genymobile.scrcpy; + +import java.util.List; + +public final class LogUtils { + + private LogUtils() { + // not instantiable + } + + public static String buildVideoEncoderListMessage() { + StringBuilder builder = new StringBuilder("List of video encoders:"); + List videoEncoders = CodecUtils.listVideoEncoders(); + if (videoEncoders.isEmpty()) { + builder.append("\n (none)"); + } else { + for (CodecUtils.DeviceEncoder encoder : videoEncoders) { + builder.append("\n --video-codec=").append(encoder.getCodec().getName()); + builder.append(" --video-encoder='").append(encoder.getInfo().getName()).append("'"); + } + } + return builder.toString(); + } + + public static String buildAudioEncoderListMessage() { + StringBuilder builder = new StringBuilder("List of audio encoders:"); + List audioEncoders = CodecUtils.listAudioEncoders(); + if (audioEncoders.isEmpty()) { + builder.append("\n (none)"); + } else { + for (CodecUtils.DeviceEncoder encoder : audioEncoders) { + builder.append("\n --audio-codec=").append(encoder.getCodec().getName()); + builder.append(" --audio-encoder='").append(encoder.getInfo().getName()).append("'"); + } + } + return builder.toString(); + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java index 668a4ed0ce..f5f996ba97 100644 --- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java @@ -202,7 +202,7 @@ private static MediaCodec createMediaCodec(Codec codec, String encoderName) thro try { return MediaCodec.createByCodecName(encoderName); } catch (IllegalArgumentException e) { - Ln.e("Encoder '" + encoderName + "' for " + codec.getName() + " not found\n" + CodecUtils.buildVideoEncoderListMessage()); + Ln.e("Encoder '" + encoderName + "' for " + codec.getName() + " not found\n" + LogUtils.buildVideoEncoderListMessage()); throw new ConfigurationException("Unknown encoder: " + encoderName); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index adfbef2af5..f46cf30813 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -359,8 +359,8 @@ public static void main(String... args) throws Exception { CleanUp.unlinkSelf(); } - Ln.i(CodecUtils.buildVideoEncoderListMessage()); - Ln.i(CodecUtils.buildAudioEncoderListMessage()); + Ln.i(LogUtils.buildVideoEncoderListMessage()); + Ln.i(LogUtils.buildAudioEncoderListMessage()); // Just print the available encoders, do not mirror return; } From b65301f672e852fe2f3fded1fac2091c85679c35 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 23 Feb 2023 23:10:15 +0100 Subject: [PATCH 070/118] Add --list-displays Add an option to list the device displays properly. --- README.md | 2 +- app/data/bash-completion/scrcpy | 1 + app/data/zsh-completion/_scrcpy | 1 + app/scrcpy.1 | 9 ++++++--- app/src/cli.c | 16 ++++++++++++---- app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 3 ++- app/src/server.c | 7 +++++-- app/src/server.h | 1 + .../main/java/com/genymobile/scrcpy/Device.java | 14 +------------- .../java/com/genymobile/scrcpy/LogUtils.java | 15 +++++++++++++++ .../java/com/genymobile/scrcpy/Options.java | 9 +++++++++ .../main/java/com/genymobile/scrcpy/Server.java | 17 +++++++++++++---- 14 files changed, 69 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 5575fc4d8a..a2e275f924 100644 --- a/README.md +++ b/README.md @@ -718,7 +718,7 @@ scrcpy --display=1 The list of display ids can be retrieved by: ```bash -adb shell dumpsys display # search "mDisplayId=" in the output +scrcpy --list-displays ``` The secondary display may only be controlled if the device runs at least Android diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 7069501964..fa95ce6eed 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -19,6 +19,7 @@ _scrcpy() { -K --hid-keyboard -h --help --legacy-paste + --list-displays --list-encoders --lock-video-orientation --lock-video-orientation= diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 268aa62647..231405ceb9 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -26,6 +26,7 @@ arguments=( {-K,--hid-keyboard}'[Simulate a physical keyboard by using HID over AOAv2]' {-h,--help}'[Print the help]' '--legacy-paste[Inject computer clipboard text as a sequence of key events on Ctrl+v]' + '--list-displays[List displays available on the device]' '--list-encoders[List video and audio encoders available on the device]' '--lock-video-orientation=[Lock video orientation]:orientation:(unlocked initial 0 1 2 3)' '--max-fps=[Limit the frame rate of screen capture]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index add263c2ef..40b8158ca5 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -73,10 +73,9 @@ Disable screensaver while scrcpy is running. .TP .BI "\-\-display " id -Specify the display id to mirror. +Specify the device display id to mirror. -The list of possible display ids can be listed by "adb shell dumpsys display" -(search "mDisplayId=" in the output). +The available display ids can be listed by \-\-list\-displays. Default is 0. @@ -134,6 +133,10 @@ This is a workaround for some devices not behaving as expected when setting the .B \-\-list\-encoders List video and audio encoders available on the device. +.TP +.B \-\-list\-displays +List displays available on the device. + .TP \fB\-\-lock\-video\-orientation\fR[=\fIvalue\fR] Lock video orientation to \fIvalue\fR. Possible values are "unlocked", "initial" (locked to the initial orientation), 0, 1, 2 and 3. Natural device orientation is 0, and each increment adds a 90 degrees rotation counterclockwise. diff --git a/app/src/cli.c b/app/src/cli.c index edb694a618..8dfcdc792b 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -69,6 +69,7 @@ enum { OPT_AUDIO_CODEC_OPTIONS, OPT_AUDIO_ENCODER, OPT_LIST_ENCODERS, + OPT_LIST_DISPLAYS, }; struct sc_option { @@ -198,10 +199,9 @@ static const struct sc_option options[] = { .longopt_id = OPT_DISPLAY_ID, .longopt = "display", .argdesc = "id", - .text = "Specify the display id to mirror.\n" - "The list of possible display ids can be listed by:\n" - " adb shell dumpsys display\n" - "(search \"mDisplayId=\" in the output)\n" + .text = "Specify the device display id to mirror.\n" + "The available display ids can be listed by:\n" + " scrcpy --list-displays\n" "Default is 0.", }, { @@ -272,6 +272,11 @@ static const struct sc_option options[] = { "This is a workaround for some devices not behaving as " "expected when setting the device clipboard programmatically.", }, + { + .longopt_id = OPT_LIST_DISPLAYS, + .longopt = "list-displays", + .text = "List device displays.", + }, { .longopt_id = OPT_LIST_ENCODERS, .longopt = "list-encoders", @@ -1803,6 +1808,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_LIST_ENCODERS: opts->list_encoders = true; break; + case OPT_LIST_DISPLAYS: + opts->list_displays = true; + break; default: // getopt prints the error message on stderr return false; diff --git a/app/src/options.c b/app/src/options.c index 1839df6e61..8560b37b9f 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -73,4 +73,5 @@ const struct scrcpy_options scrcpy_options_default = { .power_on = true, .audio = true, .list_encoders = false, + .list_displays = false, }; diff --git a/app/src/options.h b/app/src/options.h index 568b8155f1..a15d51f82a 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -155,6 +155,7 @@ struct scrcpy_options { bool power_on; bool audio; bool list_encoders; + bool list_displays; }; extern const struct scrcpy_options scrcpy_options_default; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 5739c3b29f..4d68fb2979 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -353,6 +353,7 @@ scrcpy(struct scrcpy_options *options) { .cleanup = options->cleanup, .power_on = options->power_on, .list_encoders = options->list_encoders, + .list_displays = options->list_displays, }; static const struct sc_server_callbacks cbs = { @@ -370,7 +371,7 @@ scrcpy(struct scrcpy_options *options) { server_started = true; - if (options->list_encoders) { + if (options->list_encoders || options->list_displays) { bool ok = await_for_server(NULL); ret = ok ? SCRCPY_EXIT_SUCCESS : SCRCPY_EXIT_FAILURE; goto end; diff --git a/app/src/server.c b/app/src/server.c index 077614a894..9d4fb098a7 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -303,6 +303,9 @@ execute_server(struct sc_server *server, if (params->list_encoders) { ADD_PARAM("list_encoders=true"); } + if (params->list_displays) { + ADD_PARAM("list_displays=true"); + } #undef ADD_PARAM @@ -856,9 +859,9 @@ run_server(void *data) { goto error_connection_failed; } - // If --list-encoders is passed, then the server just prints the encoders + // If --list-* is passed, then the server just prints the requested data // then exits. - if (params->list_encoders) { + if (params->list_encoders || params->list_displays) { sc_pid pid = execute_server(server, params); if (pid == SC_PROCESS_NONE) { goto error_connection_failed; diff --git a/app/src/server.h b/app/src/server.h index ada04baade..8edf26661f 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -56,6 +56,7 @@ struct sc_server_params { bool cleanup; bool power_on; bool list_encoders; + bool list_displays; }; struct sc_server { diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.java index c7f7c1f8b7..b66474b7e3 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Device.java +++ b/server/src/main/java/com/genymobile/scrcpy/Device.java @@ -65,7 +65,7 @@ public Device(Options options) throws ConfigurationException { displayId = options.getDisplayId(); DisplayInfo displayInfo = ServiceManager.getDisplayManager().getDisplayInfo(displayId); if (displayInfo == null) { - Ln.e(buildUnknownDisplayIdMessage(displayId)); + Ln.e("Display " + displayId + " not found\n" + LogUtils.buildDisplayListMessage()); throw new ConfigurationException("Unknown display id: " + displayId); } @@ -130,18 +130,6 @@ public void dispatchPrimaryClipChanged() { } } - private static String buildUnknownDisplayIdMessage(int displayId) { - StringBuilder msg = new StringBuilder("Display ").append(displayId).append(" not found"); - int[] displayIds = ServiceManager.getDisplayManager().getDisplayIds(); - if (displayIds != null && displayIds.length > 0) { - msg.append("\nTry to use one of the available display ids:"); - for (int id : displayIds) { - msg.append("\n scrcpy --display=").append(id); - } - } - return msg.toString(); - } - public synchronized void setMaxSize(int newMaxSize) { maxSize = newMaxSize; screenInfo = ScreenInfo.computeScreenInfo(screenInfo.getReverseVideoRotation(), deviceSize, crop, newMaxSize, lockVideoOrientation); diff --git a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java index e74b7e9707..c073336de6 100644 --- a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java +++ b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java @@ -1,5 +1,7 @@ package com.genymobile.scrcpy; +import com.genymobile.scrcpy.wrappers.ServiceManager; + import java.util.List; public final class LogUtils { @@ -35,4 +37,17 @@ public static String buildAudioEncoderListMessage() { } return builder.toString(); } + + public static String buildDisplayListMessage() { + StringBuilder builder = new StringBuilder("List of displays:"); + int[] displayIds = ServiceManager.getDisplayManager().getDisplayIds(); + if (displayIds == null || displayIds.length == 0) { + builder.append("\n (none)"); + } else { + for (int id : displayIds) { + builder.append("\n --display=").append(id); + } + } + return builder.toString(); + } } diff --git a/server/src/main/java/com/genymobile/scrcpy/Options.java b/server/src/main/java/com/genymobile/scrcpy/Options.java index 8cac5e2c56..bcf235edba 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Options.java +++ b/server/src/main/java/com/genymobile/scrcpy/Options.java @@ -34,6 +34,7 @@ public class Options { private boolean powerOn = true; private boolean listEncoders; + private boolean listDisplays; // Options not used by the scrcpy client, but useful to use scrcpy-server directly private boolean sendDeviceMeta = true; // send device name and size @@ -249,6 +250,14 @@ public void setListEncoders(boolean listEncoders) { this.listEncoders = listEncoders; } + public boolean getListDisplays() { + return listDisplays; + } + + public void setListDisplays(boolean listDisplays) { + this.listDisplays = listDisplays; + } + public boolean getSendDeviceMeta() { return sendDeviceMeta; } diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index f46cf30813..35da6965f8 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -295,6 +295,10 @@ private static Options createOptions(String... args) { boolean listEncoders = Boolean.parseBoolean(value); options.setListEncoders(listEncoders); break; + case "list_displays": + boolean listDisplays = Boolean.parseBoolean(value); + options.setListDisplays(listDisplays); + break; case "send_device_meta": boolean sendDeviceMeta = Boolean.parseBoolean(value); options.setSendDeviceMeta(sendDeviceMeta); @@ -354,14 +358,19 @@ public static void main(String... args) throws Exception { Ln.initLogLevel(options.getLogLevel()); - if (options.getListEncoders()) { + if (options.getListEncoders() || options.getListDisplays()) { if (options.getCleanup()) { CleanUp.unlinkSelf(); } - Ln.i(LogUtils.buildVideoEncoderListMessage()); - Ln.i(LogUtils.buildAudioEncoderListMessage()); - // Just print the available encoders, do not mirror + if (options.getListEncoders()) { + Ln.i(LogUtils.buildVideoEncoderListMessage()); + Ln.i(LogUtils.buildAudioEncoderListMessage()); + } + if (options.getListDisplays()) { + Ln.i(LogUtils.buildDisplayListMessage()); + } + // Just print the requested data, do not mirror return; } From a205ff6c8b5ac1cae5a5b2763742247da84cd4b5 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 23 Feb 2023 23:12:24 +0100 Subject: [PATCH 071/118] Log display sizes in display list This is more convenient than just the display id alone. --- .../main/java/com/genymobile/scrcpy/LogUtils.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java index c073336de6..243a156bab 100644 --- a/server/src/main/java/com/genymobile/scrcpy/LogUtils.java +++ b/server/src/main/java/com/genymobile/scrcpy/LogUtils.java @@ -1,5 +1,6 @@ package com.genymobile.scrcpy; +import com.genymobile.scrcpy.wrappers.DisplayManager; import com.genymobile.scrcpy.wrappers.ServiceManager; import java.util.List; @@ -40,12 +41,21 @@ public static String buildAudioEncoderListMessage() { public static String buildDisplayListMessage() { StringBuilder builder = new StringBuilder("List of displays:"); - int[] displayIds = ServiceManager.getDisplayManager().getDisplayIds(); + DisplayManager displayManager = ServiceManager.getDisplayManager(); + int[] displayIds = displayManager.getDisplayIds(); if (displayIds == null || displayIds.length == 0) { builder.append("\n (none)"); } else { for (int id : displayIds) { - builder.append("\n --display=").append(id); + builder.append("\n --display=").append(id).append(" ("); + DisplayInfo displayInfo = displayManager.getDisplayInfo(id); + if (displayInfo != null) { + Size size = displayInfo.getSize(); + builder.append(size.getWidth()).append("x").append(size.getHeight()); + } else { + builder.append("size unknown"); + } + builder.append(")"); } } return builder.toString(); From 99837fa600af9f128fa0830b9b007a6944b9808f Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 24 Feb 2023 21:13:56 +0100 Subject: [PATCH 072/118] Rename decoder to video_decoder This prepares the introduction of audio_decoder. PR #3757 --- app/src/scrcpy.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 4d68fb2979..578943f916 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -42,7 +42,7 @@ struct scrcpy { struct sc_screen screen; struct sc_demuxer video_demuxer; struct sc_demuxer audio_demuxer; - struct sc_decoder decoder; + struct sc_decoder video_decoder; struct sc_recorder recorder; #ifdef HAVE_V4L2 struct sc_v4l2_sink v4l2_sink; @@ -436,13 +436,13 @@ scrcpy(struct scrcpy_options *options) { &audio_demuxer_cbs, NULL); } - bool needs_decoder = options->display; + bool needs_video_decoder = options->display; #ifdef HAVE_V4L2 - needs_decoder |= !!options->v4l2_device; + needs_video_decoder |= !!options->v4l2_device; #endif - if (needs_decoder) { - sc_decoder_init(&s->decoder); - sc_demuxer_add_sink(&s->video_demuxer, &s->decoder.packet_sink); + if (needs_video_decoder) { + sc_decoder_init(&s->video_decoder); + sc_demuxer_add_sink(&s->video_demuxer, &s->video_decoder.packet_sink); } if (options->record_filename) { @@ -656,7 +656,7 @@ scrcpy(struct scrcpy_options *options) { } screen_initialized = true; - sc_decoder_add_sink(&s->decoder, &s->screen.frame_sink); + sc_decoder_add_sink(&s->video_decoder, &s->screen.frame_sink); } #ifdef HAVE_V4L2 @@ -666,7 +666,7 @@ scrcpy(struct scrcpy_options *options) { goto end; } - sc_decoder_add_sink(&s->decoder, &s->v4l2_sink.frame_sink); + sc_decoder_add_sink(&s->video_decoder, &s->v4l2_sink.frame_sink); v4l2_sink_initialized = true; } From 05f0e35d2a9074ecb214d70303f3f271631af645 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 24 Feb 2023 21:22:35 +0100 Subject: [PATCH 073/118] Give a name to decoder instances This will be useful in logs. PR #3757 --- app/src/decoder.c | 11 +++++++---- app/src/decoder.h | 5 ++++- app/src/scrcpy.c | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index 337aa329e5..d750253cf0 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -48,7 +48,7 @@ sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec) { decoder->codec_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY; if (avcodec_open2(decoder->codec_ctx, codec, NULL) < 0) { - LOGE("Could not open codec"); + LOGE("Decoder '%s': could not open codec", decoder->name); avcodec_free_context(&decoder->codec_ctx); return false; } @@ -101,7 +101,8 @@ sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) { int ret = avcodec_send_packet(decoder->codec_ctx, packet); if (ret < 0 && ret != AVERROR(EAGAIN)) { - LOGE("Could not send video packet: %d", ret); + LOGE("Decoder '%s': could not send video packet: %d", + decoder->name, ret); return false; } ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame); @@ -114,7 +115,8 @@ sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) { av_frame_unref(decoder->frame); } else if (ret != AVERROR(EAGAIN)) { - LOGE("Could not receive video frame: %d", ret); + LOGE("Decoder '%s', could not receive video frame: %d", + decoder->name, ret); return false; } return true; @@ -140,7 +142,8 @@ sc_decoder_packet_sink_push(struct sc_packet_sink *sink, } void -sc_decoder_init(struct sc_decoder *decoder) { +sc_decoder_init(struct sc_decoder *decoder, const char *name) { + decoder->name = name; // statically allocated decoder->sink_count = 0; static const struct sc_packet_sink_ops ops = { diff --git a/app/src/decoder.h b/app/src/decoder.h index 16adc5ec03..aace1af6fd 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -14,6 +14,8 @@ struct sc_decoder { struct sc_packet_sink packet_sink; // packet sink trait + const char *name; // must be statically allocated (e.g. a string literal) + struct sc_frame_sink *sinks[SC_DECODER_MAX_SINKS]; unsigned sink_count; @@ -21,8 +23,9 @@ struct sc_decoder { AVFrame *frame; }; +// The name must be statically allocated (e.g. a string literal) void -sc_decoder_init(struct sc_decoder *decoder); +sc_decoder_init(struct sc_decoder *decoder, const char *name); void sc_decoder_add_sink(struct sc_decoder *decoder, struct sc_frame_sink *sink); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 578943f916..944d5f0536 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -441,7 +441,7 @@ scrcpy(struct scrcpy_options *options) { needs_video_decoder |= !!options->v4l2_device; #endif if (needs_video_decoder) { - sc_decoder_init(&s->video_decoder); + sc_decoder_init(&s->video_decoder, "video"); sc_demuxer_add_sink(&s->video_demuxer, &s->video_decoder.packet_sink); } From e22660d698900f949cfc6afdb41d49cb371f7cff Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 24 Feb 2023 21:31:39 +0100 Subject: [PATCH 074/118] Add an audio decoder PR #3757 --- app/src/scrcpy.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 944d5f0536..eb70749a57 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -43,6 +43,7 @@ struct scrcpy { struct sc_demuxer video_demuxer; struct sc_demuxer audio_demuxer; struct sc_decoder video_decoder; + struct sc_decoder audio_decoder; struct sc_recorder recorder; #ifdef HAVE_V4L2 struct sc_v4l2_sink v4l2_sink; @@ -437,6 +438,7 @@ scrcpy(struct scrcpy_options *options) { } bool needs_video_decoder = options->display; + bool needs_audio_decoder = options->audio && options->display; #ifdef HAVE_V4L2 needs_video_decoder |= !!options->v4l2_device; #endif @@ -444,6 +446,10 @@ scrcpy(struct scrcpy_options *options) { sc_decoder_init(&s->video_decoder, "video"); sc_demuxer_add_sink(&s->video_demuxer, &s->video_decoder.packet_sink); } + if (needs_audio_decoder) { + sc_decoder_init(&s->audio_decoder, "audio"); + sc_demuxer_add_sink(&s->audio_demuxer, &s->audio_decoder.packet_sink); + } if (options->record_filename) { static const struct sc_recorder_callbacks recorder_cbs = { From 619730edafe7ab2fd2fb022f9eb6e350bfe1bb52 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 25 Feb 2023 16:19:58 +0100 Subject: [PATCH 075/118] Pass AVCodecContext to frame sinks Frame consumers may need details about the frame format. PR #3757 --- app/src/decoder.c | 11 ++++++++--- app/src/screen.c | 6 +++++- app/src/trait/frame_sink.h | 3 ++- app/src/v4l2_sink.c | 9 ++++++--- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index d750253cf0..96d4a01043 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -25,10 +25,10 @@ sc_decoder_close_sinks(struct sc_decoder *decoder) { } static bool -sc_decoder_open_sinks(struct sc_decoder *decoder) { +sc_decoder_open_sinks(struct sc_decoder *decoder, const AVCodecContext *ctx) { for (unsigned i = 0; i < decoder->sink_count; ++i) { struct sc_frame_sink *sink = decoder->sinks[i]; - if (!sink->ops->open(sink)) { + if (!sink->ops->open(sink, ctx)) { sc_decoder_close_first_sinks(decoder, i); return false; } @@ -47,6 +47,11 @@ sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec) { decoder->codec_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY; + if (codec->type == AVMEDIA_TYPE_VIDEO) { + // Hardcoded video properties + decoder->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P; + } + if (avcodec_open2(decoder->codec_ctx, codec, NULL) < 0) { LOGE("Decoder '%s': could not open codec", decoder->name); avcodec_free_context(&decoder->codec_ctx); @@ -61,7 +66,7 @@ sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec) { return false; } - if (!sc_decoder_open_sinks(decoder)) { + if (!sc_decoder_open_sinks(decoder, decoder->codec_ctx)) { av_frame_free(&decoder->frame); avcodec_close(decoder->codec_ctx); avcodec_free_context(&decoder->codec_ctx); diff --git a/app/src/screen.c b/app/src/screen.c index 425ba2c366..a9a48eae82 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -330,7 +330,11 @@ event_watcher(void *data, SDL_Event *event) { #endif static bool -sc_screen_frame_sink_open(struct sc_frame_sink *sink) { +sc_screen_frame_sink_open(struct sc_frame_sink *sink, + const AVCodecContext *ctx) { + assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P); + (void) ctx; + struct sc_screen *screen = DOWNCAST(sink); (void) screen; #ifndef NDEBUG diff --git a/app/src/trait/frame_sink.h b/app/src/trait/frame_sink.h index 0214ab3e76..30bf0d3769 100644 --- a/app/src/trait/frame_sink.h +++ b/app/src/trait/frame_sink.h @@ -5,6 +5,7 @@ #include #include +#include typedef struct AVFrame AVFrame; @@ -18,7 +19,7 @@ struct sc_frame_sink { }; struct sc_frame_sink_ops { - bool (*open)(struct sc_frame_sink *sink); + bool (*open)(struct sc_frame_sink *sink, const AVCodecContext *ctx); void (*close)(struct sc_frame_sink *sink); bool (*push)(struct sc_frame_sink *sink, const AVFrame *frame); }; diff --git a/app/src/v4l2_sink.c b/app/src/v4l2_sink.c index 9a0011f2a3..ba876b2b88 100644 --- a/app/src/v4l2_sink.c +++ b/app/src/v4l2_sink.c @@ -156,7 +156,10 @@ sc_video_buffer_on_new_frame(struct sc_video_buffer *vb, bool previous_skipped, } static bool -sc_v4l2_sink_open(struct sc_v4l2_sink *vs) { +sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) { + assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P); + (void) ctx; + static const struct sc_video_buffer_callbacks cbs = { .on_new_frame = sc_video_buffer_on_new_frame, }; @@ -336,9 +339,9 @@ sc_v4l2_sink_push(struct sc_v4l2_sink *vs, const AVFrame *frame) { } static bool -sc_v4l2_frame_sink_open(struct sc_frame_sink *sink) { +sc_v4l2_frame_sink_open(struct sc_frame_sink *sink, const AVCodecContext *ctx) { struct sc_v4l2_sink *vs = DOWNCAST(sink); - return sc_v4l2_sink_open(vs); + return sc_v4l2_sink_open(vs, ctx); } static void From 20d41fdd7e236b33fc70a12758dacdf47099c83e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 25 Feb 2023 14:32:02 +0100 Subject: [PATCH 076/118] Introduce bytebuf util Add a ring-buffer for bytes. It will be useful for audio buffering. PR #3757 --- app/meson.build | 5 +++ app/src/util/bytebuf.c | 75 +++++++++++++++++++++++++++++++++ app/src/util/bytebuf.h | 90 ++++++++++++++++++++++++++++++++++++++++ app/tests/test_bytebuf.c | 82 ++++++++++++++++++++++++++++++++++++ 4 files changed, 252 insertions(+) create mode 100644 app/src/util/bytebuf.c create mode 100644 app/src/util/bytebuf.h create mode 100644 app/tests/test_bytebuf.c diff --git a/app/meson.build b/app/meson.build index 2ea3b317a1..8be917e9e0 100644 --- a/app/meson.build +++ b/app/meson.build @@ -30,6 +30,7 @@ src = [ 'src/version.c', 'src/video_buffer.c', 'src/util/acksync.c', + 'src/util/bytebuf.c', 'src/util/file.c', 'src/util/intmap.c', 'src/util/intr.c', @@ -254,6 +255,10 @@ if get_option('buildtype') == 'debug' ['test_binary', [ 'tests/test_binary.c', ]], + ['test_bytebuf', [ + 'tests/test_bytebuf.c', + 'src/util/bytebuf.c', + ]], ['test_cbuf', [ 'tests/test_cbuf.c', ]], diff --git a/app/src/util/bytebuf.c b/app/src/util/bytebuf.c new file mode 100644 index 0000000000..6181437691 --- /dev/null +++ b/app/src/util/bytebuf.c @@ -0,0 +1,75 @@ +#include "bytebuf.h" + +#include +#include +#include + +#include "util/log.h" + +bool +sc_bytebuf_init(struct sc_bytebuf *buf, size_t alloc_size) { + assert(alloc_size); + buf->data = malloc(alloc_size); + if (!buf->data) { + LOG_OOM(); + return false; + } + + buf->alloc_size = alloc_size; + buf->head = 0; + buf->tail = 0; + + return true; +} + +void +sc_bytebuf_destroy(struct sc_bytebuf *buf) { + free(buf->data); +} + +void +sc_bytebuf_read(struct sc_bytebuf *buf, uint8_t *to, size_t len) { + assert(len); + assert(len <= sc_bytebuf_read_available(buf)); + assert(buf->tail != buf->head); // the buffer could not be empty + + size_t right_limit = buf->tail < buf->head ? buf->head : buf->alloc_size; + size_t right_len = right_limit - buf->tail; + if (len < right_len) { + right_len = len; + } + memcpy(to, buf->data + buf->tail, right_len); + + if (len > right_len) { + memcpy(to + right_len, buf->data, len - right_len); + } + + buf->tail = (buf->tail + len) % buf->alloc_size; +} + +void +sc_bytebuf_skip(struct sc_bytebuf *buf, size_t len) { + assert(len); + assert(len <= sc_bytebuf_read_available(buf)); + assert(buf->tail != buf->head); // the buffer could not be empty + + buf->tail = (buf->tail + len) % buf->alloc_size; +} + +void +sc_bytebuf_write(struct sc_bytebuf *buf, const uint8_t *from, size_t len) { + assert(len); + assert(len <= sc_bytebuf_write_available(buf)); + + size_t right_len = buf->alloc_size - buf->head; + if (len < right_len) { + right_len = len; + } + memcpy(buf->data + buf->head, from, right_len); + + if (len > right_len) { + memcpy(buf->data, from + right_len, len - right_len); + } + + buf->head = (buf->head + len) % buf->alloc_size; +} diff --git a/app/src/util/bytebuf.h b/app/src/util/bytebuf.h new file mode 100644 index 0000000000..fcebc2d38c --- /dev/null +++ b/app/src/util/bytebuf.h @@ -0,0 +1,90 @@ +#ifndef SC_BYTEBUF_H +#define SC_BYTEBUF_H + +#include "common.h" + +#include +#include + +struct sc_bytebuf { + uint8_t *data; + // The actual capacity is (allocated - 1) so that head == tail is + // non-ambiguous + size_t alloc_size; + size_t head; // writter cursor + size_t tail; // reader cursor + // empty: tail == head + // full: ((tail + 1) % alloc_size) == head +}; + +bool +sc_bytebuf_init(struct sc_bytebuf *buf, size_t alloc_size); + +/** + * Copy from the bytebuf to a user-provided array + * + * The caller must check that len <= sc_bytebuf_read_available() (it is an + * error to attempt to read more bytes than available). + * + * This function is guaranteed not to write to buf->head. + */ +void +sc_bytebuf_read(struct sc_bytebuf *buf, uint8_t *to, size_t len); + +/** + * Drop len bytes from the buffer + * + * The caller must check that len <= sc_bytebuf_read_available() (it is an + * error to attempt to skip more bytes than available). + * + * This function is guaranteed not to write to buf->head. + * + * It is equivalent to call sc_bytebuf_read() to some array and discard the + * array (but this function is more efficient since there is no copy). + */ +void +sc_bytebuf_skip(struct sc_bytebuf *buf, size_t len); + +/** + * Copy the user-provided array to the bytebuf + * + * The caller must check that len <= sc_bytebuf_write_available() (it is an + * error to write more bytes than the remaining available space). + * + * This function is guaranteed not to write to buf->tail. + */ +void +sc_bytebuf_write(struct sc_bytebuf *buf, const uint8_t *from, size_t len); + +/** + * Return the number of bytes which can be read + * + * It is an error to read more bytes than available. + */ +static inline size_t +sc_bytebuf_read_available(struct sc_bytebuf *buf) { + return (buf->alloc_size + buf->head - buf->tail) % buf->alloc_size; +} + +/** + * Return the number of bytes which can be written + * + * It is an error to write more bytes than available. + */ +static inline size_t +sc_bytebuf_write_available(struct sc_bytebuf *buf) { + return (buf->alloc_size + buf->tail - buf->head - 1) % buf->alloc_size; +} + +/** + * Return the actual capacity of the buffer (read available + write available) + */ +static inline size_t +sc_bytebuf_capacity(struct sc_bytebuf *buf) { + return buf->alloc_size - 1; +} + +void +sc_bytebuf_destroy(struct sc_bytebuf *buf); + +#endif diff --git a/app/tests/test_bytebuf.c b/app/tests/test_bytebuf.c new file mode 100644 index 0000000000..fbb33765eb --- /dev/null +++ b/app/tests/test_bytebuf.c @@ -0,0 +1,82 @@ +#include "common.h" + +#include +#include + +#include "util/bytebuf.h" + +void test_bytebuf_simple(void) { + struct sc_bytebuf buf; + uint8_t data[20]; + + bool ok = sc_bytebuf_init(&buf, 20); + assert(ok); + + sc_bytebuf_write(&buf, (uint8_t *) "hello", sizeof("hello") - 1); + assert(sc_bytebuf_read_available(&buf) == 5); + + sc_bytebuf_read(&buf, data, 4); + assert(!strncmp((char *) data, "hell", 4)); + + sc_bytebuf_write(&buf, (uint8_t *) " world", sizeof(" world") - 1); + assert(sc_bytebuf_read_available(&buf) == 7); + + sc_bytebuf_write(&buf, (uint8_t *) "!", 1); + assert(sc_bytebuf_read_available(&buf) == 8); + + sc_bytebuf_read(&buf, &data[4], 8); + assert(sc_bytebuf_read_available(&buf) == 0); + + data[12] = '\0'; + assert(!strcmp((char *) data, "hello world!")); + assert(sc_bytebuf_read_available(&buf) == 0); + + sc_bytebuf_destroy(&buf); +} + +void test_bytebuf_boundaries(void) { + struct sc_bytebuf buf; + uint8_t data[20]; + + bool ok = sc_bytebuf_init(&buf, 20); + assert(ok); + + sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); + assert(sc_bytebuf_read_available(&buf) == 6); + + sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); + assert(sc_bytebuf_read_available(&buf) == 12); + + sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); + assert(sc_bytebuf_read_available(&buf) == 18); + + sc_bytebuf_read(&buf, data, 9); + assert(!strncmp((char *) data, "hello hel", 9)); + assert(sc_bytebuf_read_available(&buf) == 9); + + sc_bytebuf_write(&buf, (uint8_t *) "world", sizeof("world") - 1); + assert(sc_bytebuf_read_available(&buf) == 14); + + sc_bytebuf_write(&buf, (uint8_t *) "!", 1); + assert(sc_bytebuf_read_available(&buf) == 15); + + sc_bytebuf_skip(&buf, 3); + assert(sc_bytebuf_read_available(&buf) == 12); + + sc_bytebuf_read(&buf, data, 12); + data[12] = '\0'; + assert(!strcmp((char *) data, "hello world!")); + assert(sc_bytebuf_read_available(&buf) == 0); + + sc_bytebuf_destroy(&buf); +} + +int main(int argc, char *argv[]) { + (void) argc; + (void) argv; + + test_bytebuf_simple(); + test_bytebuf_boundaries(); + + return 0; +} From b60a8aa657f62a596e528f244f1c101dc2f054b7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 25 Feb 2023 18:45:05 +0100 Subject: [PATCH 077/118] Add two-step write feature to bytebuf If there is exactly one producer, then it can assume that the remaining space in the buffer will only increase until it writes something. This assumption may allow the producer to write to the buffer (up to a known safe size) without any synchronization mechanism, thus allowing to read and write different parts of the buffer in parallel. The producer can then commit the write with a lock held, and update its knowledge of the safe empty remaining space. PR #3757 --- app/src/util/bytebuf.c | 39 ++++++++++++++++++++++++++++++----- app/src/util/bytebuf.h | 24 ++++++++++++++++++++++ app/tests/test_bytebuf.c | 44 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/app/src/util/bytebuf.c b/app/src/util/bytebuf.c index 6181437691..eac69e9cc6 100644 --- a/app/src/util/bytebuf.c +++ b/app/src/util/bytebuf.c @@ -56,11 +56,9 @@ sc_bytebuf_skip(struct sc_bytebuf *buf, size_t len) { buf->tail = (buf->tail + len) % buf->alloc_size; } -void -sc_bytebuf_write(struct sc_bytebuf *buf, const uint8_t *from, size_t len) { - assert(len); - assert(len <= sc_bytebuf_write_available(buf)); - +static inline void +sc_bytebuf_write_step0(struct sc_bytebuf *buf, const uint8_t *from, + size_t len) { size_t right_len = buf->alloc_size - buf->head; if (len < right_len) { right_len = len; @@ -70,6 +68,37 @@ sc_bytebuf_write(struct sc_bytebuf *buf, const uint8_t *from, size_t len) { if (len > right_len) { memcpy(buf->data, from + right_len, len - right_len); } +} +static inline void +sc_bytebuf_write_step1(struct sc_bytebuf *buf, size_t len) { buf->head = (buf->head + len) % buf->alloc_size; } + +void +sc_bytebuf_write(struct sc_bytebuf *buf, const uint8_t *from, size_t len) { + assert(len); + assert(len <= sc_bytebuf_write_available(buf)); + + sc_bytebuf_write_step0(buf, from, len); + sc_bytebuf_write_step1(buf, len); +} + +void +sc_bytebuf_prepare_write(struct sc_bytebuf *buf, const uint8_t *from, + size_t len) { + // *This function MUST NOT access buf->tail (even in assert()).* + // The purpose of this function is to allow a reader and a writer to access + // different parts of the buffer in parallel simultaneously. It is intended + // to be called without lock (only sc_bytebuf_commit_write() is intended to + // be called with lock held). + + assert(len < buf->alloc_size - 1); + sc_bytebuf_write_step0(buf, from, len); +} + +void +sc_bytebuf_commit_write(struct sc_bytebuf *buf, size_t len) { + assert(len <= sc_bytebuf_write_available(buf)); + sc_bytebuf_write_step1(buf, len); +} diff --git a/app/src/util/bytebuf.h b/app/src/util/bytebuf.h index fcebc2d38c..e8279ef888 100644 --- a/app/src/util/bytebuf.h +++ b/app/src/util/bytebuf.h @@ -56,6 +56,30 @@ sc_bytebuf_skip(struct sc_bytebuf *buf, size_t len); void sc_bytebuf_write(struct sc_bytebuf *buf, const uint8_t *from, size_t len); +/** + * Copy the user-provided array to the bytebuf, but do not advance the cursor + * + * The caller must check that len <= sc_bytebuf_write_available() (it is an + * error to write more bytes than the remaining available space). + * + * After this function is called, the write must be committed with + * sc_bytebuf_commit_write(). + * + * The purpose of this mechanism is to acquire a lock only to commit the write, + * but not to perform the actual copy. + * + * This function is guaranteed not to access buf->tail. + */ +void +sc_bytebuf_prepare_write(struct sc_bytebuf *buf, const uint8_t *from, + size_t len); + +/** + * Commit a prepared write + */ +void +sc_bytebuf_commit_write(struct sc_bytebuf *buf, size_t len); + /** * Return the number of bytes which can be read * diff --git a/app/tests/test_bytebuf.c b/app/tests/test_bytebuf.c index fbb33765eb..75af3073d5 100644 --- a/app/tests/test_bytebuf.c +++ b/app/tests/test_bytebuf.c @@ -71,12 +71,56 @@ void test_bytebuf_boundaries(void) { sc_bytebuf_destroy(&buf); } +void test_bytebuf_two_steps_write(void) { + struct sc_bytebuf buf; + uint8_t data[20]; + + bool ok = sc_bytebuf_init(&buf, 20); + assert(ok); + + sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); + assert(sc_bytebuf_read_available(&buf) == 6); + + sc_bytebuf_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); + assert(sc_bytebuf_read_available(&buf) == 12); + + sc_bytebuf_prepare_write(&buf, (uint8_t *) "hello ", sizeof("hello ") - 1); + assert(sc_bytebuf_read_available(&buf) == 12); // write not committed yet + + sc_bytebuf_read(&buf, data, 9); + assert(!strncmp((char *) data, "hello hel", 3)); + assert(sc_bytebuf_read_available(&buf) == 3); + + sc_bytebuf_commit_write(&buf, sizeof("hello ") - 1); + assert(sc_bytebuf_read_available(&buf) == 9); + + sc_bytebuf_prepare_write(&buf, (uint8_t *) "world", sizeof("world") - 1); + assert(sc_bytebuf_read_available(&buf) == 9); // write not committed yet + + sc_bytebuf_commit_write(&buf, sizeof("world") - 1); + assert(sc_bytebuf_read_available(&buf) == 14); + + sc_bytebuf_write(&buf, (uint8_t *) "!", 1); + assert(sc_bytebuf_read_available(&buf) == 15); + + sc_bytebuf_skip(&buf, 3); + assert(sc_bytebuf_read_available(&buf) == 12); + + sc_bytebuf_read(&buf, data, 12); + data[12] = '\0'; + assert(!strcmp((char *) data, "hello world!")); + assert(sc_bytebuf_read_available(&buf) == 0); + + sc_bytebuf_destroy(&buf); +} + int main(int argc, char *argv[]) { (void) argc; (void) argv; test_bytebuf_simple(); test_bytebuf_boundaries(); + test_bytebuf_two_steps_write(); return 0; } From de40cac6ad929f1d150b332f0e987c0d70ba5d38 Mon Sep 17 00:00:00 2001 From: Simon Chan <1330321+yume-chan@users.noreply.github.com> Date: Mon, 30 Jan 2023 00:42:09 +0800 Subject: [PATCH 078/118] Add workaround to capture audio on Android 11 On Android 11, it is possible to start the capture only when the running app is in foreground. But scrcpy is not an app, it's a Java application started from shell. As a workaround, start an existing Android shell existing activity just to start the capture, then close it immediately. PR #3757 Co-authored-by: Romain Vimont Signed-off-by: Romain Vimont --- .../AudioCaptureForegroundException.java | 7 +++ .../com/genymobile/scrcpy/AudioEncoder.java | 51 ++++++++++++++-- .../scrcpy/wrappers/ActivityManager.java | 60 +++++++++++++++++++ 3 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/AudioCaptureForegroundException.java diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCaptureForegroundException.java b/server/src/main/java/com/genymobile/scrcpy/AudioCaptureForegroundException.java new file mode 100644 index 0000000000..baa7d84649 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCaptureForegroundException.java @@ -0,0 +1,7 @@ +package com.genymobile.scrcpy; + +/** + * Exception thrown if audio capture failed on Android 11 specifically because the running App (shell) was not in foreground. + */ +public class AudioCaptureForegroundException extends Exception { +} diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 1ce4107f56..cc786bdbfb 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -1,7 +1,11 @@ package com.genymobile.scrcpy; +import com.genymobile.scrcpy.wrappers.ServiceManager; + import android.annotation.SuppressLint; import android.annotation.TargetApi; +import android.content.ComponentName; +import android.content.Intent; import android.media.AudioFormat; import android.media.AudioRecord; import android.media.AudioTimestamp; @@ -12,6 +16,7 @@ import android.os.Handler; import android.os.HandlerThread; import android.os.Looper; +import android.os.SystemClock; import java.io.IOException; import java.nio.ByteBuffer; @@ -179,7 +184,7 @@ public void start() { thread = new Thread(() -> { try { encode(); - } catch (ConfigurationException e) { + } catch (ConfigurationException | AudioCaptureForegroundException e) { // Do not print stack trace, a user-friendly error-message has already been logged } catch (IOException e) { Ln.e("Audio encoding error", e); @@ -218,8 +223,34 @@ private synchronized void waitEnded() { } } + private static void startWorkaroundAndroid11() { + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + // Android 11 requires Apps to be at foreground to record audio. + // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground. + // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android + // shell ("com.android.shell"). + // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the + // foreground. + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + Intent intent = new Intent(Intent.ACTION_MAIN); + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + intent.addCategory(Intent.CATEGORY_LAUNCHER); + intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity")); + ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent); + // Wait for activity to start + SystemClock.sleep(150); + } + } + } + + private static void stopWorkaroundAndroid11() { + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME); + } + } + @TargetApi(Build.VERSION_CODES.M) - public void encode() throws IOException, ConfigurationException { + public void encode() throws IOException, ConfigurationException, AudioCaptureForegroundException { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { Ln.w("Audio disabled: it is not supported before Android 11"); streamer.writeDisableStream(false); @@ -242,8 +273,20 @@ public void encode() throws IOException, ConfigurationException { mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper())); mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - recorder = createAudioRecord(); - recorder.startRecording(); + startWorkaroundAndroid11(); + try { + recorder = createAudioRecord(); + recorder.startRecording(); + } catch (UnsupportedOperationException e) { + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + Ln.e("Failed to start audio capture"); + Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy."); + throw new AudioCaptureForegroundException(); + } + throw e; + } finally { + stopWorkaroundAndroid11(); + } recorderStarted = true; final MediaCodec mediaCodecRef = mediaCodec; diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java index 76aab5f1bb..aaf83d6669 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/ActivityManager.java @@ -3,7 +3,12 @@ import com.genymobile.scrcpy.FakeContext; import com.genymobile.scrcpy.Ln; +import android.annotation.SuppressLint; +import android.annotation.TargetApi; +import android.content.Intent; import android.os.Binder; +import android.os.Build; +import android.os.Bundle; import android.os.IBinder; import android.os.IInterface; @@ -11,12 +16,15 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +@SuppressLint("PrivateApi,DiscouragedPrivateApi") public class ActivityManager { private final IInterface manager; private Method getContentProviderExternalMethod; private boolean getContentProviderExternalMethodNewVersion = true; private Method removeContentProviderExternalMethod; + private Method startActivityAsUserWithFeatureMethod; + private Method forceStopPackageMethod; public ActivityManager(IInterface manager) { this.manager = manager; @@ -43,6 +51,7 @@ private Method getRemoveContentProviderExternalMethod() throws NoSuchMethodExcep return removeContentProviderExternalMethod; } + @TargetApi(Build.VERSION_CODES.Q) private ContentProvider getContentProviderExternal(String name, IBinder token) { try { Method method = getGetContentProviderExternalMethod(); @@ -85,4 +94,55 @@ void removeContentProviderExternal(String name, IBinder token) { public ContentProvider createSettingsProvider() { return getContentProviderExternal("settings", new Binder()); } + + private Method getStartActivityAsUserWithFeatureMethod() throws NoSuchMethodException, ClassNotFoundException { + if (startActivityAsUserWithFeatureMethod == null) { + Class iApplicationThreadClass = Class.forName("android.app.IApplicationThread"); + Class profilerInfo = Class.forName("android.app.ProfilerInfo"); + startActivityAsUserWithFeatureMethod = manager.getClass() + .getMethod("startActivityAsUserWithFeature", iApplicationThreadClass, String.class, String.class, Intent.class, String.class, + IBinder.class, String.class, int.class, int.class, profilerInfo, Bundle.class, int.class); + } + return startActivityAsUserWithFeatureMethod; + } + + @SuppressWarnings("ConstantConditions") + public int startActivityAsUserWithFeature(Intent intent) { + try { + Method method = getStartActivityAsUserWithFeatureMethod(); + return (int) method.invoke( + /* this */ manager, + /* caller */ null, + /* callingPackage */ FakeContext.PACKAGE_NAME, + /* callingFeatureId */ null, + /* intent */ intent, + /* resolvedType */ null, + /* resultTo */ null, + /* resultWho */ null, + /* requestCode */ 0, + /* startFlags */ 0, + /* profilerInfo */ null, + /* bOptions */ null, + /* userId */ /* UserHandle.USER_CURRENT */ -2); + } catch (Throwable e) { + Ln.e("Could not invoke method", e); + return 0; + } + } + + private Method getForceStopPackageMethod() throws NoSuchMethodException { + if (forceStopPackageMethod == null) { + forceStopPackageMethod = manager.getClass().getMethod("forceStopPackage", String.class, int.class); + } + return forceStopPackageMethod; + } + + public void forceStopPackage(String packageName) { + try { + Method method = getForceStopPackageMethod(); + method.invoke(manager, packageName, /* userId */ /* UserHandle.USER_CURRENT */ -2); + } catch (Throwable e) { + Ln.e("Could not invoke method", e); + } + } } From c1528cdca92d72d98f8f0d1b5cd088a1dd79467c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 28 Feb 2023 21:19:43 +0100 Subject: [PATCH 079/118] Add --require-audio By default, scrcpy mirrors only the video when audio capture fails on the device. Add an option to force scrcpy to fail if audio is enabled but does not work. PR #3757 --- app/data/bash-completion/scrcpy | 1 + app/data/zsh-completion/_scrcpy | 1 + app/scrcpy.1 | 4 ++++ app/src/cli.c | 11 +++++++++++ app/src/demuxer.c | 9 ++++----- app/src/demuxer.h | 9 ++++++++- app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 30 +++++++++++++++--------------- 9 files changed, 46 insertions(+), 21 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index fa95ce6eed..74c3ee57de 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -45,6 +45,7 @@ _scrcpy() { -r --record= --record-format= --render-driver= + --require-audio --rotation= -s --serial= --shortcut-mod= diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index 231405ceb9..b28201a4d3 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -51,6 +51,7 @@ arguments=( {-r,--record=}'[Record screen to file]:record file:_files' '--record-format=[Force recording format]:format:(mp4 mkv)' '--render-driver=[Request SDL to use the given render driver]:driver name:(direct3d opengl opengles2 opengles metal software)' + '--require-audio=[Make scrcpy fail if audio is enabled but does not work]' '--rotation=[Set the initial display rotation]:rotation values:(0 1 2 3)' {-s,--serial=}'[The device serial number \(mandatory for multiple devices only\)]:serial:($("${ADB-adb}" devices | awk '\''$2 == "device" {print $1}'\''))' '--shortcut-mod=[\[key1,key2+key3,...\] Specify the modifiers to use for scrcpy shortcuts]:shortcut mod:(lctrl rctrl lalt ralt lsuper rsuper)' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 40b8158ca5..9125841496 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -272,6 +272,10 @@ Supported names are currently "direct3d", "opengl", "opengles2", "opengles", "me .UR https://wiki.libsdl.org/SDL_HINT_RENDER_DRIVER .UE +.TP +.B \-\-require\-audio +By default, scrcpy mirrors only the video if audio capture fails on the device. This option makes scrcpy fail if audio is enabled but does not work. + .TP .BI "\-\-rotation " value Set the initial display rotation. Possibles values are 0, 1, 2 and 3. Each increment adds a 90 degrees rotation counterclockwise. diff --git a/app/src/cli.c b/app/src/cli.c index 8dfcdc792b..18f3b83bb5 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -70,6 +70,7 @@ enum { OPT_AUDIO_ENCODER, OPT_LIST_ENCODERS, OPT_LIST_DISPLAYS, + OPT_REQUIRE_AUDIO, }; struct sc_option { @@ -465,6 +466,13 @@ static const struct sc_option options[] = { .longopt_id = OPT_RENDER_EXPIRED_FRAMES, .longopt = "render-expired-frames", }, + { + .longopt_id = OPT_REQUIRE_AUDIO, + .longopt = "require-audio", + .text = "By default, scrcpy mirrors only the video when audio capture " + "fails on the device. This option makes scrcpy fail if audio " + "is enabled but does not work." + }, { .longopt_id = OPT_ROTATION, .longopt = "rotation", @@ -1811,6 +1819,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_LIST_DISPLAYS: opts->list_displays = true; break; + case OPT_REQUIRE_AUDIO: + opts->require_audio = true; + break; default: // getopt prints the error message on stderr return false; diff --git a/app/src/demuxer.c b/app/src/demuxer.c index d80a5dda8e..5977a28a67 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -176,14 +176,13 @@ run_demuxer(void *data) { struct sc_demuxer *demuxer = data; // Flag to report end-of-stream (i.e. device disconnected) - bool eos = false; + enum sc_demuxer_status status = SC_DEMUXER_STATUS_ERROR; uint32_t raw_codec_id; bool ok = sc_demuxer_recv_codec_id(demuxer, &raw_codec_id); if (!ok) { LOGE("Demuxer '%s': stream disabled due to connection error", demuxer->name); - eos = true; goto end; } @@ -191,7 +190,7 @@ run_demuxer(void *data) { LOGW("Demuxer '%s': stream explicitly disabled by the device", demuxer->name); sc_demuxer_disable_sinks(demuxer); - eos = true; + status = SC_DEMUXER_STATUS_DISABLED; goto end; } @@ -241,7 +240,7 @@ run_demuxer(void *data) { bool ok = sc_demuxer_recv_packet(demuxer, packet); if (!ok) { // end of stream - eos = true; + status = SC_DEMUXER_STATUS_EOS; break; } @@ -272,7 +271,7 @@ run_demuxer(void *data) { finally_close_sinks: sc_demuxer_close_sinks(demuxer); end: - demuxer->cbs->on_ended(demuxer, eos, demuxer->cbs_userdata); + demuxer->cbs->on_ended(demuxer, status, demuxer->cbs_userdata); return 0; } diff --git a/app/src/demuxer.h b/app/src/demuxer.h index 73166b41ef..d0e41add2d 100644 --- a/app/src/demuxer.h +++ b/app/src/demuxer.h @@ -27,8 +27,15 @@ struct sc_demuxer { void *cbs_userdata; }; +enum sc_demuxer_status { + SC_DEMUXER_STATUS_EOS, + SC_DEMUXER_STATUS_DISABLED, + SC_DEMUXER_STATUS_ERROR, +}; + struct sc_demuxer_callbacks { - void (*on_ended)(struct sc_demuxer *demuxer, bool eos, void *userdata); + void (*on_ended)(struct sc_demuxer *demuxer, enum sc_demuxer_status, + void *userdata); }; // The name must be statically allocated (e.g. a string literal) diff --git a/app/src/options.c b/app/src/options.c index 8560b37b9f..5dd655ce74 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -72,6 +72,7 @@ const struct scrcpy_options scrcpy_options_default = { .start_fps_counter = false, .power_on = true, .audio = true, + .require_audio = false, .list_encoders = false, .list_displays = false, }; diff --git a/app/src/options.h b/app/src/options.h index a15d51f82a..5fcaf016a7 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -154,6 +154,7 @@ struct scrcpy_options { bool start_fps_counter; bool power_on; bool audio; + bool require_audio; bool list_encoders; bool list_displays; }; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index eb70749a57..4355d71bb9 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -216,12 +216,15 @@ sc_recorder_on_ended(struct sc_recorder *recorder, bool success, } static void -sc_video_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, - void *userdata) { +sc_video_demuxer_on_ended(struct sc_demuxer *demuxer, + enum sc_demuxer_status status, void *userdata) { (void) demuxer; (void) userdata; - if (eos) { + // The device may not decide to disable the video + assert(status != SC_DEMUXER_STATUS_DISABLED); + + if (status == SC_DEMUXER_STATUS_EOS) { PUSH_EVENT(SC_EVENT_DEVICE_DISCONNECTED); } else { PUSH_EVENT(SC_EVENT_DEMUXER_ERROR); @@ -229,20 +232,17 @@ sc_video_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, } static void -sc_audio_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, - void *userdata) { +sc_audio_demuxer_on_ended(struct sc_demuxer *demuxer, + enum sc_demuxer_status status, void *userdata) { (void) demuxer; - (void) userdata; - // Contrary to the video demuxer, keep mirroring if only the audio fails. - // 'eos' is true on end-of-stream, including when audio capture is not - // possible on the device (so that scrcpy continue to mirror video without - // failing). - // However, if an audio configuration failure occurs (for example the user - // explicitly selected an unknown audio encoder), 'eos' is false and scrcpy - // must exit. + const struct scrcpy_options *options = userdata; - if (!eos) { + // Contrary to the video demuxer, keep mirroring if only the audio fails + // (unless --require-audio is set). + if (status == SC_DEMUXER_STATUS_ERROR + || (status == SC_DEMUXER_STATUS_DISABLED + && options->require_audio)) { PUSH_EVENT(SC_EVENT_DEMUXER_ERROR); } } @@ -434,7 +434,7 @@ scrcpy(struct scrcpy_options *options) { .on_ended = sc_audio_demuxer_on_ended, }; sc_demuxer_init(&s->audio_demuxer, "audio", s->server.audio_socket, - &audio_demuxer_cbs, NULL); + &audio_demuxer_cbs, options); } bool needs_video_decoder = options->display; From 6e05d7047a9914bdba0bd833e909bc781d5b1a81 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 28 Feb 2023 21:20:28 +0100 Subject: [PATCH 080/118] Call avcodec_receive_frame() in a loop Since in scrcpy a video packet passed to avcodec_send_packet() is always a complete video frame, it is sufficient to call avcodec_receive_frame() exactly once. In practice, it also works for audio packets: the decoder produces exactly 1 frame for 1 input packet. In theory, it is an implementation detail though, so avcodec_receive_frame() should be called in a loop. --- app/src/decoder.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index 96d4a01043..e4d596281e 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -110,8 +110,19 @@ sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) { decoder->name, ret); return false; } - ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame); - if (!ret) { + + for (;;) { + ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } + + if (ret) { + LOGE("Decoder '%s', could not receive video frame: %d", + decoder->name, ret); + return false; + } + // a frame was received bool ok = push_frame_to_sinks(decoder, decoder->frame); // A frame lost should not make the whole pipeline fail. The error, if @@ -119,11 +130,8 @@ sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) { (void) ok; av_frame_unref(decoder->frame); - } else if (ret != AVERROR(EAGAIN)) { - LOGE("Decoder '%s', could not receive video frame: %d", - decoder->name, ret); - return false; } + return true; } From 6dceb328173ed9dc9c7fcdfc250621b4c57ae415 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 28 Feb 2023 21:43:19 +0100 Subject: [PATCH 081/118] Add compat for reallocarray() This function fails safely in the case where the multiplication would overflow. --- app/meson.build | 1 + app/src/compat.c | 13 +++++++++++++ app/src/compat.h | 4 ++++ 3 files changed, 18 insertions(+) diff --git a/app/meson.build b/app/meson.build index 8be917e9e0..7bdd288dd1 100644 --- a/app/meson.build +++ b/app/meson.build @@ -169,6 +169,7 @@ check_functions = [ 'vasprintf', 'nrand48', 'jrand48', + 'reallocarray', ] foreach f : check_functions diff --git a/app/src/compat.c b/app/src/compat.c index bb0152aaf7..785f843ca1 100644 --- a/app/src/compat.c +++ b/app/src/compat.c @@ -3,6 +3,9 @@ #include "config.h" #include +#ifndef HAVE_REALLOCARRAY +# include +#endif #include #include #include @@ -93,5 +96,15 @@ long jrand48(unsigned short xsubi[3]) { return v.i; } #endif +#endif +#ifndef HAVE_REALLOCARRAY +void *reallocarray(void *ptr, size_t nmemb, size_t size) { + size_t bytes; + if (__builtin_mul_overflow(nmemb, size, &bytes)) { + errno = ENOMEM; + return NULL; + } + return realloc(ptr, bytes); +} #endif diff --git a/app/src/compat.h b/app/src/compat.h index 857623e6b0..ea44437d83 100644 --- a/app/src/compat.h +++ b/app/src/compat.h @@ -67,4 +67,8 @@ long nrand48(unsigned short xsubi[3]); long jrand48(unsigned short xsubi[3]); #endif +#ifndef HAVE_REALLOCARRAY +void *reallocarray(void *ptr, size_t nmemb, size_t size); +#endif + #endif From c735b8c127bba489df8d45e397fc47089ae9f00b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 28 Feb 2023 21:49:44 +0100 Subject: [PATCH 082/118] Use reallocarray() in sc_vector This fails safely in case of overflow. --- app/src/util/vector.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/util/vector.h b/app/src/util/vector.h index 0c6cab98af..97d7c38970 100644 --- a/app/src/util/vector.h +++ b/app/src/util/vector.h @@ -118,7 +118,7 @@ static inline void * sc_vector_reallocdata_(void *ptr, size_t count, size_t size, size_t *restrict pcap, size_t *restrict psize) { - void *p = realloc(ptr, count * size); + void *p = reallocarray(ptr, count, size); if (!p) { return NULL; } From 457385d5f468f87c31b44ad8d0fec1cd743c627c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 28 Feb 2023 21:48:18 +0100 Subject: [PATCH 083/118] Add sc_allocarray() util Add a function to allocate an array, which fails safely in the case where the multiplication would overflow. --- app/meson.build | 1 + app/src/util/memory.c | 14 ++++++++++++++ app/src/util/memory.h | 15 +++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 app/src/util/memory.c create mode 100644 app/src/util/memory.h diff --git a/app/meson.build b/app/meson.build index 7bdd288dd1..c24a17dee5 100644 --- a/app/meson.build +++ b/app/meson.build @@ -35,6 +35,7 @@ src = [ 'src/util/intmap.c', 'src/util/intr.c', 'src/util/log.c', + 'src/util/memory.c', 'src/util/net.c', 'src/util/net_intr.c', 'src/util/process.c', diff --git a/app/src/util/memory.c b/app/src/util/memory.c new file mode 100644 index 0000000000..64ee616e5c --- /dev/null +++ b/app/src/util/memory.c @@ -0,0 +1,14 @@ +#include "memory.h" + +#include +#include + +void * +sc_allocarray(size_t nmemb, size_t size) { + size_t bytes; + if (__builtin_mul_overflow(nmemb, size, &bytes)) { + errno = ENOMEM; + return NULL; + } + return malloc(bytes); +} diff --git a/app/src/util/memory.h b/app/src/util/memory.h new file mode 100644 index 0000000000..0fb6bc64d5 --- /dev/null +++ b/app/src/util/memory.h @@ -0,0 +1,15 @@ +#ifndef SC_MEMORY_H +#define SC_MEMORY_H + +#include + +/** + * Allocate an array of `nmemb` items of `size` bytes each + * + * Like calloc(), but without initialization. + * Like reallocarray(), but without reallocation. + */ +void * +sc_allocarray(size_t nmemb, size_t size); + +#endif From 33df484912f8ac4af759a1cf865a2cebbdecbf34 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 28 Feb 2023 22:56:37 +0100 Subject: [PATCH 084/118] Introduce VecDeque Introduce a double-ended queue implemented with a growable ring buffer. Inspired from the Rust VecDeque type: --- app/meson.build | 4 + app/src/util/vecdeque.h | 379 ++++++++++++++++++++++++++++++++++++++ app/tests/test_vecdeque.c | 197 ++++++++++++++++++++ 3 files changed, 580 insertions(+) create mode 100644 app/src/util/vecdeque.h create mode 100644 app/tests/test_vecdeque.c diff --git a/app/meson.build b/app/meson.build index c24a17dee5..a238eb8fd5 100644 --- a/app/meson.build +++ b/app/meson.build @@ -300,6 +300,10 @@ if get_option('buildtype') == 'debug' 'src/util/str.c', 'src/util/strbuf.c', ]], + ['test_vecdeque', [ + 'tests/test_vecdeque.c', + 'src/util/memory.c', + ]], ['test_vector', [ 'tests/test_vector.c', ]], diff --git a/app/src/util/vecdeque.h b/app/src/util/vecdeque.h new file mode 100644 index 0000000000..e5372e027b --- /dev/null +++ b/app/src/util/vecdeque.h @@ -0,0 +1,379 @@ +#ifndef SC_VECDEQUE_H +#define SC_VECDEQUE_H + +#include "common.h" + +#include +#include +#include +#include +#include + +#include "util/memory.h" + +/** + * A double-ended queue implemented with a growable ring buffer. + * + * Inspired from the Rust VecDeque type: + * + */ + +/** + * VecDeque struct body + * + * A VecDeque is a dynamic ring-buffer, managed by the sc_vecdeque_* helpers. + * + * It is generic over the type of its items, so it is implemented via macros. + * + * To use a VecDeque, a new type must be defined: + * + * struct vecdeque_int SC_VECDEQUE(int); + * + * The struct may be anonymous: + * + * struct SC_VECDEQUE(const char *) names; + * + * Functions and macros having name ending with '_' are private. + */ +#define SC_VECDEQUE(type) { \ + size_t cap; \ + size_t origin; \ + size_t size; \ + type *data; \ +} + +/** + * Static initializer for a VecDeque + */ +#define SC_VECDEQUE_INITIALIZER { 0, 0, 0, NULL } + +/** + * Initialize an empty VecDeque + */ +#define sc_vecdeque_init(pv) \ +({ \ + (pv)->cap = 0; \ + (pv)->origin = 0; \ + (pv)->size = 0; \ + (pv)->data = NULL; \ +}) + +/** + * Destroy a VecDeque + */ +#define sc_vecdeque_destroy(pv) \ + free((pv)->data) + +/** + * Clear a VecDeque + * + * Remove all items. + */ +#define sc_vecdeque_clear(pv) \ +(void) ({ \ + sc_vecdeque_destroy(pv); \ + sc_vecdeque_init(pv); \ +}) + +/** + * Returns the content size + */ +#define sc_vecdeque_size(pv) \ + (pv)->size + +/** + * Return whether the VecDeque is empty (i.e. its size is 0) + */ +#define sc_vecdeque_is_empty(pv) \ + ((pv)->size == 0) + +/** + * Return whether the VecDeque is full + * + * A VecDeque is full when its size equals its current capacity. However, it + * does not prevent to push a new item (with sc_vecdeque_push()), since this + * will increase its capacity. + */ +#define sc_vecdeque_is_full(pv) \ + ((pv)->size == (pv)->cap) + +/** + * The minimal allocation size, in number of items + * + * Private. + */ +#define SC_VECDEQUE_MINCAP_ ((size_t) 10) + +/** + * The maximal allocation size, in number of items + * + * Use SIZE_MAX/2 to fit in ssize_t, and so that cap*1.5 does not overflow. + * + * Private. + */ +#define sc_vecdeque_max_cap_(pv) (SIZE_MAX / 2 / sizeof(*(pv)->data)) + +/** + * Realloc the internal array to a specific capacity + * + * On reallocation success, update the VecDeque capacity (`*pcap`) and origin + * (`*porigin`), and return the reallocated data. + * + * On reallocation failure, return NULL without any change. + * + * Private. + * + * \param ptr the current `data` field of the SC_VECDEQUE to realloc + * \param newcap the requested capacity, in number of items + * \param item_size the size of one item (the generic type is unknown from this + * function) + * \param pcap a pointer to the `cap` field of the SC_VECDEQUE [IN/OUT] + * \param porigin a pointer to pv->origin [IN/OUT] + * \param size the `size` field of the SC_VECDEQUE + * \return the new array to assign to the `data` field of the SC_VECDEQUE (if + * not NULL) + */ +static inline void * +sc_vecdeque_reallocdata_(void *ptr, size_t newcap, size_t item_size, + size_t *pcap, size_t *porigin, size_t size) { + + size_t oldcap = *pcap; + size_t oldorigin = *porigin; + + assert(newcap > oldcap); // Could only grow + + if (oldorigin + size <= oldcap) { + // The current content will stay in place, just realloc + // + // As an example, here is the content of a ring-buffer (oldcap=10) + // before the realloc: + // + // _ _ 2 3 4 5 6 7 _ _ + // ^ + // origin + // + // It is resized (newcap=15), e.g. with sc_vecdeque_reserve(): + // + // _ _ 2 3 4 5 6 7 _ _ _ _ _ _ _ + // ^ + // origin + + void *newptr = reallocarray(ptr, newcap, item_size); + if (!newptr) { + return NULL; + } + + *pcap = newcap; + return newptr; + } + + // Copy the current content to the new array + // + // As an example, here is the content of a ring-buffer (oldcap=10) before + // the realloc: + // + // 5 6 7 _ _ 0 1 2 3 4 + // ^ + // origin + // + // It is resized (newcap=15), e.g. with sc_vecdeque_reserve(): + // + // 0 1 2 3 4 5 6 7 _ _ _ _ _ _ _ + // ^ + // origin + + assert(size); + void *newptr = sc_allocarray(newcap, item_size); + if (!newptr) { + return NULL; + } + + size_t right_len = MIN(size, oldcap - oldorigin); + assert(right_len); + memcpy(newptr, ptr + (oldorigin * item_size), right_len * item_size); + + if (size > right_len) { + memcpy(newptr + (right_len * item_size), ptr, + (size - right_len) * item_size); + } + + free(ptr); + + *pcap = newcap; + *porigin = 0; + return newptr; +} + +/** + * Macro to realloc the internal data to a new capacity + * + * Private. + * + * \retval true on success + * \retval false on allocation failure (the VecDeque is left untouched) + */ +#define sc_vecdeque_realloc_(pv, newcap) \ +({ \ + void *p = sc_vecdeque_reallocdata_((pv)->data, newcap, \ + sizeof(*(pv)->data), &(pv)->cap, \ + &(pv)->origin, (pv)->size); \ + if (p) { \ + (pv)->data = p; \ + } \ + (bool) p; \ +}); + +static inline size_t +sc_vecdeque_growsize_(size_t value) +{ + /* integer multiplication by 1.5 */ + return value + (value >> 1); +} + +/** + * Increase the capacity of the VecDeque to at least `mincap` + * + * \param pv a pointer to the VecDeque + * \param mincap (`size_t`) the requested capacity + * \retval true on success + * \retval false on allocation failure (the VecDeque is left untouched) + */ +#define sc_vecdeque_reserve(pv, mincap) \ +({ \ + assert(mincap <= sc_vecdeque_max_cap_(pv)); \ + bool ok; \ + /* avoid to allocate tiny arrays (< SC_VECDEQUE_MINCAP_) */ \ + size_t mincap_ = MAX(mincap, SC_VECDEQUE_MINCAP_); \ + if (mincap_ <= (pv)->cap) { \ + /* nothing to do */ \ + ok = true; \ + } else if (mincap_ <= sc_vecdeque_max_cap_(pv)) { \ + /* not too big */ \ + size_t newsize = sc_vecdeque_growsize_((pv)->cap); \ + newsize = CLAMP(newsize, mincap_, sc_vecdeque_max_cap_(pv)); \ + ok = sc_vecdeque_realloc_(pv, newsize); \ + } else { \ + ok = false; \ + } \ + ok; \ +}) + +/** + * Automatically grow the VecDeque capacity + * + * Private. + * + * \retval true on success + * \retval false on allocation failure (the VecDeque is left untouched) + */ +#define sc_vecdeque_grow_(pv) \ +({ \ + bool ok; \ + if ((pv)->cap < sc_vecdeque_max_cap_(pv)) { \ + size_t newsize = sc_vecdeque_growsize_((pv)->cap); \ + newsize = CLAMP(newsize, SC_VECDEQUE_MINCAP_, \ + sc_vecdeque_max_cap_(pv)); \ + ok = sc_vecdeque_realloc_(pv, newsize); \ + } else { \ + ok = false; \ + } \ + ok; \ +}) + +/** + * Grow the VecDeque capacity if it is full + * + * Private. + * + * \retval true on success + * \retval false on allocation failure (the VecDeque is left untouched) + */ +#define sc_vecdeque_grow_if_needed_(pv) \ + (!sc_vecdeque_is_full(pv) || sc_vecdeque_grow_(pv)) + +/** + * Push an uninitialized item, and return a pointer to it + * + * It does not attempt to resize the VecDeque. It is an error to this function + * if the VecDeque is full. + * + * This function may not fail. It returns a valid non-NULL pointer to the + * uninitialized item just pushed. + */ +#define sc_vecdeque_push_hole_noresize(pv) \ +({ \ + assert(!sc_vecdeque_is_full(pv)); \ + ++(pv)->size; \ + &(pv)->data[((pv)->origin + (pv)->size - 1) % (pv)->cap]; \ +}) + +/** + * Push an uninitialized item, and return a pointer to it + * + * If the VecDeque is full, it is resized. + * + * This function returns either a valid non-NULL pointer to the uninitialized + * item just pushed, or NULL on reallocation failure. + */ +#define sc_vecdeque_push_hole(pv) \ + (sc_vecdeque_grow_if_needed_(pv) ? \ + sc_vecdeque_push_hole_noresize(pv) : NULL) + +/** + * Push an item + * + * It does not attempt to resize the VecDeque. It is an error to this function + * if the VecDeque is full. + * + * This function may not fail. + */ +#define sc_vecdeque_push_noresize(pv, item) \ +(void) ({ \ + assert(!sc_vecdeque_is_full(pv)); \ + ++(pv)->size; \ + (pv)->data[((pv)->origin + (pv)->size - 1) % (pv)->cap] = item; \ +}) + +/** + * Push an item + * + * If the VecDeque is full, it is resized. + * + * \retval true on success + * \retval false on allocation failure (the VecDeque is left untouched) + */ +#define sc_vecdeque_push(pv, item) \ +({ \ + bool ok = sc_vecdeque_grow_if_needed_(pv); \ + if (ok) { \ + sc_vecdeque_push_noresize(pv, item); \ + } \ + ok; \ +}) + +/** + * Pop an item and return a pointer to it (still in the VecDeque) + * + * Returning a pointer allows the caller to destroy it in place without copy + * (especially if the item type is big). + * + * It is an error to call this function if the VecDeque is empty. + */ +#define sc_vecdeque_popref(pv) \ +({ \ + assert(!sc_vecdeque_is_empty(pv)); \ + size_t pos = (pv)->origin; \ + (pv)->origin = ((pv)->origin + 1) % (pv)->cap; \ + --(pv)->size; \ + &(pv)->data[pos]; \ +}) + +/** + * Pop an item and return it + * + * It is an error to call this function if the VecDeque is empty. + */ +#define sc_vecdeque_pop(pv) \ + (*sc_vecdeque_popref(pv)) + +#endif diff --git a/app/tests/test_vecdeque.c b/app/tests/test_vecdeque.c new file mode 100644 index 0000000000..fa3ba96372 --- /dev/null +++ b/app/tests/test_vecdeque.c @@ -0,0 +1,197 @@ +#include "common.h" + +#include + +#include "util/vecdeque.h" + +#define pr(pv) \ +({ \ + fprintf(stderr, "cap=%lu origin=%lu size=%lu\n", (pv)->cap, (pv)->origin, (pv)->size); \ + for (size_t i = 0; i < (pv)->cap; ++i) \ + fprintf(stderr, "%d ", (pv)->data[i]); \ + fprintf(stderr, "\n"); \ +}) + +static void test_vecdeque_push_pop(void) { + struct SC_VECDEQUE(int) vdq = SC_VECDEQUE_INITIALIZER; + + assert(sc_vecdeque_is_empty(&vdq)); + assert(sc_vecdeque_size(&vdq) == 0); + + bool ok = sc_vecdeque_push(&vdq, 5); + assert(ok); + assert(sc_vecdeque_size(&vdq) == 1); + + ok = sc_vecdeque_push(&vdq, 12); + assert(ok); + assert(sc_vecdeque_size(&vdq) == 2); + + int v = sc_vecdeque_pop(&vdq); + assert(v == 5); + assert(sc_vecdeque_size(&vdq) == 1); + + ok = sc_vecdeque_push(&vdq, 7); + assert(ok); + assert(sc_vecdeque_size(&vdq) == 2); + + int *p = sc_vecdeque_popref(&vdq); + assert(p); + assert(*p == 12); + assert(sc_vecdeque_size(&vdq) == 1); + + v = sc_vecdeque_pop(&vdq); + assert(v == 7); + assert(sc_vecdeque_size(&vdq) == 0); + assert(sc_vecdeque_is_empty(&vdq)); + + sc_vecdeque_destroy(&vdq); +} + +static void test_vecdeque_reserve(void) { + struct SC_VECDEQUE(int) vdq = SC_VECDEQUE_INITIALIZER; + + bool ok = sc_vecdeque_reserve(&vdq, 20); + assert(ok); + assert(vdq.cap == 20); + + assert(sc_vecdeque_size(&vdq) == 0); + + for (size_t i = 0; i < 20; ++i) { + ok = sc_vecdeque_push(&vdq, i); + assert(ok); + } + + assert(sc_vecdeque_size(&vdq) == 20); + + // It is now full + + for (int i = 0; i < 5; ++i) { + int v = sc_vecdeque_pop(&vdq); + assert(v == i); + } + assert(sc_vecdeque_size(&vdq) == 15); + + for (int i = 20; i < 25; ++i) { + ok = sc_vecdeque_push(&vdq, i); + assert(ok); + } + + assert(sc_vecdeque_size(&vdq) == 20); + assert(vdq.cap == 20); + + // Now, the content wraps around the ring buffer: + // 20 21 22 23 24 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 + // ^ + // origin + + // It is now full, let's reserve some space + ok = sc_vecdeque_reserve(&vdq, 30); + assert(ok); + assert(vdq.cap == 30); + + assert(sc_vecdeque_size(&vdq) == 20); + + for (int i = 0; i < 20; ++i) { + // We should retrieve the items we inserted in order + int v = sc_vecdeque_pop(&vdq); + assert(v == i + 5); + } + + assert(sc_vecdeque_size(&vdq) == 0); + + sc_vecdeque_destroy(&vdq); +} + +static void test_vecdeque_grow() { + struct SC_VECDEQUE(int) vdq = SC_VECDEQUE_INITIALIZER; + + bool ok = sc_vecdeque_reserve(&vdq, 20); + assert(ok); + assert(vdq.cap == 20); + + assert(sc_vecdeque_size(&vdq) == 0); + + for (int i = 0; i < 500; ++i) { + ok = sc_vecdeque_push(&vdq, i); + assert(ok); + } + + assert(sc_vecdeque_size(&vdq) == 500); + + for (int i = 0; i < 100; ++i) { + int v = sc_vecdeque_pop(&vdq); + assert(v == i); + } + + assert(sc_vecdeque_size(&vdq) == 400); + + for (int i = 500; i < 1000; ++i) { + ok = sc_vecdeque_push(&vdq, i); + assert(ok); + } + + assert(sc_vecdeque_size(&vdq) == 900); + + for (int i = 100; i < 1000; ++i) { + int v = sc_vecdeque_pop(&vdq); + assert(v == i); + } + + assert(sc_vecdeque_size(&vdq) == 0); + + sc_vecdeque_destroy(&vdq); +} + +static void test_vecdeque_push_hole() { + struct SC_VECDEQUE(int) vdq = SC_VECDEQUE_INITIALIZER; + + bool ok = sc_vecdeque_reserve(&vdq, 20); + assert(ok); + assert(vdq.cap == 20); + + assert(sc_vecdeque_size(&vdq) == 0); + + for (int i = 0; i < 20; ++i) { + int *p = sc_vecdeque_push_hole(&vdq); + assert(p); + *p = i * 10; + } + + assert(sc_vecdeque_size(&vdq) == 20); + + for (int i = 0; i < 10; ++i) { + int v = sc_vecdeque_pop(&vdq); + assert(v == i * 10); + } + + assert(sc_vecdeque_size(&vdq) == 10); + + for (int i = 20; i < 30; ++i) { + int *p = sc_vecdeque_push_hole(&vdq); + assert(p); + *p = i * 10; + } + + assert(sc_vecdeque_size(&vdq) == 20); + + for (int i = 10; i < 30; ++i) { + int v = sc_vecdeque_pop(&vdq); + assert(v == i * 10); + } + + assert(sc_vecdeque_size(&vdq) == 0); + + sc_vecdeque_destroy(&vdq); +} + +int main(int argc, char *argv[]) { + (void) argc; + (void) argv; + + test_vecdeque_push_pop(); + test_vecdeque_reserve(); + test_vecdeque_grow(); + test_vecdeque_push_hole(); + + return 0; +} From efc15744da576206f407099927bceea8421b79bb Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 21:42:51 +0100 Subject: [PATCH 085/118] Use VecDeque in recorder The packets queued for recording were wrapped in a dynamically allocated structure with a "next" field. To avoid this additional layer of allocation and indirection, use a VecDeque. --- app/src/recorder.c | 169 ++++++++++++++++++++++----------------------- app/src/recorder.h | 9 +-- 2 files changed, 84 insertions(+), 94 deletions(-) diff --git a/app/src/recorder.c b/app/src/recorder.c index 9fc15dacf8..bd7c50f208 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -33,41 +33,27 @@ find_muxer(const char *name) { return oformat; } -static struct sc_record_packet * -sc_record_packet_new(const AVPacket *packet) { - struct sc_record_packet *rec = malloc(sizeof(*rec)); - if (!rec) { - LOG_OOM(); - return NULL; - } - - rec->packet = av_packet_alloc(); - if (!rec->packet) { +static AVPacket * +sc_recorder_packet_ref(const AVPacket *packet) { + AVPacket *p = av_packet_alloc(); + if (!p) { LOG_OOM(); - free(rec); return NULL; } - if (av_packet_ref(rec->packet, packet)) { - av_packet_free(&rec->packet); - free(rec); + if (av_packet_ref(p, packet)) { + av_packet_free(&p); return NULL; } - return rec; -} -static void -sc_record_packet_delete(struct sc_record_packet *rec) { - av_packet_free(&rec->packet); - free(rec); + return p; } static void sc_recorder_queue_clear(struct sc_recorder_queue *queue) { - while (!sc_queue_is_empty(queue)) { - struct sc_record_packet *rec; - sc_queue_take(queue, next, &rec); - sc_record_packet_delete(rec); + while (!sc_vecdeque_is_empty(queue)) { + AVPacket *p = sc_vecdeque_pop(queue); + av_packet_free(&p); } } @@ -227,12 +213,12 @@ sc_recorder_wait_audio_stream(struct sc_recorder *recorder) { static inline bool sc_recorder_has_empty_queues(struct sc_recorder *recorder) { - if (sc_queue_is_empty(&recorder->video_queue)) { + if (sc_vecdeque_is_empty(&recorder->video_queue)) { // The video queue is empty return true; } - if (recorder->audio && sc_queue_is_empty(&recorder->audio_queue)) { + if (recorder->audio && sc_vecdeque_is_empty(&recorder->audio_queue)) { // The audio queue is empty (when audio is enabled) return true; } @@ -249,28 +235,27 @@ sc_recorder_process_header(struct sc_recorder *recorder) { sc_cond_wait(&recorder->queue_cond, &recorder->mutex); } - if (sc_queue_is_empty(&recorder->video_queue)) { + if (sc_vecdeque_is_empty(&recorder->video_queue)) { assert(recorder->stopped); - // Don't process anything if there are not at least video packets (when - // the recorder is stopped) + // If the recorder is stopped, don't process anything if there are not + // at least video packets sc_mutex_unlock(&recorder->mutex); return false; } - struct sc_record_packet *video_pkt; - sc_queue_take(&recorder->video_queue, next, &video_pkt); + AVPacket *video_pkt = sc_vecdeque_pop(&recorder->video_queue); - struct sc_record_packet *audio_pkt = NULL; - if (!sc_queue_is_empty(&recorder->audio_queue)) { + AVPacket *audio_pkt = NULL; + if (!sc_vecdeque_is_empty(&recorder->audio_queue)) { assert(recorder->audio); - sc_queue_take(&recorder->audio_queue, next, &audio_pkt); + audio_pkt = sc_vecdeque_pop(&recorder->audio_queue); } sc_mutex_unlock(&recorder->mutex); int ret = false; - if (video_pkt->packet->pts != AV_NOPTS_VALUE) { + if (video_pkt->pts != AV_NOPTS_VALUE) { LOGE("The first video packet is not a config packet"); goto end; } @@ -278,13 +263,13 @@ sc_recorder_process_header(struct sc_recorder *recorder) { assert(recorder->video_stream_index >= 0); AVStream *video_stream = recorder->ctx->streams[recorder->video_stream_index]; - bool ok = sc_recorder_set_extradata(video_stream, video_pkt->packet); + bool ok = sc_recorder_set_extradata(video_stream, video_pkt); if (!ok) { goto end; } if (audio_pkt) { - if (audio_pkt->packet->pts != AV_NOPTS_VALUE) { + if (audio_pkt->pts != AV_NOPTS_VALUE) { LOGE("The first audio packet is not a config packet"); goto end; } @@ -292,7 +277,7 @@ sc_recorder_process_header(struct sc_recorder *recorder) { assert(recorder->audio_stream_index >= 0); AVStream *audio_stream = recorder->ctx->streams[recorder->audio_stream_index]; - ok = sc_recorder_set_extradata(audio_stream, audio_pkt->packet); + ok = sc_recorder_set_extradata(audio_stream, audio_pkt); if (!ok) { goto end; } @@ -307,9 +292,9 @@ sc_recorder_process_header(struct sc_recorder *recorder) { ret = true; end: - sc_record_packet_delete(video_pkt); + av_packet_free(&video_pkt); if (audio_pkt) { - sc_record_packet_delete(audio_pkt); + av_packet_free(&audio_pkt); } return ret; @@ -324,12 +309,12 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { return false; } - struct sc_record_packet *video_pkt = NULL; - struct sc_record_packet *audio_pkt = NULL; + AVPacket *video_pkt = NULL; + AVPacket *audio_pkt = NULL; // We can write a video packet only once we received the next one so that // we can set its duration (next_pts - current_pts) - struct sc_record_packet *video_pkt_previous = NULL; + AVPacket *video_pkt_previous = NULL; bool error = false; @@ -337,12 +322,12 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { sc_mutex_lock(&recorder->mutex); while (!recorder->stopped) { - if (!video_pkt && !sc_queue_is_empty(&recorder->video_queue)) { + if (!video_pkt && !sc_vecdeque_is_empty(&recorder->video_queue)) { // A new packet may be assigned to video_pkt and be processed break; } if (recorder->audio && !audio_pkt - && !sc_queue_is_empty(&recorder->audio_queue)) { + && !sc_vecdeque_is_empty(&recorder->audio_queue)) { // A new packet may be assigned to audio_pkt and be processed break; } @@ -354,20 +339,20 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { // If there is no audio, then the audio_queue will remain empty forever // and audio_pkt will always be NULL. - assert(recorder->audio - || (!audio_pkt && sc_queue_is_empty(&recorder->audio_queue))); + assert(recorder->audio || (!audio_pkt + && sc_vecdeque_is_empty(&recorder->audio_queue))); - if (!video_pkt && !sc_queue_is_empty(&recorder->video_queue)) { - sc_queue_take(&recorder->video_queue, next, &video_pkt); + if (!video_pkt && !sc_vecdeque_is_empty(&recorder->video_queue)) { + video_pkt = sc_vecdeque_pop(&recorder->video_queue); } - if (!audio_pkt && !sc_queue_is_empty(&recorder->audio_queue)) { - sc_queue_take(&recorder->audio_queue, next, &audio_pkt); + if (!audio_pkt && !sc_vecdeque_is_empty(&recorder->audio_queue)) { + audio_pkt = sc_vecdeque_pop(&recorder->audio_queue); } if (recorder->stopped && !video_pkt && !audio_pkt) { - assert(sc_queue_is_empty(&recorder->video_queue)); - assert(sc_queue_is_empty(&recorder->audio_queue)); + assert(sc_vecdeque_is_empty(&recorder->video_queue)); + assert(sc_vecdeque_is_empty(&recorder->audio_queue)); sc_mutex_unlock(&recorder->mutex); break; } @@ -379,28 +364,27 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { // Ignore further config packets (e.g. on device orientation // change). The next non-config packet will have the config packet // data prepended. - if (video_pkt && video_pkt->packet->pts == AV_NOPTS_VALUE) { - sc_record_packet_delete(video_pkt); + if (video_pkt && video_pkt->pts == AV_NOPTS_VALUE) { + av_packet_free(&video_pkt); video_pkt = NULL; } - if (audio_pkt && audio_pkt->packet->pts == AV_NOPTS_VALUE) { - sc_record_packet_delete(audio_pkt); - audio_pkt= NULL; + if (audio_pkt && audio_pkt->pts == AV_NOPTS_VALUE) { + av_packet_free(&audio_pkt); + audio_pkt = NULL; } if (pts_origin == AV_NOPTS_VALUE) { if (!recorder->audio) { assert(video_pkt); - pts_origin = video_pkt->packet->pts; + pts_origin = video_pkt->pts; } else if (video_pkt && audio_pkt) { - pts_origin = - MIN(video_pkt->packet->pts, audio_pkt->packet->pts); + pts_origin = MIN(video_pkt->pts, audio_pkt->pts); } else if (recorder->stopped) { if (video_pkt) { // The recorder is stopped without audio, record the video // packets - pts_origin = video_pkt->packet->pts; + pts_origin = video_pkt->pts; } else { // Fail if there is no video error = true; @@ -415,17 +399,16 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { assert(pts_origin != AV_NOPTS_VALUE); if (video_pkt) { - video_pkt->packet->pts -= pts_origin; - video_pkt->packet->dts = video_pkt->packet->pts; + video_pkt->pts -= pts_origin; + video_pkt->dts = video_pkt->pts; if (video_pkt_previous) { // we now know the duration of the previous packet - video_pkt_previous->packet->duration = - video_pkt->packet->pts - video_pkt_previous->packet->pts; + video_pkt_previous->duration = video_pkt->pts + - video_pkt_previous->pts; - bool ok = sc_recorder_write_video(recorder, - video_pkt_previous->packet); - sc_record_packet_delete(video_pkt_previous); + bool ok = sc_recorder_write_video(recorder, video_pkt_previous); + av_packet_free(&video_pkt_previous); if (!ok) { LOGE("Could not record video packet"); error = true; @@ -438,34 +421,34 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { } if (audio_pkt) { - audio_pkt->packet->pts -= pts_origin; - audio_pkt->packet->dts = audio_pkt->packet->pts; + audio_pkt->pts -= pts_origin; + audio_pkt->dts = audio_pkt->pts; - bool ok = sc_recorder_write_audio(recorder, audio_pkt->packet); + bool ok = sc_recorder_write_audio(recorder, audio_pkt); if (!ok) { LOGE("Could not record audio packet"); error = true; goto end; } - sc_record_packet_delete(audio_pkt); + av_packet_free(&audio_pkt); audio_pkt = NULL; } } // Write the last video packet - struct sc_record_packet *last = video_pkt_previous; + AVPacket *last = video_pkt_previous; if (last) { // assign an arbitrary duration to the last packet - last->packet->duration = 100000; - bool ok = sc_recorder_write_video(recorder, last->packet); + last->duration = 100000; + bool ok = sc_recorder_write_video(recorder, last); if (!ok) { // failing to write the last frame is not very serious, no // future frame may depend on it, so the resulting file // will still be valid LOGW("Could not record last packet"); } - sc_record_packet_delete(last); + av_packet_free(&last); } int ret = av_write_trailer(recorder->ctx); @@ -476,10 +459,10 @@ sc_recorder_process_packets(struct sc_recorder *recorder) { end: if (video_pkt) { - sc_record_packet_delete(video_pkt); + av_packet_free(&video_pkt); } if (audio_pkt) { - sc_record_packet_delete(audio_pkt); + av_packet_free(&audio_pkt); } return !error; @@ -585,16 +568,22 @@ sc_recorder_video_packet_sink_push(struct sc_packet_sink *sink, return false; } - struct sc_record_packet *rec = sc_record_packet_new(packet); + AVPacket *rec = sc_recorder_packet_ref(packet); if (!rec) { LOG_OOM(); sc_mutex_unlock(&recorder->mutex); return false; } - rec->packet->stream_index = recorder->video_stream_index; + rec->stream_index = recorder->video_stream_index; + + bool ok = sc_vecdeque_push(&recorder->video_queue, rec); + if (!ok) { + LOG_OOM(); + sc_mutex_unlock(&recorder->mutex); + return false; + } - sc_queue_push(&recorder->video_queue, next, rec); sc_cond_signal(&recorder->queue_cond); sc_mutex_unlock(&recorder->mutex); @@ -648,16 +637,22 @@ sc_recorder_audio_packet_sink_push(struct sc_packet_sink *sink, return false; } - struct sc_record_packet *rec = sc_record_packet_new(packet); + AVPacket *rec = sc_recorder_packet_ref(packet); if (!rec) { LOG_OOM(); sc_mutex_unlock(&recorder->mutex); return false; } - rec->packet->stream_index = recorder->audio_stream_index; + rec->stream_index = recorder->audio_stream_index; + + bool ok = sc_vecdeque_push(&recorder->audio_queue, rec); + if (!ok) { + LOG_OOM(); + sc_mutex_unlock(&recorder->mutex); + return false; + } - sc_queue_push(&recorder->audio_queue, next, rec); sc_cond_signal(&recorder->queue_cond); sc_mutex_unlock(&recorder->mutex); @@ -708,8 +703,8 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename, recorder->audio = audio; - sc_queue_init(&recorder->video_queue); - sc_queue_init(&recorder->audio_queue); + sc_vecdeque_init(&recorder->video_queue); + sc_vecdeque_init(&recorder->audio_queue); recorder->stopped = false; recorder->video_codec = NULL; diff --git a/app/src/recorder.h b/app/src/recorder.h index 6fe72401df..e3d5f018b3 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -9,15 +9,10 @@ #include "coords.h" #include "options.h" #include "trait/packet_sink.h" -#include "util/queue.h" #include "util/thread.h" +#include "util/vecdeque.h" -struct sc_record_packet { - AVPacket *packet; - struct sc_record_packet *next; -}; - -struct sc_recorder_queue SC_QUEUE(struct sc_record_packet); +struct sc_recorder_queue SC_VECDEQUE(AVPacket *); struct sc_recorder { struct sc_packet_sink video_packet_sink; From f25a67f3424406c9cbf33b7b11956ad7dcf2ddb3 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 22:21:43 +0100 Subject: [PATCH 086/118] Use VecDeque in video_buffer The packets queued for buffering were wrapped in a dynamically allocated structure with a "next" field. To avoid this additional layer of allocation and indirection, use a VecDeque. --- app/src/video_buffer.c | 63 ++++++++++++++++++++---------------------- app/src/video_buffer.h | 5 ++-- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index 11f7647907..b3b29098a9 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -10,35 +10,26 @@ #define SC_BUFFERING_NDEBUG // comment to debug -static struct sc_video_buffer_frame * -sc_video_buffer_frame_new(const AVFrame *frame) { - struct sc_video_buffer_frame *vb_frame = malloc(sizeof(*vb_frame)); - if (!vb_frame) { - LOG_OOM(); - return NULL; - } - +static bool +sc_video_buffer_frame_init(struct sc_video_buffer_frame *vb_frame, + const AVFrame *frame) { vb_frame->frame = av_frame_alloc(); if (!vb_frame->frame) { - LOG_OOM(); - free(vb_frame); - return NULL; + return false; } if (av_frame_ref(vb_frame->frame, frame)) { av_frame_free(&vb_frame->frame); - free(vb_frame); - return NULL; + return false; } - return vb_frame; + return true; } static void -sc_video_buffer_frame_delete(struct sc_video_buffer_frame *vb_frame) { +sc_video_buffer_frame_destroy(struct sc_video_buffer_frame *vb_frame) { av_frame_unref(vb_frame->frame); av_frame_free(&vb_frame->frame); - free(vb_frame); } static bool @@ -62,7 +53,7 @@ run_buffering(void *data) { for (;;) { sc_mutex_lock(&vb->b.mutex); - while (!vb->b.stopped && sc_queue_is_empty(&vb->b.queue)) { + while (!vb->b.stopped && sc_vecdeque_is_empty(&vb->b.queue)) { sc_cond_wait(&vb->b.queue_cond, &vb->b.mutex); } @@ -71,12 +62,11 @@ run_buffering(void *data) { goto stopped; } - struct sc_video_buffer_frame *vb_frame; - sc_queue_take(&vb->b.queue, next, &vb_frame); + struct sc_video_buffer_frame vb_frame = sc_vecdeque_pop(&vb->b.queue); sc_tick max_deadline = sc_tick_now() + vb->buffering_time; // PTS (written by the server) are expressed in microseconds - sc_tick pts = SC_TICK_TO_US(vb_frame->frame->pts); + sc_tick pts = SC_TICK_TO_US(vb_frame.frame->pts); bool timed_out = false; while (!vb->b.stopped && !timed_out) { @@ -91,7 +81,7 @@ run_buffering(void *data) { } if (vb->b.stopped) { - sc_video_buffer_frame_delete(vb_frame); + sc_video_buffer_frame_destroy(&vb_frame); sc_mutex_unlock(&vb->b.mutex); goto stopped; } @@ -100,20 +90,19 @@ run_buffering(void *data) { #ifndef SC_BUFFERING_NDEBUG LOGD("Buffering: %" PRItick ";%" PRItick ";%" PRItick, - pts, vb_frame->push_date, sc_tick_now()); + pts, vb_frame.push_date, sc_tick_now()); #endif - sc_video_buffer_offer(vb, vb_frame->frame); + sc_video_buffer_offer(vb, vb_frame.frame); - sc_video_buffer_frame_delete(vb_frame); + sc_video_buffer_frame_destroy(&vb_frame); } stopped: // Flush queue - while (!sc_queue_is_empty(&vb->b.queue)) { - struct sc_video_buffer_frame *vb_frame; - sc_queue_take(&vb->b.queue, next, &vb_frame); - sc_video_buffer_frame_delete(vb_frame); + while (!sc_vecdeque_is_empty(&vb->b.queue)) { + struct sc_video_buffer_frame *p = sc_vecdeque_popref(&vb->b.queue); + sc_video_buffer_frame_destroy(p); } LOGD("Buffering thread ended"); @@ -154,7 +143,7 @@ sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick buffering_time, } sc_clock_init(&vb->b.clock); - sc_queue_init(&vb->b.queue); + sc_vecdeque_init(&vb->b.queue); } assert(cbs); @@ -230,17 +219,25 @@ sc_video_buffer_push(struct sc_video_buffer *vb, const AVFrame *frame) { return sc_video_buffer_offer(vb, frame); } - struct sc_video_buffer_frame *vb_frame = sc_video_buffer_frame_new(frame); - if (!vb_frame) { + struct sc_video_buffer_frame vb_frame; + bool ok = sc_video_buffer_frame_init(&vb_frame, frame); + if (!ok) { sc_mutex_unlock(&vb->b.mutex); LOG_OOM(); return false; } #ifndef SC_BUFFERING_NDEBUG - vb_frame->push_date = sc_tick_now(); + vb_frame.push_date = sc_tick_now(); #endif - sc_queue_push(&vb->b.queue, next, vb_frame); + + ok = sc_vecdeque_push(&vb->b.queue, vb_frame); + if (!ok) { + sc_mutex_unlock(&vb->b.mutex); + LOG_OOM(); + return false; + } + sc_cond_signal(&vb->b.queue_cond); sc_mutex_unlock(&vb->b.mutex); diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h index 4877770328..41b09434be 100644 --- a/app/src/video_buffer.h +++ b/app/src/video_buffer.h @@ -7,22 +7,21 @@ #include "clock.h" #include "frame_buffer.h" -#include "util/queue.h" #include "util/thread.h" #include "util/tick.h" +#include "util/vecdeque.h" // forward declarations typedef struct AVFrame AVFrame; struct sc_video_buffer_frame { AVFrame *frame; - struct sc_video_buffer_frame *next; #ifndef NDEBUG sc_tick push_date; #endif }; -struct sc_video_buffer_frame_queue SC_QUEUE(struct sc_video_buffer_frame); +struct sc_video_buffer_frame_queue SC_VECDEQUE(struct sc_video_buffer_frame); struct sc_video_buffer { struct sc_frame_buffer fb; From 4d989de9ae27a88b08677d2cb44c0cce86380ee4 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 22:39:11 +0100 Subject: [PATCH 087/118] Use VecDeque in controller Replace cbuf by VecDeque in controller. --- app/src/controller.c | 48 ++++++++++++++++++++++++++++++-------------- app/src/controller.h | 4 ++-- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/app/src/controller.c b/app/src/controller.c index 4a1d2b1df9..0139e42cc4 100644 --- a/app/src/controller.c +++ b/app/src/controller.c @@ -4,19 +4,28 @@ #include "util/log.h" +#define SC_CONTROL_MSG_QUEUE_MAX 64 + bool sc_controller_init(struct sc_controller *controller, sc_socket control_socket, struct sc_acksync *acksync) { - cbuf_init(&controller->queue); + sc_vecdeque_init(&controller->queue); + + bool ok = sc_vecdeque_reserve(&controller->queue, SC_CONTROL_MSG_QUEUE_MAX); + if (!ok) { + return false; + } - bool ok = sc_receiver_init(&controller->receiver, control_socket, acksync); + ok = sc_receiver_init(&controller->receiver, control_socket, acksync); if (!ok) { + sc_vecdeque_destroy(&controller->queue); return false; } ok = sc_mutex_init(&controller->mutex); if (!ok) { sc_receiver_destroy(&controller->receiver); + sc_vecdeque_destroy(&controller->queue); return false; } @@ -24,6 +33,7 @@ sc_controller_init(struct sc_controller *controller, sc_socket control_socket, if (!ok) { sc_receiver_destroy(&controller->receiver); sc_mutex_destroy(&controller->mutex); + sc_vecdeque_destroy(&controller->queue); return false; } @@ -38,10 +48,12 @@ sc_controller_destroy(struct sc_controller *controller) { sc_cond_destroy(&controller->msg_cond); sc_mutex_destroy(&controller->mutex); - struct sc_control_msg msg; - while (cbuf_take(&controller->queue, &msg)) { - sc_control_msg_destroy(&msg); + while (!sc_vecdeque_is_empty(&controller->queue)) { + struct sc_control_msg *msg = sc_vecdeque_popref(&controller->queue); + assert(msg); + sc_control_msg_destroy(msg); } + sc_vecdeque_destroy(&controller->queue); sc_receiver_destroy(&controller->receiver); } @@ -54,13 +66,19 @@ sc_controller_push_msg(struct sc_controller *controller, } sc_mutex_lock(&controller->mutex); - bool was_empty = cbuf_is_empty(&controller->queue); - bool res = cbuf_push(&controller->queue, *msg); - if (was_empty) { - sc_cond_signal(&controller->msg_cond); + bool full = sc_vecdeque_is_full(&controller->queue); + if (!full) { + bool was_empty = sc_vecdeque_is_empty(&controller->queue); + sc_vecdeque_push_noresize(&controller->queue, *msg); + if (was_empty) { + sc_cond_signal(&controller->msg_cond); + } } + // Otherwise (if the queue is full), the msg is discarded + sc_mutex_unlock(&controller->mutex); - return res; + + return !full; } static bool @@ -82,7 +100,8 @@ run_controller(void *data) { for (;;) { sc_mutex_lock(&controller->mutex); - while (!controller->stopped && cbuf_is_empty(&controller->queue)) { + while (!controller->stopped + && sc_vecdeque_is_empty(&controller->queue)) { sc_cond_wait(&controller->msg_cond, &controller->mutex); } if (controller->stopped) { @@ -90,10 +109,9 @@ run_controller(void *data) { sc_mutex_unlock(&controller->mutex); break; } - struct sc_control_msg msg; - bool non_empty = cbuf_take(&controller->queue, &msg); - assert(non_empty); - (void) non_empty; + + assert(!sc_vecdeque_is_empty(&controller->queue)); + struct sc_control_msg msg = sc_vecdeque_pop(&controller->queue); sc_mutex_unlock(&controller->mutex); bool ok = process_msg(controller, &msg); diff --git a/app/src/controller.h b/app/src/controller.h index 67c3c58d9d..a044b2bf00 100644 --- a/app/src/controller.h +++ b/app/src/controller.h @@ -8,11 +8,11 @@ #include "control_msg.h" #include "receiver.h" #include "util/acksync.h" -#include "util/cbuf.h" #include "util/net.h" #include "util/thread.h" +#include "util/vecdeque.h" -struct sc_control_msg_queue CBUF(struct sc_control_msg, 64); +struct sc_control_msg_queue SC_VECDEQUE(struct sc_control_msg); struct sc_controller { sc_socket control_socket; From a0a65b3c4da6b56ca320959451fdf2979a34738d Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 22:46:10 +0100 Subject: [PATCH 088/118] Use VecDeque in file_pusher Replace cbuf by VecDeque in file_pusher. As a side-effect, the new implementation does not limit the queue to an arbitrary value. --- app/src/file_pusher.c | 31 +++++++++++++++++++------------ app/src/file_pusher.h | 6 +++--- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/app/src/file_pusher.c b/app/src/file_pusher.c index f6757870bb..b49e93e5c1 100644 --- a/app/src/file_pusher.c +++ b/app/src/file_pusher.c @@ -19,7 +19,7 @@ sc_file_pusher_init(struct sc_file_pusher *fp, const char *serial, const char *push_target) { assert(serial); - cbuf_init(&fp->queue); + sc_vecdeque_init(&fp->queue); bool ok = sc_mutex_init(&fp->mutex); if (!ok) { @@ -65,9 +65,10 @@ sc_file_pusher_destroy(struct sc_file_pusher *fp) { sc_intr_destroy(&fp->intr); free(fp->serial); - struct sc_file_pusher_request req; - while (cbuf_take(&fp->queue, &req)) { - sc_file_pusher_request_destroy(&req); + while (!sc_vecdeque_is_empty(&fp->queue)) { + struct sc_file_pusher_request *req = sc_vecdeque_popref(&fp->queue); + assert(req); + sc_file_pusher_request_destroy(req); } } @@ -91,13 +92,20 @@ sc_file_pusher_request(struct sc_file_pusher *fp, }; sc_mutex_lock(&fp->mutex); - bool was_empty = cbuf_is_empty(&fp->queue); - bool res = cbuf_push(&fp->queue, req); + bool was_empty = sc_vecdeque_is_empty(&fp->queue); + bool res = sc_vecdeque_push(&fp->queue, req); + if (!res) { + LOG_OOM(); + sc_mutex_unlock(&fp->mutex); + return false; + } + if (was_empty) { sc_cond_signal(&fp->event_cond); } sc_mutex_unlock(&fp->mutex); - return res; + + return true; } static int @@ -113,7 +121,7 @@ run_file_pusher(void *data) { for (;;) { sc_mutex_lock(&fp->mutex); - while (!fp->stopped && cbuf_is_empty(&fp->queue)) { + while (!fp->stopped && sc_vecdeque_is_empty(&fp->queue)) { sc_cond_wait(&fp->event_cond, &fp->mutex); } if (fp->stopped) { @@ -121,10 +129,9 @@ run_file_pusher(void *data) { sc_mutex_unlock(&fp->mutex); break; } - struct sc_file_pusher_request req; - bool non_empty = cbuf_take(&fp->queue, &req); - assert(non_empty); - (void) non_empty; + + assert(!sc_vecdeque_is_empty(&fp->queue)); + struct sc_file_pusher_request req = sc_vecdeque_pop(&fp->queue); sc_mutex_unlock(&fp->mutex); if (req.action == SC_FILE_PUSHER_ACTION_INSTALL_APK) { diff --git a/app/src/file_pusher.h b/app/src/file_pusher.h index 0d934d6c17..0ffb372191 100644 --- a/app/src/file_pusher.h +++ b/app/src/file_pusher.h @@ -5,9 +5,9 @@ #include -#include "util/cbuf.h" -#include "util/thread.h" #include "util/intr.h" +#include "util/thread.h" +#include "util/vecdeque.h" enum sc_file_pusher_action { SC_FILE_PUSHER_ACTION_INSTALL_APK, @@ -19,7 +19,7 @@ struct sc_file_pusher_request { char *file; }; -struct sc_file_pusher_request_queue CBUF(struct sc_file_pusher_request, 16); +struct sc_file_pusher_request_queue SC_VECDEQUE(struct sc_file_pusher_request); struct sc_file_pusher { char *serial; From f978e4d6dea2958b449906e33fdb965ef5bc26d4 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 22:50:56 +0100 Subject: [PATCH 089/118] Use VecDeque in aoa_hid Replace cbuf by VecDeque in aoa_hid --- app/src/usb/aoa_hid.c | 41 +++++++++++++++++++++++++++-------------- app/src/usb/aoa_hid.h | 4 ++-- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/app/src/usb/aoa_hid.c b/app/src/usb/aoa_hid.c index 0007169d86..fb64e57c2b 100644 --- a/app/src/usb/aoa_hid.c +++ b/app/src/usb/aoa_hid.c @@ -14,6 +14,8 @@ #define DEFAULT_TIMEOUT 1000 +#define SC_HID_EVENT_QUEUE_MAX 64 + static void sc_hid_event_log(const struct sc_hid_event *event) { // HID Event: [00] FF FF FF FF... @@ -48,14 +50,20 @@ sc_hid_event_destroy(struct sc_hid_event *hid_event) { bool sc_aoa_init(struct sc_aoa *aoa, struct sc_usb *usb, struct sc_acksync *acksync) { - cbuf_init(&aoa->queue); + sc_vecdeque_init(&aoa->queue); + + if (!sc_vecdeque_reserve(&aoa->queue, SC_HID_EVENT_QUEUE_MAX)) { + return false; + } if (!sc_mutex_init(&aoa->mutex)) { + sc_vecdeque_destroy(&aoa->queue); return false; } if (!sc_cond_init(&aoa->event_cond)) { sc_mutex_destroy(&aoa->mutex); + sc_vecdeque_destroy(&aoa->queue); return false; } @@ -69,9 +77,10 @@ sc_aoa_init(struct sc_aoa *aoa, struct sc_usb *usb, void sc_aoa_destroy(struct sc_aoa *aoa) { // Destroy remaining events - struct sc_hid_event event; - while (cbuf_take(&aoa->queue, &event)) { - sc_hid_event_destroy(&event); + while (!sc_vecdeque_is_empty(&aoa->queue)) { + struct sc_hid_event *event = sc_vecdeque_popref(&aoa->queue); + assert(event); + sc_hid_event_destroy(event); } sc_cond_destroy(&aoa->event_cond); @@ -212,13 +221,19 @@ sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) { } sc_mutex_lock(&aoa->mutex); - bool was_empty = cbuf_is_empty(&aoa->queue); - bool res = cbuf_push(&aoa->queue, *event); - if (was_empty) { - sc_cond_signal(&aoa->event_cond); + bool full = sc_vecdeque_is_full(&aoa->queue); + if (!full) { + bool was_empty = sc_vecdeque_is_empty(&aoa->queue); + sc_vecdeque_push_noresize(&aoa->queue, *event); + if (was_empty) { + sc_cond_signal(&aoa->event_cond); + } } + // Otherwise (if the queue is full), the event is discarded + sc_mutex_unlock(&aoa->mutex); - return res; + + return !full; } static int @@ -227,7 +242,7 @@ run_aoa_thread(void *data) { for (;;) { sc_mutex_lock(&aoa->mutex); - while (!aoa->stopped && cbuf_is_empty(&aoa->queue)) { + while (!aoa->stopped && sc_vecdeque_is_empty(&aoa->queue)) { sc_cond_wait(&aoa->event_cond, &aoa->mutex); } if (aoa->stopped) { @@ -235,11 +250,9 @@ run_aoa_thread(void *data) { sc_mutex_unlock(&aoa->mutex); break; } - struct sc_hid_event event; - bool non_empty = cbuf_take(&aoa->queue, &event); - assert(non_empty); - (void) non_empty; + assert(!sc_vecdeque_is_empty(&aoa->queue)); + struct sc_hid_event event = sc_vecdeque_pop(&aoa->queue); uint64_t ack_to_wait = event.ack_to_wait; sc_mutex_unlock(&aoa->mutex); diff --git a/app/src/usb/aoa_hid.h b/app/src/usb/aoa_hid.h index d785a0e96f..8803c1d94b 100644 --- a/app/src/usb/aoa_hid.h +++ b/app/src/usb/aoa_hid.h @@ -8,9 +8,9 @@ #include "usb.h" #include "util/acksync.h" -#include "util/cbuf.h" #include "util/thread.h" #include "util/tick.h" +#include "util/vecdeque.h" struct sc_hid_event { uint16_t accessory_id; @@ -27,7 +27,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, void sc_hid_event_destroy(struct sc_hid_event *hid_event); -struct sc_hid_event_queue CBUF(struct sc_hid_event, 64); +struct sc_hid_event_queue SC_VECDEQUE(struct sc_hid_event); struct sc_aoa { struct sc_usb *usb; From 338310677e2ef441d09048e19a30c94ecf4fbca4 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 22:54:08 +0100 Subject: [PATCH 090/118] Remove cbuf All uses have been replaced by VecDeque. --- app/meson.build | 3 -- app/src/util/cbuf.h | 52 ----------------------------- app/tests/test_cbuf.c | 78 ------------------------------------------- 3 files changed, 133 deletions(-) delete mode 100644 app/src/util/cbuf.h delete mode 100644 app/tests/test_cbuf.c diff --git a/app/meson.build b/app/meson.build index a238eb8fd5..35934131f2 100644 --- a/app/meson.build +++ b/app/meson.build @@ -261,9 +261,6 @@ if get_option('buildtype') == 'debug' 'tests/test_bytebuf.c', 'src/util/bytebuf.c', ]], - ['test_cbuf', [ - 'tests/test_cbuf.c', - ]], ['test_cli', [ 'tests/test_cli.c', 'src/cli.c', diff --git a/app/src/util/cbuf.h b/app/src/util/cbuf.h deleted file mode 100644 index 2a75617143..0000000000 --- a/app/src/util/cbuf.h +++ /dev/null @@ -1,52 +0,0 @@ -// generic circular buffer (bounded queue) implementation -#ifndef SC_CBUF_H -#define SC_CBUF_H - -#include "common.h" - -#include -#include - -// To define a circular buffer type of 20 ints: -// struct cbuf_int CBUF(int, 20); -// -// data has length CAP + 1 to distinguish empty vs full. -#define CBUF(TYPE, CAP) { \ - TYPE data[(CAP) + 1]; \ - size_t head; \ - size_t tail; \ -} - -#define cbuf_size_(PCBUF) \ - (sizeof((PCBUF)->data) / sizeof(*(PCBUF)->data)) - -#define cbuf_is_empty(PCBUF) \ - ((PCBUF)->head == (PCBUF)->tail) - -#define cbuf_is_full(PCBUF) \ - (((PCBUF)->head + 1) % cbuf_size_(PCBUF) == (PCBUF)->tail) - -#define cbuf_init(PCBUF) \ - (void) ((PCBUF)->head = (PCBUF)->tail = 0) - -#define cbuf_push(PCBUF, ITEM) \ - ({ \ - bool ok = !cbuf_is_full(PCBUF); \ - if (ok) { \ - (PCBUF)->data[(PCBUF)->head] = (ITEM); \ - (PCBUF)->head = ((PCBUF)->head + 1) % cbuf_size_(PCBUF); \ - } \ - ok; \ - }) - -#define cbuf_take(PCBUF, PITEM) \ - ({ \ - bool ok = !cbuf_is_empty(PCBUF); \ - if (ok) { \ - *(PITEM) = (PCBUF)->data[(PCBUF)->tail]; \ - (PCBUF)->tail = ((PCBUF)->tail + 1) % cbuf_size_(PCBUF); \ - } \ - ok; \ - }) - -#endif diff --git a/app/tests/test_cbuf.c b/app/tests/test_cbuf.c deleted file mode 100644 index 16674e926f..0000000000 --- a/app/tests/test_cbuf.c +++ /dev/null @@ -1,78 +0,0 @@ -#include "common.h" - -#include -#include - -#include "util/cbuf.h" - -struct int_queue CBUF(int, 32); - -static void test_cbuf_empty(void) { - struct int_queue queue; - cbuf_init(&queue); - - assert(cbuf_is_empty(&queue)); - - bool push_ok = cbuf_push(&queue, 42); - assert(push_ok); - assert(!cbuf_is_empty(&queue)); - - int item; - bool take_ok = cbuf_take(&queue, &item); - assert(take_ok); - assert(cbuf_is_empty(&queue)); - - bool take_empty_ok = cbuf_take(&queue, &item); - assert(!take_empty_ok); // the queue is empty -} - -static void test_cbuf_full(void) { - struct int_queue queue; - cbuf_init(&queue); - - assert(!cbuf_is_full(&queue)); - - // fill the queue - for (int i = 0; i < 32; ++i) { - bool ok = cbuf_push(&queue, i); - assert(ok); - } - bool ok = cbuf_push(&queue, 42); - assert(!ok); // the queue if full - - int item; - bool take_ok = cbuf_take(&queue, &item); - assert(take_ok); - assert(!cbuf_is_full(&queue)); -} - -static void test_cbuf_push_take(void) { - struct int_queue queue; - cbuf_init(&queue); - - bool push1_ok = cbuf_push(&queue, 42); - assert(push1_ok); - - bool push2_ok = cbuf_push(&queue, 35); - assert(push2_ok); - - int item; - - bool take1_ok = cbuf_take(&queue, &item); - assert(take1_ok); - assert(item == 42); - - bool take2_ok = cbuf_take(&queue, &item); - assert(take2_ok); - assert(item == 35); -} - -int main(int argc, char *argv[]) { - (void) argc; - (void) argv; - - test_cbuf_empty(); - test_cbuf_full(); - test_cbuf_push_take(); - return 0; -} From 6f38c6311b4db17858e221f66cbe9be1c7da12ca Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 23:05:17 +0100 Subject: [PATCH 091/118] Remove sc_queue All uses have been replaced by VecDeque. --- app/meson.build | 3 -- app/src/util/queue.h | 77 ------------------------------------------ app/tests/test_queue.c | 43 ----------------------- 3 files changed, 123 deletions(-) delete mode 100644 app/src/util/queue.h delete mode 100644 app/tests/test_queue.c diff --git a/app/meson.build b/app/meson.build index 35934131f2..e34f7cc1a9 100644 --- a/app/meson.build +++ b/app/meson.build @@ -285,9 +285,6 @@ if get_option('buildtype') == 'debug' 'tests/test_device_msg_deserialize.c', 'src/device_msg.c', ]], - ['test_queue', [ - 'tests/test_queue.c', - ]], ['test_strbuf', [ 'tests/test_strbuf.c', 'src/util/strbuf.c', diff --git a/app/src/util/queue.h b/app/src/util/queue.h deleted file mode 100644 index 2233eca022..0000000000 --- a/app/src/util/queue.h +++ /dev/null @@ -1,77 +0,0 @@ -// generic intrusive FIFO queue -#ifndef SC_QUEUE_H -#define SC_QUEUE_H - -#include "common.h" - -#include -#include -#include - -// To define a queue type of "struct foo": -// struct queue_foo QUEUE(struct foo); -#define SC_QUEUE(TYPE) { \ - TYPE *first; \ - TYPE *last; \ -} - -#define sc_queue_init(PQ) \ - (void) ((PQ)->first = (PQ)->last = NULL) - -#define sc_queue_is_empty(PQ) \ - !(PQ)->first - -// NEXTFIELD is the field in the ITEM type used for intrusive linked-list -// -// For example: -// struct foo { -// int value; -// struct foo *next; -// }; -// -// // define the type "struct my_queue" -// struct my_queue SC_QUEUE(struct foo); -// -// struct my_queue queue; -// sc_queue_init(&queue); -// -// struct foo v1 = { .value = 42 }; -// struct foo v2 = { .value = 27 }; -// -// sc_queue_push(&queue, next, v1); -// sc_queue_push(&queue, next, v2); -// -// struct foo *foo; -// sc_queue_take(&queue, next, &foo); -// assert(foo->value == 42); -// sc_queue_take(&queue, next, &foo); -// assert(foo->value == 27); -// assert(sc_queue_is_empty(&queue)); -// - -// push a new item into the queue -#define sc_queue_push(PQ, NEXTFIELD, ITEM) \ - (void) ({ \ - (ITEM)->NEXTFIELD = NULL; \ - if (sc_queue_is_empty(PQ)) { \ - (PQ)->first = (PQ)->last = (ITEM); \ - } else { \ - (PQ)->last->NEXTFIELD = (ITEM); \ - (PQ)->last = (ITEM); \ - } \ - }) - -// take the next item and remove it from the queue (the queue must not be empty) -// the result is stored in *(PITEM) -// (without typeof(), we could not store a local variable having the correct -// type so that we can "return" it) -#define sc_queue_take(PQ, NEXTFIELD, PITEM) \ - (void) ({ \ - assert(!sc_queue_is_empty(PQ)); \ - *(PITEM) = (PQ)->first; \ - (PQ)->first = (PQ)->first->NEXTFIELD; \ - }) - // no need to update (PQ)->last if the queue is left empty: - // (PQ)->last is undefined if !(PQ)->first anyway - -#endif diff --git a/app/tests/test_queue.c b/app/tests/test_queue.c deleted file mode 100644 index d8b2b4eca8..0000000000 --- a/app/tests/test_queue.c +++ /dev/null @@ -1,43 +0,0 @@ -#include "common.h" - -#include - -#include "util/queue.h" - -struct foo { - int value; - struct foo *next; -}; - -static void test_queue(void) { - struct my_queue SC_QUEUE(struct foo) queue; - sc_queue_init(&queue); - - assert(sc_queue_is_empty(&queue)); - - struct foo v1 = { .value = 42 }; - struct foo v2 = { .value = 27 }; - - sc_queue_push(&queue, next, &v1); - sc_queue_push(&queue, next, &v2); - - struct foo *foo; - - assert(!sc_queue_is_empty(&queue)); - sc_queue_take(&queue, next, &foo); - assert(foo->value == 42); - - assert(!sc_queue_is_empty(&queue)); - sc_queue_take(&queue, next, &foo); - assert(foo->value == 27); - - assert(sc_queue_is_empty(&queue)); -} - -int main(int argc, char *argv[]) { - (void) argc; - (void) argv; - - test_queue(); - return 0; -} From a3703340fc5f2d10d8b8d1a248200f302831c432 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 18:24:31 +0100 Subject: [PATCH 092/118] Fix possible race condition on video_buffer end The video_buffer thread clears the queue once it is stopped, but new frames might still be pushed asynchronously. To avoid the problem, do not push any frame once the video_buffer is stopped. --- app/src/video_buffer.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index b3b29098a9..a8f7f20af7 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -99,6 +99,8 @@ run_buffering(void *data) { } stopped: + assert(vb->b.stopped); + // Flush queue while (!sc_vecdeque_is_empty(&vb->b.queue)) { struct sc_video_buffer_frame *p = sc_vecdeque_popref(&vb->b.queue); @@ -206,6 +208,11 @@ sc_video_buffer_push(struct sc_video_buffer *vb, const AVFrame *frame) { sc_mutex_lock(&vb->b.mutex); + if (vb->b.stopped) { + sc_mutex_unlock(&vb->b.mutex); + return false; + } + sc_tick pts = SC_TICK_FROM_US(frame->pts); sc_clock_update(&vb->b.clock, sc_tick_now(), pts); sc_cond_signal(&vb->b.wait_cond); From ad94ccca0bbed9cd8ddaaa931da2c7ed18bf060c Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 18:33:03 +0100 Subject: [PATCH 093/118] Stop the video buffer on error If an error occurs from the video buffer thread (typically an out-of-memory error), then stop. --- app/src/video_buffer.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index a8f7f20af7..49c01839e6 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -93,9 +93,16 @@ run_buffering(void *data) { pts, vb_frame.push_date, sc_tick_now()); #endif - sc_video_buffer_offer(vb, vb_frame.frame); - + bool ok = sc_video_buffer_offer(vb, vb_frame.frame); sc_video_buffer_frame_destroy(&vb_frame); + if (!ok) { + LOGE("Delayed frame could not be pushed, stopping"); + sc_mutex_lock(&vb->b.mutex); + // Prevent to push any new packet + vb->b.stopped = true; + sc_mutex_unlock(&vb->b.mutex); + goto stopped; + } } stopped: From 4540f1d69e7435b78af2c968c3b07b06729a714f Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 1 Mar 2023 18:45:22 +0100 Subject: [PATCH 094/118] Report video buffer downstream errors Make the video buffer stop if its consumer could not receive a frame. --- app/src/screen.c | 21 +++++---------------- app/src/screen.h | 2 -- app/src/v4l2_sink.c | 4 +++- app/src/video_buffer.c | 3 +-- app/src/video_buffer.h | 2 +- 5 files changed, 10 insertions(+), 22 deletions(-) diff --git a/app/src/screen.c b/app/src/screen.c index a9a48eae82..ce2e74bbd9 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -362,27 +362,17 @@ sc_screen_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { return sc_video_buffer_push(&screen->vb, frame); } -static void +static bool sc_video_buffer_on_new_frame(struct sc_video_buffer *vb, bool previous_skipped, void *userdata) { (void) vb; struct sc_screen *screen = userdata; - // event_failed implies previous_skipped (the previous frame may not have - // been consumed if the event was not sent) - assert(!screen->event_failed || previous_skipped); - - bool need_new_event; if (previous_skipped) { sc_fps_counter_add_skipped_frame(&screen->fps_counter); // The SC_EVENT_NEW_FRAME triggered for the previous frame will consume - // this new frame instead, unless the previous event failed - need_new_event = screen->event_failed; + // this new frame instead } else { - need_new_event = true; - } - - if (need_new_event) { static SDL_Event new_frame_event = { .type = SC_EVENT_NEW_FRAME, }; @@ -391,11 +381,11 @@ sc_video_buffer_on_new_frame(struct sc_video_buffer *vb, bool previous_skipped, int ret = SDL_PushEvent(&new_frame_event); if (ret < 0) { LOGW("Could not post new frame event: %s", SDL_GetError()); - screen->event_failed = true; - } else { - screen->event_failed = false; + return false; } } + + return true; } bool @@ -405,7 +395,6 @@ sc_screen_init(struct sc_screen *screen, screen->has_frame = false; screen->fullscreen = false; screen->maximized = false; - screen->event_failed = false; screen->mouse_capture_key_pressed = 0; screen->req.x = params->window_x; diff --git a/app/src/screen.h b/app/src/screen.h index 222e418f80..0952c79c36 100644 --- a/app/src/screen.h +++ b/app/src/screen.h @@ -59,8 +59,6 @@ struct sc_screen { bool maximized; bool mipmaps; - bool event_failed; // in case SDL_PushEvent() returned an error - // To enable/disable mouse capture, a mouse capture key (LALT, LGUI or // RGUI) must be pressed. This variable tracks the pressed capture key. SDL_Keycode mouse_capture_key_pressed; diff --git a/app/src/v4l2_sink.c b/app/src/v4l2_sink.c index ba876b2b88..5dfe37bc45 100644 --- a/app/src/v4l2_sink.c +++ b/app/src/v4l2_sink.c @@ -141,7 +141,7 @@ run_v4l2_sink(void *data) { return 0; } -static void +static bool sc_video_buffer_on_new_frame(struct sc_video_buffer *vb, bool previous_skipped, void *userdata) { (void) vb; @@ -153,6 +153,8 @@ sc_video_buffer_on_new_frame(struct sc_video_buffer *vb, bool previous_skipped, sc_cond_signal(&vs->cond); sc_mutex_unlock(&vs->mutex); } + + return true; } static bool diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index 49c01839e6..7f77117938 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -40,8 +40,7 @@ sc_video_buffer_offer(struct sc_video_buffer *vb, const AVFrame *frame) { return false; } - vb->cbs->on_new_frame(vb, previous_skipped, vb->cbs_userdata); - return true; + return vb->cbs->on_new_frame(vb, previous_skipped, vb->cbs_userdata); } static int diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h index 41b09434be..d183a48450 100644 --- a/app/src/video_buffer.h +++ b/app/src/video_buffer.h @@ -45,7 +45,7 @@ struct sc_video_buffer { }; struct sc_video_buffer_callbacks { - void (*on_new_frame)(struct sc_video_buffer *vb, bool previous_skipped, + bool (*on_new_frame)(struct sc_video_buffer *vb, bool previous_skipped, void *userdata); }; From 6379c08012454300e399006e665a5e4f7d436202 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 9 Mar 2023 09:05:46 +0100 Subject: [PATCH 095/118] Fix buffering pts conversion The mistake had no effect, because tick is also internally expressed in microseconds. --- app/src/video_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index 7f77117938..74a4b0425a 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -65,7 +65,7 @@ run_buffering(void *data) { sc_tick max_deadline = sc_tick_now() + vb->buffering_time; // PTS (written by the server) are expressed in microseconds - sc_tick pts = SC_TICK_TO_US(vb_frame.frame->pts); + sc_tick pts = SC_TICK_FROM_US(vb_frame.frame->pts); bool timed_out = false; while (!vb->b.stopped && !timed_out) { From f410f2bdc468cd32dd099679471af08ca6a76a54 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 00:31:43 +0100 Subject: [PATCH 096/118] Extract sc_delay_buffer A video buffer had 2 responsibilities: - handle the frame delaying mechanism (queuing packets and pushing them after the expected delay); - keep only the most recent frame (using a sc_frame_buffer). In order to be able to reuse only the frame delaying mechanism, extract it to a separate component, sc_delay_buffer. --- app/meson.build | 1 + app/src/delay_buffer.c | 246 +++++++++++++++++++++++++++++++++++++++++ app/src/delay_buffer.h | 69 ++++++++++++ app/src/video_buffer.c | 216 +++--------------------------------- app/src/video_buffer.h | 34 +----- 5 files changed, 337 insertions(+), 229 deletions(-) create mode 100644 app/src/delay_buffer.c create mode 100644 app/src/delay_buffer.h diff --git a/app/meson.build b/app/meson.build index e34f7cc1a9..7749d664a8 100644 --- a/app/meson.build +++ b/app/meson.build @@ -10,6 +10,7 @@ src = [ 'src/control_msg.c', 'src/controller.c', 'src/decoder.c', + 'src/delay_buffer.c', 'src/demuxer.c', 'src/device_msg.c', 'src/icon.c', diff --git a/app/src/delay_buffer.c b/app/src/delay_buffer.c new file mode 100644 index 0000000000..95d47c9c85 --- /dev/null +++ b/app/src/delay_buffer.c @@ -0,0 +1,246 @@ +#include "delay_buffer.h" + +#include +#include + +#include +#include + +#include "util/log.h" + +#define SC_BUFFERING_NDEBUG // comment to debug + +static bool +sc_delayed_frame_init(struct sc_delayed_frame *dframe, const AVFrame *frame) { + dframe->frame = av_frame_alloc(); + if (!dframe->frame) { + LOG_OOM(); + return false; + } + + if (av_frame_ref(dframe->frame, frame)) { + LOG_OOM(); + av_frame_free(&dframe->frame); + return false; + } + + return true; +} + +static void +sc_delayed_frame_destroy(struct sc_delayed_frame *dframe) { + av_frame_unref(dframe->frame); + av_frame_free(&dframe->frame); +} + +static bool +sc_delay_buffer_offer(struct sc_delay_buffer *db, const AVFrame *frame) { + return db->cbs->on_new_frame(db, frame, db->cbs_userdata); +} + +static int +run_buffering(void *data) { + struct sc_delay_buffer *db = data; + + assert(db->delay > 0); + + for (;;) { + sc_mutex_lock(&db->b.mutex); + + while (!db->b.stopped && sc_vecdeque_is_empty(&db->b.queue)) { + sc_cond_wait(&db->b.queue_cond, &db->b.mutex); + } + + if (db->b.stopped) { + sc_mutex_unlock(&db->b.mutex); + goto stopped; + } + + struct sc_delayed_frame dframe = sc_vecdeque_pop(&db->b.queue); + + sc_tick max_deadline = sc_tick_now() + db->delay; + // PTS (written by the server) are expressed in microseconds + sc_tick pts = SC_TICK_FROM_US(dframe.frame->pts); + + bool timed_out = false; + while (!db->b.stopped && !timed_out) { + sc_tick deadline = sc_clock_to_system_time(&db->b.clock, pts) + + db->delay; + if (deadline > max_deadline) { + deadline = max_deadline; + } + + timed_out = + !sc_cond_timedwait(&db->b.wait_cond, &db->b.mutex, deadline); + } + + bool stopped = db->b.stopped; + sc_mutex_unlock(&db->b.mutex); + + if (stopped) { + sc_delayed_frame_destroy(&dframe); + goto stopped; + } + +#ifndef SC_BUFFERING_NDEBUG + LOGD("Buffering: %" PRItick ";%" PRItick ";%" PRItick, + pts, dframe.push_date, sc_tick_now()); +#endif + + bool ok = sc_delay_buffer_offer(db, dframe.frame); + sc_delayed_frame_destroy(&dframe); + if (!ok) { + LOGE("Delayed frame could not be pushed, stopping"); + sc_mutex_lock(&db->b.mutex); + // Prevent to push any new packet + db->b.stopped = true; + sc_mutex_unlock(&db->b.mutex); + goto stopped; + } + } + +stopped: + assert(db->b.stopped); + + // Flush queue + while (!sc_vecdeque_is_empty(&db->b.queue)) { + struct sc_delayed_frame *dframe = sc_vecdeque_popref(&db->b.queue); + sc_delayed_frame_destroy(dframe); + } + + LOGD("Buffering thread ended"); + + return 0; +} + +bool +sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay, + const struct sc_delay_buffer_callbacks *cbs, + void *cbs_userdata) { + assert(delay >= 0); + + if (delay) { + bool ok = sc_mutex_init(&db->b.mutex); + if (!ok) { + return false; + } + + ok = sc_cond_init(&db->b.queue_cond); + if (!ok) { + sc_mutex_destroy(&db->b.mutex); + return false; + } + + ok = sc_cond_init(&db->b.wait_cond); + if (!ok) { + sc_cond_destroy(&db->b.queue_cond); + sc_mutex_destroy(&db->b.mutex); + return false; + } + + sc_clock_init(&db->b.clock); + sc_vecdeque_init(&db->b.queue); + } + + assert(cbs); + assert(cbs->on_new_frame); + + db->delay = delay; + db->cbs = cbs; + db->cbs_userdata = cbs_userdata; + + return true; +} + +bool +sc_delay_buffer_start(struct sc_delay_buffer *db) { + if (db->delay) { + bool ok = + sc_thread_create(&db->b.thread, run_buffering, "scrcpy-dbuf", db); + if (!ok) { + LOGE("Could not start buffering thread"); + return false; + } + } + + return true; +} + +void +sc_delay_buffer_stop(struct sc_delay_buffer *db) { + if (db->delay) { + sc_mutex_lock(&db->b.mutex); + db->b.stopped = true; + sc_cond_signal(&db->b.queue_cond); + sc_cond_signal(&db->b.wait_cond); + sc_mutex_unlock(&db->b.mutex); + } +} + +void +sc_delay_buffer_join(struct sc_delay_buffer *db) { + if (db->delay) { + sc_thread_join(&db->b.thread, NULL); + } +} + +void +sc_delay_buffer_destroy(struct sc_delay_buffer *db) { + if (db->delay) { + sc_cond_destroy(&db->b.wait_cond); + sc_cond_destroy(&db->b.queue_cond); + sc_mutex_destroy(&db->b.mutex); + } +} + +bool +sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame) { + if (!db->delay) { + // No buffering + return sc_delay_buffer_offer(db, frame); + } + + sc_mutex_lock(&db->b.mutex); + + if (db->b.stopped) { + sc_mutex_unlock(&db->b.mutex); + return false; + } + + sc_tick pts = SC_TICK_FROM_US(frame->pts); + sc_clock_update(&db->b.clock, sc_tick_now(), pts); + sc_cond_signal(&db->b.wait_cond); + + if (db->b.clock.count == 1) { + sc_mutex_unlock(&db->b.mutex); + // First frame, offer it immediately, for two reasons: + // - not to delay the opening of the scrcpy window + // - the buffering estimation needs at least two clock points, so it + // could not handle the first frame + return sc_delay_buffer_offer(db, frame); + } + + struct sc_delayed_frame dframe; + bool ok = sc_delayed_frame_init(&dframe, frame); + if (!ok) { + sc_mutex_unlock(&db->b.mutex); + return false; + } + +#ifndef SC_BUFFERING_NDEBUG + dframe.push_date = sc_tick_now(); +#endif + + ok = sc_vecdeque_push(&db->b.queue, dframe); + if (!ok) { + sc_mutex_unlock(&db->b.mutex); + LOG_OOM(); + return false; + } + + sc_cond_signal(&db->b.queue_cond); + + sc_mutex_unlock(&db->b.mutex); + + return true; +} diff --git a/app/src/delay_buffer.h b/app/src/delay_buffer.h new file mode 100644 index 0000000000..9e5347c7d3 --- /dev/null +++ b/app/src/delay_buffer.h @@ -0,0 +1,69 @@ +#ifndef SC_DELAY_BUFFER_H +#define SC_DELAY_BUFFER_H + +#include "common.h" + +#include + +#include "clock.h" +#include "util/thread.h" +#include "util/tick.h" +#include "util/vecdeque.h" + +// forward declarations +typedef struct AVFrame AVFrame; + +struct sc_delayed_frame { + AVFrame *frame; +#ifndef NDEBUG + sc_tick push_date; +#endif +}; + +struct sc_delayed_frame_queue SC_VECDEQUE(struct sc_delayed_frame); + +struct sc_delay_buffer { + sc_tick delay; + + // only if delay > 0 + struct { + sc_thread thread; + sc_mutex mutex; + sc_cond queue_cond; + sc_cond wait_cond; + + struct sc_clock clock; + struct sc_delayed_frame_queue queue; + bool stopped; + } b; // buffering + + const struct sc_delay_buffer_callbacks *cbs; + void *cbs_userdata; +}; + +struct sc_delay_buffer_callbacks { + bool (*on_new_frame)(struct sc_delay_buffer *db, const AVFrame *frame, + void *userdata); +}; + +bool +sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay, + const struct sc_delay_buffer_callbacks *cbs, + void *cbs_userdata); + +bool +sc_delay_buffer_start(struct sc_delay_buffer *db); + +void +sc_delay_buffer_stop(struct sc_delay_buffer *db); + +void +sc_delay_buffer_join(struct sc_delay_buffer *db); + +void +sc_delay_buffer_destroy(struct sc_delay_buffer *db); + +bool +sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame); + +#endif diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index 74a4b0425a..da47a0b521 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -8,32 +8,13 @@ #include "util/log.h" -#define SC_BUFFERING_NDEBUG // comment to debug - static bool -sc_video_buffer_frame_init(struct sc_video_buffer_frame *vb_frame, - const AVFrame *frame) { - vb_frame->frame = av_frame_alloc(); - if (!vb_frame->frame) { - return false; - } - - if (av_frame_ref(vb_frame->frame, frame)) { - av_frame_free(&vb_frame->frame); - return false; - } - - return true; -} +sc_delay_buffer_on_new_frame(struct sc_delay_buffer *db, const AVFrame *frame, + void *userdata) { + (void) db; -static void -sc_video_buffer_frame_destroy(struct sc_video_buffer_frame *vb_frame) { - av_frame_unref(vb_frame->frame); - av_frame_free(&vb_frame->frame); -} + struct sc_video_buffer *vb = userdata; -static bool -sc_video_buffer_offer(struct sc_video_buffer *vb, const AVFrame *frame) { bool previous_skipped; bool ok = sc_frame_buffer_push(&vb->fb, frame, &previous_skipped); if (!ok) { @@ -43,83 +24,8 @@ sc_video_buffer_offer(struct sc_video_buffer *vb, const AVFrame *frame) { return vb->cbs->on_new_frame(vb, previous_skipped, vb->cbs_userdata); } -static int -run_buffering(void *data) { - struct sc_video_buffer *vb = data; - - assert(vb->buffering_time > 0); - - for (;;) { - sc_mutex_lock(&vb->b.mutex); - - while (!vb->b.stopped && sc_vecdeque_is_empty(&vb->b.queue)) { - sc_cond_wait(&vb->b.queue_cond, &vb->b.mutex); - } - - if (vb->b.stopped) { - sc_mutex_unlock(&vb->b.mutex); - goto stopped; - } - - struct sc_video_buffer_frame vb_frame = sc_vecdeque_pop(&vb->b.queue); - - sc_tick max_deadline = sc_tick_now() + vb->buffering_time; - // PTS (written by the server) are expressed in microseconds - sc_tick pts = SC_TICK_FROM_US(vb_frame.frame->pts); - - bool timed_out = false; - while (!vb->b.stopped && !timed_out) { - sc_tick deadline = sc_clock_to_system_time(&vb->b.clock, pts) - + vb->buffering_time; - if (deadline > max_deadline) { - deadline = max_deadline; - } - - timed_out = - !sc_cond_timedwait(&vb->b.wait_cond, &vb->b.mutex, deadline); - } - - if (vb->b.stopped) { - sc_video_buffer_frame_destroy(&vb_frame); - sc_mutex_unlock(&vb->b.mutex); - goto stopped; - } - - sc_mutex_unlock(&vb->b.mutex); - -#ifndef SC_BUFFERING_NDEBUG - LOGD("Buffering: %" PRItick ";%" PRItick ";%" PRItick, - pts, vb_frame.push_date, sc_tick_now()); -#endif - - bool ok = sc_video_buffer_offer(vb, vb_frame.frame); - sc_video_buffer_frame_destroy(&vb_frame); - if (!ok) { - LOGE("Delayed frame could not be pushed, stopping"); - sc_mutex_lock(&vb->b.mutex); - // Prevent to push any new packet - vb->b.stopped = true; - sc_mutex_unlock(&vb->b.mutex); - goto stopped; - } - } - -stopped: - assert(vb->b.stopped); - - // Flush queue - while (!sc_vecdeque_is_empty(&vb->b.queue)) { - struct sc_video_buffer_frame *p = sc_vecdeque_popref(&vb->b.queue); - sc_video_buffer_frame_destroy(p); - } - - LOGD("Buffering thread ended"); - - return 0; -} - bool -sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick buffering_time, +sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick delay, const struct sc_video_buffer_callbacks *cbs, void *cbs_userdata) { bool ok = sc_frame_buffer_init(&vb->fb); @@ -127,135 +33,49 @@ sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick buffering_time, return false; } - assert(buffering_time >= 0); - if (buffering_time) { - ok = sc_mutex_init(&vb->b.mutex); - if (!ok) { - sc_frame_buffer_destroy(&vb->fb); - return false; - } - - ok = sc_cond_init(&vb->b.queue_cond); - if (!ok) { - sc_mutex_destroy(&vb->b.mutex); - sc_frame_buffer_destroy(&vb->fb); - return false; - } + static const struct sc_delay_buffer_callbacks db_cbs = { + .on_new_frame = sc_delay_buffer_on_new_frame, + }; - ok = sc_cond_init(&vb->b.wait_cond); - if (!ok) { - sc_cond_destroy(&vb->b.queue_cond); - sc_mutex_destroy(&vb->b.mutex); - sc_frame_buffer_destroy(&vb->fb); - return false; - } - - sc_clock_init(&vb->b.clock); - sc_vecdeque_init(&vb->b.queue); + ok = sc_delay_buffer_init(&vb->db, delay, &db_cbs, vb); + if (!ok) { + sc_frame_buffer_destroy(&vb->fb); + return false; } assert(cbs); assert(cbs->on_new_frame); - vb->buffering_time = buffering_time; vb->cbs = cbs; vb->cbs_userdata = cbs_userdata; + return true; } bool sc_video_buffer_start(struct sc_video_buffer *vb) { - if (vb->buffering_time) { - bool ok = - sc_thread_create(&vb->b.thread, run_buffering, "scrcpy-vbuf", vb); - if (!ok) { - LOGE("Could not start buffering thread"); - return false; - } - } - - return true; + return sc_delay_buffer_start(&vb->db); } void sc_video_buffer_stop(struct sc_video_buffer *vb) { - if (vb->buffering_time) { - sc_mutex_lock(&vb->b.mutex); - vb->b.stopped = true; - sc_cond_signal(&vb->b.queue_cond); - sc_cond_signal(&vb->b.wait_cond); - sc_mutex_unlock(&vb->b.mutex); - } + return sc_delay_buffer_stop(&vb->db); } void sc_video_buffer_join(struct sc_video_buffer *vb) { - if (vb->buffering_time) { - sc_thread_join(&vb->b.thread, NULL); - } + return sc_delay_buffer_join(&vb->db); } void sc_video_buffer_destroy(struct sc_video_buffer *vb) { sc_frame_buffer_destroy(&vb->fb); - if (vb->buffering_time) { - sc_cond_destroy(&vb->b.wait_cond); - sc_cond_destroy(&vb->b.queue_cond); - sc_mutex_destroy(&vb->b.mutex); - } + sc_delay_buffer_destroy(&vb->db); } bool sc_video_buffer_push(struct sc_video_buffer *vb, const AVFrame *frame) { - if (!vb->buffering_time) { - // No buffering - return sc_video_buffer_offer(vb, frame); - } - - sc_mutex_lock(&vb->b.mutex); - - if (vb->b.stopped) { - sc_mutex_unlock(&vb->b.mutex); - return false; - } - - sc_tick pts = SC_TICK_FROM_US(frame->pts); - sc_clock_update(&vb->b.clock, sc_tick_now(), pts); - sc_cond_signal(&vb->b.wait_cond); - - if (vb->b.clock.count == 1) { - sc_mutex_unlock(&vb->b.mutex); - // First frame, offer it immediately, for two reasons: - // - not to delay the opening of the scrcpy window - // - the buffering estimation needs at least two clock points, so it - // could not handle the first frame - return sc_video_buffer_offer(vb, frame); - } - - struct sc_video_buffer_frame vb_frame; - bool ok = sc_video_buffer_frame_init(&vb_frame, frame); - if (!ok) { - sc_mutex_unlock(&vb->b.mutex); - LOG_OOM(); - return false; - } - -#ifndef SC_BUFFERING_NDEBUG - vb_frame.push_date = sc_tick_now(); -#endif - - ok = sc_vecdeque_push(&vb->b.queue, vb_frame); - if (!ok) { - sc_mutex_unlock(&vb->b.mutex); - LOG_OOM(); - return false; - } - - sc_cond_signal(&vb->b.queue_cond); - - sc_mutex_unlock(&vb->b.mutex); - - return true; + return sc_delay_buffer_push(&vb->db, frame); } void diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h index d183a48450..ebca3915ad 100644 --- a/app/src/video_buffer.h +++ b/app/src/video_buffer.h @@ -5,41 +5,13 @@ #include -#include "clock.h" +#include "delay_buffer.h" #include "frame_buffer.h" -#include "util/thread.h" -#include "util/tick.h" -#include "util/vecdeque.h" - -// forward declarations -typedef struct AVFrame AVFrame; - -struct sc_video_buffer_frame { - AVFrame *frame; -#ifndef NDEBUG - sc_tick push_date; -#endif -}; - -struct sc_video_buffer_frame_queue SC_VECDEQUE(struct sc_video_buffer_frame); struct sc_video_buffer { + struct sc_delay_buffer db; struct sc_frame_buffer fb; - sc_tick buffering_time; - - // only if buffering_time > 0 - struct { - sc_thread thread; - sc_mutex mutex; - sc_cond queue_cond; - sc_cond wait_cond; - - struct sc_clock clock; - struct sc_video_buffer_frame_queue queue; - bool stopped; - } b; // buffering - const struct sc_video_buffer_callbacks *cbs; void *cbs_userdata; }; @@ -50,7 +22,7 @@ struct sc_video_buffer_callbacks { }; bool -sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick buffering_time, +sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick delay, const struct sc_video_buffer_callbacks *cbs, void *cbs_userdata); From c39054a63da52ab266ed26d78e2329c48e4e0216 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 09:07:25 +0100 Subject: [PATCH 097/118] Introduce packet source trait There was a packet sink trait, implemented by components able to receive AVPackets, but each packet source had to manually send packets to sinks. In order to mutualise sink management, add a packet source trait. --- app/meson.build | 1 + app/src/trait/packet_source.c | 70 +++++++++++++++++++++++++++++++++++ app/src/trait/packet_source.h | 41 ++++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 app/src/trait/packet_source.c create mode 100644 app/src/trait/packet_source.h diff --git a/app/meson.build b/app/meson.build index 7749d664a8..821058769a 100644 --- a/app/meson.build +++ b/app/meson.build @@ -30,6 +30,7 @@ src = [ 'src/server.c', 'src/version.c', 'src/video_buffer.c', + 'src/trait/packet_source.c', 'src/util/acksync.c', 'src/util/bytebuf.c', 'src/util/file.c', diff --git a/app/src/trait/packet_source.c b/app/src/trait/packet_source.c new file mode 100644 index 0000000000..df678e16d0 --- /dev/null +++ b/app/src/trait/packet_source.c @@ -0,0 +1,70 @@ +#include "packet_source.h" + +void +sc_packet_source_init(struct sc_packet_source *source) { + source->sink_count = 0; +} + +void +sc_packet_source_add_sink(struct sc_packet_source *source, + struct sc_packet_sink *sink) { + assert(source->sink_count < SC_PACKET_SOURCE_MAX_SINKS); + assert(sink); + assert(sink->ops); + source->sinks[source->sink_count++] = sink; +} + +static void +sc_packet_source_sinks_close_firsts(struct sc_packet_source *source, + unsigned count) { + while (count) { + struct sc_packet_sink *sink = source->sinks[--count]; + sink->ops->close(sink); + } +} + +bool +sc_packet_source_sinks_open(struct sc_packet_source *source, + const AVCodec *codec) { + assert(source->sink_count); + for (unsigned i = 0; i < source->sink_count; ++i) { + struct sc_packet_sink *sink = source->sinks[i]; + if (!sink->ops->open(sink, codec)) { + sc_packet_source_sinks_close_firsts(source, i); + return false; + } + } + + return true; +} + +void +sc_packet_source_sinks_close(struct sc_packet_source *source) { + assert(source->sink_count); + sc_packet_source_sinks_close_firsts(source, source->sink_count); +} + +bool +sc_packet_source_sinks_push(struct sc_packet_source *source, + const AVPacket *packet) { + assert(source->sink_count); + for (unsigned i = 0; i < source->sink_count; ++i) { + struct sc_packet_sink *sink = source->sinks[i]; + if (!sink->ops->push(sink, packet)) { + return false; + } + } + + return true; +} + +void +sc_packet_source_sinks_disable(struct sc_packet_source *source) { + assert(source->sink_count); + for (unsigned i = 0; i < source->sink_count; ++i) { + struct sc_packet_sink *sink = source->sinks[i]; + if (sink->ops->disable) { + sink->ops->disable(sink); + } + } +} diff --git a/app/src/trait/packet_source.h b/app/src/trait/packet_source.h new file mode 100644 index 0000000000..c34aa5d3e1 --- /dev/null +++ b/app/src/trait/packet_source.h @@ -0,0 +1,41 @@ +#ifndef SC_PACKET_SOURCE_H +#define SC_PACKET_SOURCE_H + +#include "common.h" + +#include "packet_sink.h" + +#define SC_PACKET_SOURCE_MAX_SINKS 2 + +/** + * Packet source trait + * + * Component able to send AVPackets should implement this trait. + */ +struct sc_packet_source { + struct sc_packet_sink *sinks[SC_PACKET_SOURCE_MAX_SINKS]; + unsigned sink_count; +}; + +void +sc_packet_source_init(struct sc_packet_source *source); + +void +sc_packet_source_add_sink(struct sc_packet_source *source, + struct sc_packet_sink *sink); + +bool +sc_packet_source_sinks_open(struct sc_packet_source *source, + const AVCodec *codec); + +void +sc_packet_source_sinks_close(struct sc_packet_source *source); + +bool +sc_packet_source_sinks_push(struct sc_packet_source *source, + const AVPacket *packet); + +void +sc_packet_source_sinks_disable(struct sc_packet_source *source); + +#endif From f3197e178d297544774c37c766907b1992929e67 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 09:20:37 +0100 Subject: [PATCH 098/118] Use packet source trait in demuxer --- app/src/demuxer.c | 83 +++++------------------------------------------ app/src/demuxer.h | 11 ++----- app/src/scrcpy.c | 13 +++++--- 3 files changed, 19 insertions(+), 88 deletions(-) diff --git a/app/src/demuxer.c b/app/src/demuxer.c index 5977a28a67..15a595a00c 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -112,65 +112,6 @@ sc_demuxer_recv_packet(struct sc_demuxer *demuxer, AVPacket *packet) { return true; } -static bool -push_packet_to_sinks(struct sc_demuxer *demuxer, const AVPacket *packet) { - for (unsigned i = 0; i < demuxer->sink_count; ++i) { - struct sc_packet_sink *sink = demuxer->sinks[i]; - if (!sink->ops->push(sink, packet)) { - return false; - } - } - - return true; -} - -static bool -sc_demuxer_push_packet(struct sc_demuxer *demuxer, AVPacket *packet) { - bool ok = push_packet_to_sinks(demuxer, packet); - if (!ok) { - LOGE("Demuxer '%s': could not process packet", demuxer->name); - return false; - } - - return true; -} - -static void -sc_demuxer_close_first_sinks(struct sc_demuxer *demuxer, unsigned count) { - while (count) { - struct sc_packet_sink *sink = demuxer->sinks[--count]; - sink->ops->close(sink); - } -} - -static inline void -sc_demuxer_close_sinks(struct sc_demuxer *demuxer) { - sc_demuxer_close_first_sinks(demuxer, demuxer->sink_count); -} - -static bool -sc_demuxer_open_sinks(struct sc_demuxer *demuxer, const AVCodec *codec) { - for (unsigned i = 0; i < demuxer->sink_count; ++i) { - struct sc_packet_sink *sink = demuxer->sinks[i]; - if (!sink->ops->open(sink, codec)) { - sc_demuxer_close_first_sinks(demuxer, i); - return false; - } - } - - return true; -} - -static void -sc_demuxer_disable_sinks(struct sc_demuxer *demuxer) { - for (unsigned i = 0; i < demuxer->sink_count; ++i) { - struct sc_packet_sink *sink = demuxer->sinks[i]; - if (sink->ops->disable) { - sink->ops->disable(sink); - } - } -} - static int run_demuxer(void *data) { struct sc_demuxer *demuxer = data; @@ -189,7 +130,7 @@ run_demuxer(void *data) { if (raw_codec_id == 0) { LOGW("Demuxer '%s': stream explicitly disabled by the device", demuxer->name); - sc_demuxer_disable_sinks(demuxer); + sc_packet_source_sinks_disable(&demuxer->packet_source); status = SC_DEMUXER_STATUS_DISABLED; goto end; } @@ -204,7 +145,7 @@ run_demuxer(void *data) { if (codec_id == AV_CODEC_ID_NONE) { LOGE("Demuxer '%s': stream disabled due to unsupported codec", demuxer->name); - sc_demuxer_disable_sinks(demuxer); + sc_packet_source_sinks_disable(&demuxer->packet_source); goto end; } @@ -212,11 +153,11 @@ run_demuxer(void *data) { if (!codec) { LOGE("Demuxer '%s': stream disabled due to missing decoder", demuxer->name); - sc_demuxer_disable_sinks(demuxer); + sc_packet_source_sinks_disable(&demuxer->packet_source); goto end; } - if (!sc_demuxer_open_sinks(demuxer, codec)) { + if (!sc_packet_source_sinks_open(&demuxer->packet_source, codec)) { goto end; } @@ -253,10 +194,10 @@ run_demuxer(void *data) { } } - ok = sc_demuxer_push_packet(demuxer, packet); + ok = sc_packet_source_sinks_push(&demuxer->packet_source, packet); av_packet_unref(packet); if (!ok) { - // cannot process packet (error already logged) + // The sink already logged its concrete error break; } } @@ -269,7 +210,7 @@ run_demuxer(void *data) { av_packet_free(&packet); finally_close_sinks: - sc_demuxer_close_sinks(demuxer); + sc_packet_source_sinks_close(&demuxer->packet_source); end: demuxer->cbs->on_ended(demuxer, status, demuxer->cbs_userdata); @@ -283,7 +224,7 @@ sc_demuxer_init(struct sc_demuxer *demuxer, const char *name, sc_socket socket, demuxer->name = name; // statically allocated demuxer->socket = socket; - demuxer->sink_count = 0; + sc_packet_source_init(&demuxer->packet_source); assert(cbs && cbs->on_ended); @@ -291,14 +232,6 @@ sc_demuxer_init(struct sc_demuxer *demuxer, const char *name, sc_socket socket, demuxer->cbs_userdata = cbs_userdata; } -void -sc_demuxer_add_sink(struct sc_demuxer *demuxer, struct sc_packet_sink *sink) { - assert(demuxer->sink_count < SC_DEMUXER_MAX_SINKS); - assert(sink); - assert(sink->ops); - demuxer->sinks[demuxer->sink_count++] = sink; -} - bool sc_demuxer_start(struct sc_demuxer *demuxer) { LOGD("Demuxer '%s': starting thread", demuxer->name); diff --git a/app/src/demuxer.h b/app/src/demuxer.h index d0e41add2d..5587d12dcb 100644 --- a/app/src/demuxer.h +++ b/app/src/demuxer.h @@ -8,21 +8,19 @@ #include #include +#include "trait/packet_source.h" #include "trait/packet_sink.h" #include "util/net.h" #include "util/thread.h" -#define SC_DEMUXER_MAX_SINKS 2 - struct sc_demuxer { + struct sc_packet_source packet_source; // packet source trait + const char *name; // must be statically allocated (e.g. a string literal) sc_socket socket; sc_thread thread; - struct sc_packet_sink *sinks[SC_DEMUXER_MAX_SINKS]; - unsigned sink_count; - const struct sc_demuxer_callbacks *cbs; void *cbs_userdata; }; @@ -43,9 +41,6 @@ void sc_demuxer_init(struct sc_demuxer *demuxer, const char *name, sc_socket socket, const struct sc_demuxer_callbacks *cbs, void *cbs_userdata); -void -sc_demuxer_add_sink(struct sc_demuxer *demuxer, struct sc_packet_sink *sink); - bool sc_demuxer_start(struct sc_demuxer *demuxer); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 4355d71bb9..d9625a44d4 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -444,11 +444,13 @@ scrcpy(struct scrcpy_options *options) { #endif if (needs_video_decoder) { sc_decoder_init(&s->video_decoder, "video"); - sc_demuxer_add_sink(&s->video_demuxer, &s->video_decoder.packet_sink); + sc_packet_source_add_sink(&s->video_demuxer.packet_source, + &s->video_decoder.packet_sink); } if (needs_audio_decoder) { sc_decoder_init(&s->audio_decoder, "audio"); - sc_demuxer_add_sink(&s->audio_demuxer, &s->audio_decoder.packet_sink); + sc_packet_source_add_sink(&s->audio_demuxer.packet_source, + &s->audio_decoder.packet_sink); } if (options->record_filename) { @@ -467,10 +469,11 @@ scrcpy(struct scrcpy_options *options) { } recorder_started = true; - sc_demuxer_add_sink(&s->video_demuxer, &s->recorder.video_packet_sink); + sc_packet_source_add_sink(&s->video_demuxer.packet_source, + &s->recorder.video_packet_sink); if (options->audio) { - sc_demuxer_add_sink(&s->audio_demuxer, - &s->recorder.audio_packet_sink); + sc_packet_source_add_sink(&s->audio_demuxer.packet_source, + &s->recorder.audio_packet_sink); } } From 6543964f12b9089d2b5cf57a9aab7e7fe624842f Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 09:25:25 +0100 Subject: [PATCH 099/118] Introduce frame source trait There was a frame sink trait, implemented by components able to receive AVFrames, but each frame source had to manually send frame to sinks. In order to mutualise sink management, add a frame sink trait. --- app/meson.build | 1 + app/src/trait/frame_source.c | 59 ++++++++++++++++++++++++++++++++++++ app/src/trait/frame_source.h | 38 +++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 app/src/trait/frame_source.c create mode 100644 app/src/trait/frame_source.h diff --git a/app/meson.build b/app/meson.build index 821058769a..9f73b43419 100644 --- a/app/meson.build +++ b/app/meson.build @@ -30,6 +30,7 @@ src = [ 'src/server.c', 'src/version.c', 'src/video_buffer.c', + 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/util/acksync.c', 'src/util/bytebuf.c', diff --git a/app/src/trait/frame_source.c b/app/src/trait/frame_source.c new file mode 100644 index 0000000000..416eccd9b0 --- /dev/null +++ b/app/src/trait/frame_source.c @@ -0,0 +1,59 @@ +#include "frame_source.h" + +void +sc_frame_source_init(struct sc_frame_source *source) { + source->sink_count = 0; +} + +void +sc_frame_source_add_sink(struct sc_frame_source *source, + struct sc_frame_sink *sink) { + assert(source->sink_count < SC_FRAME_SOURCE_MAX_SINKS); + assert(sink); + assert(sink->ops); + source->sinks[source->sink_count++] = sink; +} + +static void +sc_frame_source_sinks_close_firsts(struct sc_frame_source *source, + unsigned count) { + while (count) { + struct sc_frame_sink *sink = source->sinks[--count]; + sink->ops->close(sink); + } +} + +bool +sc_frame_source_sinks_open(struct sc_frame_source *source, + const AVCodecContext *ctx) { + assert(source->sink_count); + for (unsigned i = 0; i < source->sink_count; ++i) { + struct sc_frame_sink *sink = source->sinks[i]; + if (!sink->ops->open(sink, ctx)) { + sc_frame_source_sinks_close_firsts(source, i); + return false; + } + } + + return true; +} + +void +sc_frame_source_sinks_close(struct sc_frame_source *source) { + assert(source->sink_count); + sc_frame_source_sinks_close_firsts(source, source->sink_count); +} + +bool +sc_frame_source_sinks_push(struct sc_frame_source *source, + const AVFrame *frame) { + assert(source->sink_count); + for (unsigned i = 0; i < source->sink_count; ++i) { + struct sc_frame_sink *sink = source->sinks[i]; + if (!sink->ops->push(sink, frame)) { + return false; + } + } + + return true; +} diff --git a/app/src/trait/frame_source.h b/app/src/trait/frame_source.h new file mode 100644 index 0000000000..94222af039 --- /dev/null +++ b/app/src/trait/frame_source.h @@ -0,0 +1,38 @@ +#ifndef SC_FRAME_SOURCE_H +#define SC_FRAME_SOURCE_H + +#include "common.h" + +#include "frame_sink.h" + +#define SC_FRAME_SOURCE_MAX_SINKS 2 + +/** + * Frame source trait + * + * Component able to send AVFrames should implement this trait. + */ +struct sc_frame_source { + struct sc_frame_sink *sinks[SC_FRAME_SOURCE_MAX_SINKS]; + unsigned sink_count; +}; + +void +sc_frame_source_init(struct sc_frame_source *source); + +void +sc_frame_source_add_sink(struct sc_frame_source *source, + struct sc_frame_sink *sink); + +bool +sc_frame_source_sinks_open(struct sc_frame_source *source, + const AVCodecContext *ctx); + +void +sc_frame_source_sinks_close(struct sc_frame_source *source); + +bool +sc_frame_source_sinks_push(struct sc_frame_source *source, + const AVFrame *frame); + +#endif From 974227a3fcfa5b60921aeb71d43da4ac84c73d91 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 09:37:36 +0100 Subject: [PATCH 100/118] Use frame source trait in decoder --- app/src/decoder.c | 56 +++++------------------------------------------ app/src/decoder.h | 10 ++------- app/src/scrcpy.c | 6 +++-- 3 files changed, 12 insertions(+), 60 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index e4d596281e..2931c1eccb 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -11,32 +11,6 @@ /** Downcast packet_sink to decoder */ #define DOWNCAST(SINK) container_of(SINK, struct sc_decoder, packet_sink) -static void -sc_decoder_close_first_sinks(struct sc_decoder *decoder, unsigned count) { - while (count) { - struct sc_frame_sink *sink = decoder->sinks[--count]; - sink->ops->close(sink); - } -} - -static inline void -sc_decoder_close_sinks(struct sc_decoder *decoder) { - sc_decoder_close_first_sinks(decoder, decoder->sink_count); -} - -static bool -sc_decoder_open_sinks(struct sc_decoder *decoder, const AVCodecContext *ctx) { - for (unsigned i = 0; i < decoder->sink_count; ++i) { - struct sc_frame_sink *sink = decoder->sinks[i]; - if (!sink->ops->open(sink, ctx)) { - sc_decoder_close_first_sinks(decoder, i); - return false; - } - } - - return true; -} - static bool sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec) { decoder->codec_ctx = avcodec_alloc_context3(codec); @@ -66,7 +40,8 @@ sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec) { return false; } - if (!sc_decoder_open_sinks(decoder, decoder->codec_ctx)) { + if (!sc_frame_source_sinks_open(&decoder->frame_source, + decoder->codec_ctx)) { av_frame_free(&decoder->frame); avcodec_close(decoder->codec_ctx); avcodec_free_context(&decoder->codec_ctx); @@ -78,24 +53,12 @@ sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec) { static void sc_decoder_close(struct sc_decoder *decoder) { - sc_decoder_close_sinks(decoder); + sc_frame_source_sinks_close(&decoder->frame_source); av_frame_free(&decoder->frame); avcodec_close(decoder->codec_ctx); avcodec_free_context(&decoder->codec_ctx); } -static bool -push_frame_to_sinks(struct sc_decoder *decoder, const AVFrame *frame) { - for (unsigned i = 0; i < decoder->sink_count; ++i) { - struct sc_frame_sink *sink = decoder->sinks[i]; - if (!sink->ops->push(sink, frame)) { - return false; - } - } - - return true; -} - static bool sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) { bool is_config = packet->pts == AV_NOPTS_VALUE; @@ -124,7 +87,8 @@ sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) { } // a frame was received - bool ok = push_frame_to_sinks(decoder, decoder->frame); + bool ok = sc_frame_source_sinks_push(&decoder->frame_source, + decoder->frame); // A frame lost should not make the whole pipeline fail. The error, if // any, is already logged. (void) ok; @@ -157,7 +121,7 @@ sc_decoder_packet_sink_push(struct sc_packet_sink *sink, void sc_decoder_init(struct sc_decoder *decoder, const char *name) { decoder->name = name; // statically allocated - decoder->sink_count = 0; + sc_frame_source_init(&decoder->frame_source); static const struct sc_packet_sink_ops ops = { .open = sc_decoder_packet_sink_open, @@ -167,11 +131,3 @@ sc_decoder_init(struct sc_decoder *decoder, const char *name) { decoder->packet_sink.ops = &ops; } - -void -sc_decoder_add_sink(struct sc_decoder *decoder, struct sc_frame_sink *sink) { - assert(decoder->sink_count < SC_DECODER_MAX_SINKS); - assert(sink); - assert(sink->ops); - decoder->sinks[decoder->sink_count++] = sink; -} diff --git a/app/src/decoder.h b/app/src/decoder.h index aace1af6fd..87aaf6a2e3 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -3,22 +3,19 @@ #include "common.h" +#include "trait/frame_source.h" #include "trait/packet_sink.h" #include #include #include -#define SC_DECODER_MAX_SINKS 2 - struct sc_decoder { struct sc_packet_sink packet_sink; // packet sink trait + struct sc_frame_source frame_source; // frame source trait const char *name; // must be statically allocated (e.g. a string literal) - struct sc_frame_sink *sinks[SC_DECODER_MAX_SINKS]; - unsigned sink_count; - AVCodecContext *codec_ctx; AVFrame *frame; }; @@ -27,7 +24,4 @@ struct sc_decoder { void sc_decoder_init(struct sc_decoder *decoder, const char *name); -void -sc_decoder_add_sink(struct sc_decoder *decoder, struct sc_frame_sink *sink); - #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index d9625a44d4..54858c018a 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -665,7 +665,8 @@ scrcpy(struct scrcpy_options *options) { } screen_initialized = true; - sc_decoder_add_sink(&s->video_decoder, &s->screen.frame_sink); + sc_frame_source_add_sink(&s->video_decoder.frame_source, + &s->screen.frame_sink); } #ifdef HAVE_V4L2 @@ -675,7 +676,8 @@ scrcpy(struct scrcpy_options *options) { goto end; } - sc_decoder_add_sink(&s->video_decoder, &s->v4l2_sink.frame_sink); + sc_frame_source_add_sink(&s->video_decoder.frame_source, + &s->v4l2_sink.frame_sink); v4l2_sink_initialized = true; } From 1230149fdd73e9e90230b0f592612edbe078650a Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 21:30:24 +0100 Subject: [PATCH 101/118] Use delay buffer as a frame source/sink The components needing delayed frames (sc_screen and sc_v4l2_sink) managed a sc_video_buffer instance, which itself embedded a sc_frame_buffer instance (to keep only the most recent frame). In theory, these components should not be aware of delaying: they should just receive AVFrames later, and only handle a sc_frame_buffer. Therefore, refactor sc_delay_buffer as a frame source (it consumes) frames) and a frame sink (it produces frames, after some delay), and plug an instance in the pipeline only when a delay is requested. This also removes the need for a specific sc_video_buffer. PR #3757 --- app/meson.build | 1 - app/src/decoder.c | 1 - app/src/delay_buffer.c | 158 ++++++++++++++++++++--------------------- app/src/delay_buffer.h | 33 +++------ app/src/scrcpy.c | 26 +++++-- app/src/screen.c | 40 +++-------- app/src/screen.h | 6 +- app/src/v4l2_sink.c | 61 ++++++---------- app/src/v4l2_sink.h | 7 +- app/src/video_buffer.c | 84 ---------------------- app/src/video_buffer.h | 47 ------------ 11 files changed, 147 insertions(+), 317 deletions(-) delete mode 100644 app/src/video_buffer.c delete mode 100644 app/src/video_buffer.h diff --git a/app/meson.build b/app/meson.build index 9f73b43419..392fa6d0cf 100644 --- a/app/meson.build +++ b/app/meson.build @@ -29,7 +29,6 @@ src = [ 'src/screen.c', 'src/server.c', 'src/version.c', - 'src/video_buffer.c', 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/util/acksync.c', diff --git a/app/src/decoder.c b/app/src/decoder.c index 2931c1eccb..a8168f6616 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -4,7 +4,6 @@ #include #include "events.h" -#include "video_buffer.h" #include "trait/frame_sink.h" #include "util/log.h" diff --git a/app/src/delay_buffer.c b/app/src/delay_buffer.c index 95d47c9c85..72af367214 100644 --- a/app/src/delay_buffer.c +++ b/app/src/delay_buffer.c @@ -10,6 +10,9 @@ #define SC_BUFFERING_NDEBUG // comment to debug +/** Downcast frame_sink to sc_delay_buffer */ +#define DOWNCAST(SINK) container_of(SINK, struct sc_delay_buffer, frame_sink) + static bool sc_delayed_frame_init(struct sc_delayed_frame *dframe, const AVFrame *frame) { dframe->frame = av_frame_alloc(); @@ -33,11 +36,6 @@ sc_delayed_frame_destroy(struct sc_delayed_frame *dframe) { av_frame_free(&dframe->frame); } -static bool -sc_delay_buffer_offer(struct sc_delay_buffer *db, const AVFrame *frame) { - return db->cbs->on_new_frame(db, frame, db->cbs_userdata); -} - static int run_buffering(void *data) { struct sc_delay_buffer *db = data; @@ -87,12 +85,12 @@ run_buffering(void *data) { pts, dframe.push_date, sc_tick_now()); #endif - bool ok = sc_delay_buffer_offer(db, dframe.frame); + bool ok = sc_frame_source_sinks_push(&db->frame_source, dframe.frame); sc_delayed_frame_destroy(&dframe); if (!ok) { LOGE("Delayed frame could not be pushed, stopping"); sc_mutex_lock(&db->b.mutex); - // Prevent to push any new packet + // Prevent to push any new frame db->b.stopped = true; sc_mutex_unlock(&db->b.mutex); goto stopped; @@ -113,92 +111,77 @@ run_buffering(void *data) { return 0; } -bool -sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay, - const struct sc_delay_buffer_callbacks *cbs, - void *cbs_userdata) { - assert(delay >= 0); - - if (delay) { - bool ok = sc_mutex_init(&db->b.mutex); - if (!ok) { - return false; - } - - ok = sc_cond_init(&db->b.queue_cond); - if (!ok) { - sc_mutex_destroy(&db->b.mutex); - return false; - } +static bool +sc_delay_buffer_frame_sink_open(struct sc_frame_sink *sink, + const AVCodecContext *ctx) { + struct sc_delay_buffer *db = DOWNCAST(sink); + (void) ctx; - ok = sc_cond_init(&db->b.wait_cond); - if (!ok) { - sc_cond_destroy(&db->b.queue_cond); - sc_mutex_destroy(&db->b.mutex); - return false; - } + bool ok = sc_mutex_init(&db->b.mutex); + if (!ok) { + return false; + } - sc_clock_init(&db->b.clock); - sc_vecdeque_init(&db->b.queue); + ok = sc_cond_init(&db->b.queue_cond); + if (!ok) { + goto error_destroy_mutex; } - assert(cbs); - assert(cbs->on_new_frame); + ok = sc_cond_init(&db->b.wait_cond); + if (!ok) { + goto error_destroy_queue_cond; + } - db->delay = delay; - db->cbs = cbs; - db->cbs_userdata = cbs_userdata; + sc_clock_init(&db->b.clock); + sc_vecdeque_init(&db->b.queue); - return true; -} + if (!sc_frame_source_sinks_open(&db->frame_source, ctx)) { + goto error_destroy_wait_cond; + } -bool -sc_delay_buffer_start(struct sc_delay_buffer *db) { - if (db->delay) { - bool ok = - sc_thread_create(&db->b.thread, run_buffering, "scrcpy-dbuf", db); - if (!ok) { - LOGE("Could not start buffering thread"); - return false; - } + ok = sc_thread_create(&db->b.thread, run_buffering, "scrcpy-dbuf", db); + if (!ok) { + LOGE("Could not start buffering thread"); + goto error_close_sinks; } return true; -} -void -sc_delay_buffer_stop(struct sc_delay_buffer *db) { - if (db->delay) { - sc_mutex_lock(&db->b.mutex); - db->b.stopped = true; - sc_cond_signal(&db->b.queue_cond); - sc_cond_signal(&db->b.wait_cond); - sc_mutex_unlock(&db->b.mutex); - } -} +error_close_sinks: + sc_frame_source_sinks_close(&db->frame_source); +error_destroy_wait_cond: + sc_cond_destroy(&db->b.wait_cond); +error_destroy_queue_cond: + sc_cond_destroy(&db->b.queue_cond); +error_destroy_mutex: + sc_mutex_destroy(&db->b.mutex); -void -sc_delay_buffer_join(struct sc_delay_buffer *db) { - if (db->delay) { - sc_thread_join(&db->b.thread, NULL); - } + return false; } -void -sc_delay_buffer_destroy(struct sc_delay_buffer *db) { - if (db->delay) { - sc_cond_destroy(&db->b.wait_cond); - sc_cond_destroy(&db->b.queue_cond); - sc_mutex_destroy(&db->b.mutex); - } +static void +sc_delay_buffer_frame_sink_close(struct sc_frame_sink *sink) { + struct sc_delay_buffer *db = DOWNCAST(sink); + + sc_mutex_lock(&db->b.mutex); + db->b.stopped = true; + sc_cond_signal(&db->b.queue_cond); + sc_cond_signal(&db->b.wait_cond); + sc_mutex_unlock(&db->b.mutex); + + sc_thread_join(&db->b.thread, NULL); + + sc_frame_source_sinks_close(&db->frame_source); + + sc_cond_destroy(&db->b.wait_cond); + sc_cond_destroy(&db->b.queue_cond); + sc_mutex_destroy(&db->b.mutex); } -bool -sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame) { - if (!db->delay) { - // No buffering - return sc_delay_buffer_offer(db, frame); - } +static bool +sc_delay_buffer_frame_sink_push(struct sc_frame_sink *sink, + const AVFrame *frame) { + struct sc_delay_buffer *db = DOWNCAST(sink); sc_mutex_lock(&db->b.mutex); @@ -213,11 +196,11 @@ sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame) { if (db->b.clock.count == 1) { sc_mutex_unlock(&db->b.mutex); - // First frame, offer it immediately, for two reasons: + // First frame, push it immediately, for two reasons: // - not to delay the opening of the scrcpy window // - the buffering estimation needs at least two clock points, so it // could not handle the first frame - return sc_delay_buffer_offer(db, frame); + return sc_frame_source_sinks_push(&db->frame_source, frame); } struct sc_delayed_frame dframe; @@ -244,3 +227,20 @@ sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame) { return true; } + +void +sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay) { + assert(delay > 0); + + db->delay = delay; + + sc_frame_source_init(&db->frame_source); + + static const struct sc_frame_sink_ops ops = { + .open = sc_delay_buffer_frame_sink_open, + .close = sc_delay_buffer_frame_sink_close, + .push = sc_delay_buffer_frame_sink_push, + }; + + db->frame_sink.ops = &ops; +} diff --git a/app/src/delay_buffer.h b/app/src/delay_buffer.h index 9e5347c7d3..4cb981c867 100644 --- a/app/src/delay_buffer.h +++ b/app/src/delay_buffer.h @@ -6,6 +6,8 @@ #include #include "clock.h" +#include "trait/frame_source.h" +#include "trait/frame_sink.h" #include "util/thread.h" #include "util/tick.h" #include "util/vecdeque.h" @@ -23,9 +25,11 @@ struct sc_delayed_frame { struct sc_delayed_frame_queue SC_VECDEQUE(struct sc_delayed_frame); struct sc_delay_buffer { + struct sc_frame_source frame_source; // frame source trait + struct sc_frame_sink frame_sink; // frame sink trait + sc_tick delay; - // only if delay > 0 struct { sc_thread thread; sc_mutex mutex; @@ -36,9 +40,6 @@ struct sc_delay_buffer { struct sc_delayed_frame_queue queue; bool stopped; } b; // buffering - - const struct sc_delay_buffer_callbacks *cbs; - void *cbs_userdata; }; struct sc_delay_buffer_callbacks { @@ -46,24 +47,12 @@ struct sc_delay_buffer_callbacks { void *userdata); }; -bool -sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay, - const struct sc_delay_buffer_callbacks *cbs, - void *cbs_userdata); - -bool -sc_delay_buffer_start(struct sc_delay_buffer *db); - +/** + * Initialize a delay buffer. + * + * \param delay a (strictly) positive delay + */ void -sc_delay_buffer_stop(struct sc_delay_buffer *db); - -void -sc_delay_buffer_join(struct sc_delay_buffer *db); - -void -sc_delay_buffer_destroy(struct sc_delay_buffer *db); - -bool -sc_delay_buffer_push(struct sc_delay_buffer *db, const AVFrame *frame); +sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay); #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 54858c018a..2688cab6f7 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -15,6 +15,7 @@ #include "controller.h" #include "decoder.h" +#include "delay_buffer.h" #include "demuxer.h" #include "events.h" #include "file_pusher.h" @@ -45,8 +46,10 @@ struct scrcpy { struct sc_decoder video_decoder; struct sc_decoder audio_decoder; struct sc_recorder recorder; + struct sc_delay_buffer display_buffer; #ifdef HAVE_V4L2 struct sc_v4l2_sink v4l2_sink; + struct sc_delay_buffer v4l2_buffer; #endif struct sc_controller controller; struct sc_file_pusher file_pusher; @@ -657,7 +660,6 @@ scrcpy(struct scrcpy_options *options) { .mipmaps = options->mipmaps, .fullscreen = options->fullscreen, .start_fps_counter = options->start_fps_counter, - .buffering_time = options->display_buffer, }; if (!sc_screen_init(&s->screen, &screen_params)) { @@ -665,19 +667,31 @@ scrcpy(struct scrcpy_options *options) { } screen_initialized = true; - sc_frame_source_add_sink(&s->video_decoder.frame_source, - &s->screen.frame_sink); + struct sc_frame_source *src = &s->video_decoder.frame_source; + if (options->display_buffer) { + sc_delay_buffer_init(&s->display_buffer, options->display_buffer); + sc_frame_source_add_sink(src, &s->display_buffer.frame_sink); + src = &s->display_buffer.frame_source; + } + + sc_frame_source_add_sink(src, &s->screen.frame_sink); } #ifdef HAVE_V4L2 if (options->v4l2_device) { if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device, - info->frame_size, options->v4l2_buffer)) { + info->frame_size)) { goto end; } - sc_frame_source_add_sink(&s->video_decoder.frame_source, - &s->v4l2_sink.frame_sink); + struct sc_frame_source *src = &s->video_decoder.frame_source; + if (options->v4l2_buffer) { + sc_delay_buffer_init(&s->v4l2_buffer, options->v4l2_buffer); + sc_frame_source_add_sink(src, &s->v4l2_buffer.frame_sink); + src = &s->v4l2_buffer.frame_source; + } + + sc_frame_source_add_sink(src, &s->v4l2_sink.frame_sink); v4l2_sink_initialized = true; } diff --git a/app/src/screen.c b/app/src/screen.c index ce2e74bbd9..b814ada1c3 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -7,7 +7,6 @@ #include "events.h" #include "icon.h" #include "options.h" -#include "video_buffer.h" #include "util/log.h" #define DISPLAY_MARGINS 96 @@ -359,14 +358,12 @@ sc_screen_frame_sink_close(struct sc_frame_sink *sink) { static bool sc_screen_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { struct sc_screen *screen = DOWNCAST(sink); - return sc_video_buffer_push(&screen->vb, frame); -} -static bool -sc_video_buffer_on_new_frame(struct sc_video_buffer *vb, bool previous_skipped, - void *userdata) { - (void) vb; - struct sc_screen *screen = userdata; + bool previous_skipped; + bool ok = sc_frame_buffer_push(&screen->fb, frame, &previous_skipped); + if (!ok) { + return false; + } if (previous_skipped) { sc_fps_counter_add_skipped_frame(&screen->fps_counter); @@ -404,23 +401,13 @@ sc_screen_init(struct sc_screen *screen, screen->req.fullscreen = params->fullscreen; screen->req.start_fps_counter = params->start_fps_counter; - static const struct sc_video_buffer_callbacks cbs = { - .on_new_frame = sc_video_buffer_on_new_frame, - }; - - bool ok = sc_video_buffer_init(&screen->vb, params->buffering_time, &cbs, - screen); + bool ok = sc_frame_buffer_init(&screen->fb); if (!ok) { return false; } - ok = sc_video_buffer_start(&screen->vb); - if (!ok) { - goto error_destroy_video_buffer; - } - if (!sc_fps_counter_init(&screen->fps_counter)) { - goto error_stop_and_join_video_buffer; + goto error_destroy_frame_buffer; } screen->frame_size = params->frame_size; @@ -552,11 +539,8 @@ sc_screen_init(struct sc_screen *screen, SDL_DestroyWindow(screen->window); error_destroy_fps_counter: sc_fps_counter_destroy(&screen->fps_counter); -error_stop_and_join_video_buffer: - sc_video_buffer_stop(&screen->vb); - sc_video_buffer_join(&screen->vb); -error_destroy_video_buffer: - sc_video_buffer_destroy(&screen->vb); +error_destroy_frame_buffer: + sc_frame_buffer_destroy(&screen->fb); return false; } @@ -593,13 +577,11 @@ sc_screen_hide_window(struct sc_screen *screen) { void sc_screen_interrupt(struct sc_screen *screen) { - sc_video_buffer_stop(&screen->vb); sc_fps_counter_interrupt(&screen->fps_counter); } void sc_screen_join(struct sc_screen *screen) { - sc_video_buffer_join(&screen->vb); sc_fps_counter_join(&screen->fps_counter); } @@ -613,7 +595,7 @@ sc_screen_destroy(struct sc_screen *screen) { SDL_DestroyRenderer(screen->renderer); SDL_DestroyWindow(screen->window); sc_fps_counter_destroy(&screen->fps_counter); - sc_video_buffer_destroy(&screen->vb); + sc_frame_buffer_destroy(&screen->fb); } static void @@ -719,7 +701,7 @@ update_texture(struct sc_screen *screen, const AVFrame *frame) { static bool sc_screen_update_frame(struct sc_screen *screen) { av_frame_unref(screen->frame); - sc_video_buffer_consume(&screen->vb, screen->frame); + sc_frame_buffer_consume(&screen->fb, screen->frame); AVFrame *frame = screen->frame; sc_fps_counter_add_rendered_frame(&screen->fps_counter); diff --git a/app/src/screen.h b/app/src/screen.h index 0952c79c36..28afea4075 100644 --- a/app/src/screen.h +++ b/app/src/screen.h @@ -10,12 +10,12 @@ #include "controller.h" #include "coords.h" #include "fps_counter.h" +#include "frame_buffer.h" #include "input_manager.h" #include "opengl.h" #include "trait/key_processor.h" #include "trait/frame_sink.h" #include "trait/mouse_processor.h" -#include "video_buffer.h" struct sc_screen { struct sc_frame_sink frame_sink; // frame sink trait @@ -25,7 +25,7 @@ struct sc_screen { #endif struct sc_input_manager im; - struct sc_video_buffer vb; + struct sc_frame_buffer fb; struct sc_fps_counter fps_counter; // The initial requested window properties @@ -93,8 +93,6 @@ struct sc_screen_params { bool fullscreen; bool start_fps_counter; - - sc_tick buffering_time; }; // initialize screen, create window, renderer and texture (window is hidden) diff --git a/app/src/v4l2_sink.c b/app/src/v4l2_sink.c index 5dfe37bc45..fe11614a91 100644 --- a/app/src/v4l2_sink.c +++ b/app/src/v4l2_sink.c @@ -126,7 +126,7 @@ run_v4l2_sink(void *data) { vs->has_frame = false; sc_mutex_unlock(&vs->mutex); - sc_video_buffer_consume(&vs->vb, vs->frame); + sc_frame_buffer_consume(&vs->fb, vs->frame); bool ok = encode_and_write_frame(vs, vs->frame); av_frame_unref(vs->frame); @@ -141,44 +141,19 @@ run_v4l2_sink(void *data) { return 0; } -static bool -sc_video_buffer_on_new_frame(struct sc_video_buffer *vb, bool previous_skipped, - void *userdata) { - (void) vb; - struct sc_v4l2_sink *vs = userdata; - - if (!previous_skipped) { - sc_mutex_lock(&vs->mutex); - vs->has_frame = true; - sc_cond_signal(&vs->cond); - sc_mutex_unlock(&vs->mutex); - } - - return true; -} - static bool sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) { assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P); (void) ctx; - static const struct sc_video_buffer_callbacks cbs = { - .on_new_frame = sc_video_buffer_on_new_frame, - }; - - bool ok = sc_video_buffer_init(&vs->vb, vs->buffering_time, &cbs, vs); + bool ok = sc_frame_buffer_init(&vs->fb); if (!ok) { return false; } - ok = sc_video_buffer_start(&vs->vb); - if (!ok) { - goto error_video_buffer_destroy; - } - ok = sc_mutex_init(&vs->mutex); if (!ok) { - goto error_video_buffer_stop_and_join; + goto error_frame_buffer_destroy; } ok = sc_cond_init(&vs->cond); @@ -303,11 +278,8 @@ sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) { sc_cond_destroy(&vs->cond); error_mutex_destroy: sc_mutex_destroy(&vs->mutex); -error_video_buffer_stop_and_join: - sc_video_buffer_stop(&vs->vb); - sc_video_buffer_join(&vs->vb); -error_video_buffer_destroy: - sc_video_buffer_destroy(&vs->vb); +error_frame_buffer_destroy: + sc_frame_buffer_destroy(&vs->fb); return false; } @@ -319,10 +291,7 @@ sc_v4l2_sink_close(struct sc_v4l2_sink *vs) { sc_cond_signal(&vs->cond); sc_mutex_unlock(&vs->mutex); - sc_video_buffer_stop(&vs->vb); - sc_thread_join(&vs->thread, NULL); - sc_video_buffer_join(&vs->vb); av_packet_free(&vs->packet); av_frame_free(&vs->frame); @@ -332,12 +301,25 @@ sc_v4l2_sink_close(struct sc_v4l2_sink *vs) { avformat_free_context(vs->format_ctx); sc_cond_destroy(&vs->cond); sc_mutex_destroy(&vs->mutex); - sc_video_buffer_destroy(&vs->vb); + sc_frame_buffer_destroy(&vs->fb); } static bool sc_v4l2_sink_push(struct sc_v4l2_sink *vs, const AVFrame *frame) { - return sc_video_buffer_push(&vs->vb, frame); + bool previous_skipped; + bool ok = sc_frame_buffer_push(&vs->fb, frame, &previous_skipped); + if (!ok) { + return false; + } + + if (!previous_skipped) { + sc_mutex_lock(&vs->mutex); + vs->has_frame = true; + sc_cond_signal(&vs->cond); + sc_mutex_unlock(&vs->mutex); + } + + return true; } static bool @@ -360,7 +342,7 @@ sc_v4l2_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { bool sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name, - struct sc_size frame_size, sc_tick buffering_time) { + struct sc_size frame_size) { vs->device_name = strdup(device_name); if (!vs->device_name) { LOGE("Could not strdup v4l2 device name"); @@ -368,7 +350,6 @@ sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name, } vs->frame_size = frame_size; - vs->buffering_time = buffering_time; static const struct sc_frame_sink_ops ops = { .open = sc_v4l2_frame_sink_open, diff --git a/app/src/v4l2_sink.h b/app/src/v4l2_sink.h index 339a61f20b..789e31c365 100644 --- a/app/src/v4l2_sink.h +++ b/app/src/v4l2_sink.h @@ -8,19 +8,18 @@ #include "coords.h" #include "trait/frame_sink.h" -#include "video_buffer.h" +#include "frame_buffer.h" #include "util/tick.h" struct sc_v4l2_sink { struct sc_frame_sink frame_sink; // frame sink trait - struct sc_video_buffer vb; + struct sc_frame_buffer fb; AVFormatContext *format_ctx; AVCodecContext *encoder_ctx; char *device_name; struct sc_size frame_size; - sc_tick buffering_time; sc_thread thread; sc_mutex mutex; @@ -35,7 +34,7 @@ struct sc_v4l2_sink { bool sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name, - struct sc_size frame_size, sc_tick buffering_time); + struct sc_size frame_size); void sc_v4l2_sink_destroy(struct sc_v4l2_sink *vs); diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c deleted file mode 100644 index da47a0b521..0000000000 --- a/app/src/video_buffer.c +++ /dev/null @@ -1,84 +0,0 @@ -#include "video_buffer.h" - -#include -#include - -#include -#include - -#include "util/log.h" - -static bool -sc_delay_buffer_on_new_frame(struct sc_delay_buffer *db, const AVFrame *frame, - void *userdata) { - (void) db; - - struct sc_video_buffer *vb = userdata; - - bool previous_skipped; - bool ok = sc_frame_buffer_push(&vb->fb, frame, &previous_skipped); - if (!ok) { - return false; - } - - return vb->cbs->on_new_frame(vb, previous_skipped, vb->cbs_userdata); -} - -bool -sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick delay, - const struct sc_video_buffer_callbacks *cbs, - void *cbs_userdata) { - bool ok = sc_frame_buffer_init(&vb->fb); - if (!ok) { - return false; - } - - static const struct sc_delay_buffer_callbacks db_cbs = { - .on_new_frame = sc_delay_buffer_on_new_frame, - }; - - ok = sc_delay_buffer_init(&vb->db, delay, &db_cbs, vb); - if (!ok) { - sc_frame_buffer_destroy(&vb->fb); - return false; - } - - assert(cbs); - assert(cbs->on_new_frame); - - vb->cbs = cbs; - vb->cbs_userdata = cbs_userdata; - - return true; -} - -bool -sc_video_buffer_start(struct sc_video_buffer *vb) { - return sc_delay_buffer_start(&vb->db); -} - -void -sc_video_buffer_stop(struct sc_video_buffer *vb) { - return sc_delay_buffer_stop(&vb->db); -} - -void -sc_video_buffer_join(struct sc_video_buffer *vb) { - return sc_delay_buffer_join(&vb->db); -} - -void -sc_video_buffer_destroy(struct sc_video_buffer *vb) { - sc_frame_buffer_destroy(&vb->fb); - sc_delay_buffer_destroy(&vb->db); -} - -bool -sc_video_buffer_push(struct sc_video_buffer *vb, const AVFrame *frame) { - return sc_delay_buffer_push(&vb->db, frame); -} - -void -sc_video_buffer_consume(struct sc_video_buffer *vb, AVFrame *dst) { - sc_frame_buffer_consume(&vb->fb, dst); -} diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h deleted file mode 100644 index ebca3915ad..0000000000 --- a/app/src/video_buffer.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef SC_VIDEO_BUFFER_H -#define SC_VIDEO_BUFFER_H - -#include "common.h" - -#include - -#include "delay_buffer.h" -#include "frame_buffer.h" - -struct sc_video_buffer { - struct sc_delay_buffer db; - struct sc_frame_buffer fb; - - const struct sc_video_buffer_callbacks *cbs; - void *cbs_userdata; -}; - -struct sc_video_buffer_callbacks { - bool (*on_new_frame)(struct sc_video_buffer *vb, bool previous_skipped, - void *userdata); -}; - -bool -sc_video_buffer_init(struct sc_video_buffer *vb, sc_tick delay, - const struct sc_video_buffer_callbacks *cbs, - void *cbs_userdata); - -bool -sc_video_buffer_start(struct sc_video_buffer *vb); - -void -sc_video_buffer_stop(struct sc_video_buffer *vb); - -void -sc_video_buffer_join(struct sc_video_buffer *vb); - -void -sc_video_buffer_destroy(struct sc_video_buffer *vb); - -bool -sc_video_buffer_push(struct sc_video_buffer *vb, const AVFrame *frame); - -void -sc_video_buffer_consume(struct sc_video_buffer *vb, AVFrame *dst); - -#endif From 48a537d45c48bdff45c85197584f5570e9d9dc57 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 21:30:24 +0100 Subject: [PATCH 102/118] Remove anonymous struct in delay buffer For clarity, the fields used only when a delay was set were wrapped in an anonymous structure. Now that the delay buffer has been extracted to a separate component, the delay is necessarily set (it may not be 0), so the fields are always used. PR #3757 --- app/src/delay_buffer.c | 94 +++++++++++++++++++++--------------------- app/src/delay_buffer.h | 18 ++++---- 2 files changed, 55 insertions(+), 57 deletions(-) diff --git a/app/src/delay_buffer.c b/app/src/delay_buffer.c index 72af367214..2694eb019a 100644 --- a/app/src/delay_buffer.c +++ b/app/src/delay_buffer.c @@ -43,37 +43,37 @@ run_buffering(void *data) { assert(db->delay > 0); for (;;) { - sc_mutex_lock(&db->b.mutex); + sc_mutex_lock(&db->mutex); - while (!db->b.stopped && sc_vecdeque_is_empty(&db->b.queue)) { - sc_cond_wait(&db->b.queue_cond, &db->b.mutex); + while (!db->stopped && sc_vecdeque_is_empty(&db->queue)) { + sc_cond_wait(&db->queue_cond, &db->mutex); } - if (db->b.stopped) { - sc_mutex_unlock(&db->b.mutex); + if (db->stopped) { + sc_mutex_unlock(&db->mutex); goto stopped; } - struct sc_delayed_frame dframe = sc_vecdeque_pop(&db->b.queue); + struct sc_delayed_frame dframe = sc_vecdeque_pop(&db->queue); sc_tick max_deadline = sc_tick_now() + db->delay; // PTS (written by the server) are expressed in microseconds sc_tick pts = SC_TICK_FROM_US(dframe.frame->pts); bool timed_out = false; - while (!db->b.stopped && !timed_out) { - sc_tick deadline = sc_clock_to_system_time(&db->b.clock, pts) + while (!db->stopped && !timed_out) { + sc_tick deadline = sc_clock_to_system_time(&db->clock, pts) + db->delay; if (deadline > max_deadline) { deadline = max_deadline; } timed_out = - !sc_cond_timedwait(&db->b.wait_cond, &db->b.mutex, deadline); + !sc_cond_timedwait(&db->wait_cond, &db->mutex, deadline); } - bool stopped = db->b.stopped; - sc_mutex_unlock(&db->b.mutex); + bool stopped = db->stopped; + sc_mutex_unlock(&db->mutex); if (stopped) { sc_delayed_frame_destroy(&dframe); @@ -89,20 +89,20 @@ run_buffering(void *data) { sc_delayed_frame_destroy(&dframe); if (!ok) { LOGE("Delayed frame could not be pushed, stopping"); - sc_mutex_lock(&db->b.mutex); + sc_mutex_lock(&db->mutex); // Prevent to push any new frame - db->b.stopped = true; - sc_mutex_unlock(&db->b.mutex); + db->stopped = true; + sc_mutex_unlock(&db->mutex); goto stopped; } } stopped: - assert(db->b.stopped); + assert(db->stopped); // Flush queue - while (!sc_vecdeque_is_empty(&db->b.queue)) { - struct sc_delayed_frame *dframe = sc_vecdeque_popref(&db->b.queue); + while (!sc_vecdeque_is_empty(&db->queue)) { + struct sc_delayed_frame *dframe = sc_vecdeque_popref(&db->queue); sc_delayed_frame_destroy(dframe); } @@ -117,29 +117,29 @@ sc_delay_buffer_frame_sink_open(struct sc_frame_sink *sink, struct sc_delay_buffer *db = DOWNCAST(sink); (void) ctx; - bool ok = sc_mutex_init(&db->b.mutex); + bool ok = sc_mutex_init(&db->mutex); if (!ok) { return false; } - ok = sc_cond_init(&db->b.queue_cond); + ok = sc_cond_init(&db->queue_cond); if (!ok) { goto error_destroy_mutex; } - ok = sc_cond_init(&db->b.wait_cond); + ok = sc_cond_init(&db->wait_cond); if (!ok) { goto error_destroy_queue_cond; } - sc_clock_init(&db->b.clock); - sc_vecdeque_init(&db->b.queue); + sc_clock_init(&db->clock); + sc_vecdeque_init(&db->queue); if (!sc_frame_source_sinks_open(&db->frame_source, ctx)) { goto error_destroy_wait_cond; } - ok = sc_thread_create(&db->b.thread, run_buffering, "scrcpy-dbuf", db); + ok = sc_thread_create(&db->thread, run_buffering, "scrcpy-dbuf", db); if (!ok) { LOGE("Could not start buffering thread"); goto error_close_sinks; @@ -150,11 +150,11 @@ sc_delay_buffer_frame_sink_open(struct sc_frame_sink *sink, error_close_sinks: sc_frame_source_sinks_close(&db->frame_source); error_destroy_wait_cond: - sc_cond_destroy(&db->b.wait_cond); + sc_cond_destroy(&db->wait_cond); error_destroy_queue_cond: - sc_cond_destroy(&db->b.queue_cond); + sc_cond_destroy(&db->queue_cond); error_destroy_mutex: - sc_mutex_destroy(&db->b.mutex); + sc_mutex_destroy(&db->mutex); return false; } @@ -163,19 +163,19 @@ static void sc_delay_buffer_frame_sink_close(struct sc_frame_sink *sink) { struct sc_delay_buffer *db = DOWNCAST(sink); - sc_mutex_lock(&db->b.mutex); - db->b.stopped = true; - sc_cond_signal(&db->b.queue_cond); - sc_cond_signal(&db->b.wait_cond); - sc_mutex_unlock(&db->b.mutex); + sc_mutex_lock(&db->mutex); + db->stopped = true; + sc_cond_signal(&db->queue_cond); + sc_cond_signal(&db->wait_cond); + sc_mutex_unlock(&db->mutex); - sc_thread_join(&db->b.thread, NULL); + sc_thread_join(&db->thread, NULL); sc_frame_source_sinks_close(&db->frame_source); - sc_cond_destroy(&db->b.wait_cond); - sc_cond_destroy(&db->b.queue_cond); - sc_mutex_destroy(&db->b.mutex); + sc_cond_destroy(&db->wait_cond); + sc_cond_destroy(&db->queue_cond); + sc_mutex_destroy(&db->mutex); } static bool @@ -183,19 +183,19 @@ sc_delay_buffer_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { struct sc_delay_buffer *db = DOWNCAST(sink); - sc_mutex_lock(&db->b.mutex); + sc_mutex_lock(&db->mutex); - if (db->b.stopped) { - sc_mutex_unlock(&db->b.mutex); + if (db->stopped) { + sc_mutex_unlock(&db->mutex); return false; } sc_tick pts = SC_TICK_FROM_US(frame->pts); - sc_clock_update(&db->b.clock, sc_tick_now(), pts); - sc_cond_signal(&db->b.wait_cond); + sc_clock_update(&db->clock, sc_tick_now(), pts); + sc_cond_signal(&db->wait_cond); - if (db->b.clock.count == 1) { - sc_mutex_unlock(&db->b.mutex); + if (db->clock.count == 1) { + sc_mutex_unlock(&db->mutex); // First frame, push it immediately, for two reasons: // - not to delay the opening of the scrcpy window // - the buffering estimation needs at least two clock points, so it @@ -206,7 +206,7 @@ sc_delay_buffer_frame_sink_push(struct sc_frame_sink *sink, struct sc_delayed_frame dframe; bool ok = sc_delayed_frame_init(&dframe, frame); if (!ok) { - sc_mutex_unlock(&db->b.mutex); + sc_mutex_unlock(&db->mutex); return false; } @@ -214,16 +214,16 @@ sc_delay_buffer_frame_sink_push(struct sc_frame_sink *sink, dframe.push_date = sc_tick_now(); #endif - ok = sc_vecdeque_push(&db->b.queue, dframe); + ok = sc_vecdeque_push(&db->queue, dframe); if (!ok) { - sc_mutex_unlock(&db->b.mutex); + sc_mutex_unlock(&db->mutex); LOG_OOM(); return false; } - sc_cond_signal(&db->b.queue_cond); + sc_cond_signal(&db->queue_cond); - sc_mutex_unlock(&db->b.mutex); + sc_mutex_unlock(&db->mutex); return true; } diff --git a/app/src/delay_buffer.h b/app/src/delay_buffer.h index 4cb981c867..96fbaa3d0d 100644 --- a/app/src/delay_buffer.h +++ b/app/src/delay_buffer.h @@ -30,16 +30,14 @@ struct sc_delay_buffer { sc_tick delay; - struct { - sc_thread thread; - sc_mutex mutex; - sc_cond queue_cond; - sc_cond wait_cond; - - struct sc_clock clock; - struct sc_delayed_frame_queue queue; - bool stopped; - } b; // buffering + sc_thread thread; + sc_mutex mutex; + sc_cond queue_cond; + sc_cond wait_cond; + + struct sc_clock clock; + struct sc_delayed_frame_queue queue; + bool stopped; }; struct sc_delay_buffer_callbacks { From 9b3ca208bfe21ceb303798c2f05e2364aa364880 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 22:13:48 +0100 Subject: [PATCH 103/118] Accept clock estimation with a single point If there is only one point, assume the slope is 1. PR #3757 --- app/src/clock.c | 21 +++++++++++++-------- app/src/delay_buffer.c | 6 ++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/app/src/clock.c b/app/src/clock.c index bb2430fdf7..3e1a794dcb 100644 --- a/app/src/clock.c +++ b/app/src/clock.c @@ -18,7 +18,15 @@ sc_clock_init(struct sc_clock *clock) { static void sc_clock_estimate(struct sc_clock *clock, double *out_slope, sc_tick *out_offset) { - assert(clock->count > 1); // two points are necessary + assert(clock->count); + + if (clock->count == 1) { + // If there is only 1 point, we can't compute a slope. Assume it is 1. + struct sc_clock_point *single_point = &clock->right_sum; + *out_slope = 1; + *out_offset = single_point->system - single_point->stream; + return; + } struct sc_clock_point left_avg = { .system = clock->left_sum.system / (clock->count / 2), @@ -93,19 +101,16 @@ sc_clock_update(struct sc_clock *clock, sc_tick system, sc_tick stream) { clock->head = (clock->head + 1) % SC_CLOCK_RANGE; - if (clock->count > 1) { - // Update estimation - sc_clock_estimate(clock, &clock->slope, &clock->offset); + // Update estimation + sc_clock_estimate(clock, &clock->slope, &clock->offset); #ifndef SC_CLOCK_NDEBUG - LOGD("Clock estimation: %f * pts + %" PRItick, - clock->slope, clock->offset); + LOGD("Clock estimation: %f * pts + %" PRItick, clock->slope, clock->offset); #endif - } } sc_tick sc_clock_to_system_time(struct sc_clock *clock, sc_tick stream) { - assert(clock->count > 1); // sc_clock_update() must have been called + assert(clock->count); // sc_clock_update() must have been called return (sc_tick) (stream * clock->slope) + clock->offset; } diff --git a/app/src/delay_buffer.c b/app/src/delay_buffer.c index 2694eb019a..360e2b66a1 100644 --- a/app/src/delay_buffer.c +++ b/app/src/delay_buffer.c @@ -196,10 +196,8 @@ sc_delay_buffer_frame_sink_push(struct sc_frame_sink *sink, if (db->clock.count == 1) { sc_mutex_unlock(&db->mutex); - // First frame, push it immediately, for two reasons: - // - not to delay the opening of the scrcpy window - // - the buffering estimation needs at least two clock points, so it - // could not handle the first frame + // First frame, push it immediately, not to delay the opening of the + // scrcpy window return sc_frame_source_sinks_push(&db->frame_source, frame); } From e1333f6f3b29fd68239f3991271e63bff316abc0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 22:33:31 +0100 Subject: [PATCH 104/118] Optionally do not delay the first frame A delay buffer delayed all the frames except the first one, to open the scrcpy window immediately and get a picture. Make this feature optional, so that the delay buffer might also be used for audio (especially for simulating a high delay for debugging). PR #3757 --- app/src/delay_buffer.c | 8 ++++---- app/src/delay_buffer.h | 6 +++++- app/src/scrcpy.c | 5 +++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/app/src/delay_buffer.c b/app/src/delay_buffer.c index 360e2b66a1..9d4690a220 100644 --- a/app/src/delay_buffer.c +++ b/app/src/delay_buffer.c @@ -194,10 +194,8 @@ sc_delay_buffer_frame_sink_push(struct sc_frame_sink *sink, sc_clock_update(&db->clock, sc_tick_now(), pts); sc_cond_signal(&db->wait_cond); - if (db->clock.count == 1) { + if (db->first_frame_asap && db->clock.count == 1) { sc_mutex_unlock(&db->mutex); - // First frame, push it immediately, not to delay the opening of the - // scrcpy window return sc_frame_source_sinks_push(&db->frame_source, frame); } @@ -227,10 +225,12 @@ sc_delay_buffer_frame_sink_push(struct sc_frame_sink *sink, } void -sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay) { +sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay, + bool first_frame_asap) { assert(delay > 0); db->delay = delay; + db->first_frame_asap = first_frame_asap; sc_frame_source_init(&db->frame_source); diff --git a/app/src/delay_buffer.h b/app/src/delay_buffer.h index 96fbaa3d0d..53592372c7 100644 --- a/app/src/delay_buffer.h +++ b/app/src/delay_buffer.h @@ -29,6 +29,7 @@ struct sc_delay_buffer { struct sc_frame_sink frame_sink; // frame sink trait sc_tick delay; + bool first_frame_asap; sc_thread thread; sc_mutex mutex; @@ -49,8 +50,11 @@ struct sc_delay_buffer_callbacks { * Initialize a delay buffer. * * \param delay a (strictly) positive delay + * \param first_frame_asap if true, do not delay the first frame (useful for + a video stream). */ void -sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay); +sc_delay_buffer_init(struct sc_delay_buffer *db, sc_tick delay, + bool first_frame_asap); #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 2688cab6f7..dba1bad9e3 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -669,7 +669,8 @@ scrcpy(struct scrcpy_options *options) { struct sc_frame_source *src = &s->video_decoder.frame_source; if (options->display_buffer) { - sc_delay_buffer_init(&s->display_buffer, options->display_buffer); + sc_delay_buffer_init(&s->display_buffer, options->display_buffer, + true); sc_frame_source_add_sink(src, &s->display_buffer.frame_sink); src = &s->display_buffer.frame_source; } @@ -686,7 +687,7 @@ scrcpy(struct scrcpy_options *options) { struct sc_frame_source *src = &s->video_decoder.frame_source; if (options->v4l2_buffer) { - sc_delay_buffer_init(&s->v4l2_buffer, options->v4l2_buffer); + sc_delay_buffer_init(&s->v4l2_buffer, options->v4l2_buffer, true); sc_frame_source_add_sink(src, &s->v4l2_buffer.frame_sink); src = &s->v4l2_buffer.frame_source; } From fbe0f951e113e6055c4365c2986663cb3ef16d0e Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 00:43:20 +0100 Subject: [PATCH 105/118] Add audio player Play the decoded audio using SDL. The audio player frame sink receives the audio frames, resample them and write them to a byte buffer (introduced by this commit). On SDL audio callback (from an internal SDL thread), copy samples from this byte buffer to the SDL audio buffer. The byte buffer is protected by the SDL_AudioDeviceLock(), but it has been designed so that the producer and the consumer may write and read in parallel, provided that they don't access the same slices of the ring-buffer buffer. PR #3757 Co-authored-by: Simon Chan <1330321+yume-chan@users.noreply.github.com> --- BUILD.md | 4 +- app/meson.build | 4 + app/src/audio_player.c | 409 +++++++++++++++++++++++++++++++++++++++++ app/src/audio_player.h | 78 ++++++++ app/src/decoder.c | 6 + app/src/scrcpy.c | 21 ++- app/src/util/average.c | 26 +++ app/src/util/average.h | 40 ++++ 8 files changed, 583 insertions(+), 5 deletions(-) create mode 100644 app/src/audio_player.c create mode 100644 app/src/audio_player.h create mode 100644 app/src/util/average.c create mode 100644 app/src/util/average.h diff --git a/BUILD.md b/BUILD.md index 0c708bdeeb..51f8141e15 100644 --- a/BUILD.md +++ b/BUILD.md @@ -15,7 +15,7 @@ First, you need to install the required packages: sudo apt install ffmpeg libsdl2-2.0-0 adb wget \ gcc git pkg-config meson ninja-build libsdl2-dev \ libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \ - libusb-1.0-0 libusb-1.0-0-dev + libswresample-dev libusb-1.0-0 libusb-1.0-0-dev ``` Then clone the repo and execute the installation script @@ -94,7 +94,7 @@ sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0 # client build dependencies sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \ libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \ - libusb-1.0-0-dev + libswresample-dev libusb-1.0-0-dev # server build dependencies sudo apt install openjdk-11-jdk diff --git a/app/meson.build b/app/meson.build index 392fa6d0cf..723274c930 100644 --- a/app/meson.build +++ b/app/meson.build @@ -4,6 +4,7 @@ src = [ 'src/adb/adb_device.c', 'src/adb/adb_parser.c', 'src/adb/adb_tunnel.c', + 'src/audio_player.c', 'src/cli.c', 'src/clock.c', 'src/compat.c', @@ -32,6 +33,7 @@ src = [ 'src/trait/frame_source.c', 'src/trait/packet_source.c', 'src/util/acksync.c', + 'src/util/average.c', 'src/util/bytebuf.c', 'src/util/file.c', 'src/util/intmap.c', @@ -103,6 +105,7 @@ if not crossbuild_windows dependency('libavformat', version: '>= 57.33'), dependency('libavcodec', version: '>= 57.37'), dependency('libavutil'), + dependency('libswresample'), dependency('sdl2', version: '>= 2.0.5'), ] @@ -138,6 +141,7 @@ else cc.find_library('avcodec-60', dirs: ffmpeg_bin_dir), cc.find_library('avformat-60', dirs: ffmpeg_bin_dir), cc.find_library('avutil-58', dirs: ffmpeg_bin_dir), + cc.find_library('swresample-4', dirs: ffmpeg_bin_dir), ], include_directories: include_directories(ffmpeg_include_dir) ) diff --git a/app/src/audio_player.c b/app/src/audio_player.c new file mode 100644 index 0000000000..78a8ffe1af --- /dev/null +++ b/app/src/audio_player.c @@ -0,0 +1,409 @@ +#include "audio_player.h" + +#include + +#include "util/log.h" + +#define SC_AUDIO_PLAYER_NDEBUG // comment to debug + +/** Downcast frame_sink to sc_audio_player */ +#define DOWNCAST(SINK) container_of(SINK, struct sc_audio_player, frame_sink) + +#define SC_AV_SAMPLE_FMT AV_SAMPLE_FMT_FLT +#define SC_SDL_SAMPLE_FMT AUDIO_F32 + +#define SC_AUDIO_OUTPUT_BUFFER_SAMPLES 240 // 5ms at 48000Hz + +static inline uint32_t +bytes_to_samples(struct sc_audio_player *ap, size_t bytes) { + assert(bytes % (ap->nb_channels * ap->out_bytes_per_sample) == 0); + return bytes / (ap->nb_channels * ap->out_bytes_per_sample); +} + +static inline size_t +samples_to_bytes(struct sc_audio_player *ap, uint32_t samples) { + return samples * ap->nb_channels * ap->out_bytes_per_sample; +} + +static void SDLCALL +sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) { + struct sc_audio_player *ap = userdata; + + // This callback is called with the lock used by SDL_AudioDeviceLock(), so + // the bytebuf is protected + + assert(len_int > 0); + size_t len = len_int; + +#ifndef SC_AUDIO_PLAYER_NDEBUG + LOGD("[Audio] SDL callback requests %" PRIu32 " samples", + bytes_to_samples(ap, len)); +#endif + + size_t read_avail = sc_bytebuf_read_available(&ap->buf); + if (!ap->played) { + uint32_t buffered_samples = bytes_to_samples(ap, read_avail); + + // Part of the buffering is handled by inserting initial silence. The + // remaining (margin) last samples will be handled by compensation. + uint32_t margin = 30 * ap->sample_rate / 1000; // 30ms + if (buffered_samples + margin < ap->target_buffering) { + LOGV("[Audio] Inserting initial buffering silence: %" PRIu32 + " samples", bytes_to_samples(ap, len)); + // Delay playback starting to reach the target buffering. Fill the + // whole buffer with silence (len is small compared to the + // arbitrary margin value). + memset(stream, 0, len); + return; + } + } + + size_t read = MIN(read_avail, len); + if (read) { + sc_bytebuf_read(&ap->buf, stream, read); + } + + if (read < len) { + size_t silence_bytes = len - read; + uint32_t silence_samples = bytes_to_samples(ap, silence_bytes); + // Insert silence. In theory, the inserted silent samples replace the + // missing real samples, which will arrive later, so they should be + // dropped to keep the latency minimal. However, this would cause very + // audible glitches, so let the clock compensation restore the target + // latency. + LOGD("[Audio] Buffer underflow, inserting silence: %" PRIu32 " samples", + silence_samples); + memset(stream + read, 0, silence_bytes); + + if (ap->received) { + // Inserting additional samples immediately increases buffering + ap->avg_buffering.avg += silence_samples; + } + } + + ap->played = true; +} + +static uint8_t * +sc_audio_player_get_swr_buf(struct sc_audio_player *ap, uint32_t min_samples) { + size_t min_buf_size = samples_to_bytes(ap, min_samples); + if (min_buf_size > ap->swr_buf_alloc_size) { + size_t new_size = min_buf_size + 4096; + uint8_t *buf = realloc(ap->swr_buf, new_size); + if (!buf) { + LOG_OOM(); + // Could not realloc to the requested size + return NULL; + } + ap->swr_buf = buf; + ap->swr_buf_alloc_size = new_size; + } + + return ap->swr_buf; +} + +static bool +sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, + const AVFrame *frame) { + struct sc_audio_player *ap = DOWNCAST(sink); + + SwrContext *swr_ctx = ap->swr_ctx; + + int64_t swr_delay = swr_get_delay(swr_ctx, ap->sample_rate); + // No need to av_rescale_rnd(), input and output sample rates are the same. + // Add more space (256) for clock compensation. + int dst_nb_samples = swr_delay + frame->nb_samples + 256; + + uint8_t *swr_buf = sc_audio_player_get_swr_buf(ap, dst_nb_samples); + if (!swr_buf) { + return false; + } + + int ret = swr_convert(swr_ctx, &swr_buf, dst_nb_samples, + (const uint8_t **) frame->data, frame->nb_samples); + if (ret < 0) { + LOGE("Resampling failed: %d", ret); + return false; + } + + // swr_convert() returns the number of samples which would have been + // written if the buffer was big enough. + uint32_t samples_written = MIN(ret, dst_nb_samples); + size_t swr_buf_size = samples_to_bytes(ap, samples_written); +#ifndef SC_AUDIO_PLAYER_NDEBUG + LOGD("[Audio] %" PRIu32 " samples written to buffer", samples_written); +#endif + + // Since this function is the only writer, the current available space is + // at least the previous available space. In practice, it should almost + // always be possible to write without lock. + bool lockless_write = swr_buf_size <= ap->previous_write_avail; + if (lockless_write) { + sc_bytebuf_prepare_write(&ap->buf, swr_buf, swr_buf_size); + } + + SDL_LockAudioDevice(ap->device); + + size_t read_avail = sc_bytebuf_read_available(&ap->buf); + uint32_t buffered_samples = bytes_to_samples(ap, read_avail); + + if (lockless_write) { + sc_bytebuf_commit_write(&ap->buf, swr_buf_size); + } else { + // Take care to keep full samples + size_t align = ap->nb_channels * ap->out_bytes_per_sample; + size_t write_avail = + sc_bytebuf_write_available(&ap->buf) / align * align; + if (swr_buf_size > write_avail) { + // Entering this branch is very unlikely, the ring-buffer (bytebuf) + // is allocated with a size sufficient to store 1 second more than + // the target buffering. If this happens, though, we have to skip + // old samples. + size_t cap = sc_bytebuf_capacity(&ap->buf) / align * align; + if (swr_buf_size > cap) { + // Very very unlikely: a single resampled frame should never + // exceed the ring-buffer size (or something is very wrong). + // Ignore the first bytes in swr_buf + swr_buf += swr_buf_size - cap; + swr_buf_size = cap; + // This change in samples_written will impact the + // instant_compensation below + samples_written -= bytes_to_samples(ap, swr_buf_size - cap); + } + + assert(swr_buf_size >= write_avail); + if (swr_buf_size > write_avail) { + sc_bytebuf_skip(&ap->buf, swr_buf_size - write_avail); + uint32_t skip_samples = + bytes_to_samples(ap, swr_buf_size - write_avail); + assert(buffered_samples >= skip_samples); + buffered_samples -= skip_samples; + if (ap->played) { + // Dropping input samples instantly decreases buffering + ap->avg_buffering.avg -= skip_samples; + } + } + + // It should remain exactly the expected size to write the new + // samples. + assert((sc_bytebuf_write_available(&ap->buf) / align * align) + == swr_buf_size); + } + + sc_bytebuf_write(&ap->buf, swr_buf, swr_buf_size); + } + + buffered_samples += samples_written; + assert(samples_to_bytes(ap, buffered_samples) + == sc_bytebuf_read_available(&ap->buf)); + + // Read with lock held, to be used after unlocking + bool played = ap->played; + if (played) { + uint32_t max_buffered_samples = ap->target_buffering + + 12 * SC_AUDIO_OUTPUT_BUFFER_SAMPLES + + ap->target_buffering / 10; + if (buffered_samples > max_buffered_samples) { + uint32_t skip_samples = buffered_samples - max_buffered_samples; + size_t skip_bytes = samples_to_bytes(ap, skip_samples); + sc_bytebuf_skip(&ap->buf, skip_bytes); +#ifndef SC_AUDIO_PLAYER_NDEBUG + LOGD("[Audio] Buffering threshold exceeded, skipping %" PRIu32 + " samples", skip_samples); +#endif + } + + // Number of samples added (or removed, if negative) for compensation + int32_t instant_compensation = + (int32_t) samples_written - frame->nb_samples; + + // The compensation must apply instantly, it must not be smoothed + ap->avg_buffering.avg += instant_compensation; + + // However, the buffering level must be smoothed + sc_average_push(&ap->avg_buffering, buffered_samples); + +#ifndef SC_AUDIO_PLAYER_NDEBUG + LOGD("[Audio] buffered_samples=%" PRIu32 " avg_buffering=%f", + buffered_samples, sc_average_get(&ap->avg_buffering)); +#endif + } else { + // SDL playback not started yet, do not accumulate more than + // max_initial_buffering samples, this would cause unnecessary delay + // (and glitches to compensate) on start. + uint32_t max_initial_buffering = ap->target_buffering + + 2 * SC_AUDIO_OUTPUT_BUFFER_SAMPLES; + if (buffered_samples > max_initial_buffering) { + uint32_t skip_samples = buffered_samples - max_initial_buffering; + size_t skip_bytes = samples_to_bytes(ap, skip_samples); + sc_bytebuf_skip(&ap->buf, skip_bytes); +#ifndef SC_AUDIO_PLAYER_NDEBUG + LOGD("[Audio] Playback not started, skipping %" PRIu32 " samples", + skip_samples); +#endif + } + } + + ap->previous_write_avail = sc_bytebuf_write_available(&ap->buf); + ap->received = true; + + SDL_UnlockAudioDevice(ap->device); + + if (played) { + ap->samples_since_resync += samples_written; + if (ap->samples_since_resync >= ap->sample_rate) { + // Recompute compensation every second + ap->samples_since_resync = 0; + + float avg = sc_average_get(&ap->avg_buffering); + int diff = ap->target_buffering - avg; + if (diff < 0 && buffered_samples < ap->target_buffering) { + // Do not accelerate if the instant buffering level is below + // the average, this would increase underflow + diff = 0; + } + // Compensate the diff over 4 seconds (but will be recomputed after + // 1 second) + int distance = 4 * ap->sample_rate; + // Limit compensation rate to 2% + int abs_max_diff = distance / 50; + diff = CLAMP(diff, -abs_max_diff, abs_max_diff); + LOGV("[Audio] Buffering: target=%" PRIu32 " avg=%f cur=%" PRIu32 + " compensation=%d", ap->target_buffering, avg, + buffered_samples, diff); + int ret = swr_set_compensation(swr_ctx, diff, distance); + if (ret < 0) { + LOGW("Resampling compensation failed: %d", ret); + // not fatal + } + } + } + + return true; +} + +static bool +sc_audio_player_frame_sink_open(struct sc_frame_sink *sink, + const AVCodecContext *ctx) { + struct sc_audio_player *ap = DOWNCAST(sink); + + SDL_AudioSpec desired = { + .freq = ctx->sample_rate, + .format = SC_SDL_SAMPLE_FMT, + .channels = ctx->ch_layout.nb_channels, + .samples = SC_AUDIO_OUTPUT_BUFFER_SAMPLES, + .callback = sc_audio_player_sdl_callback, + .userdata = ap, + }; + SDL_AudioSpec obtained; + + ap->device = SDL_OpenAudioDevice(NULL, 0, &desired, &obtained, 0); + if (!ap->device) { + LOGE("Could not open audio device: %s", SDL_GetError()); + return false; + } + + SwrContext *swr_ctx = swr_alloc(); + if (!swr_ctx) { + LOG_OOM(); + goto error_close_audio_device; + } + ap->swr_ctx = swr_ctx; + + assert(ctx->sample_rate > 0); + assert(ctx->ch_layout.nb_channels > 0); + assert(!av_sample_fmt_is_planar(SC_AV_SAMPLE_FMT)); + int out_bytes_per_sample = av_get_bytes_per_sample(SC_AV_SAMPLE_FMT); + assert(out_bytes_per_sample > 0); + + av_opt_set_chlayout(swr_ctx, "in_chlayout", &ctx->ch_layout, 0); + av_opt_set_chlayout(swr_ctx, "out_chlayout", &ctx->ch_layout, 0); + + av_opt_set_int(swr_ctx, "in_sample_rate", ctx->sample_rate, 0); + av_opt_set_int(swr_ctx, "out_sample_rate", ctx->sample_rate, 0); + + av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", ctx->sample_fmt, 0); + av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", SC_AV_SAMPLE_FMT, 0); + + int ret = swr_init(swr_ctx); + if (ret) { + LOGE("Failed to initialize the resampling context"); + goto error_free_swr_ctx; + } + + ap->sample_rate = ctx->sample_rate; + ap->nb_channels = ctx->ch_layout.nb_channels; + ap->out_bytes_per_sample = out_bytes_per_sample; + + ap->target_buffering = ap->target_buffering_delay * ap->sample_rate + / SC_TICK_FREQ; + + // Use a ring-buffer of the target buffering size plus 1 second between the + // producer and the consumer. It's too big on purpose, to guarantee that + // the producer and the consumer will be able to access it in parallel + // without locking. + size_t bytebuf_samples = ap->target_buffering + ap->sample_rate; + size_t bytebuf_size = samples_to_bytes(ap, bytebuf_samples); + + bool ok = sc_bytebuf_init(&ap->buf, bytebuf_size); + if (!ok) { + goto error_free_swr_ctx; + } + + size_t initial_swr_buf_size = samples_to_bytes(ap, 4096); + ap->swr_buf = malloc(initial_swr_buf_size); + if (!ap->swr_buf) { + LOG_OOM(); + goto error_destroy_bytebuf; + } + ap->swr_buf_alloc_size = initial_swr_buf_size; + + ap->previous_write_avail = sc_bytebuf_write_available(&ap->buf); + + // Samples are produced and consumed by blocks, so the buffering must be + // smoothed to get a relatively stable value. + sc_average_init(&ap->avg_buffering, 32); + ap->samples_since_resync = 0; + + ap->received = false; + ap->played = false; + + SDL_PauseAudioDevice(ap->device, 0); + + return true; + +error_destroy_bytebuf: + sc_bytebuf_destroy(&ap->buf); +error_free_swr_ctx: + swr_free(&ap->swr_ctx); +error_close_audio_device: + SDL_CloseAudioDevice(ap->device); + + return false; +} + +static void +sc_audio_player_frame_sink_close(struct sc_frame_sink *sink) { + struct sc_audio_player *ap = DOWNCAST(sink); + + assert(ap->device); + SDL_PauseAudioDevice(ap->device, 1); + SDL_CloseAudioDevice(ap->device); + + free(ap->swr_buf); + sc_bytebuf_destroy(&ap->buf); + swr_free(&ap->swr_ctx); +} + +void +sc_audio_player_init(struct sc_audio_player *ap, sc_tick target_buffering) { + ap->target_buffering_delay = target_buffering; + + static const struct sc_frame_sink_ops ops = { + .open = sc_audio_player_frame_sink_open, + .close = sc_audio_player_frame_sink_close, + .push = sc_audio_player_frame_sink_push, + }; + + ap->frame_sink.ops = &ops; +} diff --git a/app/src/audio_player.h b/app/src/audio_player.h new file mode 100644 index 0000000000..c64760eca2 --- /dev/null +++ b/app/src/audio_player.h @@ -0,0 +1,78 @@ +#ifndef SC_AUDIO_PLAYER_H +#define SC_AUDIO_PLAYER_H + +#include "common.h" + +#include +#include "trait/frame_sink.h" +#include +#include +#include +#include + +#include +#include +#include + +struct sc_audio_player { + struct sc_frame_sink frame_sink; + + SDL_AudioDeviceID device; + + // The target buffering between the producer and the consumer. This value + // is directly use for compensation. + // Since audio capture and/or encoding on the device typically produce + // blocks of 960 samples (20ms) or 1024 samples (~21.3ms), this target + // value should be higher. + sc_tick target_buffering_delay; + uint32_t target_buffering; // in samples + + // Audio buffer to communicate between the receiver and the SDL audio + // callback (protected by SDL_AudioDeviceLock()) + struct sc_bytebuf buf; + + // The previous number of bytes available in the buffer (only used by the + // receiver thread) + size_t previous_write_avail; + + // Resampler (only used from the receiver thread) + struct SwrContext *swr_ctx; + + // The sample rate is the same for input and output + unsigned sample_rate; + // The number of channels is the same for input and output + unsigned nb_channels; + // The number of bytes per sample for a single channel + unsigned out_bytes_per_sample; + + // Target buffer for resampling (only used by the receiver thread) + uint8_t *swr_buf; + size_t swr_buf_alloc_size; + + // Number of buffered samples (may be negative on underflow) (only used by + // the receiver thread) + struct sc_average avg_buffering; + // Count the number of samples to trigger a compensation update regularly + // (only used by the receiver thread) + uint32_t samples_since_resync; + + // Set to true the first time a sample is received (protected by + // SDL_AudioDeviceLock()) + bool received; + + // Set to true the first time the SDL callback is called (protected by + // SDL_AudioDeviceLock()) + bool played; + + const struct sc_audio_player_callbacks *cbs; + void *cbs_userdata; +}; + +struct sc_audio_player_callbacks { + void (*on_ended)(struct sc_audio_player *ap, bool success, void *userdata); +}; + +void +sc_audio_player_init(struct sc_audio_player *ap, sc_tick target_buffering); + +#endif diff --git a/app/src/decoder.c b/app/src/decoder.c index a8168f6616..4384186dc1 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -2,6 +2,7 @@ #include #include +#include #include "events.h" #include "trait/frame_sink.h" @@ -23,6 +24,11 @@ sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec) { if (codec->type == AVMEDIA_TYPE_VIDEO) { // Hardcoded video properties decoder->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P; + } else { + // Hardcoded audio properties + decoder->codec_ctx->ch_layout = + (AVChannelLayout) AV_CHANNEL_LAYOUT_STEREO; + decoder->codec_ctx->sample_rate = 48000; } if (avcodec_open2(decoder->codec_ctx, codec, NULL) < 0) { diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index dba1bad9e3..3f3a34f00f 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -13,6 +13,7 @@ # include #endif +#include "audio_player.h" #include "controller.h" #include "decoder.h" #include "delay_buffer.h" @@ -41,6 +42,7 @@ struct scrcpy { struct sc_server server; struct sc_screen screen; + struct sc_audio_player audio_player; struct sc_demuxer video_demuxer; struct sc_demuxer audio_demuxer; struct sc_decoder video_decoder; @@ -386,9 +388,16 @@ scrcpy(struct scrcpy_options *options) { } // Initialize SDL video in addition if display is enabled - if (options->display && SDL_Init(SDL_INIT_VIDEO)) { - LOGE("Could not initialize SDL: %s", SDL_GetError()); - goto end; + if (options->display) { + if (SDL_Init(SDL_INIT_VIDEO)) { + LOGE("Could not initialize SDL video: %s", SDL_GetError()); + goto end; + } + + if (options->audio && SDL_Init(SDL_INIT_AUDIO)) { + LOGE("Could not initialize SDL audio: %s", SDL_GetError()); + goto end; + } } sdl_configure(options->display, options->disable_screensaver); @@ -676,6 +685,12 @@ scrcpy(struct scrcpy_options *options) { } sc_frame_source_add_sink(src, &s->screen.frame_sink); + + if (options->audio) { + sc_audio_player_init(&s->audio_player, SC_TICK_FROM_MS(50)); + sc_frame_source_add_sink(&s->audio_decoder.frame_source, + &s->audio_player.frame_sink); + } } #ifdef HAVE_V4L2 diff --git a/app/src/util/average.c b/app/src/util/average.c new file mode 100644 index 0000000000..ace23d456c --- /dev/null +++ b/app/src/util/average.c @@ -0,0 +1,26 @@ +#include "average.h" + +#include + +void +sc_average_init(struct sc_average *avg, unsigned range) { + avg->range = range; + avg->avg = 0; + avg->count = 0; +} + +void +sc_average_push(struct sc_average *avg, float value) { + if (avg->count < avg->range) { + ++avg->count; + } + + assert(avg->count); + avg->avg = ((avg->count - 1) * avg->avg + value) / avg->count; +} + +float +sc_average_get(struct sc_average *avg) { + assert(avg->count); + return avg->avg; +} diff --git a/app/src/util/average.h b/app/src/util/average.h new file mode 100644 index 0000000000..59fae7d1e8 --- /dev/null +++ b/app/src/util/average.h @@ -0,0 +1,40 @@ +#ifndef SC_AVERAGE +#define SC_AVERAGE + +#include "common.h" + +#include +#include + +struct sc_average { + // Current average value + float avg; + + // Target range, to update the average as follow: + // avg = ((range - 1) * avg + new_value) / range + unsigned range; + + // Number of values pushed when less than range (count <= range). + // The purpose is to handle the first (range - 1) values properly. + unsigned count; +}; + +void +sc_average_init(struct sc_average *avg, unsigned range); + +/** + * Push a new value to update the "rolling" average + */ +void +sc_average_push(struct sc_average *avg, float value); + +/** + * Get the current average value + * + * It is an error to call this function if sc_average_push() has not been + * called at least once. + */ +float +sc_average_get(struct sc_average *avg); + +#endif From d66b0b3dccc118600b83988173e6631ad661df51 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 27 Feb 2023 21:41:27 +0100 Subject: [PATCH 106/118] Add compat support for FFmpeg < 5.1 The new chlayout API has been introduced in FFmpeg 5.1. Use the old channel_layout API on older versions. PR #3757 --- app/src/audio_player.c | 20 +++++++++++++++++--- app/src/compat.h | 7 +++++++ app/src/decoder.c | 5 +++++ app/src/recorder.c | 5 +++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/app/src/audio_player.c b/app/src/audio_player.c index 78a8ffe1af..de218f1ecd 100644 --- a/app/src/audio_player.c +++ b/app/src/audio_player.c @@ -286,11 +286,19 @@ static bool sc_audio_player_frame_sink_open(struct sc_frame_sink *sink, const AVCodecContext *ctx) { struct sc_audio_player *ap = DOWNCAST(sink); +#ifdef SCRCPY_LAVU_HAS_CHLAYOUT + assert(ctx->ch_layout.nb_channels > 0); + unsigned nb_channels = ctx->ch_layout.nb_channels; +#else + int tmp = av_get_channel_layout_nb_channels(ctx->channel_layout); + assert(tmp > 0); + unsigned nb_channels = tmp; +#endif SDL_AudioSpec desired = { .freq = ctx->sample_rate, .format = SC_SDL_SAMPLE_FMT, - .channels = ctx->ch_layout.nb_channels, + .channels = nb_channels, .samples = SC_AUDIO_OUTPUT_BUFFER_SAMPLES, .callback = sc_audio_player_sdl_callback, .userdata = ap, @@ -311,13 +319,19 @@ sc_audio_player_frame_sink_open(struct sc_frame_sink *sink, ap->swr_ctx = swr_ctx; assert(ctx->sample_rate > 0); - assert(ctx->ch_layout.nb_channels > 0); assert(!av_sample_fmt_is_planar(SC_AV_SAMPLE_FMT)); int out_bytes_per_sample = av_get_bytes_per_sample(SC_AV_SAMPLE_FMT); assert(out_bytes_per_sample > 0); +#ifdef SCRCPY_LAVU_HAS_CHLAYOUT av_opt_set_chlayout(swr_ctx, "in_chlayout", &ctx->ch_layout, 0); av_opt_set_chlayout(swr_ctx, "out_chlayout", &ctx->ch_layout, 0); +#else + av_opt_set_channel_layout(swr_ctx, "in_channel_layout", + ctx->channel_layout, 0); + av_opt_set_channel_layout(swr_ctx, "out_channel_layout", + ctx->channel_layout, 0); +#endif av_opt_set_int(swr_ctx, "in_sample_rate", ctx->sample_rate, 0); av_opt_set_int(swr_ctx, "out_sample_rate", ctx->sample_rate, 0); @@ -332,7 +346,7 @@ sc_audio_player_frame_sink_open(struct sc_frame_sink *sink, } ap->sample_rate = ctx->sample_rate; - ap->nb_channels = ctx->ch_layout.nb_channels; + ap->nb_channels = nb_channels; ap->out_bytes_per_sample = out_bytes_per_sample; ap->target_buffering = ap->target_buffering_delay * ap->sample_rate diff --git a/app/src/compat.h b/app/src/compat.h index ea44437d83..22563421cb 100644 --- a/app/src/compat.h +++ b/app/src/compat.h @@ -37,6 +37,13 @@ # define SCRCPY_LAVF_HAS_AVFORMATCONTEXT_URL #endif +// Not documented in ffmpeg/doc/APIchanges, but the channel_layout API +// has been replaced by chlayout in FFmpeg commit +// f423497b455da06c1337846902c770028760e094. +#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 23, 100) +# define SCRCPY_LAVU_HAS_CHLAYOUT +#endif + #if SDL_VERSION_ATLEAST(2, 0, 6) // # define SCRCPY_SDL_HAS_HINT_TOUCH_MOUSE_EVENTS diff --git a/app/src/decoder.c b/app/src/decoder.c index 4384186dc1..e87cfd6baf 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -26,8 +26,13 @@ sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec) { decoder->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P; } else { // Hardcoded audio properties +#ifdef SCRCPY_LAVU_HAS_CHLAYOUT decoder->codec_ctx->ch_layout = (AVChannelLayout) AV_CHANNEL_LAYOUT_STEREO; +#else + decoder->codec_ctx->channel_layout = AV_CH_LAYOUT_STEREO; + decoder->codec_ctx->channels = 2; +#endif decoder->codec_ctx->sample_rate = 48000; } diff --git a/app/src/recorder.c b/app/src/recorder.c index bd7c50f208..af5fe51084 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -202,7 +202,12 @@ sc_recorder_wait_audio_stream(struct sc_recorder *recorder) { stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; stream->codecpar->codec_id = codec->id; +#ifdef SCRCPY_LAVU_HAS_CHLAYOUT stream->codecpar->ch_layout.nb_channels = 2; +#else + stream->codecpar->channel_layout = AV_CH_LAYOUT_STEREO; + stream->codecpar->channels = 2; +#endif stream->codecpar->sample_rate = 48000; recorder->audio_stream_index = stream->index; From df55bc2683e2ca2aec105d6e435cb0b170037ed7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Thu, 2 Mar 2023 23:14:01 +0100 Subject: [PATCH 107/118] Add --audio-buffer Expose an option to add a buffering delay (in milliseconds) before playing audio. This is similar to the options --display-buffer and --v4l2-buffer for video frames. PR #3757 --- app/scrcpy.1 | 8 ++++++++ app/src/cli.c | 15 +++++++++++++++ app/src/options.c | 1 + app/src/options.h | 1 + app/src/scrcpy.c | 2 +- 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 9125841496..120ea19220 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -25,6 +25,14 @@ Encode the audio at the given bit\-rate, expressed in bits/s. Unit suffixes are Default is 128K (128000). +.TP +.BI "\-\-audio\-buffer ms +Configure the audio buffering delay (in milliseconds). + +Lower values decrease the latency, but increase the likelyhood of buffer underrun (causing audio glitches). + +Default is 50. + .TP .BI "\-\-audio\-codec " name Select an audio codec (opus or aac). diff --git a/app/src/cli.c b/app/src/cli.c index 18f3b83bb5..122a58918b 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -71,6 +71,7 @@ enum { OPT_LIST_ENCODERS, OPT_LIST_DISPLAYS, OPT_REQUIRE_AUDIO, + OPT_AUDIO_BUFFER, }; struct sc_option { @@ -120,6 +121,15 @@ static const struct sc_option options[] = { "Unit suffixes are supported: 'K' (x1000) and 'M' (x1000000).\n" "Default is 128K (128000).", }, + { + .longopt_id = OPT_AUDIO_BUFFER, + .longopt = "audio-buffer", + .argdesc = "ms", + .text = "Configure the audio buffering delay (in milliseconds).\n" + "Lower values decrease the latency, but increase the " + "likelyhood of buffer underrun (causing audio glitches).\n" + "Default is 50.", + }, { .longopt_id = OPT_AUDIO_CODEC, .longopt = "audio-codec", @@ -1822,6 +1832,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], case OPT_REQUIRE_AUDIO: opts->require_audio = true; break; + case OPT_AUDIO_BUFFER: + if (!parse_buffering_time(optarg, &opts->audio_buffer)) { + return false; + } + break; default: // getopt prints the error message on stderr return false; diff --git a/app/src/options.c b/app/src/options.c index 5dd655ce74..68c16d5316 100644 --- a/app/src/options.c +++ b/app/src/options.c @@ -43,6 +43,7 @@ const struct scrcpy_options scrcpy_options_default = { .display_id = 0, .display_buffer = 0, .v4l2_buffer = 0, + .audio_buffer = SC_TICK_FROM_MS(50), #ifdef HAVE_USB .otg = false, #endif diff --git a/app/src/options.h b/app/src/options.h index 5fcaf016a7..d9c2d22812 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -125,6 +125,7 @@ struct scrcpy_options { uint32_t display_id; sc_tick display_buffer; sc_tick v4l2_buffer; + sc_tick audio_buffer; #ifdef HAVE_USB bool otg; #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 3f3a34f00f..ce045c9746 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -687,7 +687,7 @@ scrcpy(struct scrcpy_options *options) { sc_frame_source_add_sink(src, &s->screen.frame_sink); if (options->audio) { - sc_audio_player_init(&s->audio_player, SC_TICK_FROM_MS(50)); + sc_audio_player_init(&s->audio_player, options->audio_buffer); sc_frame_source_add_sink(&s->audio_decoder.frame_source, &s->audio_player.frame_sink); } From 02dd1be4a1ab24795a10ca769e2998dec28806f7 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 00:42:51 +0100 Subject: [PATCH 108/118] Stop on decoder frame push error On push, frame sinks report downstream errors to stop upstream components. Do not ignore the error. --- app/src/decoder.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/decoder.c b/app/src/decoder.c index e87cfd6baf..ecad837316 100644 --- a/app/src/decoder.c +++ b/app/src/decoder.c @@ -99,11 +99,11 @@ sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) { // a frame was received bool ok = sc_frame_source_sinks_push(&decoder->frame_source, decoder->frame); - // A frame lost should not make the whole pipeline fail. The error, if - // any, is already logged. - (void) ok; - av_frame_unref(decoder->frame); + if (!ok) { + // Error already logged + return false; + } } return true; From 65cc9d765d8d0feb26a41c2287b3abe13dc37d12 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 18:46:59 +0100 Subject: [PATCH 109/118] Extract audio capture The audio capture was implemented in AudioEncoder. In order to reuse it without encoding, extract it to a separate class. PR #3757 --- .../com/genymobile/scrcpy/AudioCapture.java | 148 ++++++++++++++++++ .../com/genymobile/scrcpy/AudioEncoder.java | 136 ++-------------- 2 files changed, 161 insertions(+), 123 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/AudioCapture.java diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java new file mode 100644 index 0000000000..3cef7801b2 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java @@ -0,0 +1,148 @@ +package com.genymobile.scrcpy; + +import com.genymobile.scrcpy.wrappers.ServiceManager; + +import android.annotation.SuppressLint; +import android.annotation.TargetApi; +import android.content.ComponentName; +import android.content.Intent; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.AudioTimestamp; +import android.media.MediaCodec; +import android.media.MediaRecorder; +import android.os.Build; +import android.os.SystemClock; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public final class AudioCapture { + + public static final int SAMPLE_RATE = 48000; + public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO; + public static final int CHANNELS = 2; + public static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT; + public static final int BYTES_PER_SAMPLE = 2; + + private AudioRecord recorder; + + private final AudioTimestamp timestamp = new AudioTimestamp(); + private long previousPts = 0; + private long nextPts = 0; + + public static int millisToBytes(int millis) { + return SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * millis / 1000; + } + + private static AudioFormat createAudioFormat() { + AudioFormat.Builder builder = new AudioFormat.Builder(); + builder.setEncoding(FORMAT); + builder.setSampleRate(SAMPLE_RATE); + builder.setChannelMask(CHANNEL_CONFIG); + return builder.build(); + } + + @TargetApi(Build.VERSION_CODES.M) + @SuppressLint({"WrongConstant", "MissingPermission"}) + private static AudioRecord createAudioRecord() { + AudioRecord.Builder builder = new AudioRecord.Builder(); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + // On older APIs, Workarounds.fillAppInfo() must be called beforehand + builder.setContext(FakeContext.get()); + } + builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX); + builder.setAudioFormat(createAudioFormat()); + int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT); + // This buffer size does not impact latency + builder.setBufferSizeInBytes(8 * minBufferSize); + return builder.build(); + } + + private static void startWorkaroundAndroid11() { + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + // Android 11 requires Apps to be at foreground to record audio. + // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground. + // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android + // shell ("com.android.shell"). + // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the + // foreground. + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + Intent intent = new Intent(Intent.ACTION_MAIN); + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + intent.addCategory(Intent.CATEGORY_LAUNCHER); + intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity")); + ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent); + // Wait for activity to start + SystemClock.sleep(150); + } + } + } + + private static void stopWorkaroundAndroid11() { + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME); + } + } + + public void start() throws AudioCaptureForegroundException { + startWorkaroundAndroid11(); + try { + recorder = createAudioRecord(); + recorder.startRecording(); + } catch (UnsupportedOperationException e) { + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) { + Ln.e("Failed to start audio capture"); + Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy."); + throw new AudioCaptureForegroundException(); + } + throw e; + } finally { + stopWorkaroundAndroid11(); + } + } + + public void stop() { + if (recorder != null) { + // Will call .stop() if necessary, without throwing an IllegalStateException + recorder.release(); + } + } + + @TargetApi(Build.VERSION_CODES.N) + public int read(ByteBuffer directBuffer, int size, MediaCodec.BufferInfo outBufferInfo) throws IOException { + int r = recorder.read(directBuffer, size); + if (r < 0) { + return r; + } + + long pts; + + int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC); + if (ret == AudioRecord.SUCCESS) { + pts = timestamp.nanoTime / 1000; + } else { + if (nextPts == 0) { + Ln.w("Could not get any audio timestamp"); + } + // compute from previous timestamp and packet size + pts = nextPts; + } + + long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE); + nextPts = pts + durationUs; + + if (previousPts != 0 && pts < previousPts) { + // Audio PTS may come from two sources: + // - recorder.getTimestamp() if the call works; + // - an estimation from the previous PTS and the packet size as a fallback. + // + // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it. + pts = previousPts + 1; + } + previousPts = pts; + + outBufferInfo.set(0, r, pts, 0); + return r; + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index cc786bdbfb..8b60d37ec8 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -1,22 +1,12 @@ package com.genymobile.scrcpy; -import com.genymobile.scrcpy.wrappers.ServiceManager; - -import android.annotation.SuppressLint; import android.annotation.TargetApi; -import android.content.ComponentName; -import android.content.Intent; -import android.media.AudioFormat; -import android.media.AudioRecord; -import android.media.AudioTimestamp; import android.media.MediaCodec; import android.media.MediaFormat; -import android.media.MediaRecorder; import android.os.Build; import android.os.Handler; import android.os.HandlerThread; import android.os.Looper; -import android.os.SystemClock; import java.io.IOException; import java.nio.ByteBuffer; @@ -44,14 +34,11 @@ private static class OutputTask { } } - private static final int SAMPLE_RATE = 48000; - private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO; - private static final int CHANNELS = 2; - private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT; - private static final int BYTES_PER_SAMPLE = 2; + private static final int SAMPLE_RATE = AudioCapture.SAMPLE_RATE; + private static final int CHANNELS = AudioCapture.CHANNELS; private static final int READ_MS = 5; // milliseconds - private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000; + private static final int READ_SIZE = AudioCapture.millisToBytes(READ_MS); private final Streamer streamer; private final int bitRate; @@ -78,30 +65,6 @@ public AudioEncoder(Streamer streamer, int bitRate, List codecOptio this.encoderName = encoderName; } - private static AudioFormat createAudioFormat() { - AudioFormat.Builder builder = new AudioFormat.Builder(); - builder.setEncoding(FORMAT); - builder.setSampleRate(SAMPLE_RATE); - builder.setChannelMask(CHANNEL_CONFIG); - return builder.build(); - } - - @TargetApi(Build.VERSION_CODES.M) - @SuppressLint({"WrongConstant", "MissingPermission"}) - private static AudioRecord createAudioRecord() { - AudioRecord.Builder builder = new AudioRecord.Builder(); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - // On older APIs, Workarounds.fillAppInfo() must be called beforehand - builder.setContext(FakeContext.get()); - } - builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX); - builder.setAudioFormat(createAudioFormat()); - int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT); - // This buffer size does not impact latency - builder.setBufferSizeInBytes(8 * minBufferSize); - return builder.build(); - } - private static MediaFormat createFormat(String mimeType, int bitRate, List codecOptions) { MediaFormat format = new MediaFormat(); format.setString(MediaFormat.KEY_MIME, mimeType); @@ -122,47 +85,18 @@ private static MediaFormat createFormat(String mimeType, int bitRate, List { try { - inputThread(mediaCodecRef, recorderRef); + inputThread(mediaCodecRef, captureRef); } catch (IOException | InterruptedException e) { Ln.e("Audio capture error", e); } finally { @@ -366,11 +259,8 @@ public void encode() throws IOException, ConfigurationException, AudioCaptureFor } mediaCodec.release(); } - if (recorder != null) { - if (recorderStarted) { - recorder.stop(); - } - recorder.release(); + if (capture != null) { + capture.stop(); } } } From dc228eaad0ea53f57c27e9aba2b51a422ed3aa94 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 18:49:05 +0100 Subject: [PATCH 110/118] Extract async processor interface On the server side, several components are started, stopped and joined. Extract an interface to handle them generically. This will help to support both encoded and raw audio stream, because they will be two different concrete components, but implementing the same interface. PR #3757 --- .../com/genymobile/scrcpy/AsyncProcessor.java | 7 ++++ .../com/genymobile/scrcpy/AudioEncoder.java | 2 +- .../com/genymobile/scrcpy/Controller.java | 2 +- .../java/com/genymobile/scrcpy/Server.java | 35 +++++++++---------- 4 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/AsyncProcessor.java diff --git a/server/src/main/java/com/genymobile/scrcpy/AsyncProcessor.java b/server/src/main/java/com/genymobile/scrcpy/AsyncProcessor.java new file mode 100644 index 0000000000..cbc435b0d3 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AsyncProcessor.java @@ -0,0 +1,7 @@ +package com.genymobile.scrcpy; + +public interface AsyncProcessor { + void start(); + void stop(); + void join() throws InterruptedException; +} diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 8b60d37ec8..0ba424ca44 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -14,7 +14,7 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; -public final class AudioEncoder { +public final class AudioEncoder implements AsyncProcessor { private static class InputTask { private final int index; diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index 02d77cb1bc..59fae60246 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -14,7 +14,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -public class Controller { +public class Controller implements AsyncProcessor { private static final int DEFAULT_DEVICE_ID = 0; diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 35da6965f8..3d3e02fdbb 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -5,6 +5,7 @@ import android.os.Build; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -91,8 +92,7 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc Workarounds.fillAppInfo(); } - Controller controller = null; - AudioEncoder audioEncoder = null; + List asyncProcessors = new ArrayList<>(); try (DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, audio, control, sendDummyByte)) { if (options.getSendDeviceMeta()) { @@ -101,24 +101,27 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc } if (control) { - controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn()); - controller.start(); - - final Controller controllerRef = controller; - device.setClipboardListener(text -> controllerRef.getSender().pushClipboardText(text)); + Controller controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn()); + device.setClipboardListener(text -> controller.getSender().pushClipboardText(text)); + asyncProcessors.add(controller); } if (audio) { Streamer audioStreamer = new Streamer(connection.getAudioFd(), options.getAudioCodec(), options.getSendCodecId(), options.getSendFrameMeta()); - audioEncoder = new AudioEncoder(audioStreamer, options.getAudioBitRate(), options.getAudioCodecOptions(), options.getAudioEncoder()); - audioEncoder.start(); + AudioEncoder audioRecorder = new AudioEncoder(audioStreamer, options.getAudioBitRate(), options.getAudioCodecOptions(), options.getAudioEncoder()); + asyncProcessors.add(audioRecorder); } Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecId(), options.getSendFrameMeta()); ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(), options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError()); + + for (AsyncProcessor asyncProcessor : asyncProcessors) { + asyncProcessor.start(); + } + try { // synchronous screenEncoder.streamScreen(); @@ -131,20 +134,14 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc } finally { Ln.d("Screen streaming stopped"); initThread.interrupt(); - if (audioEncoder != null) { - audioEncoder.stop(); - } - if (controller != null) { - controller.stop(); + for (AsyncProcessor asyncProcessor : asyncProcessors) { + asyncProcessor.stop(); } try { initThread.join(); - if (audioEncoder != null) { - audioEncoder.join(); - } - if (controller != null) { - controller.join(); + for (AsyncProcessor asyncProcessor : asyncProcessors) { + asyncProcessor.join(); } } catch (InterruptedException e) { // ignore From 66b6c06443130a8a11984ca2cd6d80f5cd3db6ec Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 21:14:28 +0100 Subject: [PATCH 111/118] Add raw audio recorder Add an alternative AudioRecorder to stream raw packets without encoding. PR #3757 --- .../com/genymobile/scrcpy/AudioCodec.java | 3 +- .../genymobile/scrcpy/AudioRawRecorder.java | 75 +++++++++++++++++++ .../java/com/genymobile/scrcpy/Server.java | 11 ++- 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java b/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java index dc000e98dc..1f3b07a032 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java @@ -4,7 +4,8 @@ public enum AudioCodec implements Codec { OPUS(0x6f_70_75_73, "opus", MediaFormat.MIMETYPE_AUDIO_OPUS), - AAC(0x00_61_61_63, "aac", MediaFormat.MIMETYPE_AUDIO_AAC); + AAC(0x00_61_61_63, "aac", MediaFormat.MIMETYPE_AUDIO_AAC), + RAW(0x00_72_61_77, "raw", MediaFormat.MIMETYPE_AUDIO_RAW); private final int id; // 4-byte ASCII representation of the name private final String name; diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java b/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java new file mode 100644 index 0000000000..2e483daa80 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java @@ -0,0 +1,75 @@ +package com.genymobile.scrcpy; + +import android.media.MediaCodec; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public final class AudioRawRecorder implements AsyncProcessor { + + private final Streamer streamer; + + private Thread thread; + + private static final int READ_MS = 5; // milliseconds + private static final int READ_SIZE = AudioCapture.millisToBytes(READ_MS); + + public AudioRawRecorder(Streamer streamer) { + this.streamer = streamer; + } + + private void record() throws IOException, AudioCaptureForegroundException { + final ByteBuffer buffer = ByteBuffer.allocateDirect(READ_SIZE); + final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + + AudioCapture capture = new AudioCapture(); + try { + capture.start(); + + streamer.writeHeader(); + while (!Thread.currentThread().isInterrupted()) { + buffer.position(0); + int r = capture.read(buffer, READ_SIZE, bufferInfo); + if (r < 0) { + throw new IOException("Could not read audio: " + r); + } + buffer.limit(r); + + streamer.writePacket(buffer, bufferInfo); + } + } catch (Throwable e) { + // Notify the client that the audio could not be captured + streamer.writeDisableStream(false); + throw e; + } finally { + capture.stop(); + } + } + + public void start() { + thread = new Thread(() -> { + try { + record(); + } catch (AudioCaptureForegroundException e) { + // Do not print stack trace, a user-friendly error-message has already been logged + } catch (IOException e) { + Ln.e("Audio recording error", e); + } finally { + Ln.d("Audio recorder stopped"); + } + }); + thread.start(); + } + + public void stop() { + if (thread != null) { + thread.interrupt(); + } + } + + public void join() throws InterruptedException { + if (thread != null) { + thread.join(); + } + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index 3d3e02fdbb..86555e3b5b 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -107,9 +107,16 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc } if (audio) { - Streamer audioStreamer = new Streamer(connection.getAudioFd(), options.getAudioCodec(), options.getSendCodecId(), + AudioCodec audioCodec = options.getAudioCodec(); + Streamer audioStreamer = new Streamer(connection.getAudioFd(), audioCodec, options.getSendCodecId(), options.getSendFrameMeta()); - AudioEncoder audioRecorder = new AudioEncoder(audioStreamer, options.getAudioBitRate(), options.getAudioCodecOptions(), options.getAudioEncoder()); + AsyncProcessor audioRecorder; + if (audioCodec == AudioCodec.RAW) { + audioRecorder = new AudioRawRecorder(audioStreamer); + } else { + audioRecorder = new AudioEncoder(audioStreamer, options.getAudioBitRate(), options.getAudioCodecOptions(), + options.getAudioEncoder()); + } asyncProcessors.add(audioRecorder); } From d2952c7e93f151d773f38aa368b7163b0ff31cf2 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 21:19:37 +0100 Subject: [PATCH 112/118] Add --audio-codec=raw option Add support for raw (PCM S16 LE) audio codec (a raw decoder is included in FFmpeg). PR #3757 --- app/data/bash-completion/scrcpy | 2 +- app/data/zsh-completion/_scrcpy | 2 +- app/scrcpy.1 | 2 +- app/src/cli.c | 13 +++++++++++-- app/src/demuxer.c | 3 +++ app/src/options.h | 1 + app/src/server.c | 2 ++ 7 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpy index 74c3ee57de..c364936453 100644 --- a/app/data/bash-completion/scrcpy +++ b/app/data/bash-completion/scrcpy @@ -78,7 +78,7 @@ _scrcpy() { return ;; --audio-codec) - COMPREPLY=($(compgen -W 'opus aac' -- "$cur")) + COMPREPLY=($(compgen -W 'opus aac raw' -- "$cur")) return ;; --lock-video-orientation) diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpy index b28201a4d3..d713761c2e 100644 --- a/app/data/zsh-completion/_scrcpy +++ b/app/data/zsh-completion/_scrcpy @@ -10,7 +10,7 @@ local arguments arguments=( '--always-on-top[Make scrcpy window always on top \(above other windows\)]' '--audio-bit-rate=[Encode the audio at the given bit-rate]' - '--audio-codec=[Select the audio codec]:codec:(opus aac)' + '--audio-codec=[Select the audio codec]:codec:(opus aac raw)' '--audio-codec-options=[Set a list of comma-separated key\:type=value options for the device audio encoder]' '--audio-encoder=[Use a specific MediaCodec audio encoder]' {-b,--video-bit-rate=}'[Encode the video at the given bit-rate]' diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 120ea19220..e8e3618844 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -35,7 +35,7 @@ Default is 50. .TP .BI "\-\-audio\-codec " name -Select an audio codec (opus or aac). +Select an audio codec (opus, aac or raw). Default is opus. diff --git a/app/src/cli.c b/app/src/cli.c index 122a58918b..d45be8780a 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -134,7 +134,7 @@ static const struct sc_option options[] = { .longopt_id = OPT_AUDIO_CODEC, .longopt = "audio-codec", .argdesc = "name", - .text = "Select an audio codec (opus or aac).\n" + .text = "Select an audio codec (opus, aac or raw).\n" "Default is opus.", }, { @@ -1522,7 +1522,11 @@ parse_audio_codec(const char *optarg, enum sc_codec *codec) { *codec = SC_CODEC_AAC; return true; } - LOGE("Unsupported audio codec: %s (expected opus or aac)", optarg); + if (!strcmp(optarg, "raw")) { + *codec = SC_CODEC_RAW; + return true; + } + LOGE("Unsupported audio codec: %s (expected opus, aac or raw)", optarg); return false; } @@ -1923,6 +1927,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], } } + if (opts->record_filename && opts->audio_codec == SC_CODEC_RAW) { + LOGW("Recording does not support RAW audio codec"); + return false; + } + if (!opts->control) { if (opts->turn_screen_off) { LOGE("Could not request to turn screen off if control is disabled"); diff --git a/app/src/demuxer.c b/app/src/demuxer.c index 15a595a00c..a4fa19f459 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -25,6 +25,7 @@ sc_demuxer_to_avcodec_id(uint32_t codec_id) { #define SC_CODEC_ID_AV1 UINT32_C(0x00617631) // "av1" in ASCII #define SC_CODEC_ID_OPUS UINT32_C(0x6f707573) // "opus" in ASCII #define SC_CODEC_ID_AAC UINT32_C(0x00616163) // "aac in ASCII" +#define SC_CODEC_ID_RAW UINT32_C(0x00726177) // "raw" in ASCII switch (codec_id) { case SC_CODEC_ID_H264: return AV_CODEC_ID_H264; @@ -36,6 +37,8 @@ sc_demuxer_to_avcodec_id(uint32_t codec_id) { return AV_CODEC_ID_OPUS; case SC_CODEC_ID_AAC: return AV_CODEC_ID_AAC; + case SC_CODEC_ID_RAW: + return AV_CODEC_ID_PCM_S16LE; default: LOGE("Unknown codec id 0x%08" PRIx32, codec_id); return AV_CODEC_ID_NONE; diff --git a/app/src/options.h b/app/src/options.h index d9c2d22812..06b4ddfa9b 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -29,6 +29,7 @@ enum sc_codec { SC_CODEC_AV1, SC_CODEC_OPUS, SC_CODEC_AAC, + SC_CODEC_RAW, }; enum sc_lock_video_orientation { diff --git a/app/src/server.c b/app/src/server.c index 9d4fb098a7..7b50342777 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -173,6 +173,8 @@ sc_server_get_codec_name(enum sc_codec codec) { return "opus"; case SC_CODEC_AAC: return "aac"; + case SC_CODEC_RAW: + return "raw"; default: return NULL; } From 7da45c246e919386f7d4fd3facc4597bb056e752 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 3 Mar 2023 21:58:57 +0100 Subject: [PATCH 113/118] Warn on ignored audio options For raw audio codec, some audio options are ignored. PR #3757 --- app/src/cli.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/src/cli.c b/app/src/cli.c index d45be8780a..8a0b6aa4a3 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -1932,6 +1932,18 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } + if (opts->audio_codec == SC_CODEC_RAW) { + if (opts->audio_bit_rate) { + LOGW("--audio-bit-rate is ignored for raw audio codec"); + } + if (opts->audio_codec_options) { + LOGW("--audio-codec-options is ignored for raw audio codec"); + } + if (opts->audio_encoder) { + LOGW("--audio-encoder is ignored for raw audio codec"); + } + } + if (!opts->control) { if (opts->turn_screen_off) { LOGE("Could not request to turn screen off if control is disabled"); From bb56472d4eb15d7c318a05a8236af6d0f5a46a04 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 8 Mar 2023 20:07:03 +0100 Subject: [PATCH 114/118] Print server logs and newline in one call System.out.println() first prints the message, then the new line. Between these two calls, the client might print a message, breaking formatting. Instead, call System.out.print() with '\n' appended to the message. --- server/src/main/java/com/genymobile/scrcpy/Ln.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Ln.java b/server/src/main/java/com/genymobile/scrcpy/Ln.java index c39fc621c7..291f26ff08 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Ln.java +++ b/server/src/main/java/com/genymobile/scrcpy/Ln.java @@ -39,28 +39,28 @@ public static boolean isEnabled(Level level) { public static void v(String message) { if (isEnabled(Level.VERBOSE)) { Log.v(TAG, message); - System.out.println(PREFIX + "VERBOSE: " + message); + System.out.print(PREFIX + "VERBOSE: " + message + '\n'); } } public static void d(String message) { if (isEnabled(Level.DEBUG)) { Log.d(TAG, message); - System.out.println(PREFIX + "DEBUG: " + message); + System.out.print(PREFIX + "DEBUG: " + message + '\n'); } } public static void i(String message) { if (isEnabled(Level.INFO)) { Log.i(TAG, message); - System.out.println(PREFIX + "INFO: " + message); + System.out.print(PREFIX + "INFO: " + message + '\n'); } } public static void w(String message, Throwable throwable) { if (isEnabled(Level.WARN)) { Log.w(TAG, message, throwable); - System.out.println(PREFIX + "WARN: " + message); + System.out.print(PREFIX + "WARN: " + message + '\n'); if (throwable != null) { throwable.printStackTrace(); } @@ -74,7 +74,7 @@ public static void w(String message) { public static void e(String message, Throwable throwable) { if (isEnabled(Level.ERROR)) { Log.e(TAG, message, throwable); - System.out.println(PREFIX + "ERROR: " + message); + System.out.print(PREFIX + "ERROR: " + message + "\n"); if (throwable != null) { throwable.printStackTrace(); } From 4a25f3e53bdac8c2510cc1c7f06e479a17a42e6b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 8 Mar 2023 20:13:08 +0100 Subject: [PATCH 115/118] Print info logs to stdout All server logs were printed to stdout, while all client logs were printed to stderr. Instead, use stderr for warnings and errors, stdout for the others: - stdout: verbose, debug, info - stderr: warn, error --- app/src/util/log.c | 22 +++++++++++++++++++ .../main/java/com/genymobile/scrcpy/Ln.java | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/app/src/util/log.c b/app/src/util/log.c index 25b1f26e8c..0975e54ab0 100644 --- a/app/src/util/log.c +++ b/app/src/util/log.c @@ -125,8 +125,30 @@ sc_av_log_callback(void *avcl, int level, const char *fmt, va_list vl) { free(local_fmt); } +static const char *const sc_sdl_log_priority_names[SDL_NUM_LOG_PRIORITIES] = { + [SDL_LOG_PRIORITY_VERBOSE] = "VERBOSE", + [SDL_LOG_PRIORITY_DEBUG] = "DEBUG", + [SDL_LOG_PRIORITY_INFO] = "INFO", + [SDL_LOG_PRIORITY_WARN] = "WARN", + [SDL_LOG_PRIORITY_ERROR] = "ERROR", + [SDL_LOG_PRIORITY_CRITICAL] = "CRITICAL", +}; + +static void SDLCALL +sc_sdl_log_print(void *userdata, int category, SDL_LogPriority priority, + const char *message) { + (void) userdata; + (void) category; + + FILE *out = priority < SDL_LOG_PRIORITY_WARN ? stdout : stderr; + assert(priority < SDL_NUM_LOG_PRIORITIES); + const char *prio_name = sc_sdl_log_priority_names[priority]; + fprintf(out, "%s: %s\n", prio_name, message); +} + void sc_log_configure() { + SDL_LogSetOutputFunction(sc_sdl_log_print, NULL); // Redirect FFmpeg logs to SDL logs av_log_set_callback(sc_av_log_callback); } diff --git a/server/src/main/java/com/genymobile/scrcpy/Ln.java b/server/src/main/java/com/genymobile/scrcpy/Ln.java index 291f26ff08..199c29bec2 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Ln.java +++ b/server/src/main/java/com/genymobile/scrcpy/Ln.java @@ -60,7 +60,7 @@ public static void i(String message) { public static void w(String message, Throwable throwable) { if (isEnabled(Level.WARN)) { Log.w(TAG, message, throwable); - System.out.print(PREFIX + "WARN: " + message + '\n'); + System.err.print(PREFIX + "WARN: " + message + '\n'); if (throwable != null) { throwable.printStackTrace(); } @@ -74,7 +74,7 @@ public static void w(String message) { public static void e(String message, Throwable throwable) { if (isEnabled(Level.ERROR)) { Log.e(TAG, message, throwable); - System.out.print(PREFIX + "ERROR: " + message + "\n"); + System.err.print(PREFIX + "ERROR: " + message + "\n"); if (throwable != null) { throwable.printStackTrace(); } From 5ee59e0f133b079f0652f10b80bb6320b41be278 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 8 Mar 2023 21:34:42 +0100 Subject: [PATCH 116/118] Add thread priority API Expose an API to change the priority of the current thread. --- app/src/compat.h | 4 ++++ app/src/util/thread.c | 33 +++++++++++++++++++++++++++++++++ app/src/util/thread.h | 10 ++++++++++ 3 files changed, 47 insertions(+) diff --git a/app/src/compat.h b/app/src/compat.h index 22563421cb..00cb720458 100644 --- a/app/src/compat.h +++ b/app/src/compat.h @@ -54,6 +54,10 @@ # define SCRCPY_SDL_HAS_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR #endif +#if SDL_VERSION_ATLEAST(2, 0, 16) +# define SCRCPY_SDL_HAS_THREAD_PRIORITY_TIME_CRITICAL +#endif + #ifndef HAVE_STRDUP char *strdup(const char *s); #endif diff --git a/app/src/util/thread.c b/app/src/util/thread.c index f9687add3f..94921fb7ba 100644 --- a/app/src/util/thread.c +++ b/app/src/util/thread.c @@ -23,6 +23,39 @@ sc_thread_create(sc_thread *thread, sc_thread_fn fn, const char *name, return true; } +static SDL_ThreadPriority +to_sdl_thread_priority(enum sc_thread_priority priority) { + switch (priority) { + case SC_THREAD_PRIORITY_TIME_CRITICAL: +#ifdef SCRCPY_SDL_HAS_THREAD_PRIORITY_TIME_CRITICAL + return SDL_THREAD_PRIORITY_TIME_CRITICAL; +#else + // fall through +#endif + case SC_THREAD_PRIORITY_HIGH: + return SDL_THREAD_PRIORITY_HIGH; + case SC_THREAD_PRIORITY_NORMAL: + return SDL_THREAD_PRIORITY_NORMAL; + case SC_THREAD_PRIORITY_LOW: + return SDL_THREAD_PRIORITY_LOW; + default: + assert(!"Unknown thread priority"); + return 0; + } +} + +bool +sc_thread_set_priority(enum sc_thread_priority priority) { + SDL_ThreadPriority sdl_priority = to_sdl_thread_priority(priority); + int r = SDL_SetThreadPriority(sdl_priority); + if (r) { + LOGD("Could not set thread priority: %s", SDL_GetError()); + return false; + } + + return true; +} + void sc_thread_join(sc_thread *thread, int *status) { SDL_WaitThread(thread->thread, status); diff --git a/app/src/util/thread.h b/app/src/util/thread.h index 7add6f1c2a..4183adacc5 100644 --- a/app/src/util/thread.h +++ b/app/src/util/thread.h @@ -21,6 +21,13 @@ typedef struct sc_thread { SDL_Thread *thread; } sc_thread; +enum sc_thread_priority { + SC_THREAD_PRIORITY_LOW, + SC_THREAD_PRIORITY_NORMAL, + SC_THREAD_PRIORITY_HIGH, + SC_THREAD_PRIORITY_TIME_CRITICAL, +}; + typedef struct sc_mutex { SDL_mutex *mutex; #ifndef NDEBUG @@ -39,6 +46,9 @@ sc_thread_create(sc_thread *thread, sc_thread_fn fn, const char *name, void sc_thread_join(sc_thread *thread, int *status); +bool +sc_thread_set_priority(enum sc_thread_priority priority); + bool sc_mutex_init(sc_mutex *mutex); From aa450ffc3f618a286c0eb7a5f60244b53d650e75 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 8 Mar 2023 21:37:27 +0100 Subject: [PATCH 117/118] Increase audio thread priority The audio demuxer thread is the one filling the audio buffer read by the SDL audio thread. It is time critical to avoid buffer underflow. --- app/src/audio_player.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/src/audio_player.c b/app/src/audio_player.c index de218f1ecd..85de0620f1 100644 --- a/app/src/audio_player.c +++ b/app/src/audio_player.c @@ -382,6 +382,14 @@ sc_audio_player_frame_sink_open(struct sc_frame_sink *sink, ap->received = false; ap->played = false; + // The thread calling open() is the thread calling push(), which fills the + // audio buffer consumed by the SDL audio thread. + ok = sc_thread_set_priority(SC_THREAD_PRIORITY_TIME_CRITICAL); + if (!ok) { + ok = sc_thread_set_priority(SC_THREAD_PRIORITY_HIGH); + (void) ok; // We don't care if it worked, at least we tried + } + SDL_PauseAudioDevice(ap->device, 0); return true; From 408f45863617465cee36453fb9ce2c7403e159e0 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Wed, 8 Mar 2023 21:40:39 +0100 Subject: [PATCH 118/118] Decrease recorder thread priority Recording is background task, writing the packets to a file is not urgent. --- app/src/recorder.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/src/recorder.c b/app/src/recorder.c index af5fe51084..572d3e24fd 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -505,6 +505,10 @@ static int run_recorder(void *data) { struct sc_recorder *recorder = data; + // Recording is a background task + bool ok = sc_thread_set_priority(SC_THREAD_PRIORITY_LOW); + (void) ok; // We don't care if it worked + bool success = sc_recorder_record(recorder); sc_mutex_lock(&recorder->mutex);