Skip to content

Commit

Permalink
Capture device audio
Browse files Browse the repository at this point in the history 
Create an AudioRecorder to capture the audio source REMOTE_SUBMIX.

For now, the captured packets are just logged into the console.

PR #3757 <#3757>

Co-authored-by: Romain Vimont <[email protected]>
Signed-off-by: Romain Vimont <[email protected]>
  • Loading branch information
@yume-chan @rom1v
yume-chan and rom1v committed Mar 3, 2023
1 parent 87bb26f commit 2655312
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 9 deletions.
80 changes: 80 additions & 0 deletions server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package com.genymobile.scrcpy;

import android.annotation.SuppressLint;
import android.annotation.TargetApi;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.Build;

public final class AudioEncoder {

private static final int SAMPLE_RATE = 48000;
private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
private static final int CHANNELS = 2;
private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
private static final int BYTES_PER_SAMPLE = 2;

private static final int BUFFER_MS = 5; // milliseconds
private static final int BUFFER_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * BUFFER_MS / 1000;

private Thread thread;

private static AudioFormat createAudioFormat() {
AudioFormat.Builder builder = new AudioFormat.Builder();
builder.setEncoding(FORMAT);
builder.setSampleRate(SAMPLE_RATE);
builder.setChannelMask(CHANNEL_CONFIG);
return builder.build();
}

@TargetApi(Build.VERSION_CODES.M)
@SuppressLint({"WrongConstant", "MissingPermission"})
private static AudioRecord createAudioRecord() {
AudioRecord.Builder builder = new AudioRecord.Builder();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
// On older APIs, Workarounds.fillAppInfo() must be called beforehand
builder.setContext(FakeContext.get());
}
builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
builder.setAudioFormat(createAudioFormat());
int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT);
// This buffer size does not impact latency
builder.setBufferSizeInBytes(8 * minBufferSize);
return builder.build();
}

public void start() {
AudioRecord recorder = createAudioRecord();

thread = new Thread(() -> {
recorder.startRecording();
try {
byte[] buf = new byte[BUFFER_SIZE];
while (!Thread.currentThread().isInterrupted()) {
int r = recorder.read(buf, 0, buf.length);
if (r > 0) {
Ln.i("Audio captured: " + r + " bytes");
} else {
Ln.e("Audio capture error: " + r);
}
}
} finally {
recorder.stop();
}
});
thread.start();
}

public void stop() {
if (thread != null) {
thread.interrupt();
}
}

public void join() throws InterruptedException {
if (thread != null) {
thread.join();
}
}
}
38 changes: 29 additions & 9 deletions server/src/main/java/com/genymobile/scrcpy/Server.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,19 +72,28 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc
boolean sendDummyByte = options.getSendDummyByte();

Workarounds.prepareMainLooper();
if (Build.BRAND.equalsIgnoreCase("meizu")) {
// Workarounds must be applied for Meizu phones:
// - <https://github.com/Genymobile/scrcpy/issues/240>
// - <https://github.com/Genymobile/scrcpy/issues/365>
// - <https://github.com/Genymobile/scrcpy/issues/2656>
//
// But only apply when strictly necessary, since workarounds can cause other issues:
// - <https://github.com/Genymobile/scrcpy/issues/940>
// - <https://github.com/Genymobile/scrcpy/issues/994>

// Workarounds must be applied for Meizu phones:
// - <https://github.com/Genymobile/scrcpy/issues/240>
// - <https://github.com/Genymobile/scrcpy/issues/365>
// - <https://github.com/Genymobile/scrcpy/issues/2656>
//
// But only apply when strictly necessary, since workarounds can cause other issues:
// - <https://github.com/Genymobile/scrcpy/issues/940>
// - <https://github.com/Genymobile/scrcpy/issues/994>
boolean mustFillAppInfo = Build.BRAND.equalsIgnoreCase("meizu");

// Before Android 11, audio is not supported.
// Since Android 12, we can properly set a context on the AudioRecord.
// Only on Android 11 we must fill app info for the AudioRecord to work.
mustFillAppInfo |= audio && Build.VERSION.SDK_INT == Build.VERSION_CODES.R;

if (mustFillAppInfo) {
Workarounds.fillAppInfo();
}

Controller controller = null;
AudioEncoder audioEncoder = null;

try (DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, audio, control, sendDummyByte)) {
VideoCodec codec = options.getCodec();
Expand All @@ -101,6 +110,11 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc
device.setClipboardListener(text -> controllerRef.getSender().pushClipboardText(text));
}

if (audio) {
audioEncoder = new AudioEncoder();
audioEncoder.start();
}

Streamer videoStreamer = new Streamer(connection.getVideoFd(), codec, options.getSendCodecId(), options.getSendFrameMeta());
ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getBitRate(), options.getMaxFps(),
codecOptions, options.getEncoderName(), options.getDownsizeOnError());
Expand All @@ -116,12 +130,18 @@ private static void scrcpy(Options options) throws IOException, ConfigurationExc
} finally {
Ln.d("Screen streaming stopped");
initThread.interrupt();
if (audioEncoder != null) {
audioEncoder.stop();
}
if (controller != null) {
controller.stop();
}

try {
initThread.join();
if (audioEncoder != null) {
audioEncoder.join();
}
if (controller != null) {
controller.join();
}
Expand Down

0 comments on commit 2655312

Please sign in to comment.