[android] Handle mediacodec callback with a handler thread

Now the MediaCodec callbacks and the activity lifecycle callbacks
are called on the main thread. When pausing YouTube, the MediaCodec
callbacks may be blocked until the lifecycle callbacks are done.
This may cause frame drops on a resource-limited device when the
MediaCodec callback to notify a newly decoded frame is blocked and
we run out of decoded frames. Running the MediaCodec callbacks on a
handler thread could avoid this problem.

Change-Id: I5ffdb1f5a582c3d01964b3f98c99d7aed211674a

starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java

@@ -31,6 +31,8 @@
 import android.media.MediaFormat;
 import android.os.Build;
 import android.os.Bundle;
+import android.os.Handler;
+import android.os.HandlerThread;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import dev.cobalt.util.Log;
@@ -93,6 +95,9 @@ class MediaCodecBridge {
   // which would cause GC cycles long enough to impact playback.
   private final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
 
+  private Handler mHandler = null;
+  private HandlerThread mCallbackThread = null;
+
   // Type of bitrate adjustment for video encoder.
   public enum BitrateAdjustmentTypes {
     // No adjustment - video encoder has no known bitrate problem.
@@ -451,7 +456,8 @@ private MediaCodecBridge(
       MediaCodec mediaCodec,
       String mime,
       BitrateAdjustmentTypes bitrateAdjustmentType,
-      int tunnelModeAudioSessionId) {
+      int tunnelModeAudioSessionId,
+      boolean useCallbackThread) {
     if (mediaCodec == null) {
       throw new IllegalArgumentException();
     }
@@ -461,6 +467,11 @@ private MediaCodecBridge(
     mLastPresentationTimeUs = 0;
     mFlushed = true;
     mBitrateAdjustmentType = bitrateAdjustmentType;
+    if (useCallbackThread) {
+      mCallbackThread = new HandlerThread("MediaCodec:Callback:Handler");
+      mCallbackThread.start();
+      mHandler = new Handler(mCallbackThread.getLooper());
+    }
     mCallback =
         new MediaCodec.Callback() {
           @Override
@@ -525,7 +536,7 @@ public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
             }
           }
         };
-    mMediaCodec.setCallback(mCallback);
+    mMediaCodec.setCallback(mCallback, mHandler);
 
     // TODO: support OnFrameRenderedListener for non tunnel mode
     if (tunnelModeAudioSessionId != -1) {
@@ -555,6 +566,7 @@ public static MediaCodecBridge createAudioMediaCodecBridge(
       int sampleRate,
       int channelCount,
       MediaCrypto crypto,
+      boolean useCallbackThread,
       @Nullable byte[] configurationData) {
     if (decoderName.equals("")) {
       Log.e(TAG, "Invalid decoder name.");
@@ -573,7 +585,12 @@ public static MediaCodecBridge createAudioMediaCodecBridge(
     }
     MediaCodecBridge bridge =
         new MediaCodecBridge(
-            nativeMediaCodecBridge, mediaCodec, mime, BitrateAdjustmentTypes.NO_ADJUSTMENT, -1);
+            nativeMediaCodecBridge,
+            mediaCodec,
+            mime,
+            BitrateAdjustmentTypes.NO_ADJUSTMENT,
+            -1,
+            useCallbackThread);
 
     MediaFormat mediaFormat = createAudioFormat(mime, sampleRate, channelCount);
 
@@ -618,6 +635,7 @@ public static void createVideoMediaCodecBridge(
       MediaCrypto crypto,
       ColorInfo colorInfo,
       int tunnelModeAudioSessionId,
+      boolean useCallbackThread,
       CreateMediaCodecBridgeResult outCreateMediaCodecBridgeResult) {
     MediaCodec mediaCodec = null;
     outCreateMediaCodecBridgeResult.mMediaCodecBridge = null;
@@ -671,7 +689,8 @@ public static void createVideoMediaCodecBridge(
             mediaCodec,
             mime,
             BitrateAdjustmentTypes.NO_ADJUSTMENT,
-            tunnelModeAudioSessionId);
+            tunnelModeAudioSessionId,
+            useCallbackThread);
     MediaFormat mediaFormat =
         createVideoDecoderFormat(mime, widthHint, heightHint, videoCapabilities);
 
@@ -806,6 +825,11 @@ public void release() {
       Log.e(TAG, "Cannot release media codec", e);
     }
     mMediaCodec = null;
+    if (mCallbackThread != null) {
+      mCallbackThread.quitSafely();
+      mCallbackThread = null;
+      mHandler = null;
+    }
   }
 
   @SuppressWarnings("unused")

starboard/android/shared/audio_decoder.cc

@@ -70,12 +70,14 @@ void* IncrementPointerByBytes(void* pointer, int offset) {
 }  // namespace
 
 AudioDecoder::AudioDecoder(const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system)
+                           SbDrmSystem drm_system,
+                           bool use_mediacodec_callback_thread)
     : audio_stream_info_(audio_stream_info),
       sample_type_(GetSupportedSampleType()),
       output_sample_rate_(audio_stream_info.samples_per_second),
       output_channel_count_(audio_stream_info.number_of_channels),
-      drm_system_(static_cast<DrmSystem*>(drm_system)) {
+      drm_system_(static_cast<DrmSystem*>(drm_system)),
+      use_mediacodec_callback_thread_(use_mediacodec_callback_thread) {
   if (!InitializeCodec()) {
     SB_LOG(ERROR) << "Failed to initialize audio decoder.";
   }
@@ -186,7 +188,8 @@ void AudioDecoder::Reset() {
 
 bool AudioDecoder::InitializeCodec() {
   SB_DCHECK(!media_decoder_);
-  media_decoder_.reset(new MediaDecoder(this, audio_stream_info_, drm_system_));
+  media_decoder_.reset(new MediaDecoder(this, audio_stream_info_, drm_system_,
+                                        use_mediacodec_callback_thread_));
   if (media_decoder_->is_valid()) {
     if (error_cb_) {
       media_decoder_->Initialize(

starboard/android/shared/audio_decoder.h

@@ -45,7 +45,8 @@ class AudioDecoder
       AudioStreamInfo;
 
   AudioDecoder(const AudioStreamInfo& audio_stream_info,
-               SbDrmSystem drm_system);
+               SbDrmSystem drm_system,
+               bool use_mediacodec_callback_thread);
   ~AudioDecoder() override;
 
   void Initialize(const OutputCB& output_cb, const ErrorCB& error_cb) override;
@@ -83,6 +84,10 @@ class AudioDecoder
 
   DrmSystem* drm_system_;
 
+  // Set mediacodec callback with a handler on another thread to avoid running
+  // callbacks on the main thread and being blocked by other main thread tasks.
+  const bool use_mediacodec_callback_thread_;
+
   OutputCB output_cb_;
   ErrorCB error_cb_;
   ConsumedCB consumed_cb_;

starboard/android/shared/audio_renderer_passthrough.cc

@@ -75,14 +75,15 @@ int ParseAc3SyncframeAudioSampleCount(const uint8_t* buffer, int size) {
 
 AudioRendererPassthrough::AudioRendererPassthrough(
     const AudioStreamInfo& audio_stream_info,
-    SbDrmSystem drm_system)
+    SbDrmSystem drm_system,
+    bool use_mediacodec_callback_thread)
     : audio_stream_info_(audio_stream_info) {
   SB_DCHECK(audio_stream_info_.codec == kSbMediaAudioCodecAc3 ||
             audio_stream_info_.codec == kSbMediaAudioCodecEac3);
   if (SbDrmSystemIsValid(drm_system)) {
     SB_LOG(INFO) << "Creating AudioDecoder as decryptor.";
-    scoped_ptr<AudioDecoder> audio_decoder(
-        new AudioDecoder(audio_stream_info, drm_system));
+    scoped_ptr<AudioDecoder> audio_decoder(new AudioDecoder(
+        audio_stream_info, drm_system, use_mediacodec_callback_thread));
     if (audio_decoder->is_valid()) {
       decoder_.reset(audio_decoder.release());
     }

starboard/android/shared/audio_renderer_passthrough.h

@@ -53,7 +53,8 @@ class AudioRendererPassthrough
       AudioStreamInfo;
 
   AudioRendererPassthrough(const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system);
+                           SbDrmSystem drm_system,
+                           bool use_mediacodec_callback_thread);
   ~AudioRendererPassthrough() override;
 
   bool is_valid() const { return decoder_ != nullptr; }

starboard/android/shared/media_codec_bridge.cc

@@ -158,7 +158,8 @@ Java_dev_cobalt_media_MediaCodecBridge_nativeOnMediaCodecOutputFormatChanged(
 scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateAudioMediaCodecBridge(
     const AudioStreamInfo& audio_stream_info,
     Handler* handler,
-    jobject j_media_crypto) {
+    jobject j_media_crypto,
+    bool use_callback_thread) {
   bool is_passthrough = false;
   const char* mime =
       SupportedAudioCodecToMimeType(audio_stream_info.codec, &is_passthrough);
@@ -193,11 +194,11 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateAudioMediaCodecBridge(
       new MediaCodecBridge(handler));
   jobject j_media_codec_bridge = env->CallStaticObjectMethodOrAbort(
       "dev/cobalt/media/MediaCodecBridge", "createAudioMediaCodecBridge",
-      "(JLjava/lang/String;Ljava/lang/String;IILandroid/media/MediaCrypto;"
+      "(JLjava/lang/String;Ljava/lang/String;IILandroid/media/MediaCrypto;Z;"
       "[B)Ldev/cobalt/media/MediaCodecBridge;",
       reinterpret_cast<jlong>(native_media_codec_bridge.get()), j_mime.Get(),
       j_decoder_name.Get(), audio_stream_info.samples_per_second,
-      audio_stream_info.number_of_channels, j_media_crypto,
+      audio_stream_info.number_of_channels, j_media_crypto, use_callback_thread,
       configuration_data.Get());
 
   if (!j_media_codec_bridge) {
@@ -227,6 +228,7 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateVideoMediaCodecBridge(
     bool require_software_codec,
     int tunnel_mode_audio_session_id,
     bool force_big_endian_hdr_metadata,
+    bool use_callback_thread,
     std::string* error_message) {
   SB_DCHECK(error_message);
   SB_DCHECK(max_width.has_engaged() == max_height.has_engaged());
@@ -316,14 +318,14 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateVideoMediaCodecBridge(
       "(JLjava/lang/String;Ljava/lang/String;IIIIILandroid/view/Surface;"
       "Landroid/media/MediaCrypto;"
       "Ldev/cobalt/media/MediaCodecBridge$ColorInfo;"
-      "I"
+      "IZ"
       "Ldev/cobalt/media/MediaCodecBridge$CreateMediaCodecBridgeResult;)"
       "V",
       reinterpret_cast<jlong>(native_media_codec_bridge.get()), j_mime.Get(),
       j_decoder_name.Get(), width_hint, height_hint, fps,
       max_width.value_or(-1), max_height.value_or(-1), j_surface,
       j_media_crypto, j_color_info.Get(), tunnel_mode_audio_session_id,
-      j_create_media_codec_bridge_result.Get());
+      use_callback_thread, j_create_media_codec_bridge_result.Get());
 
   jobject j_media_codec_bridge = env->CallObjectMethodOrAbort(
       j_create_media_codec_bridge_result.Get(), "mediaCodecBridge",

starboard/android/shared/media_codec_bridge.h

@@ -147,7 +147,8 @@ class MediaCodecBridge {
   static scoped_ptr<MediaCodecBridge> CreateAudioMediaCodecBridge(
       const AudioStreamInfo& audio_stream_info,
       Handler* handler,
-      jobject j_media_crypto);
+      jobject j_media_crypto,
+      bool use_callback_thread);
 
   // `max_width` and `max_height` can be set to positive values to specify the
   // maximum resolutions the video can be adapted to.
@@ -173,6 +174,7 @@ class MediaCodecBridge {
       bool require_software_codec,
       int tunnel_mode_audio_session_id,
       bool force_big_endian_hdr_metadata,
+      bool use_callback_thread,
       std::string* error_message);
 
   ~MediaCodecBridge();

starboard/android/shared/media_decoder.cc

@@ -76,7 +76,8 @@ const char* GetDecoderName(SbMediaType media_type) {
 
 MediaDecoder::MediaDecoder(Host* host,
                            const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system)
+                           SbDrmSystem drm_system,
+                           bool use_mediacodec_callback_thread)
     : media_type_(kSbMediaTypeAudio),
       host_(host),
       drm_system_(static_cast<DrmSystem*>(drm_system)),
@@ -87,7 +88,7 @@ MediaDecoder::MediaDecoder(Host* host,
   jobject j_media_crypto = drm_system_ ? drm_system_->GetMediaCrypto() : NULL;
   SB_DCHECK(!drm_system_ || j_media_crypto);
   media_codec_bridge_ = MediaCodecBridge::CreateAudioMediaCodecBridge(
-      audio_stream_info, this, j_media_crypto);
+      audio_stream_info, this, j_media_crypto, use_mediacodec_callback_thread);
   if (!media_codec_bridge_) {
     SB_LOG(ERROR) << "Failed to create audio media codec bridge.";
     return;
@@ -118,6 +119,7 @@ MediaDecoder::MediaDecoder(Host* host,
                            const FrameRenderedCB& frame_rendered_cb,
                            int tunnel_mode_audio_session_id,
                            bool force_big_endian_hdr_metadata,
+                           bool use_mediacodec_callback_thread,
                            std::string* error_message)
     : media_type_(kSbMediaTypeVideo),
       host_(host),
@@ -135,7 +137,8 @@ MediaDecoder::MediaDecoder(Host* host,
       video_codec, width_hint, height_hint, fps, max_width, max_height, this,
       j_output_surface, j_media_crypto, color_metadata, require_secured_decoder,
       require_software_codec, tunnel_mode_audio_session_id,
-      force_big_endian_hdr_metadata, error_message);
+      force_big_endian_hdr_metadata, use_mediacodec_callback_thread,
+      error_message);
   if (!media_codec_bridge_) {
     SB_LOG(ERROR) << "Failed to create video media codec bridge with error: "
                   << *error_message;

starboard/android/shared/media_decoder.h

@@ -79,7 +79,8 @@ class MediaDecoder
 
   MediaDecoder(Host* host,
                const AudioStreamInfo& audio_stream_info,
-               SbDrmSystem drm_system);
+               SbDrmSystem drm_system,
+               bool use_mediacodec_callback_thread);
   MediaDecoder(Host* host,
                SbMediaVideoCodec video_codec,
                // `width_hint` and `height_hint` are used to create the Android
@@ -97,6 +98,7 @@ class MediaDecoder
                const FrameRenderedCB& frame_rendered_cb,
                int tunnel_mode_audio_session_id,
                bool force_big_endian_hdr_metadata,
+               bool use_mediacodec_callback_thread,
                std::string* error_message);
   ~MediaDecoder();
 

starboard/android/shared/media_is_video_supported.cc

@@ -92,6 +92,10 @@ bool SbMediaIsVideoSupported(SbMediaVideoCodec video_codec,
                                      false)) {
       MaxMediaCodecOutputBuffersLookupTable::GetInstance()->SetEnabled(false);
     }
+
+    if (!mime_type->ValidateBoolParameter("mediacodeccallbackthread")) {
+      return false;
+    }
   }
 
   if (must_support_tunnel_mode && decode_to_texture_required) {