From 4deb881ec8b4a3d3eda492d457e576342f29aae2 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Sat, 18 Nov 2017 16:37:17 -0800 Subject: [PATCH] Enable adaptive playback on non-Intel devices --- .../video/MediaCodecDecoderRenderer.java | 88 +++++++++++++------ .../binding/video/MediaCodecHelper.java | 30 +++---- 2 files changed, 74 insertions(+), 44 deletions(-) diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index abd53c7cd..03f08aca4 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -33,6 +33,9 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer { private byte[] vpsBuffer; private byte[] spsBuffer; + private byte[] ppsBuffer; + private boolean submittedCsd; + private boolean submitCsdNextCall; private MediaCodec videoDecoder; private Thread rendererThread; @@ -151,7 +154,7 @@ public MediaCodecDecoderRenderer(PreferenceConfiguration prefs, // shared between AVC and HEVC decoders on the same device. if (avcDecoder != null) { directSubmit = MediaCodecHelper.decoderCanDirectSubmit(avcDecoder.getName()); - adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(avcDecoder.getName()); + adaptivePlayback = MediaCodecHelper.decoderSupportsAdaptivePlayback(avcDecoder); refFrameInvalidationAvc = MediaCodecHelper.decoderSupportsRefFrameInvalidationAvc(avcDecoder.getName()); refFrameInvalidationHevc = MediaCodecHelper.decoderSupportsRefFrameInvalidationHevc(avcDecoder.getName()); @@ -743,29 +746,44 @@ else if (decodeUnitType == MoonBridge.BUFFER_TYPE_SPS) { else if (decodeUnitType == MoonBridge.BUFFER_TYPE_PPS) { numPpsIn++; - inputBufferIndex = dequeueInputBuffer(); - if (inputBufferIndex < 0) { - // We're being torn down now - return MoonBridge.DR_NEED_IDR; - } + // If this is the first CSD blob or we aren't supporting + // adaptive playback, we will submit the CSD blob in a + // separate input buffer. + if (!submittedCsd || !adaptivePlayback) { + inputBufferIndex = dequeueInputBuffer(); + if (inputBufferIndex < 0) { + // We're being torn down now + return MoonBridge.DR_NEED_IDR; + } - buf = getEmptyInputBuffer(inputBufferIndex); - if (buf == null) { - // We're being torn down now - return MoonBridge.DR_NEED_IDR; - } + buf = getEmptyInputBuffer(inputBufferIndex); + if (buf == null) { + // We're being torn down now + return MoonBridge.DR_NEED_IDR; + } - // When we get the PPS, submit the VPS and SPS together with - // the PPS, as required by AOSP docs on use of MediaCodec. - if (vpsBuffer != null) { - buf.put(vpsBuffer); - } - if (spsBuffer != null) { - buf.put(spsBuffer); + // When we get the PPS, submit the VPS and SPS together with + // the PPS, as required by AOSP docs on use of MediaCodec. + if (vpsBuffer != null) { + buf.put(vpsBuffer); + } + if (spsBuffer != null) { + buf.put(spsBuffer); + } + + // This is the CSD blob + codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG; } + else { + // Batch this to submit together with the next I-frame + ppsBuffer = new byte[decodeUnitLength]; + System.arraycopy(decodeUnitData, 0, ppsBuffer, 0, decodeUnitLength); + + // Next call will be I-frame data + submitCsdNextCall = true; - // This is the CSD blob - codecFlags |= MediaCodec.BUFFER_FLAG_CODEC_CONFIG; + return MoonBridge.DR_OK; + } } else { inputBufferIndex = dequeueInputBuffer(); @@ -779,6 +797,20 @@ else if (decodeUnitType == MoonBridge.BUFFER_TYPE_PPS) { // We're being torn down now return MoonBridge.DR_NEED_IDR; } + + if (submitCsdNextCall) { + if (vpsBuffer != null) { + buf.put(vpsBuffer); + } + if (spsBuffer != null) { + buf.put(spsBuffer); + } + if (ppsBuffer != null) { + buf.put(ppsBuffer); + } + + submitCsdNextCall = false; + } } // Copy data from our buffer list into the input buffer @@ -790,14 +822,18 @@ else if (decodeUnitType == MoonBridge.BUFFER_TYPE_PPS) { return MoonBridge.DR_NEED_IDR; } - if (needsBaselineSpsHack) { - needsBaselineSpsHack = false; + if ((codecFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + submittedCsd = true; - if (!replaySps()) { - return MoonBridge.DR_NEED_IDR; - } + if (needsBaselineSpsHack) { + needsBaselineSpsHack = false; - LimeLog.info("SPS replay complete"); + if (!replaySps()) { + return MoonBridge.DR_NEED_IDR; + } + + LimeLog.info("SPS replay complete"); + } } return MoonBridge.DR_OK; diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java index c503370f8..12b53fe68 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java @@ -27,7 +27,7 @@ public class MediaCodecHelper { private static final List blacklistedDecoderPrefixes; private static final List spsFixupBitstreamFixupDecoderPrefixes; - private static final List whitelistedAdaptiveResolutionPrefixes; + private static final List blacklistedAdaptivePlaybackPrefixes; private static final List deprioritizedHevcDecoders; private static final List baselineProfileHackPrefixes; private static final List directSubmitPrefixes; @@ -93,11 +93,10 @@ public class MediaCodecHelper { baselineProfileHackPrefixes = new LinkedList<>(); baselineProfileHackPrefixes.add("omx.intel"); - whitelistedAdaptiveResolutionPrefixes = new LinkedList<>(); - whitelistedAdaptiveResolutionPrefixes.add("omx.nvidia"); - whitelistedAdaptiveResolutionPrefixes.add("omx.qcom"); - whitelistedAdaptiveResolutionPrefixes.add("omx.sec"); - whitelistedAdaptiveResolutionPrefixes.add("omx.TI"); + // The Intel decoder on Lollipop on Nexus Player would increase latency badly + // if adaptive playback was enabled so let's avoid it to be safe. + blacklistedAdaptivePlaybackPrefixes = new LinkedList<>(); + blacklistedAdaptivePlaybackPrefixes.add("omx.intel"); constrainedHighProfilePrefixes = new LinkedList<>(); constrainedHighProfilePrefixes.add("omx.intel"); @@ -208,19 +207,14 @@ public static long getMonotonicMillis() { return System.nanoTime() / 1000000L; } - @TargetApi(Build.VERSION_CODES.KITKAT) - public static boolean decoderSupportsAdaptivePlayback(String decoderName) { - /* - FIXME: Intel's decoder on Nexus Player forces the high latency path if adaptive playback is enabled - so we'll keep it off for now, since we don't know whether other devices also do the same - - if (isDecoderInList(whitelistedAdaptiveResolutionPrefixes, decoderName)) { - LimeLog.info("Adaptive playback supported (whitelist)"); - return true; - } - + public static boolean decoderSupportsAdaptivePlayback(MediaCodecInfo decoderInfo) { // Possibly enable adaptive playback on KitKat and above if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { + if (isDecoderInList(blacklistedAdaptivePlaybackPrefixes, decoderInfo.getName())) { + LimeLog.info("Decoder blacklisted for adaptive playback"); + return false; + } + try { if (decoderInfo.getCapabilitiesForType("video/avc"). isFeatureSupported(CodecCapabilities.FEATURE_AdaptivePlayback)) @@ -232,7 +226,7 @@ public static boolean decoderSupportsAdaptivePlayback(String decoderName) { } catch (Exception e) { // Tolerate buggy codecs } - }*/ + } return false; }