diff --git a/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.kt b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.kt new file mode 100644 index 0000000000..d6ff4a7909 --- /dev/null +++ b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.kt @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc + +/** + * The main difference with the standard [DefaultAlignedVideoEncoderFactory] is that this fixes + * issues with resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can + * set the alignment by setting [resolutionAdjustment]. Internally the resolution during streaming + * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the + * standard [DefaultVideoEncoderFactory] and it will use the SW encoder if HW fails + * or is not available. + * + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072 + * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnco + * derFactoryWrapper.kt#L18 + */ +class DefaultAlignedVideoEncoderFactory( + eglContext: EglBase.Context?, + enableIntelVp8Encoder: Boolean = true, + enableH264HighProfile: Boolean = false, + resolutionAdjustment: ResolutionAdjustment, +) : VideoEncoderFactory { + private val hardwareVideoEncoderFactory: VideoEncoderFactory + private val softwareVideoEncoderFactory: VideoEncoderFactory = SoftwareVideoEncoderFactory() + + init { + val defaultFactory = + HardwareVideoEncoderFactory(eglContext, enableIntelVp8Encoder, enableH264HighProfile) + + hardwareVideoEncoderFactory = if (resolutionAdjustment == ResolutionAdjustment.NONE) { + defaultFactory + } else { + HardwareVideoEncoderWrapperFactory(defaultFactory, resolutionAdjustment.value) + } + } + + override fun createEncoder(info: VideoCodecInfo): VideoEncoder? { + val softwareEncoder: VideoEncoder? = softwareVideoEncoderFactory.createEncoder(info) + val hardwareEncoder: VideoEncoder? = hardwareVideoEncoderFactory.createEncoder(info) + if (hardwareEncoder != null && softwareEncoder != null) { + return VideoEncoderFallback(softwareEncoder, hardwareEncoder) + } + return hardwareEncoder ?: softwareEncoder + } + + override fun getSupportedCodecs(): Array { + val supportedCodecInfos = LinkedHashSet() + supportedCodecInfos.addAll(listOf(*softwareVideoEncoderFactory.supportedCodecs)) + supportedCodecInfos.addAll(listOf(*hardwareVideoEncoderFactory.supportedCodecs)) + return supportedCodecInfos.toTypedArray() + } +} diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.kt b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.kt new file mode 100644 index 0000000000..d7e65640a3 --- /dev/null +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.kt @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc + +/** + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207 + * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco + * derWrapperFactory.kt + */ +internal class HardwareVideoEncoderWrapper( + private val internalEncoder: VideoEncoder, + private val alignment: Int, +) : VideoEncoder { + class CropSizeCalculator( + alignment: Int, + private val originalWidth: Int, + private val originalHeight: Int, + ) { + + companion object { + val TAG = CropSizeCalculator::class.simpleName + } + + val cropX: Int = originalWidth % alignment + val cropY: Int = originalHeight % alignment + + val croppedWidth: Int + get() = originalWidth - cropX + + val croppedHeight: Int + get() = originalHeight - cropY + + val isCropRequired: Boolean + get() = cropX != 0 || cropY != 0 + + init { + if (originalWidth != 0 && originalHeight != 0) { + Logging.v( + TAG, + "$this init(): alignment=$alignment" + + "" + + " size=${originalWidth}x$originalHeight => ${croppedWidth}x$croppedHeight", + ) + } + } + + fun hasFrameSizeChanged(nextWidth: Int, nextHeight: Int): Boolean { + return if (originalWidth == nextWidth && originalHeight == nextHeight) { + false + } else { + Logging.v( + TAG, + "frame size has changed: " + + "${originalWidth}x$originalHeight => ${nextWidth}x$nextHeight", + ) + true + } + } + } + + companion object { + val TAG = HardwareVideoEncoderWrapper::class.simpleName + } + + private var calculator = CropSizeCalculator(1, 0, 0) + + private fun retryWithoutCropping( + width: Int, + height: Int, + retryFunc: () -> VideoCodecStatus, + ): VideoCodecStatus { + Logging.v(TAG, "retrying without resolution adjustment") + + calculator = CropSizeCalculator(1, width, height) + + return retryFunc() + } + + override fun initEncode( + originalSettings: VideoEncoder.Settings, + callback: VideoEncoder.Callback?, + ): VideoCodecStatus { + calculator = CropSizeCalculator(alignment, originalSettings.width, originalSettings.height) + + if (!calculator.isCropRequired) { + return internalEncoder.initEncode(originalSettings, callback) + } else { + val croppedSettings = VideoEncoder.Settings( + originalSettings.numberOfCores, + calculator.croppedWidth, + calculator.croppedHeight, + originalSettings.startBitrate, + originalSettings.maxFramerate, + originalSettings.numberOfSimulcastStreams, + originalSettings.automaticResizeOn, + originalSettings.capabilities, + ) + + try { + val result = internalEncoder.initEncode(croppedSettings, callback) + return if (result == VideoCodecStatus.FALLBACK_SOFTWARE) { + Logging.e( + TAG, + "internalEncoder.initEncode() returned FALLBACK_SOFTWARE: " + + "croppedSettings $croppedSettings", + ) + retryWithoutCropping( + originalSettings.width, + originalSettings.height, + ) { internalEncoder.initEncode(originalSettings, callback) } + } else { + result + } + } catch (e: Exception) { + Logging.e(TAG, "internalEncoder.initEncode() failed", e) + return retryWithoutCropping( + originalSettings.width, + originalSettings.height, + ) { internalEncoder.initEncode(originalSettings, callback) } + } + } + } + + override fun release(): VideoCodecStatus { + return internalEncoder.release() + } + + override fun encode(frame: VideoFrame, encodeInfo: VideoEncoder.EncodeInfo?): VideoCodecStatus { + if (calculator.hasFrameSizeChanged(frame.buffer.width, frame.buffer.height)) { + calculator = CropSizeCalculator(alignment, frame.buffer.width, frame.buffer.height) + } + + if (!calculator.isCropRequired) { + return internalEncoder.encode(frame, encodeInfo) + } else { + // https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/sdk/android/api/org/webrtc/JavaI420Buffer.java;l=172-185;drc=02334e07c5c04c729dd3a8a279bb1fbe24ee8b7c + val croppedWidth = calculator.croppedWidth + val croppedHeight = calculator.croppedHeight + val croppedBuffer = frame.buffer.cropAndScale( + calculator.cropX / 2, + calculator.cropY / 2, + croppedWidth, + croppedHeight, + croppedWidth, + croppedHeight, + ) + + val croppedFrame = VideoFrame(croppedBuffer, frame.rotation, frame.timestampNs) + + try { + val result = internalEncoder.encode(croppedFrame, encodeInfo) + return if (result == VideoCodecStatus.FALLBACK_SOFTWARE) { + Logging.e(TAG, "internalEncoder.encode() returned FALLBACK_SOFTWARE") + retryWithoutCropping(frame.buffer.width, frame.buffer.height) { + internalEncoder.encode( + frame, + encodeInfo, + ) + } + } else { + result + } + } catch (e: Exception) { + Logging.e(TAG, "internalEncoder.encode() failed", e) + return retryWithoutCropping( + frame.buffer.width, + frame.buffer.height, + ) { internalEncoder.encode(frame, encodeInfo) } + } finally { + croppedBuffer.release() + } + } + } + + override fun setRateAllocation( + allocation: VideoEncoder.BitrateAllocation?, + frameRate: Int, + ): VideoCodecStatus { + return internalEncoder.setRateAllocation(allocation, frameRate) + } + + override fun getScalingSettings(): VideoEncoder.ScalingSettings { + return internalEncoder.scalingSettings + } + + override fun getImplementationName(): String { + return internalEncoder.implementationName + } + + override fun createNativeVideoEncoder(): Long { + return internalEncoder.createNativeVideoEncoder() + } + + override fun isHardwareEncoder(): Boolean { + return internalEncoder.isHardwareEncoder + } + + override fun setRates(rcParameters: VideoEncoder.RateControlParameters?): VideoCodecStatus { + return internalEncoder.setRates(rcParameters) + } + + override fun getResolutionBitrateLimits(): Array { + return internalEncoder.resolutionBitrateLimits + } + + override fun getEncoderInfo(): VideoEncoder.EncoderInfo { + return internalEncoder.encoderInfo + } +} + +internal class HardwareVideoEncoderWrapperFactory( + private val factory: HardwareVideoEncoderFactory, + private val resolutionPixelAlignment: Int, +) : VideoEncoderFactory { + companion object { + val TAG = HardwareVideoEncoderWrapperFactory::class.simpleName + } + + init { + if (resolutionPixelAlignment == 0) { + throw java.lang.Exception("resolutionPixelAlignment should not be 0") + } + } + + override fun createEncoder(videoCodecInfo: VideoCodecInfo?): VideoEncoder? { + try { + val encoder = factory.createEncoder(videoCodecInfo) ?: return null + return HardwareVideoEncoderWrapper(encoder, resolutionPixelAlignment) + } catch (e: Exception) { + Logging.e(TAG, "createEncoder failed", e) + return null + } + } + + override fun getSupportedCodecs(): Array { + return factory.supportedCodecs + } +} diff --git a/sdk/android/api/org/webrtc/ResolutionAdjustment.kt b/sdk/android/api/org/webrtc/ResolutionAdjustment.kt new file mode 100644 index 0000000000..959d988c99 --- /dev/null +++ b/sdk/android/api/org/webrtc/ResolutionAdjustment.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc + +/** + * Resolution alignment values. Generally the [MULTIPLE_OF_16] is recommended + * for both VP8 and H264 + */ +enum class ResolutionAdjustment(val value: Int) { + NONE(1), + MULTIPLE_OF_2(2), + MULTIPLE_OF_4(4), + MULTIPLE_OF_8(8), + MULTIPLE_OF_16(16), +} diff --git a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.kt b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.kt new file mode 100644 index 0000000000..865e79b419 --- /dev/null +++ b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.kt @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc + +import java.util.concurrent.Callable +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * The main difference with the standard [SimulcastVideoEncoderFactory] is that this fixes issues + * with simulcasting resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can + * set the alignment by setting [resolutionAdjustment]. Internally the resolutions during simulcast + * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the + * standard [SimulcastVideoEncoderFactory] and it will use the SW encoder if HW fails + * or is not available. + * + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072 + * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnc + * oderFactoryWrapper.kt#L18 + */ +class SimulcastAlignedVideoEncoderFactory( + sharedContext: EglBase.Context?, + enableIntelVp8Encoder: Boolean = true, + enableH264HighProfile: Boolean = false, + resolutionAdjustment: ResolutionAdjustment, +) : VideoEncoderFactory { + + private class StreamEncoderWrapper( + private val encoder: VideoEncoder, + ) : VideoEncoder { + companion object { + val TAG = StreamEncoderWrapper::class.simpleName + } + + private val executor: ExecutorService = Executors.newSingleThreadExecutor() + private var streamSettings: VideoEncoder.Settings? = null + + override fun initEncode( + settings: VideoEncoder.Settings, + callback: VideoEncoder.Callback?, + ): VideoCodecStatus { + streamSettings = settings + val future = executor.submit( + Callable { + Logging.v( + TAG, + """initEncode() thread=${Thread.currentThread().name} [${Thread.currentThread().id}] + | encoder=${encoder.implementationName} + | streamSettings: + | numberOfCores=${settings.numberOfCores} + | width=${settings.width} + | height=${settings.height} + | startBitrate=${settings.startBitrate} + | maxFramerate=${settings.maxFramerate} + | automaticResizeOn=${settings.automaticResizeOn} + | numberOfSimulcastStreams=${settings.numberOfSimulcastStreams} + | lossNotification=${settings.capabilities.lossNotification} + """.trimMargin(), + ) + return@Callable encoder.initEncode(settings, callback) + }, + ) + return future.get() + } + + override fun release(): VideoCodecStatus { + val future = executor.submit(Callable { return@Callable encoder.release() }) + return future.get() + } + + override fun encode(frame: VideoFrame, encodeInfo: VideoEncoder.EncodeInfo?): VideoCodecStatus { + val future = executor.submit( + Callable { + return@Callable streamSettings?.let { + if (frame.buffer.width == it.width) { + encoder.encode(frame, encodeInfo) + } else { + val originalWidth = frame.buffer.width + val originalHeight = frame.buffer.height + val scaledBuffer = frame.buffer.cropAndScale( + 0, 0, originalWidth, originalHeight, + it.width, it.height, + ) + val scaledFrame = VideoFrame(scaledBuffer, frame.rotation, frame.timestampNs) + val result = encoder.encode(scaledFrame, encodeInfo) + scaledBuffer.release() + result + } + } ?: run { + VideoCodecStatus.ERROR + } + }, + ) + return future.get() + } + + override fun setRateAllocation( + allocation: VideoEncoder.BitrateAllocation?, + frameRate: Int, + ): VideoCodecStatus { + val future = executor.submit( + Callable { + return@Callable encoder.setRateAllocation( + allocation, + frameRate, + ) + }, + ) + return future.get() + } + + override fun getScalingSettings(): VideoEncoder.ScalingSettings { + val future = executor.submit(Callable { return@Callable encoder.scalingSettings }) + return future.get() + } + + override fun getImplementationName(): String { + val future = executor.submit(Callable { return@Callable encoder.implementationName }) + return future.get() + } + } + + private class StreamEncoderWrapperFactory( + private val factory: VideoEncoderFactory, + ) : VideoEncoderFactory { + override fun createEncoder(videoCodecInfo: VideoCodecInfo?): VideoEncoder? { + val encoder = factory.createEncoder(videoCodecInfo) + if (encoder == null) { + return null + } + return StreamEncoderWrapper(encoder) + } + + override fun getSupportedCodecs(): Array { + return factory.supportedCodecs + } + } + + private val primary: VideoEncoderFactory + private val fallback: VideoEncoderFactory? + private val native: SimulcastVideoEncoderFactory + + init { + val hardwareVideoEncoderFactory = HardwareVideoEncoderFactory( + sharedContext, + enableIntelVp8Encoder, + enableH264HighProfile, + ) + + val encoderFactory = if (resolutionAdjustment == ResolutionAdjustment.NONE) { + hardwareVideoEncoderFactory + } else { + HardwareVideoEncoderWrapperFactory( + hardwareVideoEncoderFactory, + resolutionAdjustment.value, + ) + } + + primary = StreamEncoderWrapperFactory(encoderFactory) + fallback = SoftwareVideoEncoderFactory() + native = SimulcastVideoEncoderFactory(primary, fallback) + } + + override fun createEncoder(info: VideoCodecInfo?): VideoEncoder? { + return native.createEncoder(info) + } + + override fun getSupportedCodecs(): Array { + return native.supportedCodecs + } +}