Support for video described by the AV1 Dependency Descriptor (#1998)

jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt

@@ -137,13 +137,11 @@ class MediaSourceDesc
     fun getRtpLayerByQualityIdx(idx: Int): RtpLayerDesc? = layersByIndex[idx]
 
     @Synchronized
-    fun findRtpLayerDesc(videoRtpPacket: VideoRtpPacket): RtpLayerDesc? {
+    fun findRtpLayerDescs(videoRtpPacket: VideoRtpPacket): Collection<RtpLayerDesc> {
         if (ArrayUtils.isNullOrEmpty(rtpEncodings)) {
-            return null
+            return emptyList()
         }
-        val encodingId = videoRtpPacket.getEncodingId()
-        val desc = layersById[encodingId]
-        return desc
+        return videoRtpPacket.getEncodingIds().mapNotNull { layersById[it] }
     }
 
     @Synchronized
@@ -188,10 +186,14 @@ class MediaSourceDesc
  */
 fun Array<MediaSourceDesc>.copy() = Array(this.size) { i -> this[i].copy() }
 
-fun Array<MediaSourceDesc>.findRtpLayerDesc(packet: VideoRtpPacket): RtpLayerDesc? {
+fun Array<MediaSourceDesc>.findRtpLayerDescs(packet: VideoRtpPacket): Collection<RtpLayerDesc> {
+    return this.flatMap { it.findRtpLayerDescs(packet) }
+}
+
+fun Array<MediaSourceDesc>.findRtpEncodingId(packet: VideoRtpPacket): Int? {
     for (source in this) {
-        source.findRtpLayerDesc(packet)?.let {
-            return it
+        source.findRtpEncodingDesc(packet.ssrc)?.let {
+            return it.eid
         }
     }
     return null

jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt

@@ -73,9 +73,17 @@ constructor(
         validateLayerEids(initialLayers)
     }
 
+    private var nominalHeight = initialLayers.getNominalHeight()
+
     internal var layers = initialLayers
         set(newLayers) {
             validateLayerEids(newLayers)
+            /* Check if the new layer set is a single spatial layer that doesn't specify a height - if so, we
+             * want to apply the nominal height to them.
+             */
+            val useNominalHeight = nominalHeight != RtpLayerDesc.NO_HEIGHT &&
+                newLayers.all { it.sid == 0 } &&
+                newLayers.all { it.height == RtpLayerDesc.NO_HEIGHT }
             /* Copy the rate statistics objects from the old layers to the new layers
              * with matching layer IDs.
              */
@@ -89,6 +97,15 @@ constructor(
                 oldLayerMap[newLayer.layerId]?.let {
                     newLayer.inheritFrom(it)
                 }
+                if (useNominalHeight) {
+                    newLayer.height = nominalHeight
+                }
+            }
+            if (!useNominalHeight) {
+                val newNominalHeight = newLayers.getNominalHeight()
+                if (newNominalHeight != RtpLayerDesc.NO_HEIGHT) {
+                    nominalHeight = newNominalHeight
+                }
             }
             field = newLayers
         }
@@ -157,6 +174,7 @@ constructor(
         addNumber("rtx_ssrc", getSecondarySsrc(SsrcAssociationType.RTX))
         addNumber("fec_ssrc", getSecondarySsrc(SsrcAssociationType.FEC))
         addNumber("eid", eid)
+        addNumber("nominal_height", nominalHeight)
         for (layer in layers) {
             addBlock(layer.getNodeStats())
         }
@@ -167,6 +185,23 @@ constructor(
     }
 }
 
-fun VideoRtpPacket.getEncodingId(): Long {
-    return RtpEncodingDesc.calcEncodingId(ssrc, this.layerId)
+fun VideoRtpPacket.getEncodingIds(): Collection<Long> {
+    return this.layerIds.map { RtpEncodingDesc.calcEncodingId(ssrc, it) }
+}
+
+/**
+ * Get the "nominal" height of a set of layers - if they all indicate the same spatial layer and same height.
+ */
+private fun Array<RtpLayerDesc>.getNominalHeight(): Int {
+    if (isEmpty()) {
+        return RtpLayerDesc.NO_HEIGHT
+    }
+    val firstHeight = first().height
+    if (!(all { it.sid == 0 } || all { it.sid == -1 })) {
+        return RtpLayerDesc.NO_HEIGHT
+    }
+    if (any { it.height != firstHeight }) {
+        return RtpLayerDesc.NO_HEIGHT
+    }
+    return firstHeight
 }

jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt

@@ -20,106 +20,60 @@ import org.jitsi.nlj.transform.node.incoming.BitrateCalculator
 import org.jitsi.nlj.util.Bandwidth
 import org.jitsi.nlj.util.BitrateTracker
 import org.jitsi.nlj.util.DataSize
-import org.jitsi.nlj.util.sum
+import org.jitsi.utils.OrderedJsonObject
 
 /**
  * Keeps track of its subjective quality index,
  * its last stable bitrate and other useful things for adaptivity/routing.
  *
- * Note: this class and [getBitrate] are only open to allow to be overridden for testing. We found that mocking has
- * severe overhead and is not suitable for performance tests.
- *
  * @author George Politis
  */
-open class RtpLayerDesc
-@JvmOverloads
+abstract class RtpLayerDesc
 constructor(
     /**
      * The index of this instance's encoding in the source encoding array.
      */
     val eid: Int,
     /**
-     * The temporal layer ID of this instance, or negative for unknown.
+     * The temporal layer ID of this instance.
      */
     val tid: Int,
     /**
-     * The spatial layer ID of this instance, or negative for unknown.
+     * The spatial layer ID of this instance.
      */
     val sid: Int,
     /**
      * The max height of the bitstream that this instance represents. The actual
-     * height may be less due to bad network or system load.
+     * height may be less due to bad network or system load.  [NO_HEIGHT] for unknown.
      *
      * XXX we should be able to sniff the actual height from the RTP packets.
      */
-    val height: Int,
+    var height: Int,
     /**
      * The max frame rate (in fps) of the bitstream that this instance
      * represents. The actual frame rate may be less due to bad network or
-     * system load.
+     * system load.  [NO_FRAME_RATE] for unknown.
      */
     val frameRate: Double,
-    /**
-     * The [RtpLayerDesc]s on which this layer definitely depends.
-     */
-    private val dependencyLayers: Array<RtpLayerDesc> = emptyArray(),
-    /**
-     * The [RtpLayerDesc]s on which this layer possibly depends.
-     * (The intended use case is K-SVC mode.)
-     */
-    private val softDependencyLayers: Array<RtpLayerDesc> = emptyArray()
 ) {
-    init {
-        require(tid < 8) { "Invalid temporal ID $tid" }
-        require(sid < 8) { "Invalid spatial ID $sid" }
-    }
-
-    /**
-     * Clone an existing layer desc, inheriting its statistics,
-     * modifying only specific values.
-     */
-    fun copy(
-        eid: Int = this.eid,
-        tid: Int = this.tid,
-        sid: Int = this.sid,
-        height: Int = this.height,
-        frameRate: Double = this.frameRate,
-        dependencyLayers: Array<RtpLayerDesc> = this.dependencyLayers,
-        softDependencyLayers: Array<RtpLayerDesc> = this.softDependencyLayers
-    ) = RtpLayerDesc(eid, tid, sid, height, frameRate, dependencyLayers, softDependencyLayers).also {
-        it.inheritFrom(this)
-    }
-
-    /**
-     * Whether softDependencyLayers are to be used.
-     */
-    var useSoftDependencies = true
+    abstract fun copy(height: Int = this.height): RtpLayerDesc
 
     /**
      * The [BitrateTracker] instance used to calculate the receiving bitrate of this RTP layer.
      */
-    private var bitrateTracker = BitrateCalculator.createBitrateTracker()
+    protected var bitrateTracker = BitrateCalculator.createBitrateTracker()
 
     /**
      * @return the "id" of this layer within this encoding. This is a server-side id and should
      * not be confused with any encoding id defined in the client (such as the
      * rid).
      */
-    val layerId = getIndex(0, sid, tid)
+    abstract val layerId: Int
 
     /**
      * A local index of this track.
      */
-    val index = getIndex(eid, sid, tid)
-
-    /**
-     * {@inheritDoc}
-     */
-    override fun toString(): String {
-        return "subjective_quality=" + index +
-            ",temporal_id=" + tid +
-            ",spatial_id=" + sid
-    }
+    abstract val index: Int
 
     /**
      * Inherit a [BitrateTracker] object
@@ -131,9 +85,8 @@ constructor(
     /**
      * Inherit another layer description's [BitrateTracker] object.
      */
-    internal fun inheritFrom(other: RtpLayerDesc) {
+    internal open fun inheritFrom(other: RtpLayerDesc) {
         inheritStatistics(other.bitrateTracker)
-        useSoftDependencies = other.useSoftDependencies
     }
 
     /**
@@ -152,80 +105,38 @@ constructor(
     /**
      * Gets the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies.
      *
-     * This is left open for use in testing.
-     *
      * @param nowMs
      * @return the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies.
      */
-    open fun getBitrate(nowMs: Long): Bandwidth = calcBitrate(nowMs).values.sum()
+    abstract fun getBitrate(nowMs: Long): Bandwidth
 
     /**
      * Expose [getBitrate] as a [Double] in order to make it accessible from java (since [Bandwidth] is an inline
      * class).
      */
     fun getBitrateBps(nowMs: Long): Double = getBitrate(nowMs).bps
 
-    /**
-     * Recursively adds the bitrate (in bps) of this [RTPLayerDesc] and
-     * its dependencies in the map passed in as an argument.
-     *
-     * This is necessary to ensure we don't double-count layers in cases
-     * of multiple dependencies.
-     *
-     * @param nowMs
-     */
-    private fun calcBitrate(nowMs: Long, rates: MutableMap<Int, Bandwidth> = HashMap()): MutableMap<Int, Bandwidth> {
-        if (rates.containsKey(index)) {
-            return rates
-        }
-        rates[index] = bitrateTracker.getRate(nowMs)
-
-        dependencyLayers.forEach { it.calcBitrate(nowMs, rates) }
-
-        if (useSoftDependencies) {
-            softDependencyLayers.forEach { it.calcBitrate(nowMs, rates) }
-        }
-
-        return rates
-    }
-
-    /**
-     * Returns true if this layer, alone, has a zero bitrate.
-     */
-    private fun layerHasZeroBitrate(nowMs: Long) = bitrateTracker.getAccumulatedSize(nowMs).bits == 0L
-
     /**
      * Recursively checks this layer and its dependencies to see if the bitrate is zero.
      * Note that unlike [calcBitrate] this does not avoid double-visiting layers; the overhead
      * of the hash table is usually more than the cost of any double-visits.
-     *
-     * This is left open for use in testing.
      */
-    open fun hasZeroBitrate(nowMs: Long): Boolean {
-        if (!layerHasZeroBitrate(nowMs)) {
-            return false
-        }
-        if (dependencyLayers.any { !it.layerHasZeroBitrate(nowMs) }) {
-            return false
-        }
-        if (useSoftDependencies && softDependencyLayers.any { !it.layerHasZeroBitrate(nowMs) }) {
-            return false
-        }
-        return true
-    }
+    abstract fun hasZeroBitrate(nowMs: Long): Boolean
 
     /**
      * Extracts a [NodeStatsBlock] from an [RtpLayerDesc].
      */
-    fun getNodeStats() = NodeStatsBlock(indexString(index)).apply {
+    open fun getNodeStats() = NodeStatsBlock(indexString()).apply {
         addNumber("frameRate", frameRate)
         addNumber("height", height)
         addNumber("index", index)
         addNumber("bitrate_bps", getBitrate(System.currentTimeMillis()).bps)
-        addNumber("tid", tid)
-        addNumber("sid", sid)
     }
 
+    fun debugState(): OrderedJsonObject = getNodeStats().toJson().apply { put("indexString", indexString()) }
+
+    abstract fun indexString(): String
+
     companion object {
         /**
          * The index value that is used to represent that forwarding is suspended.
@@ -270,14 +181,14 @@ constructor(
         fun getEidFromIndex(index: Int) = index shr 6
 
         /**
-         * Get an spatial ID from a layer index.  If the index is [SUSPENDED_INDEX],
+         * Get a spatial ID from a layer index.  If the index is [SUSPENDED_INDEX],
          * the value is unspecified.
          */
         @JvmStatic
         fun getSidFromIndex(index: Int) = (index and 0x38) shr 3
 
         /**
-         * Get an temporal ID from a layer index.  If the index is [SUSPENDED_INDEX],
+         * Get a temporal ID from a layer index.  If the index is [SUSPENDED_INDEX],
          * the value is unspecified.
          */
         @JvmStatic

jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt

@@ -143,7 +143,7 @@ class RtpReceiverImpl @JvmOverloads constructor(
     private val videoBitrateCalculator = VideoBitrateCalculator(parentLogger)
     private val audioBitrateCalculator = BitrateCalculator("Audio bitrate calculator")
 
-    private val videoParser = VideoParser(streamInformationStore, logger)
+    private val videoParser = VideoParser(streamInformationStore, logger, diagnosticContext)
 
     override fun isReceivingAudio() = audioBitrateCalculator.active
     override fun isReceivingVideo() = videoBitrateCalculator.active

jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt

@@ -25,8 +25,8 @@ abstract class ParsedVideoPacket(
     buffer: ByteArray,
     offset: Int,
     length: Int,
-    encodingIndex: Int?
-) : VideoRtpPacket(buffer, offset, length, encodingIndex) {
+    encodingId: Int
+) : VideoRtpPacket(buffer, offset, length, encodingId) {
 
     abstract val isKeyframe: Boolean
     abstract val isStartOfFrame: Boolean

jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt

@@ -96,7 +96,14 @@ enum class RtpExtensionType(val uri: String) {
     /**
      * The URN which identifies the RTP Header Extension for Video Orientation.
      */
-    VIDEO_ORIENTATION("urn:3gpp:video-orientation");
+    VIDEO_ORIENTATION("urn:3gpp:video-orientation"),
+
+    /**
+     * The URN which identifies the AV1 Dependency Descriptor RTP Header Extension
+     */
+    AV1_DEPENDENCY_DESCRIPTOR(
+        "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension"
+    );
 
     companion object {
         private val uriMap = RtpExtensionType.values().associateBy(RtpExtensionType::uri)