Reland "Add spatial index to EncodedImage."

This is a reland of da0898d

Original change's description:
> Add spatial index to EncodedImage.
>
> Replaces the VP8 simulcast index and VP9 spatial index formely part of
> CodecSpecificInfo.
>
> Bug: webrtc:9378
> Change-Id: I80eafd63fbdee0a25864338196a690628b4bd3d2
> Reviewed-on: https://webrtc-review.googlesource.com/83161
> Commit-Queue: Niels Moller <[email protected]>
> Reviewed-by: Erik Språng <[email protected]>
> Reviewed-by: Sebastian Jansson <[email protected]>
> Reviewed-by: Magnus Jedvert <[email protected]>
> Reviewed-by: Philip Eliasson <[email protected]>
> Reviewed-by: Rasmus Brandt <[email protected]>
> Cr-Commit-Position: refs/heads/master@{#24485}

Tbr: [email protected]
Bug: webrtc:9378
Change-Id: Iff20b656581ef63317e073833d1a326f7118fdfd
Reviewed-on: https://webrtc-review.googlesource.com/96780
Commit-Queue: Niels Moller <[email protected]>
Reviewed-by: Sebastian Jansson <[email protected]>
Reviewed-by: Erik Språng <[email protected]>
Reviewed-by: Philip Eliasson <[email protected]>
Cr-Commit-Position: refs/heads/master@{#24507}

call/rtp_payload_params.cc

@@ -22,6 +22,7 @@ namespace webrtc {
 
 namespace {
 void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
+                                   absl::optional<int> spatial_index,
                                    RTPVideoHeader* rtp) {
   rtp->codec = info.codecType;
   switch (info.codecType) {
@@ -31,7 +32,7 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
       rtp->vp8().temporalIdx = info.codecSpecific.VP8.temporalIdx;
       rtp->vp8().layerSync = info.codecSpecific.VP8.layerSync;
       rtp->vp8().keyIdx = info.codecSpecific.VP8.keyIdx;
-      rtp->simulcastIdx = info.codecSpecific.VP8.simulcastIdx;
+      rtp->simulcastIdx = spatial_index.value_or(0);
       return;
     }
     case kVideoCodecVP9: {
@@ -44,13 +45,16 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
       vp9_header.non_ref_for_inter_layer_pred =
           info.codecSpecific.VP9.non_ref_for_inter_layer_pred;
       vp9_header.temporal_idx = info.codecSpecific.VP9.temporal_idx;
-      vp9_header.spatial_idx = info.codecSpecific.VP9.spatial_idx;
       vp9_header.temporal_up_switch = info.codecSpecific.VP9.temporal_up_switch;
       vp9_header.inter_layer_predicted =
           info.codecSpecific.VP9.inter_layer_predicted;
       vp9_header.gof_idx = info.codecSpecific.VP9.gof_idx;
       vp9_header.num_spatial_layers = info.codecSpecific.VP9.num_spatial_layers;
-
+      if (vp9_header.num_spatial_layers > 1) {
+        vp9_header.spatial_idx = spatial_index.value_or(kNoSpatialIdx);
+      } else {
+        vp9_header.spatial_idx = kNoSpatialIdx;
+      }
       if (info.codecSpecific.VP9.ss_data_available) {
         vp9_header.spatial_layer_resolution_present =
             info.codecSpecific.VP9.spatial_layer_resolution_present;
@@ -75,13 +79,13 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
       auto& h264_header = rtp->video_type_header.emplace<RTPVideoHeaderH264>();
       h264_header.packetization_mode =
           info.codecSpecific.H264.packetization_mode;
-      rtp->simulcastIdx = info.codecSpecific.H264.simulcast_idx;
+      rtp->simulcastIdx = spatial_index.value_or(0);
       return;
     }
     case kVideoCodecMultiplex:
     case kVideoCodecGeneric:
       rtp->codec = kVideoCodecGeneric;
-      rtp->simulcastIdx = info.codecSpecific.generic.simulcast_idx;
+      rtp->simulcastIdx = spatial_index.value_or(0);
       return;
     default:
       return;
@@ -131,7 +135,8 @@ RTPVideoHeader RtpPayloadParams::GetRtpVideoHeader(
     int64_t shared_frame_id) {
   RTPVideoHeader rtp_video_header;
   if (codec_specific_info) {
-    PopulateRtpWithCodecSpecifics(*codec_specific_info, &rtp_video_header);
+    PopulateRtpWithCodecSpecifics(*codec_specific_info, image.SpatialIndex(),
+                                  &rtp_video_header);
   }
   rtp_video_header.rotation = image.rotation_;
   rtp_video_header.content_type = image.content_type_;

call/rtp_payload_params_unittest.cc

@@ -38,11 +38,11 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp8) {
   EncodedImage encoded_image;
   encoded_image.rotation_ = kVideoRotation_90;
   encoded_image.content_type_ = VideoContentType::SCREENSHARE;
+  encoded_image.SetSpatialIndex(1);
 
   CodecSpecificInfo codec_info;
   memset(&codec_info, 0, sizeof(CodecSpecificInfo));
   codec_info.codecType = kVideoCodecVP8;
-  codec_info.codecSpecific.VP8.simulcastIdx = 1;
   codec_info.codecSpecific.VP8.temporalIdx = 0;
   codec_info.codecSpecific.VP8.keyIdx = kNoKeyIdx;
   codec_info.codecSpecific.VP8.layerSync = false;
@@ -52,7 +52,6 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp8) {
       params.GetRtpVideoHeader(encoded_image, &codec_info, kDontCare);
 
   codec_info.codecType = kVideoCodecVP8;
-  codec_info.codecSpecific.VP8.simulcastIdx = 1;
   codec_info.codecSpecific.VP8.temporalIdx = 1;
   codec_info.codecSpecific.VP8.layerSync = true;
 
@@ -79,13 +78,12 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp9) {
   EncodedImage encoded_image;
   encoded_image.rotation_ = kVideoRotation_90;
   encoded_image.content_type_ = VideoContentType::SCREENSHARE;
-
+  encoded_image.SetSpatialIndex(0);
   CodecSpecificInfo codec_info;
   memset(&codec_info, 0, sizeof(CodecSpecificInfo));
   codec_info.codecType = kVideoCodecVP9;
   codec_info.codecSpecific.VP9.num_spatial_layers = 3;
   codec_info.codecSpecific.VP9.first_frame_in_picture = true;
-  codec_info.codecSpecific.VP9.spatial_idx = 0;
   codec_info.codecSpecific.VP9.temporal_idx = 2;
   codec_info.codecSpecific.VP9.end_of_picture = false;
 
@@ -100,17 +98,17 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp9) {
   EXPECT_EQ(kPictureId + 1, vp9_header.picture_id);
   EXPECT_EQ(kTl0PicIdx, vp9_header.tl0_pic_idx);
   EXPECT_EQ(vp9_header.temporal_idx, codec_info.codecSpecific.VP9.temporal_idx);
-  EXPECT_EQ(vp9_header.spatial_idx, codec_info.codecSpecific.VP9.spatial_idx);
+  EXPECT_EQ(vp9_header.spatial_idx, encoded_image.SpatialIndex());
   EXPECT_EQ(vp9_header.num_spatial_layers,
             codec_info.codecSpecific.VP9.num_spatial_layers);
   EXPECT_EQ(vp9_header.end_of_picture,
             codec_info.codecSpecific.VP9.end_of_picture);
 
   // Next spatial layer.
   codec_info.codecSpecific.VP9.first_frame_in_picture = false;
-  codec_info.codecSpecific.VP9.spatial_idx += 1;
   codec_info.codecSpecific.VP9.end_of_picture = true;
 
+  encoded_image.SetSpatialIndex(1);
   header = params.GetRtpVideoHeader(encoded_image, &codec_info, kDontCare);
 
   EXPECT_EQ(kVideoRotation_90, header.rotation);
@@ -119,7 +117,7 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp9) {
   EXPECT_EQ(kPictureId + 1, vp9_header.picture_id);
   EXPECT_EQ(kTl0PicIdx, vp9_header.tl0_pic_idx);
   EXPECT_EQ(vp9_header.temporal_idx, codec_info.codecSpecific.VP9.temporal_idx);
-  EXPECT_EQ(vp9_header.spatial_idx, codec_info.codecSpecific.VP9.spatial_idx);
+  EXPECT_EQ(vp9_header.spatial_idx, encoded_image.SpatialIndex());
   EXPECT_EQ(vp9_header.num_spatial_layers,
             codec_info.codecSpecific.VP9.num_spatial_layers);
   EXPECT_EQ(vp9_header.end_of_picture,
@@ -154,7 +152,6 @@ TEST(RtpPayloadParamsTest, PictureIdIsSetForVp8) {
   CodecSpecificInfo codec_info;
   memset(&codec_info, 0, sizeof(CodecSpecificInfo));
   codec_info.codecType = kVideoCodecVP8;
-  codec_info.codecSpecific.VP8.simulcastIdx = 0;
 
   RtpPayloadParams params(kSsrc1, &state);
   RTPVideoHeader header =

call/rtp_video_sender.cc

@@ -95,21 +95,6 @@ std::vector<std::unique_ptr<RtpRtcp>> CreateRtpRtcpModules(
   return modules;
 }
 
-absl::optional<size_t> GetSimulcastIdx(const CodecSpecificInfo* info) {
-  if (!info)
-    return absl::nullopt;
-  switch (info->codecType) {
-    case kVideoCodecVP8:
-      return absl::optional<size_t>(info->codecSpecific.VP8.simulcastIdx);
-    case kVideoCodecH264:
-      return absl::optional<size_t>(info->codecSpecific.H264.simulcast_idx);
-    case kVideoCodecMultiplex:
-    case kVideoCodecGeneric:
-      return absl::optional<size_t>(info->codecSpecific.generic.simulcast_idx);
-    default:
-      return absl::nullopt;
-  }
-}
 bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) {
   const VideoCodecType codecType = PayloadStringToCodecType(payload_name);
   if (codecType == kVideoCodecVP8 || codecType == kVideoCodecVP9) {
@@ -320,7 +305,14 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage(
     return Result(Result::ERROR_SEND_FAILED);
 
   shared_frame_id_++;
-  size_t stream_index = GetSimulcastIdx(codec_specific_info).value_or(0);
+  size_t stream_index = 0;
+  if (codec_specific_info &&
+      (codec_specific_info->codecType == kVideoCodecVP8 ||
+       codec_specific_info->codecType == kVideoCodecH264 ||
+       codec_specific_info->codecType == kVideoCodecGeneric)) {
+    // Map spatial index to simulcast.
+    stream_index = encoded_image.SpatialIndex().value_or(0);
+  }
   RTC_DCHECK_LT(stream_index, rtp_modules_.size());
   RTPVideoHeader rtp_video_header = params_[stream_index].GetRtpVideoHeader(
       encoded_image, codec_specific_info, shared_frame_id_);

call/rtp_video_sender_unittest.cc

@@ -169,44 +169,41 @@ TEST(RtpVideoSenderTest, SendOnOneModule) {
 
 TEST(RtpVideoSenderTest, SendSimulcastSetActive) {
   uint8_t payload = 'a';
-  EncodedImage encoded_image;
-  encoded_image.SetTimestamp(1);
-  encoded_image.capture_time_ms_ = 2;
-  encoded_image._frameType = kVideoFrameKey;
-  encoded_image._buffer = &payload;
-  encoded_image._length = 1;
+  EncodedImage encoded_image_1;
+  encoded_image_1.SetTimestamp(1);
+  encoded_image_1.capture_time_ms_ = 2;
+  encoded_image_1._frameType = kVideoFrameKey;
+  encoded_image_1._buffer = &payload;
+  encoded_image_1._length = 1;
 
   RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, kPayloadType, {});
 
-  CodecSpecificInfo codec_info_1;
-  memset(&codec_info_1, 0, sizeof(CodecSpecificInfo));
-  codec_info_1.codecType = kVideoCodecVP8;
-  codec_info_1.codecSpecific.VP8.simulcastIdx = 0;
+  CodecSpecificInfo codec_info;
+  memset(&codec_info, 0, sizeof(CodecSpecificInfo));
+  codec_info.codecType = kVideoCodecVP8;
 
   test.router()->SetActive(true);
   EXPECT_EQ(EncodedImageCallback::Result::OK,
             test.router()
-                ->OnEncodedImage(encoded_image, &codec_info_1, nullptr)
+                ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
                 .error);
 
-  CodecSpecificInfo codec_info_2;
-  memset(&codec_info_2, 0, sizeof(CodecSpecificInfo));
-  codec_info_2.codecType = kVideoCodecVP8;
-  codec_info_2.codecSpecific.VP8.simulcastIdx = 1;
+  EncodedImage encoded_image_2(encoded_image_1);
+  encoded_image_2.SetSpatialIndex(1);
   EXPECT_EQ(EncodedImageCallback::Result::OK,
             test.router()
-                ->OnEncodedImage(encoded_image, &codec_info_2, nullptr)
+                ->OnEncodedImage(encoded_image_2, &codec_info, nullptr)
                 .error);
 
   // Inactive.
   test.router()->SetActive(false);
   EXPECT_NE(EncodedImageCallback::Result::OK,
             test.router()
-                ->OnEncodedImage(encoded_image, &codec_info_1, nullptr)
+                ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
                 .error);
   EXPECT_NE(EncodedImageCallback::Result::OK,
             test.router()
-                ->OnEncodedImage(encoded_image, &codec_info_2, nullptr)
+                ->OnEncodedImage(encoded_image_2, &codec_info, nullptr)
                 .error);
 }
 
@@ -216,30 +213,27 @@ TEST(RtpVideoSenderTest, SendSimulcastSetActive) {
 // be sent if both modules are inactive.
 TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) {
   uint8_t payload = 'a';
-  EncodedImage encoded_image;
-  encoded_image.SetTimestamp(1);
-  encoded_image.capture_time_ms_ = 2;
-  encoded_image._frameType = kVideoFrameKey;
-  encoded_image._buffer = &payload;
-  encoded_image._length = 1;
+  EncodedImage encoded_image_1;
+  encoded_image_1.SetTimestamp(1);
+  encoded_image_1.capture_time_ms_ = 2;
+  encoded_image_1._frameType = kVideoFrameKey;
+  encoded_image_1._buffer = &payload;
+  encoded_image_1._length = 1;
+  EncodedImage encoded_image_2(encoded_image_1);
+  encoded_image_2.SetSpatialIndex(1);
 
   RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, kPayloadType, {});
-  CodecSpecificInfo codec_info_1;
-  memset(&codec_info_1, 0, sizeof(CodecSpecificInfo));
-  codec_info_1.codecType = kVideoCodecVP8;
-  codec_info_1.codecSpecific.VP8.simulcastIdx = 0;
-  CodecSpecificInfo codec_info_2;
-  memset(&codec_info_2, 0, sizeof(CodecSpecificInfo));
-  codec_info_2.codecType = kVideoCodecVP8;
-  codec_info_2.codecSpecific.VP8.simulcastIdx = 1;
+  CodecSpecificInfo codec_info;
+  memset(&codec_info, 0, sizeof(CodecSpecificInfo));
+  codec_info.codecType = kVideoCodecVP8;
 
   // Only setting one stream to active will still set the payload router to
   // active and allow sending data on the active stream.
   std::vector<bool> active_modules({true, false});
   test.router()->SetActiveModules(active_modules);
   EXPECT_EQ(EncodedImageCallback::Result::OK,
             test.router()
-                ->OnEncodedImage(encoded_image, &codec_info_1, nullptr)
+                ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
                 .error);
 
   // Setting both streams to inactive will turn the payload router to
@@ -250,11 +244,11 @@ TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) {
   // because the payload router is inactive.
   EXPECT_NE(EncodedImageCallback::Result::OK,
             test.router()
-                ->OnEncodedImage(encoded_image, &codec_info_1, nullptr)
+                ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
                 .error);
   EXPECT_NE(EncodedImageCallback::Result::OK,
             test.router()
-                ->OnEncodedImage(encoded_image, &codec_info_2, nullptr)
+                ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
                 .error);
 }
 

media/engine/simulcast_encoder_adapter.cc

@@ -449,16 +449,14 @@ EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage(
     const EncodedImage& encodedImage,
     const CodecSpecificInfo* codecSpecificInfo,
     const RTPFragmentationHeader* fragmentation) {
+  EncodedImage stream_image(encodedImage);
   CodecSpecificInfo stream_codec_specific = *codecSpecificInfo;
   stream_codec_specific.codec_name = implementation_name_.c_str();
-  if (stream_codec_specific.codecType == webrtc::kVideoCodecVP8) {
-    stream_codec_specific.codecSpecific.VP8.simulcastIdx = stream_idx;
-  } else if (stream_codec_specific.codecType == webrtc::kVideoCodecH264) {
-    stream_codec_specific.codecSpecific.H264.simulcast_idx = stream_idx;
-  }
+
+  stream_image.SetSpatialIndex(stream_idx);
 
   return encoded_complete_callback_->OnEncodedImage(
-      encodedImage, &stream_codec_specific, fragmentation);
+      stream_image, &stream_codec_specific, fragmentation);
 }
 
 void SimulcastEncoderAdapter::PopulateStreamCodec(

media/engine/simulcast_encoder_adapter_unittest.cc

@@ -339,11 +339,10 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
                         const RTPFragmentationHeader* fragmentation) override {
     last_encoded_image_width_ = encoded_image._encodedWidth;
     last_encoded_image_height_ = encoded_image._encodedHeight;
-    if (codec_specific_info) {
-      last_encoded_image_simulcast_index_ =
-          codec_specific_info->codecSpecific.VP8.simulcastIdx;
-    }
-    return Result(Result::OK, encoded_image.Timestamp());
+    last_encoded_image_simulcast_index_ =
+        encoded_image.SpatialIndex().value_or(-1);
+
+    return Result(Result::OK, encoded_image._timeStamp);
   }
 
   bool GetLastEncodedImageInfo(int* out_width,

modules/video_coding/codecs/h264/h264_encoder_impl.cc

@@ -506,6 +506,7 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame,
             : VideoContentType::UNSPECIFIED;
     encoded_images_[i].timing_.flags = VideoSendTiming::kInvalid;
     encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
+    encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx);
 
     // Split encoded image up into fragments. This also updates
     // |encoded_image_|.
@@ -526,8 +527,6 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame,
       codec_specific.codecType = kVideoCodecH264;
       codec_specific.codecSpecific.H264.packetization_mode =
           packetization_mode_;
-      codec_specific.codecSpecific.H264.simulcast_idx =
-          configurations_[i].simulcast_idx;
       encoded_image_callback_->OnEncodedImage(encoded_images_[i],
                                               &codec_specific, &frag_header);
     }

modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc

@@ -285,7 +285,6 @@ EncodedImageCallback::Result MultiplexEncoderAdapter::OnEncodedImage(
 
       CodecSpecificInfo codec_info = *codecSpecificInfo;
       codec_info.codecType = kVideoCodecMultiplex;
-      codec_info.codecSpecific.generic.simulcast_idx = 0;
       encoded_complete_callback_->OnEncodedImage(combined_image_, &codec_info,
                                                  fragmentation);
     }

modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc

@@ -232,7 +232,7 @@ TEST_P(TestMultiplexAdapter, CheckSingleFrameEncodedBitstream) {
   CodecSpecificInfo codec_specific_info;
   ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
   EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
-  EXPECT_EQ(0, codec_specific_info.codecSpecific.generic.simulcast_idx);
+  EXPECT_FALSE(encoded_frame.SpatialIndex());
 
   const MultiplexImage& unpacked_frame =
       MultiplexEncodedImagePacker::Unpack(encoded_frame);
@@ -252,7 +252,7 @@ TEST_P(TestMultiplexAdapter, CheckDoubleFramesEncodedBitstream) {
   CodecSpecificInfo codec_specific_info;
   ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
   EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
-  EXPECT_EQ(0, codec_specific_info.codecSpecific.generic.simulcast_idx);
+  EXPECT_FALSE(encoded_frame.SpatialIndex());
 
   const MultiplexImage& unpacked_frame =
       MultiplexEncodedImagePacker::Unpack(encoded_frame);