From e7624ee7236f8b8cf17e7b489487937615f868b0 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 13 Feb 2023 13:28:55 +0800 Subject: [PATCH] fix: Use CVPixelBuffer to build DesktopCapture Frame, fix the crash caused by non-CVPixelBuffer frame in RTCVideoEncoderH264 that cannot be cropped. --- sdk/objc/native/src/objc_desktop_capture.h | 1 - sdk/objc/native/src/objc_desktop_capture.mm | 109 +++++------ .../native/src/objc_desktop_media_list.mm | 175 +++++++++--------- 3 files changed, 145 insertions(+), 140 deletions(-) diff --git a/sdk/objc/native/src/objc_desktop_capture.h b/sdk/objc/native/src/objc_desktop_capture.h index 2209620449..ea7eeb3e3d 100644 --- a/sdk/objc/native/src/objc_desktop_capture.h +++ b/sdk/objc/native/src/objc_desktop_capture.h @@ -59,7 +59,6 @@ class ObjCDesktopCapturer : public DesktopCapturer::Callback, public rtc::Messag webrtc::DesktopCaptureOptions options_; std::unique_ptr capturer_; std::unique_ptr thread_; - rtc::scoped_refptr i420_buffer_; CaptureState capture_state_ = CS_STOPPED; DesktopType type_; webrtc::DesktopCapturer::SourceId source_id_; diff --git a/sdk/objc/native/src/objc_desktop_capture.mm b/sdk/objc/native/src/objc_desktop_capture.mm index 9a69d31804..2cd9895168 100644 --- a/sdk/objc/native/src/objc_desktop_capture.mm +++ b/sdk/objc/native/src/objc_desktop_capture.mm @@ -19,14 +19,15 @@ #include "third_party/libyuv/include/libyuv.h" #import "components/capturer/RTCDesktopCapturer+Private.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" namespace webrtc { enum { kCaptureDelay = 33, kCaptureMessageId = 1000 }; ObjCDesktopCapturer::ObjCDesktopCapturer(DesktopType type, - webrtc::DesktopCapturer::SourceId source_id, - id delegate) + webrtc::DesktopCapturer::SourceId source_id, + id delegate) : thread_(rtc::Thread::Create()), source_id_(source_id), delegate_(delegate) { RTC_DCHECK(thread_); type_ = type; @@ -36,53 +37,47 @@ options_.set_allow_iosurface(true); thread_->Invoke(RTC_FROM_HERE, [this, type] { if (type == kScreen) { - capturer_ = std::make_unique(webrtc::DesktopCapturer::CreateScreenCapturer(options_), options_); - } else { - capturer_ = std::make_unique(webrtc::DesktopCapturer::CreateWindowCapturer(options_), options_); + capturer_ = std::make_unique( + webrtc::DesktopCapturer::CreateScreenCapturer(options_), options_); + } else { + capturer_ = std::make_unique( + webrtc::DesktopCapturer::CreateWindowCapturer(options_), options_); } }); } ObjCDesktopCapturer::~ObjCDesktopCapturer() { - thread_->Invoke(RTC_FROM_HERE, [this] { - capturer_.reset(); - }); + thread_->Invoke(RTC_FROM_HERE, [this] { capturer_.reset(); }); } ObjCDesktopCapturer::CaptureState ObjCDesktopCapturer::Start(uint32_t fps) { - - if(fps == 0) { - capture_state_ = CS_FAILED; - return capture_state_; + if (fps == 0) { + capture_state_ = CS_FAILED; + return capture_state_; } - if(fps >= 60) { + if (fps >= 60) { capture_delay_ = uint32_t(1000.0 / 60.0); } else { capture_delay_ = uint32_t(1000.0 / fps); } - if(source_id_ != -1) { - if(!capturer_->SelectSource(source_id_)) { - capture_state_ = CS_FAILED; - return capture_state_; + if (source_id_ != -1) { + if (!capturer_->SelectSource(source_id_)) { + capture_state_ = CS_FAILED; + return capture_state_; } - if(type_ == kWindow) { - if(!capturer_->FocusOnSelectedSource()) { + if (type_ == kWindow) { + if (!capturer_->FocusOnSelectedSource()) { capture_state_ = CS_FAILED; return capture_state_; } } } - thread_->Invoke(RTC_FROM_HERE, [this] { - capturer_->Start(this); - }); + thread_->Invoke(RTC_FROM_HERE, [this] { capturer_->Start(this); }); capture_state_ = CS_RUNNING; - thread_->PostTask(ToQueuedTask( - [this]{ - CaptureFrame(); - })); + thread_->PostTask(ToQueuedTask([this] { CaptureFrame(); })); [delegate_ didSourceCaptureStart]; return capture_state_; } @@ -97,7 +92,7 @@ } void ObjCDesktopCapturer::OnCaptureResult(webrtc::DesktopCapturer::Result result, - std::unique_ptr frame) { + std::unique_ptr frame) { if (result != result_) { if (result == webrtc::DesktopCapturer::Result::ERROR_PERMANENT) { [delegate_ didSourceCaptureError]; @@ -118,14 +113,14 @@ } if (result == webrtc::DesktopCapturer::Result::ERROR_TEMPORARY) { - return; + return; } int width = frame->size().width(); int height = frame->size().height(); int real_width = width; - if(type_ == kWindow) { + if (type_ == kWindow) { int multiple = 0; #if defined(WEBRTC_ARCH_X86_FAMILY) multiple = 16; @@ -134,24 +129,26 @@ #endif // A multiple of $multiple must be used as the width of the src frame, // and the right black border needs to be cropped during conversion. - if( multiple != 0 && (width % multiple) != 0 ) { + if (multiple != 0 && (width % multiple) != 0) { width = (width / multiple + 1) * multiple; } } - - if (!i420_buffer_ || !i420_buffer_.get() || - i420_buffer_->width() * i420_buffer_->height() != real_width * height) { - i420_buffer_ = webrtc::I420Buffer::Create(real_width, height); - } - libyuv::ConvertToI420(frame->data(), - 0, - i420_buffer_->MutableDataY(), - i420_buffer_->StrideY(), - i420_buffer_->MutableDataU(), - i420_buffer_->StrideU(), - i420_buffer_->MutableDataV(), - i420_buffer_->StrideV(), + CVPixelBufferRef pixelBuffer = NULL; + + NSDictionary *pixelAttributes = @{(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}}; + CVReturn res = CVPixelBufferCreate(kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)(pixelAttributes), + &pixelBuffer); + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + uint8_t *pxdata = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer); + libyuv::ConvertToARGB(reinterpret_cast(frame->data()), + real_width * height * 4, + reinterpret_cast(pxdata), + width * 4, 0, 0, width, @@ -160,20 +157,26 @@ height, libyuv::kRotate0, libyuv::FOURCC_ARGB); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + if (res != kCVReturnSuccess) { + NSLog(@"Unable to create cvpixelbuffer %d", res); + return; + } + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBuffer]; NSTimeInterval timeStampSeconds = CACurrentMediaTime(); int64_t timeStampNs = lroundf(timeStampSeconds * NSEC_PER_SEC); - RTCVideoFrame* rtc_video_frame = - ToObjCVideoFrame( - webrtc::VideoFrame::Builder() - .set_video_frame_buffer(i420_buffer_) - .set_rotation(webrtc::kVideoRotation_0) - .set_timestamp_us(timeStampNs / 1000) - .build() - ); - [delegate_ didCaptureVideoFrame:rtc_video_frame]; + RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:rtcPixelBuffer + rotation:RTCVideoRotation_0 + timeStampNs:timeStampNs]; + + [delegate_ didCaptureVideoFrame:videoFrame]; } -void ObjCDesktopCapturer::OnMessage(rtc::Message* msg) { +void ObjCDesktopCapturer::OnMessage(rtc::Message *msg) { if (msg->message_id == kCaptureMessageId) { CaptureFrame(); } diff --git a/sdk/objc/native/src/objc_desktop_media_list.mm b/sdk/objc/native/src/objc_desktop_media_list.mm index 6b5e6b5bba..b25f466420 100644 --- a/sdk/objc/native/src/objc_desktop_media_list.mm +++ b/sdk/objc/native/src/objc_desktop_media_list.mm @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + #include "sdk/objc/native/src/objc_desktop_media_list.h" -#include "sdk/objc/native/src/objc_video_frame.h" #include "rtc_base/checks.h" +#include "sdk/objc/native/src/objc_video_frame.h" #include "third_party/libyuv/include/libyuv.h" extern "C" { @@ -28,59 +28,54 @@ #endif } -#include #include +#include namespace webrtc { ObjCDesktopMediaList::ObjCDesktopMediaList(DesktopType type, - RTC_OBJC_TYPE(RTCDesktopMediaList)* objcMediaList) - :thread_(rtc::Thread::Create()),objcMediaList_(objcMediaList),type_(type) { + RTC_OBJC_TYPE(RTCDesktopMediaList) * objcMediaList) + : thread_(rtc::Thread::Create()), objcMediaList_(objcMediaList), type_(type) { RTC_DCHECK(thread_); thread_->Start(); options_ = webrtc::DesktopCaptureOptions::CreateDefault(); options_.set_detect_updated_region(true); options_.set_allow_iosurface(true); - + callback_ = std::make_unique(); thread_->Invoke(RTC_FROM_HERE, [this, type] { - if (type == kScreen) { + if (type == kScreen) { capturer_ = webrtc::DesktopCapturer::CreateScreenCapturer(options_); - } else { - capturer_ = webrtc::DesktopCapturer::CreateWindowCapturer(options_); + } else { + capturer_ = webrtc::DesktopCapturer::CreateWindowCapturer(options_); } capturer_->Start(callback_.get()); }); - } ObjCDesktopMediaList::~ObjCDesktopMediaList() { - thread_->Invoke(RTC_FROM_HERE, [this] { - capturer_.reset(); - }); + thread_->Invoke(RTC_FROM_HERE, [this] { capturer_.reset(); }); } int32_t ObjCDesktopMediaList::UpdateSourceList(bool force_reload, bool get_thumbnail) { - - if(force_reload) { - for( auto source : sources_) { - [objcMediaList_ mediaSourceRemoved:source.get()]; + if (force_reload) { + for (auto source : sources_) { + [objcMediaList_ mediaSourceRemoved:source.get()]; } sources_.clear(); } webrtc::DesktopCapturer::SourceList new_sources; - thread_->Invoke(RTC_FROM_HERE, [this,&new_sources] { - capturer_->GetSourceList(&new_sources); - }); + thread_->Invoke(RTC_FROM_HERE, + [this, &new_sources] { capturer_->GetSourceList(&new_sources); }); typedef std::set SourceSet; SourceSet new_source_set; for (size_t i = 0; i < new_sources.size(); ++i) { - if(type_ == kScreen && new_sources[i].title.length() == 0) { - new_sources[i].title = std::string("Screen " + std::to_string(i+1)); + if (type_ == kScreen && new_sources[i].title.length() == 0) { + new_sources[i].title = std::string("Screen " + std::to_string(i + 1)); } new_source_set.insert(new_sources[i].id); } @@ -100,7 +95,7 @@ } for (size_t i = 0; i < new_sources.size(); ++i) { if (old_source_set.find(new_sources[i].id) == old_source_set.end()) { - MediaSource* source = new MediaSource(this, new_sources[i],type_); + MediaSource *source = new MediaSource(this, new_sources[i], type_); sources_.insert(sources_.begin() + i, std::shared_ptr(source)); [objcMediaList_ mediaSourceAdded:source]; GetThumbnail(source, true); @@ -118,8 +113,7 @@ // of |sources_|, because entries before |pos| should have been sorted. size_t old_pos = pos + 1; for (; old_pos < sources_.size(); ++old_pos) { - if (sources_[old_pos]->id() == new_sources[pos].id) - break; + if (sources_[old_pos]->id() == new_sources[pos].id) break; } RTC_DCHECK(sources_[old_pos]->id() == new_sources[pos].id); @@ -137,40 +131,38 @@ ++pos; } - if(get_thumbnail) { - for( auto source : sources_) { - GetThumbnail(source.get(), true); + if (get_thumbnail) { + for (auto source : sources_) { + GetThumbnail(source.get(), true); } } return sources_.size(); } bool ObjCDesktopMediaList::GetThumbnail(MediaSource *source, bool notify) { - - thread_->PostTask(ToQueuedTask( - [this, source, notify] { - if(capturer_->SelectSource(source->id())){ - callback_->SetCallback([&](webrtc::DesktopCapturer::Result result, - std::unique_ptr frame) { - auto old_thumbnail = source->thumbnail(); - source->SaveCaptureResult(result, std::move(frame)); - if(old_thumbnail.size() != source->thumbnail().size() && notify) { - [objcMediaList_ mediaSourceThumbnailChanged:source]; - } - }); - capturer_->CaptureFrame(); - } + thread_->PostTask(ToQueuedTask([this, source, notify] { + if (capturer_->SelectSource(source->id())) { + callback_->SetCallback( + [&](webrtc::DesktopCapturer::Result result, std::unique_ptr frame) { + auto old_thumbnail = source->thumbnail(); + source->SaveCaptureResult(result, std::move(frame)); + if (old_thumbnail.size() != source->thumbnail().size() && notify) { + [objcMediaList_ mediaSourceThumbnailChanged:source]; + } + }); + capturer_->CaptureFrame(); + } })); return true; } int ObjCDesktopMediaList::GetSourceCount() const { - return sources_.size(); + return sources_.size(); } - + MediaSource *ObjCDesktopMediaList::GetSource(int index) { - return sources_[index].get(); + return sources_[index].get(); } bool MediaSource::UpdateThumbnail() { @@ -178,28 +170,15 @@ } void MediaSource::SaveCaptureResult(webrtc::DesktopCapturer::Result result, - std::unique_ptr frame) { + std::unique_ptr frame) { if (result != webrtc::DesktopCapturer::Result::SUCCESS) { return; } - int quality = 80; - const int kColorPlanes = 4; // alpha, R, G and B. - unsigned char* out_buffer = NULL; - unsigned long out_size = 0; - // Invoking LIBJPEG - struct jpeg_compress_struct cinfo; - struct jpeg_error_mgr jerr; - JSAMPROW row_pointer[1]; - cinfo.err = jpeg_std_error(&jerr); - jpeg_create_compress(&cinfo); - - jpeg_mem_dest(&cinfo, &out_buffer, &out_size); - int width = frame->size().width(); int height = frame->size().height(); int real_width = width; - if(type_ == kWindow) { + if (type_ == kWindow) { int multiple = 0; #if defined(WEBRTC_ARCH_X86_FAMILY) multiple = 16; @@ -208,39 +187,63 @@ #endif // A multiple of $multiple must be used as the width of the src frame, // and the right black border needs to be cropped during conversion. - if( multiple != 0 && (width % multiple) != 0 ) { + if (multiple != 0 && (width % multiple) != 0) { width = (width / multiple + 1) * multiple; } } - cinfo.image_width = real_width; - cinfo.image_height = height; - cinfo.input_components = kColorPlanes; - cinfo.in_color_space = JCS_EXT_BGRA; - jpeg_set_defaults(&cinfo); - jpeg_set_quality(&cinfo, quality, TRUE); - - jpeg_start_compress(&cinfo, TRUE); - int row_stride = width * kColorPlanes; - while (cinfo.next_scanline < cinfo.image_height) { - row_pointer[0] = &frame->data()[cinfo.next_scanline * row_stride]; - jpeg_write_scanlines(&cinfo, row_pointer, 1); + CVPixelBufferRef pixelBuffer = NULL; + + NSDictionary *pixelAttributes = @{(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}}; + CVReturn res = CVPixelBufferCreate(kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)(pixelAttributes), + &pixelBuffer); + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + uint8_t *pxdata = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer); + libyuv::ConvertToARGB(reinterpret_cast(frame->data()), + real_width * height * 4, + reinterpret_cast(pxdata), + width * 4, + 0, + 0, + width, + height, + real_width, + height, + libyuv::kRotate0, + libyuv::FOURCC_ARGB); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + if (res != kCVReturnSuccess) { + NSLog(@"Unable to create cvpixelbuffer %d", res); + return; } - jpeg_finish_compress(&cinfo); - jpeg_destroy_compress(&cinfo); - - thumbnail_.resize(out_size); - - std::copy(out_buffer - , out_buffer + out_size - , thumbnail_.begin()); - - free(out_buffer); + CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer]; + CGRect outputSize = CGRectMake(0, 0, width, height); + + CIContext *tempContext = [CIContext contextWithOptions:nil]; + CGImageRef cgImage = [tempContext createCGImage:ciImage fromRect:outputSize]; + NSData *imageData; + NSBitmapImageRep *newRep = [[NSBitmapImageRep alloc] initWithCGImage:cgImage]; + [newRep setSize:NSSizeToCGSize(outputSize.size)]; + imageData = [newRep representationUsingType:NSJPEGFileType + properties:@{ + NSImageCompressionFactor : @1.0f + }]; + + thumbnail_.resize(imageData.length); + const void *_Nullable rawData = [imageData bytes]; + char *src = (char *)rawData; + std::copy(src, src + imageData.length, thumbnail_.begin()); + + CGImageRelease(cgImage); + CVPixelBufferRelease(pixelBuffer); } -void ObjCDesktopMediaList::OnMessage(rtc::Message* msg) { - -} +void ObjCDesktopMediaList::OnMessage(rtc::Message *msg) {} } // namespace webrtc