Enable buffer scaling

Exports VideoFrameBuffer::Scale for every buffer by extending the livekit::*Buffer types.
livekit · Oct 23, 2024 · 42aac20 · 42aac20
1 parent 3de6fa6
commit 42aac20
Show file tree

Hide file tree

Showing 5 changed files with 189 additions and 0 deletions.
diff --git a/libwebrtc/src/native/video_frame.rs b/libwebrtc/src/native/video_frame.rs
@@ -312,6 +312,14 @@ impl I420Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420Buffer {
+        vf::I420Buffer {
+            handle: I420Buffer {
+                sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
+            },
+        }
+    }
 }
 
 impl I420ABuffer {
@@ -410,6 +418,14 @@ impl I420ABuffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420ABuffer {
+        vf::I420ABuffer {
+            handle: I420ABuffer {
+                sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
+            },
+        }
+    }
 }
 
 impl I422Buffer {
@@ -516,6 +532,14 @@ impl I422Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I422Buffer {
+        vf::I422Buffer {
+            handle: I422Buffer {
+                sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
+            },
+        }
+    }
 }
 impl I444Buffer {
     pub fn new(
@@ -621,6 +645,14 @@ impl I444Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I444Buffer {
+        vf::I444Buffer {
+            handle: I444Buffer {
+                sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
+            },
+        }
+    }
 }
 
 impl I010Buffer {
@@ -738,6 +770,14 @@ impl I010Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I010Buffer {
+        vf::I010Buffer {
+            handle: I010Buffer {
+                sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
+            },
+        }
+    }
 }
 
 impl NV12Buffer {
@@ -843,4 +883,12 @@ impl NV12Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::NV12Buffer {
+        vf::NV12Buffer {
+            handle: NV12Buffer {
+                sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
+            },
+        }
+    }
 }
diff --git a/libwebrtc/src/video_frame.rs b/libwebrtc/src/video_frame.rs
@@ -235,6 +235,10 @@ impl I420Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420Buffer {
+        self.handle.scale(scaled_width, scaled_height)
+    }
 }
 
 impl I420ABuffer {
@@ -274,6 +278,10 @@ impl I420ABuffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420ABuffer {
+        self.handle.scale(scaled_width, scaled_height)
+    }
 }
 
 impl I422Buffer {
@@ -317,6 +325,10 @@ impl I422Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I422Buffer {
+        self.handle.scale(scaled_width, scaled_height)
+    }
 }
 
 impl I444Buffer {
@@ -360,6 +372,10 @@ impl I444Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I444Buffer {
+        self.handle.scale(scaled_width, scaled_height)
+    }
 }
 
 impl I010Buffer {
@@ -403,6 +419,10 @@ impl I010Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I010Buffer {
+        self.handle.scale(scaled_width, scaled_height)
+    }
 }
 
 impl NV12Buffer {
@@ -439,6 +459,10 @@ impl NV12Buffer {
             )
         }
     }
+
+    pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> NV12Buffer {
+        self.handle.scale(scaled_width, scaled_height)
+    }
 }
 
 #[cfg(not(target_arch = "wasm32"))]

diff --git a/webrtc-sys/include/livekit/video_frame_buffer.h b/webrtc-sys/include/livekit/video_frame_buffer.h
@@ -151,6 +151,11 @@ class BiplanarYuv8Buffer : public BiplanarYuvBuffer {
 class I420Buffer : public PlanarYuv8Buffer {
  public:
   explicit I420Buffer(rtc::scoped_refptr<webrtc::I420BufferInterface> buffer);
+
+  std::unique_ptr<I420Buffer> scale(int scaled_width, int scaled_height) const;
+
+ private:
+  webrtc::I420BufferInterface* buffer() const;
 };
 
 class I420ABuffer : public I420Buffer {
@@ -160,28 +165,50 @@ class I420ABuffer : public I420Buffer {
   unsigned int stride_a() const;
   const uint8_t* data_a() const;
 
+  std::unique_ptr<I420ABuffer> scale(int scaled_width, int scaled_height) const;
+
  private:
   webrtc::I420ABufferInterface* buffer() const;
 };
 
 class I422Buffer : public PlanarYuv8Buffer {
  public:
   explicit I422Buffer(rtc::scoped_refptr<webrtc::I422BufferInterface> buffer);
+
+  std::unique_ptr<I422Buffer> scale(int scaled_width, int scaled_height) const;
+
+ private:
+  webrtc::I422BufferInterface* buffer() const;
 };
 
 class I444Buffer : public PlanarYuv8Buffer {
  public:
   explicit I444Buffer(rtc::scoped_refptr<webrtc::I444BufferInterface> buffer);
+
+  std::unique_ptr<I444Buffer> scale(int scaled_width, int scaled_height) const;
+
+ private:
+  webrtc::I444BufferInterface* buffer() const;
 };
 
 class I010Buffer : public PlanarYuv16BBuffer {
  public:
   explicit I010Buffer(rtc::scoped_refptr<webrtc::I010BufferInterface> buffer);
+
+  std::unique_ptr<I010Buffer> scale(int scaled_width, int scaled_height) const;
+
+ private:
+  webrtc::I010BufferInterface* buffer() const;
 };
 
 class NV12Buffer : public BiplanarYuv8Buffer {
  public:
   explicit NV12Buffer(rtc::scoped_refptr<webrtc::NV12BufferInterface> buffer);
+
+  std::unique_ptr<NV12Buffer> scale(int scaled_width, int scaled_height) const;
+
+ private:
+  webrtc::NV12BufferInterface* buffer() const;
 };
 
 std::unique_ptr<I420Buffer> copy_i420_buffer(

diff --git a/webrtc-sys/src/video_frame_buffer.cpp b/webrtc-sys/src/video_frame_buffer.cpp
@@ -192,6 +192,19 @@ webrtc::BiplanarYuv8Buffer* BiplanarYuv8Buffer::buffer() const {
 I420Buffer::I420Buffer(rtc::scoped_refptr<webrtc::I420BufferInterface> buffer)
     : PlanarYuv8Buffer(buffer) {}
 
+webrtc::I420BufferInterface* I420Buffer::buffer() const {
+  return static_cast<webrtc::I420BufferInterface*>(buffer_.get());
+}
+
+std::unique_ptr<I420Buffer> I420Buffer::scale(int scaled_width,
+                                              int scaled_height) const {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> result =
+      buffer()->Scale(scaled_width, scaled_height);
+  return std::make_unique<I420Buffer>(
+      rtc::scoped_refptr<webrtc::I420BufferInterface>(
+          const_cast<webrtc::I420BufferInterface*>(result->GetI420())));
+}
+
 I420ABuffer::I420ABuffer(
     rtc::scoped_refptr<webrtc::I420ABufferInterface> buffer)
     : I420Buffer(buffer) {}
@@ -208,18 +221,79 @@ webrtc::I420ABufferInterface* I420ABuffer::buffer() const {
   return static_cast<webrtc::I420ABufferInterface*>(buffer_.get());
 }
 
+std::unique_ptr<I420ABuffer> I420ABuffer::scale(int scaled_width,
+                                                int scaled_height) const {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> result =
+      buffer()->Scale(scaled_width, scaled_height);
+  return std::make_unique<I420ABuffer>(
+      rtc::scoped_refptr<webrtc::I420ABufferInterface>(
+          const_cast<webrtc::I420ABufferInterface*>(result->GetI420A())));
+}
+
 I422Buffer::I422Buffer(rtc::scoped_refptr<webrtc::I422BufferInterface> buffer)
     : PlanarYuv8Buffer(buffer) {}
 
+webrtc::I422BufferInterface* I422Buffer::buffer() const {
+  return static_cast<webrtc::I422BufferInterface*>(buffer_.get());
+}
+
+std::unique_ptr<I422Buffer> I422Buffer::scale(int scaled_width,
+                                              int scaled_height) const {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> result =
+      buffer()->Scale(scaled_width, scaled_height);
+  return std::make_unique<I422Buffer>(
+      rtc::scoped_refptr<webrtc::I422BufferInterface>(
+          const_cast<webrtc::I422BufferInterface*>(result->GetI422())));
+}
+
 I444Buffer::I444Buffer(rtc::scoped_refptr<webrtc::I444BufferInterface> buffer)
     : PlanarYuv8Buffer(buffer) {}
 
+webrtc::I444BufferInterface* I444Buffer::buffer() const {
+  return static_cast<webrtc::I444BufferInterface*>(buffer_.get());
+}
+
+std::unique_ptr<I444Buffer> I444Buffer::scale(int scaled_width,
+                                              int scaled_height) const {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> result =
+      buffer()->Scale(scaled_width, scaled_height);
+  return std::make_unique<I444Buffer>(
+      rtc::scoped_refptr<webrtc::I444BufferInterface>(
+          const_cast<webrtc::I444BufferInterface*>(result->GetI444())));
+}
+
 I010Buffer::I010Buffer(rtc::scoped_refptr<webrtc::I010BufferInterface> buffer)
     : PlanarYuv16BBuffer(buffer) {}
 
+webrtc::I010BufferInterface* I010Buffer::buffer() const {
+  return static_cast<webrtc::I010BufferInterface*>(buffer_.get());
+}
+
+std::unique_ptr<I010Buffer> I010Buffer::scale(int scaled_width,
+                                              int scaled_height) const {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> result =
+      buffer()->Scale(scaled_width, scaled_height);
+  return std::make_unique<I010Buffer>(
+      rtc::scoped_refptr<webrtc::I010BufferInterface>(
+          const_cast<webrtc::I010BufferInterface*>(result->GetI010())));
+}
+
 NV12Buffer::NV12Buffer(rtc::scoped_refptr<webrtc::NV12BufferInterface> buffer)
     : BiplanarYuv8Buffer(buffer) {}
 
+webrtc::NV12BufferInterface* NV12Buffer::buffer() const {
+  return static_cast<webrtc::NV12BufferInterface*>(buffer_.get());
+}
+
+std::unique_ptr<NV12Buffer> NV12Buffer::scale(int scaled_width,
+                                              int scaled_height) const {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> result =
+      buffer()->Scale(scaled_width, scaled_height);
+  return std::make_unique<NV12Buffer>(
+      rtc::scoped_refptr<webrtc::NV12BufferInterface>(
+          const_cast<webrtc::NV12BufferInterface*>(result->GetNV12())));
+}
+
 std::unique_ptr<I420Buffer> copy_i420_buffer(
     const std::unique_ptr<I420Buffer>& i420) {
   return std::make_unique<I420Buffer>(webrtc::I420Buffer::Copy(*i420->get()));

diff --git a/webrtc-sys/src/video_frame_buffer.rs b/webrtc-sys/src/video_frame_buffer.rs
@@ -87,6 +87,22 @@ pub mod ffi {
         fn stride_a(self: &I420ABuffer) -> u32;
         fn data_a(self: &I420ABuffer) -> *const u8;
 
+        fn scale(self: &I420Buffer, scaled_width: i32, scaled_height: i32)
+            -> UniquePtr<I420Buffer>;
+        fn scale(
+            self: &I420ABuffer,
+            scaled_width: i32,
+            scaled_height: i32,
+        ) -> UniquePtr<I420ABuffer>;
+        fn scale(self: &I422Buffer, scaled_width: i32, scaled_height: i32)
+            -> UniquePtr<I422Buffer>;
+        fn scale(self: &I444Buffer, scaled_width: i32, scaled_height: i32)
+            -> UniquePtr<I444Buffer>;
+        fn scale(self: &I010Buffer, scaled_width: i32, scaled_height: i32)
+            -> UniquePtr<I010Buffer>;
+        fn scale(self: &NV12Buffer, scaled_width: i32, scaled_height: i32)
+            -> UniquePtr<NV12Buffer>;
+
         fn copy_i420_buffer(i420: &UniquePtr<I420Buffer>) -> UniquePtr<I420Buffer>;
         fn new_i420_buffer(
             width: i32,