Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@master-of-zen
master-of-zen committed May 3, 2021
1 parent 4b23924 commit 629292f
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 61 deletions.
2 changes: 2 additions & 0 deletions src/api/internal.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ impl<T: Pixel> ContextInner<T> {

let seq = Arc::new(Sequence::new(enc));
let inter_cfg = InterConfig::new(enc);
let lookahead_distance = inter_cfg.keyframe_lookahead_distance() as usize;

ContextInner {
frame_count: 0,
Expand All @@ -288,6 +289,7 @@ impl<T: Pixel> ContextInner<T> {
keyframe_detector: SceneChangeDetector::new(
*enc,
CpuFeatureLevel::default(),
lookahead_distance,
seq.clone(),
),
config: Arc::new(*enc),
Expand Down
170 changes: 109 additions & 61 deletions src/scenechange/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,12 @@ pub struct SceneChangeDetector<T: Pixel> {
scale_factor: usize,
// Frame buffer for scaled frames
frame_buffer: Vec<Plane<T>>,
// Deque offset for current
lookahead_offset: usize,
// Start deque offset based on lookahead
deque_offset: usize,
// Scenechange results for adaptive threshold
score_deque: Vec<f64>,
score_deque: Vec<(f64, f64)>,
/// Number of pixels in scaled frame for fast mode
pixels: usize,
/// The bit depth of the video.
Expand All @@ -42,7 +46,7 @@ pub struct SceneChangeDetector<T: Pixel> {
impl<T: Pixel> SceneChangeDetector<T> {
pub fn new(
encoder_config: EncoderConfig, cpu_feature_level: CpuFeatureLevel,
sequence: Arc<Sequence>,
lookahead_distance: usize, sequence: Arc<Sequence>,
) -> Self {
// This implementation is based on a Python implementation at
// https://pyscenedetect.readthedocs.io/en/latest/reference/detection-methods/.
Expand All @@ -55,6 +59,9 @@ impl<T: Pixel> SceneChangeDetector<T> {
// This may be adjusted later.
//
// This threshold is only used for the fast scenecut implementation.
//
// Testing shown that default threshold of 12 overallocates keyframes by almost double,
// compared to other scene change implementations
const BASE_THRESHOLD: usize = 25;
let bit_depth = encoder_config.bit_depth;
let fast_mode = encoder_config.speed_settings.fast_scene_detection
Expand All @@ -64,23 +71,30 @@ impl<T: Pixel> SceneChangeDetector<T> {
let scale_factor =
if fast_mode { detect_scale_factor(&sequence) } else { 1_usize };

let score_deque = Vec::with_capacity(5);
// Pixel count for fast scenedetect
// Set lookahead offset to 5 if normal lookahead available
let lookahead_offset = if lookahead_distance >= 5 { 5 } else { 0 };
let deque_offset = lookahead_offset;

let score_deque = Vec::with_capacity(5 + lookahead_distance);

// Pixel count for fast scenedetect
let pixels = if fast_mode {
(sequence.max_frame_height as usize / scale_factor)
* (sequence.max_frame_width as usize / scale_factor)
} else {
1
};

let frame_buffer = Vec::with_capacity(2);
let frame_buffer =
if fast_mode { Vec::with_capacity(2) } else { Vec::new() };

Self {
threshold: BASE_THRESHOLD * bit_depth / 8,
fast_mode,
scale_factor,
frame_buffer,
lookahead_offset,
deque_offset,
score_deque,
pixels,
bit_depth,
Expand All @@ -103,6 +117,9 @@ impl<T: Pixel> SceneChangeDetector<T> {
&mut self, frame_set: &[Arc<Frame<T>>], input_frameno: u64,
previous_keyframe: u64,
) -> bool {
// Use score deque for adaptive threshold for scene cut
// Declare score_deque offset based on lookahead for scene change scores

// Find the distance to the previous keyframe.
let distance = input_frameno - previous_keyframe;

Expand All @@ -118,56 +135,111 @@ impl<T: Pixel> SceneChangeDetector<T> {
return false;
}

// Set our scenecut method
// Decrease deque offset if there is no more frames
if self.deque_offset > frame_set.len() && self.lookahead_offset > 0 {
self.deque_offset = frame_set.len();
}

// Initially fill score deque with forward frames
if self.deque_offset > 0 && self.score_deque.len() == 0 {
for x in 0..self.lookahead_offset {
// Filling score deque with forward frames
let result = if self.fast_mode {
self.fast_scenecut(frame_set[x].clone(), frame_set[x + 1].clone())
} else {
self.cost_scenecut(
frame_set[x].clone(),
frame_set[x + 1].clone(),
input_frameno,
previous_keyframe,
)
};
self.score_deque.push((result.inter_cost, result.intra_cost));
}
debug!("{:.0?}", self.score_deque)
}

// Running single frame comparison and adding it to deque
let result = if self.fast_mode {
self.fast_scenecut(frame_set[0].clone(), frame_set[1].clone())
self.fast_scenecut(
frame_set[0 + self.deque_offset].clone(),
frame_set[1 + self.deque_offset].clone(),
)
} else {
self.cost_scenecut(
frame_set[0].clone(),
frame_set[1].clone(),
frame_set[0 + self.deque_offset].clone(),
frame_set[1 + self.deque_offset].clone(),
input_frameno,
previous_keyframe,
)
};
self
.score_deque
.push((result.inter_cost as f64, result.intra_cost as f64));

// Adaptive scenecut check;
let scenecut = self.adaptive_scenecut();

debug!(
"[SC-Detect] Frame {}: T={:.1} P={:.1} {}",
"[SC-Detect] Frame {}: Cost={:.0} Threshold= {:.0} {}",
input_frameno,
result.threshold,
result.inter_cost,
if result.has_scenecut { "Scenecut" } else { "No cut" }
self.score_deque[self.score_deque.len() - self.deque_offset].0,
self.score_deque[self.score_deque.len() - self.deque_offset].1,
if scenecut { "Scenecut" } else { "No cut" }
);
result.has_scenecut

if scenecut {
// Reset lookahead offset
self.deque_offset = self.lookahead_offset;

// Clear buffers and deque
self.frame_buffer.clear();
self.score_deque.clear();
} else {
// Keep score deque 5 + lookahead_size frames
self
.score_deque
.push((result.inter_cost as f64, result.intra_cost as f64));
if self.score_deque.len() > 5 + self.deque_offset {
self.score_deque.remove(0);
}
}

scenecut
}

/// Compares current scene score to adapted threshold based on previous scores
/// Value of current frame is offset by lookahead, if lookahead >=5
/// Returns true if current scene score is higher than adapted threshold
fn adaptive_scenecut(&mut self, scene_score: f64) -> bool {
if self.score_deque.is_empty() {
true // we skip high delta on first frame comparision as it's probably inside flashing or high motion scene
fn adaptive_scenecut(&mut self) -> bool {
// Max cost of all available frames
let max_of_deque: f64 = self
.score_deque
.iter()
.cloned()
.map(|(_, b)| b)
.fold(-1. / 0. /* -inf */, f64::max);

// Scenecut check
let threshold = if self.fast_mode {
self.threshold as f64 + max_of_deque
} else {
let max_of_deque: f64 = self
.score_deque
.iter()
.cloned()
.fold(-1. / 0. /* -inf */, f64::max); // max of last n(5) frames

//
let scenecut = scene_score > self.threshold as f64 + max_of_deque;
debug!(
"[SC-Detect] P: {:.1} {:.1?} Cut: {}",
scene_score, self.score_deque, scenecut
);
scenecut
}
max_of_deque
};

let scene_score =
self.score_deque[self.score_deque.len() - self.deque_offset].0;

let scenecut = scene_score >= threshold;
scenecut
}

/// The fast algorithm detects fast cuts using a raw difference
/// in pixel values between the scaled frames.
#[hawktracer(fast_scenecut)]
fn fast_scenecut(
&mut self, frame1: Arc<Frame<T>>, frame2: Arc<Frame<T>>,
) -> ScenecutResult {
) -> ScenecutData {
// Downscaling both frames for comparison
// Moving scaled frames to buffer
if self.frame_buffer.is_empty() {
Expand All @@ -184,27 +256,10 @@ impl<T: Pixel> SceneChangeDetector<T> {
let delta =
self.delta_in_planes(&self.frame_buffer[0], &self.frame_buffer[1]);

// Adaptive scenecut check;
let scenecut =
delta >= self.threshold as f64 && self.adaptive_scenecut(delta);

if scenecut {
// Clear buffers
self.frame_buffer.clear();
self.score_deque.clear();
} else {
// Keep score deque 5 frames
self.score_deque.push(delta as f64);
if self.score_deque.len() > 5 {
self.score_deque.remove(0);
}
}

ScenecutResult {
ScenecutData {
intra_cost: self.threshold as f64,
threshold: self.threshold as f64,
inter_cost: delta as f64,
has_scenecut: scenecut,
}
}

Expand All @@ -217,7 +272,7 @@ impl<T: Pixel> SceneChangeDetector<T> {
fn cost_scenecut(
&self, frame1: Arc<Frame<T>>, frame2: Arc<Frame<T>>, frameno: u64,
previous_keyframe: u64,
) -> ScenecutResult {
) -> ScenecutData {
let frame2_ref2 = Arc::clone(&frame2);
let (intra_cost, inter_cost) = crate::rayon::join(
move || {
Expand Down Expand Up @@ -269,12 +324,7 @@ impl<T: Pixel> SceneChangeDetector<T> {
};
let threshold = intra_cost * (1.0 - bias);

ScenecutResult {
intra_cost,
threshold,
inter_cost,
has_scenecut: inter_cost > threshold,
}
ScenecutData { intra_cost, inter_cost, threshold }
}

/// Calculates delta beetween 2 planes
Expand All @@ -299,7 +349,7 @@ impl<T: Pixel> SceneChangeDetector<T> {
}
}

/// Scaling factor for frame in scenedetection
/// Scaling factor for frame in scene detection
fn detect_scale_factor(sequence: &Arc<Sequence>) -> usize {
let small_edge =
cmp::min(sequence.max_frame_height, sequence.max_frame_width) as usize;
Expand All @@ -324,11 +374,9 @@ fn detect_scale_factor(sequence: &Arc<Sequence>) -> usize {
}

/// This struct primarily exists for returning metrics to the caller
/// for logging debug information.
#[derive(Debug, Clone, Copy)]
struct ScenecutResult {
struct ScenecutData {
intra_cost: f64,
inter_cost: f64,
threshold: f64,
has_scenecut: bool,
}

0 comments on commit 629292f

Please sign in to comment.