Skip to content

Commit

Permalink
wip: frame flow
Browse files Browse the repository at this point in the history 
fix 16 channel issues
  • Loading branch information
@niklaspandersson
niklaspandersson committed Dec 6, 2024
1 parent 678635d commit 8de1bb0
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 22 deletions.
5 changes: 3 additions & 2 deletions src/modules/ffmpeg/util/audio_resampler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ namespace caspar { namespace ffmpeg {
AudioResampler::AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt)
{
AVChannelLayout channel_layout = AV_CHANNEL_LAYOUT_7POINT1;
AVChannelLayout channel_layout_out = AV_CHANNEL_LAYOUT_HEXADECAGONAL;

SwrContext* raw_ctx = nullptr;
FF(swr_alloc_set_opts2(&raw_ctx,
&channel_layout,
&channel_layout_out,
AV_SAMPLE_FMT_S32,
sample_rate,
&channel_layout,
Expand All @@ -30,7 +31,7 @@ AudioResampler::AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt

caspar::array<int32_t> AudioResampler::convert(int frames, const void** src)
{
auto result = caspar::array<int32_t>(frames * 8 * sizeof(int32_t));
auto result = caspar::array<int32_t>(frames * 16 * sizeof(int32_t));
auto ptr = result.data();
auto ret = swr_convert(ctx.get(), (uint8_t**)&ptr, frames, reinterpret_cast<const uint8_t**>(src), frames);

Expand Down
40 changes: 20 additions & 20 deletions src/modules/html/producer/html_producer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,22 +76,24 @@ inline std::int_least64_t now()
struct presentation_frame
{
std::int_least64_t timestamp;
core::draw_frame frame = core::draw_frame::empty();
core::draw_frame frame = core::draw_frame::empty();
bool is_empty = true;

explicit presentation_frame(core::draw_frame video = {}, std::int_least64_t ts = now())
{
timestamp = ts;
if (video) {
frame = std::move(video);
frame = std::move(video);
is_empty = false;
}
}

presentation_frame(presentation_frame&& other) noexcept
: timestamp(other.timestamp)
, frame(std::move(other.frame))
, is_empty(other.is_empty)
{
other.is_empty = true;
}

presentation_frame(const presentation_frame&) = delete;
Expand All @@ -101,6 +103,7 @@ struct presentation_frame
{
timestamp = rhs.timestamp;
frame = std::move(rhs.frame);
is_empty = rhs.is_empty;
return *this;
}

Expand Down Expand Up @@ -131,14 +134,14 @@ class html_client
std::atomic<bool> loaded_;
std::queue<presentation_frame> frames_;
std::queue<presentation_frame> audio_frames_;
core::draw_frame last_generated_frame_;
mutable std::mutex frames_mutex_;
mutable std::mutex audio_frames_mutex_;
const size_t frames_max_size_ = 4;
std::atomic<bool> closing_;

std::unique_ptr<ffmpeg::AudioResampler> audioResampler_;

core::draw_frame last_video_frame_;
core::draw_frame last_frame_;
std::int_least64_t last_frame_time_;

Expand Down Expand Up @@ -195,14 +198,17 @@ class html_client

bool try_pop(const core::video_field field)
{
bool result = false;
std::lock_guard<std::mutex> lock(frames_mutex_);

core::draw_frame audio_frame;
uint64_t audio_frame_timestamp = 0;

auto current_time = now();
{
// CASPAR_LOG(info) << "[html_producer] receive: last_frame_time_: " << last_frame_time_;
std::lock_guard<std::mutex> audio_lock(audio_frames_mutex_);
if (!audio_frames_.empty()) {
if (!audio_frames_.empty() && audio_frames_.front().timestamp <= current_time) {
audio_frame_timestamp = audio_frames_.front().timestamp;
audio_frame = core::draw_frame(std::move(audio_frames_.front().frame));
audio_frames_.pop();
}
Expand Down Expand Up @@ -235,31 +241,28 @@ class html_client
}
}

last_frame_time_ = frames_.front().timestamp;
last_frame_ = std::move(frames_.front().frame);

last_frame_time_ = frames_.front().timestamp;
last_video_frame_ = std::move(frames_.front().frame);
last_frame_ = last_video_frame_;
frames_.pop();

if (audio_frame)
last_frame_= core::draw_frame::over(last_frame_, audio_frame);

graph_->set_value("buffered-frames", (double)frames_.size() / frames_max_size_);

return true;
result = true;
}

if (audio_frame) {
last_frame_time_ = now();
last_frame_ = core::draw_frame::over(last_frame_, audio_frame);
return true;
last_frame_time_ = audio_frame_timestamp;
last_frame_ = core::draw_frame::over(last_video_frame_, audio_frame);
result = true;
}

return false;
return result;
}

core::draw_frame receive(const core::video_field field)
{

if (!try_pop(field)) {
graph_->set_tag(diagnostics::tag_severity::SILENT, "late-frame");
return core::draw_frame::still(last_frame_);
Expand Down Expand Up @@ -369,7 +372,6 @@ class html_client
std::lock_guard<std::mutex> lock(frames_mutex_);

core::draw_frame new_frame = core::draw_frame(std::move(frame));
last_generated_frame_ = new_frame;

frames_.push(presentation_frame(std::move(new_frame)));
while (frames_.size() > frames_max_size_) {
Expand Down Expand Up @@ -480,7 +482,6 @@ class html_client

void OnAudioStreamStarted(CefRefPtr<CefBrowser> browser, const CefAudioParameters& params, int channels) override
{
//48000
CASPAR_LOG(info) << "[html_producer] OnAudioStreamStarted: sample_rate: " << params.sample_rate;
audioResampler_ = std::make_unique<ffmpeg::AudioResampler>(params.sample_rate, AV_SAMPLE_FMT_FLTP);
}
Expand All @@ -497,8 +498,7 @@ class html_client
while (audio_frames_.size() >= frames_max_size_) {
audio_frames_.pop();
}
// CASPAR_LOG(info) << "[html_producer] OnAudioStreamPacket: pts: " << pts;
audio_frames_.push(presentation_frame(core::draw_frame(std::move(audio_frame))));
audio_frames_.push(presentation_frame(core::draw_frame(std::move(audio_frame)), pts + 40));
}
}
void OnAudioStreamStopped(CefRefPtr<CefBrowser> browser) override { audioResampler_ = nullptr; }
Expand Down

0 comments on commit 8de1bb0

Please sign in to comment.