Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix coalesce stream in FLV transmuxer to account for missing audio data in pending tracks #125

Merged
merged 2 commits into from
Nov 17, 2016
Merged

fix coalesce stream in FLV transmuxer to account for missing audio data in pending tracks #125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c917791
fix coalesce stream to account for missing audio data in pending tracks
mjneil Oct 18, 2016
719cd17
add test for emitting done event
mjneil Nov 3, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 143 additions & 0 deletions lib/flv/coalesce-stream.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
'use strict';

var Stream = require('../utils/stream.js');

/**
* The final stage of the transmuxer that emits the flv tags
* for audio, video, and metadata. Also tranlates in time and
* outputs caption data and id3 cues.
*/
var CoalesceStream = function(options) {
// Number of Tracks per output segment
// If greater than 1, we combine multiple
// tracks into a single segment
this.numberOfTracks = 0;
this.metadataStream = options.metadataStream;

this.videoTags = [];
this.audioTags = [];
this.videoTrack = null;
this.audioTrack = null;
this.pendingCaptions = [];
this.pendingMetadata = [];
this.pendingTracks = 0;
this.processedTracks = 0;

CoalesceStream.prototype.init.call(this);

// Take output from multiple
this.push = function(output) {
// buffer incoming captions until the associated video segment
// finishes
if (output.text) {
return this.pendingCaptions.push(output);
}
// buffer incoming id3 tags until the final flush
if (output.frames) {
return this.pendingMetadata.push(output);
}

if (output.track.type === 'video') {
this.videoTrack = output.track;
this.videoTags = output.tags;
this.pendingTracks++;
}
if (output.track.type === 'audio') {
this.audioTrack = output.track;
this.audioTags = output.tags;
this.pendingTracks++;
}
};
};

CoalesceStream.prototype = new Stream();
CoalesceStream.prototype.flush = function(flushSource) {
var
id3,
caption,
i,
timelineStartPts,
event = {
tags: {},
captions: [],
metadata: []
};

if (this.pendingTracks < this.numberOfTracks) {
if (flushSource !== 'VideoSegmentStream' &&
flushSource !== 'AudioSegmentStream') {
// Return because we haven't received a flush from a data-generating
// portion of the segment (meaning that we have only recieved meta-data
// or captions.)
return;
} else if (this.pendingTracks === 0) {
// In the case where we receive a flush without any data having been
// received we consider it an emitted track for the purposes of coalescing
// `done` events.
// We do this for the case where there is an audio and video track in the
// segment but no audio data. (seen in several playlists with alternate
// audio tracks and no audio present in the main TS segments.)
this.processedTracks++;

if (this.processedTracks < this.numberOfTracks) {
return;
}
}
}

this.processedTracks += this.pendingTracks;
this.pendingTracks = 0;

if (this.processedTracks < this.numberOfTracks) {
return;
}

if (this.videoTrack) {
timelineStartPts = this.videoTrack.timelineStartInfo.pts;
} else if (this.audioTrack) {
timelineStartPts = this.audioTrack.timelineStartInfo.pts;
}

event.tags.videoTags = this.videoTags;
event.tags.audioTags = this.audioTags;

// Translate caption PTS times into second offsets into the
// video timeline for the segment
for (i = 0; i < this.pendingCaptions.length; i++) {
caption = this.pendingCaptions[i];
caption.startTime = caption.startPts - timelineStartPts;
caption.startTime /= 90e3;
caption.endTime = caption.endPts - timelineStartPts;
caption.endTime /= 90e3;
event.captions.push(caption);
}

// Translate ID3 frame PTS times into second offsets into the
// video timeline for the segment
for (i = 0; i < this.pendingMetadata.length; i++) {
id3 = this.pendingMetadata[i];
id3.cueTime = id3.pts - timelineStartPts;
id3.cueTime /= 90e3;
event.metadata.push(id3);
}
// We add this to every single emitted segment even though we only need
// it for the first
event.metadata.dispatchType = this.metadataStream.dispatchType;

// Reset stream state
this.videoTrack = null;
this.audioTrack = null;
this.videoTags = [];
this.audioTags = [];
this.pendingCaptions.length = 0;
this.pendingMetadata.length = 0;
this.pendingTracks = 0;
this.processedTracks = 0;

// Emit the final segment
this.trigger('data', event);

this.trigger('done');
};

module.exports = CoalesceStream;
121 changes: 5 additions & 116 deletions lib/flv/transmuxer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ var FlvTag = require('./flv-tag.js');
var m2ts = require('../m2ts/m2ts.js');
var AdtsStream = require('../codecs/adts.js');
var H264Stream = require('../codecs/h264').H264Stream;
var CoalesceStream = require('./coalesce-stream.js');

var
Transmuxer,
VideoSegmentStream,
AudioSegmentStream,
CoalesceStream,
collectTimelineInfo,
metaDataTag,
extraDataTag;
Expand Down Expand Up @@ -116,7 +116,7 @@ AudioSegmentStream = function(track) {
var currentFrame, adtsFrame, lastMetaPts, tags = [];
// return early if no audio data has been observed
if (adtsFrames.length === 0) {
this.trigger('done');
this.trigger('done', 'AudioSegmentStream');
return;
}

Expand Down Expand Up @@ -171,7 +171,7 @@ AudioSegmentStream = function(track) {
oldExtraData = null;
this.trigger('data', {track: track, tags: tags});

this.trigger('done');
this.trigger('done', 'AudioSegmentStream');
};
};
AudioSegmentStream.prototype = new Stream();
Expand Down Expand Up @@ -232,7 +232,7 @@ VideoSegmentStream = function(track) {

// return early if no video data has been observed
if (nalUnits.length === 0) {
this.trigger('done');
this.trigger('done', 'VideoSegmentStream');
return;
}

Expand Down Expand Up @@ -278,123 +278,12 @@ VideoSegmentStream = function(track) {
this.trigger('data', {track: track, tags: tags});

// Continue with the flush process now
this.trigger('done');
this.trigger('done', 'VideoSegmentStream');
};
};

VideoSegmentStream.prototype = new Stream();

/**
* The final stage of the transmuxer that emits the flv tags
* for audio, video, and metadata. Also tranlates in time and
* outputs caption data and id3 cues.
*/
CoalesceStream = function(options) {
// Number of Tracks per output segment
// If greater than 1, we combine multiple
// tracks into a single segment
this.numberOfTracks = 0;
this.metadataStream = options.metadataStream;

this.videoTags = [];
this.audioTags = [];
this.videoTrack = null;
this.audioTrack = null;
this.pendingCaptions = [];
this.pendingMetadata = [];
this.pendingTracks = 0;

CoalesceStream.prototype.init.call(this);

// Take output from multiple
this.push = function(output) {
// buffer incoming captions until the associated video segment
// finishes
if (output.text) {
return this.pendingCaptions.push(output);
}
// buffer incoming id3 tags until the final flush
if (output.frames) {
return this.pendingMetadata.push(output);
}

if (output.track.type === 'video') {
this.videoTrack = output.track;
this.videoTags = output.tags;
this.pendingTracks++;
}
if (output.track.type === 'audio') {
this.audioTrack = output.track;
this.audioTags = output.tags;
this.pendingTracks++;
}
};
};

CoalesceStream.prototype = new Stream();
CoalesceStream.prototype.flush = function() {
var
id3,
caption,
i,
timelineStartPts,
event = {
tags: {},
captions: [],
metadata: []
};

if (this.pendingTracks < this.numberOfTracks) {
return;
}

if (this.videoTrack) {
timelineStartPts = this.videoTrack.timelineStartInfo.pts;
} else if (this.audioTrack) {
timelineStartPts = this.audioTrack.timelineStartInfo.pts;
}

event.tags.videoTags = this.videoTags;
event.tags.audioTags = this.audioTags;

// Translate caption PTS times into second offsets into the
// video timeline for the segment
for (i = 0; i < this.pendingCaptions.length; i++) {
caption = this.pendingCaptions[i];
caption.startTime = caption.startPts - timelineStartPts;
caption.startTime /= 90e3;
caption.endTime = caption.endPts - timelineStartPts;
caption.endTime /= 90e3;
event.captions.push(caption);
}

// Translate ID3 frame PTS times into second offsets into the
// video timeline for the segment
for (i = 0; i < this.pendingMetadata.length; i++) {
id3 = this.pendingMetadata[i];
id3.cueTime = id3.pts - timelineStartPts;
id3.cueTime /= 90e3;
event.metadata.push(id3);
}
// We add this to every single emitted segment even though we only need
// it for the first
event.metadata.dispatchType = this.metadataStream.dispatchType;

// Reset stream state
this.videoTrack = null;
this.audioTrack = null;
this.videoTags = [];
this.audioTags = [];
this.pendingCaptions.length = 0;
this.pendingMetadata.length = 0;
this.pendingTracks = 0;

// Emit the final segment
this.trigger('data', event);

this.trigger('done');
};

/**
* An object that incrementally transmuxes MPEG2 Trasport Stream
* chunks into an FLV.
Expand Down
33 changes: 33 additions & 0 deletions test/transmuxer.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3206,6 +3206,39 @@ QUnit.test('does not buffer a duplicate video sample on subsequent flushes', fun
QUnit.equal(segments[0].tags.videoTags.length, 2, 'generated two video tags');
});

QUnit.test('emits done event when no audio data is present', function() {
var segments = [];
var done = false;

transmuxer.on('data', function(data) {
segments.push(data);
});
transmuxer.on('done', function() {
done = true;
});
transmuxer.push(packetize(PAT));
transmuxer.push(packetize(generatePMT({
hasVideo: true,
hasAudio: true
})));

// buffer a NAL
transmuxer.push(packetize(videoPes([0x09, 0x01], true)));
transmuxer.push(packetize(videoPes([0x00, 0x02])));

// add an access_unit_delimiter_rbsp
transmuxer.push(packetize(videoPes([0x09, 0x03])));
transmuxer.push(packetize(videoPes([0x00, 0x04])));
transmuxer.push(packetize(videoPes([0x00, 0x05])));

// flush everything
transmuxer.flush();

QUnit.equal(segments[0].tags.audioTags.length, 0, 'generated no audio tags');
QUnit.equal(segments[0].tags.videoTags.length, 2, 'generated two video tags');
QUnit.ok(done, 'emitted done event even though no audio data was given');
});

QUnit.module('AAC Stream');
QUnit.test('parses correct ID3 tag size', function() {
var packetStream = new Uint8Array(10),
Expand Down