Skip to content

Commit

Permalink
Improve handling of RTMP timestamps.
Browse files Browse the repository at this point in the history
  • Loading branch information
@shogo4405
shogo4405 committed Apr 11, 2024
1 parent 44c324f commit 5ae4425
Show file tree
Hide file tree
Showing 6 changed files with 186 additions and 121 deletions.
2 changes: 1 addition & 1 deletion Examples/iOSSwiftUI/Model/ViewModel.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ final class ViewModel: ObservableObject {

func rotateCamera() {
let position: AVCaptureDevice.Position = currentPosition == .back ? .front : .back
rtmpStream.attachCamera(AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position)) { unit, error in
rtmpStream.attachCamera(AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position)) { _, error in
logger.error(error)
}
currentPosition = position
Expand Down
16 changes: 8 additions & 8 deletions HaishinKit.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,8 @@
BC959F1229717EDB0067BA97 /* PreferenceViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC959F1129717EDB0067BA97 /* PreferenceViewController.swift */; };
BC9CFA9323BDE8B700917EEF /* IOStreamView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9CFA9223BDE8B700917EEF /* IOStreamView.swift */; };
BC9F9C7826F8C16600B01ED0 /* Choreographer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */; };
BCA3A5252BC4ED220083BBB1 /* RTMPTimestamp.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA3A5242BC4ED220083BBB1 /* RTMPTimestamp.swift */; };
BCA3A5272BC507880083BBB1 /* RTMPTimestampTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA3A5262BC507880083BBB1 /* RTMPTimestampTests.swift */; };
BCA7C24F2A91AA0500882D85 /* IOStreamRecorderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA7C24E2A91AA0500882D85 /* IOStreamRecorderTests.swift */; };
BCAD0C18263ED67F00ADFB80 /* [email protected] in Resources */ = {isa = PBXBuildFile; fileRef = BCAD0C16263ED67F00ADFB80 /* [email protected] */; };
BCAD0C19263ED67F00ADFB80 /* SampleVideo_360x240_5mb@m4v in Resources */ = {isa = PBXBuildFile; fileRef = BCAD0C17263ED67F00ADFB80 /* SampleVideo_360x240_5mb@m4v */; };
Expand Down Expand Up @@ -636,6 +638,8 @@
BC959F1129717EDB0067BA97 /* PreferenceViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PreferenceViewController.swift; sourceTree = "<group>"; };
BC9CFA9223BDE8B700917EEF /* IOStreamView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOStreamView.swift; sourceTree = "<group>"; };
BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Choreographer.swift; sourceTree = "<group>"; };
BCA3A5242BC4ED220083BBB1 /* RTMPTimestamp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RTMPTimestamp.swift; sourceTree = "<group>"; };
BCA3A5262BC507880083BBB1 /* RTMPTimestampTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RTMPTimestampTests.swift; sourceTree = "<group>"; };
BCA7C24E2A91AA0500882D85 /* IOStreamRecorderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOStreamRecorderTests.swift; sourceTree = "<group>"; };
BCAD0C16263ED67F00ADFB80 /* [email protected] */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "[email protected]"; sourceTree = "<group>"; };
BCAD0C17263ED67F00ADFB80 /* SampleVideo_360x240_5mb@m4v */ = {isa = PBXFileReference; lastKnownFileType = folder; path = "SampleVideo_360x240_5mb@m4v"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -822,6 +826,7 @@
290686021DFDB7A6008EB7ED /* RTMPConnectionTests.swift */,
2976077E20A89FBB00DCF24F /* RTMPMessageTests.swift */,
035AFA032263868E009DD0BB /* RTMPStreamTests.swift */,
BCA3A5262BC507880083BBB1 /* RTMPTimestampTests.swift */,
);
path = RTMP;
sourceTree = "<group>";
Expand Down Expand Up @@ -1006,7 +1011,6 @@
295018191FFA196800358E10 /* Codec */,
291C2AD21CE9FF48006F042B /* Core */,
BC03945D2AA8AFDD006EDE38 /* Extension */,
BC1DC5022A02893600E928ED /* FLV */,
BC0BF4F329866FB700D72CB4 /* IO */,
291C2ACF1CE9FF2B006F042B /* ISO */,
291C2ACE1CE9FF25006F042B /* RTMP */,
Expand Down Expand Up @@ -1107,6 +1111,7 @@
29DF20652312A436004057C3 /* RTMPSocketCompatible.swift */,
29B876AA1CD70B2800FC07DA /* RTMPStream.swift */,
BC558267240BB40E00011AC0 /* RTMPStreamInfo.swift */,
BCA3A5242BC4ED220083BBB1 /* RTMPTimestamp.swift */,
294852551D84BFAD002DE492 /* RTMPTSocket.swift */,
);
path = RTMP;
Expand Down Expand Up @@ -1193,13 +1198,6 @@
path = IO;
sourceTree = "<group>";
};
BC1DC5022A02893600E928ED /* FLV */ = {
isa = PBXGroup;
children = (
);
path = FLV;
sourceTree = "<group>";
};
BC3004FA296C3FC400119932 /* Extension */ = {
isa = PBXGroup;
children = (
Expand Down Expand Up @@ -1821,6 +1819,7 @@
BC570B4828E9ACC10098A12C /* IOUnit.swift in Sources */,
2976A4861D4903C300B53EF2 /* DeviceUtil.swift in Sources */,
B34239852B9FD3E30068C3FB /* AudioNode.swift in Sources */,
BCA3A5252BC4ED220083BBB1 /* RTMPTimestamp.swift in Sources */,
BC7C56BB299E595000C41A9B /* VideoCodecSettings.swift in Sources */,
29B876881CD70AE800FC07DA /* TSPacket.swift in Sources */,
BC22EEEE2AAF50F200E3406D /* Codec.swift in Sources */,
Expand Down Expand Up @@ -1885,6 +1884,7 @@
BC03945F2AA8AFF5006EDE38 /* ExpressibleByIntegerLiteral+ExtensionTests.swift in Sources */,
290EA8AA1DFB61E700053022 /* CRC32Tests.swift in Sources */,
035AFA042263868E009DD0BB /* RTMPStreamTests.swift in Sources */,
BCA3A5272BC507880083BBB1 /* RTMPTimestampTests.swift in Sources */,
290686031DFDB7A7008EB7ED /* RTMPConnectionTests.swift in Sources */,
BCC9E9092636FF7400948774 /* DataBufferTests.swift in Sources */,
);
Expand Down
113 changes: 53 additions & 60 deletions Sources/RTMP/RTMPMuxer.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,11 @@ final class RTMPMuxer {
}
var buffer = Data([RTMPMuxer.aac, FLVAACPacketType.seq.rawValue])
buffer.append(contentsOf: AudioSpecificConfig(formatDescription: audioFormat.formatDescription).bytes)
stream?.outputAudio(buffer, withTimestamp: 0)
stream?.doOutput(
.zero,
chunkStreamId: FLVTagType.audio.streamId,
message: RTMPAudioMessage(streamId: 0, timestamp: 0, payload: buffer)
)
case .playing:
if let audioFormat {
audioBuffer = AVAudioCompressedBuffer(format: audioFormat, packetCapacity: 1, maximumPacketSize: 1024 * Int(audioFormat.channelCount))
Expand All @@ -30,24 +34,29 @@ final class RTMPMuxer {
didSet {
switch stream?.readyState {
case .publishing:
guard let videoFormat else {
return
}
switch CMFormatDescriptionGetMediaSubType(videoFormat) {
case kCMVideoCodecType_H264:
switch videoFormat?.mediaSubType {
case .h264?:
guard let avcC = AVCDecoderConfigurationRecord.getData(videoFormat) else {
return
}
var buffer = Data([FLVFrameType.key.rawValue << 4 | FLVVideoCodec.avc.rawValue, FLVAVCPacketType.seq.rawValue, 0, 0, 0])
buffer.append(avcC)
stream?.outputVideo(buffer, withTimestamp: 0)
case kCMVideoCodecType_HEVC:
stream?.doOutput(
.zero,
chunkStreamId: FLVTagType.video.streamId,
message: RTMPVideoMessage(streamId: 0, timestamp: 0, payload: buffer)
)
case .hevc?:
guard let hvcC = HEVCDecoderConfigurationRecord.getData(videoFormat) else {
return
}
var buffer = Data([0b10000000 | FLVFrameType.key.rawValue << 4 | FLVVideoPacketType.sequenceStart.rawValue, 0x68, 0x76, 0x63, 0x31])
buffer.append(hvcC)
stream?.outputVideo(buffer, withTimestamp: 0)
stream?.doOutput(
.zero,
chunkStreamId: FLVTagType.video.streamId,
message: RTMPVideoMessage(streamId: 0, timestamp: 0, payload: buffer)
)
default:
break
}
Expand All @@ -60,10 +69,9 @@ final class RTMPMuxer {
}

var isRunning: Atomic<Bool> = .init(false)
private var videoTimeStamp: CMTime = .zero
private var audioBuffer: AVAudioCompressedBuffer?
private var audioTimeStamp: AVAudioTime = .init(hostTime: 0)
private let compositiionTimeOffset: CMTime = .init(value: 3, timescale: 30)
private var audioTimestamp: RTMPTimestamp<AVAudioTime> = .init()
private var videoTimestamp: RTMPTimestamp<CMTime> = .init()
private weak var stream: RTMPStream?

init(_ stream: RTMPStream) {
Expand All @@ -74,11 +82,10 @@ final class RTMPMuxer {
let payload = message.payload
let codec = message.codec
stream?.info.byteCount.mutate { $0 += Int64(payload.count) }

audioTimestamp.update(message, chunkType: type)
guard let stream, message.codec.isSupported else {
return
}

switch payload[1] {
case FLVAACPacketType.seq.rawValue:
let config = AudioSpecificConfig(bytes: [UInt8](payload[codec.headerSize..<payload.count]))
Expand All @@ -96,37 +103,30 @@ final class RTMPMuxer {
audioBuffer.packetCount = 1
audioBuffer.byteLength = UInt32(byteCount)
audioBuffer.data.copyMemory(from: baseAddress.advanced(by: codec.headerSize), byteCount: byteCount)
stream.mixer.audioIO.append(audioBuffer, when: audioTimeStamp)
stream.mixer.audioIO.append(audioBuffer, when: audioTimestamp.value)
}
default:
break
}

switch type {
case .zero:
audioTimeStamp = .init(hostTime: AVAudioTime.hostTime(forSeconds: Double(message.timestamp) / 1000))
default:
audioTimeStamp = .init(hostTime: AVAudioTime.hostTime(forSeconds: AVAudioTime.seconds(forHostTime: audioTimeStamp.hostTime) + Double(message.timestamp) / 1000))
}
}

func append(_ message: RTMPVideoMessage, type: RTMPChunkType) {
stream?.info.byteCount.mutate { $0 += Int64( message.payload.count) }
videoTimestamp.update(message, chunkType: type)
guard let stream, FLVTagType.video.headerSize <= message.payload.count && message.isSupported else {
return
}

if message.isExHeader {
// IsExHeader for Enhancing RTMP, FLV
switch message.packetType {
case FLVVideoPacketType.sequenceStart.rawValue:
videoFormat = message.makeFormatDescription()
case FLVVideoPacketType.codedFrames.rawValue:
if let sampleBuffer = message.makeSampleBuffer(videoTimeStamp, formatDesciption: videoFormat) {
if let sampleBuffer = message.makeSampleBuffer(videoTimestamp.value, formatDesciption: videoFormat) {
stream.mixer.videoIO.append(sampleBuffer)
}
case FLVVideoPacketType.codedFramesX.rawValue:
if let sampleBuffer = message.makeSampleBuffer(videoTimeStamp, formatDesciption: videoFormat) {
if let sampleBuffer = message.makeSampleBuffer(videoTimestamp.value, formatDesciption: videoFormat) {
stream.mixer.videoIO.append(sampleBuffer)
}
default:
Expand All @@ -137,20 +137,13 @@ final class RTMPMuxer {
case FLVAVCPacketType.seq.rawValue:
videoFormat = message.makeFormatDescription()
case FLVAVCPacketType.nal.rawValue:
if let sampleBuffer = message.makeSampleBuffer(videoTimeStamp, formatDesciption: videoFormat) {
if let sampleBuffer = message.makeSampleBuffer(videoTimestamp.value, formatDesciption: videoFormat) {
stream.mixer.videoIO.append(sampleBuffer)
}
default:
break
}
}

switch type {
case .zero:
videoTimeStamp = .init(value: CMTimeValue(message.timestamp), timescale: 1000)
default:
videoTimeStamp = CMTimeAdd(videoTimeStamp, .init(value: CMTimeValue(message.timestamp), timescale: 1000))
}
}
}

Expand All @@ -160,47 +153,47 @@ extension RTMPMuxer: IOMuxer {
guard let audioBuffer = audioBuffer as? AVAudioCompressedBuffer else {
return
}
let delta = audioTimeStamp.hostTime == 0 ? 0 :
(AVAudioTime.seconds(forHostTime: when.hostTime) - AVAudioTime.seconds(forHostTime: audioTimeStamp.hostTime)) * 1000
guard 0 <= delta else {
return
}
let timedelta = audioTimestamp.update(when)
var buffer = Data([RTMPMuxer.aac, FLVAACPacketType.raw.rawValue])
buffer.append(audioBuffer.data.assumingMemoryBound(to: UInt8.self), count: Int(audioBuffer.byteLength))
stream?.outputAudio(buffer, withTimestamp: delta)
audioTimeStamp = when
stream?.doOutput(
.one,
chunkStreamId: FLVTagType.audio.streamId,
message: RTMPVideoMessage(streamId: 0, timestamp: timedelta, payload: buffer)
)
}

func append(_ sampleBuffer: CMSampleBuffer) {
let keyframe = !sampleBuffer.isNotSync
let decodeTimeStamp = sampleBuffer.decodeTimeStamp.isValid ? sampleBuffer.decodeTimeStamp : sampleBuffer.presentationTimeStamp
let compositionTime = getCompositionTime(sampleBuffer)
let delta = videoTimeStamp == .zero ? 0 : (decodeTimeStamp.seconds - videoTimeStamp.seconds) * 1000
guard let formatDescription = sampleBuffer.formatDescription, let data = sampleBuffer.dataBuffer?.data, 0 <= delta else {
guard let data = sampleBuffer.dataBuffer?.data else {
return
}
switch CMFormatDescriptionGetMediaSubType(formatDescription) {
case kCMVideoCodecType_H264:
let keyframe = !sampleBuffer.isNotSync
let decodeTimeStamp = sampleBuffer.decodeTimeStamp.isValid ? sampleBuffer.decodeTimeStamp : sampleBuffer.presentationTimeStamp
let compositionTime = videoTimestamp.getCompositionTime(sampleBuffer)
let timedelta = videoTimestamp.update(decodeTimeStamp)
stream?.frameCount += 1
switch sampleBuffer.formatDescription?.mediaSubType {
case .h264?:
var buffer = Data([((keyframe ? FLVFrameType.key.rawValue : FLVFrameType.inter.rawValue) << 4) | FLVVideoCodec.avc.rawValue, FLVAVCPacketType.nal.rawValue])
buffer.append(contentsOf: compositionTime.bigEndian.data[1..<4])
buffer.append(data)
stream?.outputVideo(buffer, withTimestamp: delta)
case kCMVideoCodecType_HEVC:
stream?.doOutput(
.one,
chunkStreamId: FLVTagType.video.streamId,
message: RTMPVideoMessage(streamId: 0, timestamp: timedelta, payload: buffer)
)
case .hevc?:
var buffer = Data([0b10000000 | ((keyframe ? FLVFrameType.key.rawValue : FLVFrameType.inter.rawValue) << 4) | FLVVideoPacketType.codedFrames.rawValue, 0x68, 0x76, 0x63, 0x31])
buffer.append(contentsOf: compositionTime.bigEndian.data[1..<4])
buffer.append(data)
stream?.outputVideo(buffer, withTimestamp: delta)
stream?.doOutput(
.one,
chunkStreamId: FLVTagType.video.streamId,
message: RTMPVideoMessage(streamId: 0, timestamp: timedelta, payload: buffer)
)
default:
break
}
videoTimeStamp = decodeTimeStamp
}

private func getCompositionTime(_ sampleBuffer: CMSampleBuffer) -> Int32 {
guard sampleBuffer.decodeTimeStamp.isValid, sampleBuffer.decodeTimeStamp != sampleBuffer.presentationTimeStamp else {
return 0
}
return Int32((sampleBuffer.presentationTimeStamp - videoTimeStamp + compositiionTimeOffset).seconds * 1000)
}
}

Expand All @@ -210,10 +203,10 @@ extension RTMPMuxer: Running {
guard !isRunning.value else {
return
}
audioTimeStamp = .init(hostTime: 0)
videoTimeStamp = .zero
audioFormat = nil
videoFormat = nil
audioTimestamp.clear()
videoTimestamp.clear()
isRunning.mutate { $0 = true }
}

Expand Down
Loading

0 comments on commit 5ae4425

Please sign in to comment.