From 570ddd0b1c72a1de8c9196e399be48733efc6e17 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Wed, 22 Jul 2020 16:20:04 -0700 Subject: [PATCH] Add simulcast example Add jsfiddle and documentation around using the example --- examples/README.md | 1 + examples/examples.json | 6 + examples/simulcast/README.md | 27 ++++ examples/simulcast/jsfiddle/demo.css | 4 + examples/simulcast/jsfiddle/demo.details | 5 + examples/simulcast/jsfiddle/demo.html | 18 +++ examples/simulcast/jsfiddle/demo.js | 92 ++++++++++++ examples/simulcast/main.go | 173 +++++++++++++++++++++++ 8 files changed, 326 insertions(+) create mode 100644 examples/simulcast/README.md create mode 100644 examples/simulcast/jsfiddle/demo.css create mode 100644 examples/simulcast/jsfiddle/demo.details create mode 100644 examples/simulcast/jsfiddle/demo.html create mode 100644 examples/simulcast/jsfiddle/demo.js create mode 100644 examples/simulcast/main.go diff --git a/examples/README.md b/examples/README.md index 5036331b152..5f685cbb0ee 100644 --- a/examples/README.md +++ b/examples/README.md @@ -16,6 +16,7 @@ For more full featured examples that use 3rd party libraries see our **[example- * [Broadcast](broadcast): The broadcast example demonstrates how to broadcast a video to multiple peers. A broadcaster uploads the video once and the server forwards it to all other peers. * [RTP Forwarder](rtp-forwarder): The rtp-forwarder example demonstrates how to forward your audio/video streams using RTP. * [RTP to WebRTC](rtp-to-webrtc): The rtp-to-webrtc example demonstrates how to take RTP packets sent to a Pion process into your browser. +* [Simulcast](simulcast): The simulcast example demonstrates how to accept and demux 1 Track that contains 3 Simulcast streams. It then returns the media as 3 independent Tracks back to the sender. #### Data Channel API * [Data Channels](data-channels): The data-channels example shows how you can send/recv DataChannel messages from a web browser. diff --git a/examples/examples.json b/examples/examples.json index 79e9223f09b..54534c6850c 100644 --- a/examples/examples.json +++ b/examples/examples.json @@ -82,5 +82,11 @@ "link": "#", "description": "Example custom-logger demonstrates how the user can override the logging and process messages instead of printing to stdout. It has no corresponding web page.", "type": "browser" + }, + { + "title": "Simulcast", + "link": "simulcast", + "description": "Example simulcast demonstrates how to accept and demux 1 Track that contains 3 Simulcast streams. It then returns the media as 3 independent Tracks back to the sender.", + "type": "browser" } ] diff --git a/examples/simulcast/README.md b/examples/simulcast/README.md new file mode 100644 index 00000000000..a19ddd04133 --- /dev/null +++ b/examples/simulcast/README.md @@ -0,0 +1,27 @@ +# simulcast +demonstrates of how to handle incoming track with multiple simulcast rtp streams and show all them back. + +## Instructions +### Download simulcast +``` +go get github.com/pion/webrtc/v3/examples/simulcast +``` + +### Open simulcast example page +[jsfiddle.net](https://jsfiddle.net/rxk4bftc) you should see two text-areas and a 'Start Session' button. + +### Run simulcast, with your browsers SessionDescription as stdin +In the jsfiddle the top textarea is your browser, copy that and: +#### Linux/macOS +Run `echo $BROWSER_SDP | simulcast` +#### Windows +1. Paste the SessionDescription into a file. +1. Run `simulcast < my_file` + +### Input simulcast's SessionDescription into your browser +Copy the text that `simulcast` just emitted and copy into second text area + +### Hit 'Start Session' in jsfiddle, enjoy your video! +Your browser should send a simulcast track to Pion, and then all 3 incoming streams will be relayed back. + +Congrats, you have used Pion WebRTC! Now start building something cool diff --git a/examples/simulcast/jsfiddle/demo.css b/examples/simulcast/jsfiddle/demo.css new file mode 100644 index 00000000000..9e43d340755 --- /dev/null +++ b/examples/simulcast/jsfiddle/demo.css @@ -0,0 +1,4 @@ +textarea { + width: 500px; + min-height: 75px; +} \ No newline at end of file diff --git a/examples/simulcast/jsfiddle/demo.details b/examples/simulcast/jsfiddle/demo.details new file mode 100644 index 00000000000..eb0b725681f --- /dev/null +++ b/examples/simulcast/jsfiddle/demo.details @@ -0,0 +1,5 @@ +--- + name: simulcast + description: Example of how to have Pion handle incoming track with multiple simulcast rtp streams and show all them back. + authors: + - Simone Gotti diff --git a/examples/simulcast/jsfiddle/demo.html b/examples/simulcast/jsfiddle/demo.html new file mode 100644 index 00000000000..1377fcd2a37 --- /dev/null +++ b/examples/simulcast/jsfiddle/demo.html @@ -0,0 +1,18 @@ + +Browser base64 Session Description
+
+ +Golang base64 Session Description
+
+
+ +
+ +
+ Browser stream
+ +
+ +
+ Video from server
+
diff --git a/examples/simulcast/jsfiddle/demo.js b/examples/simulcast/jsfiddle/demo.js new file mode 100644 index 00000000000..e8ecc0457d9 --- /dev/null +++ b/examples/simulcast/jsfiddle/demo.js @@ -0,0 +1,92 @@ +// Create peer conn +const pc = new RTCPeerConnection({ + iceServers: [ + { + urls: "stun:stun.l.google.com:19302", + }, + ], +}); + +pc.oniceconnectionstatechange = (e) => { + console.log("connection state change", pc.iceConnectionState); +}; +pc.onicecandidate = (event) => { + if (event.candidate === null) { + document.getElementById("localSessionDescription").value = btoa( + JSON.stringify(pc.localDescription) + ); + } +}; + +pc.onnegotiationneeded = (e) => + pc + .createOffer() + .then((d) => pc.setLocalDescription(d)) + .catch(console.error); + +pc.ontrack = (event) => { + console.log("Got track event", event); + let video = document.createElement("video"); + video.srcObject = event.streams[0]; + video.autoplay = true; + video.width = "500"; + let label = document.createElement("div"); + label.textContent = event.streams[0].id; + document.getElementById("serverVideos").appendChild(label); + document.getElementById("serverVideos").appendChild(video); +}; + +navigator.mediaDevices + .getUserMedia({ + video: { + width: { + ideal: 4096, + }, + height: { + ideal: 2160, + }, + frameRate: { + ideal: 60, + min: 10, + }, + }, + audio: false, + }) + .then((stream) => { + document.getElementById("browserVideo").srcObject = stream; + pc.addTransceiver(stream.getVideoTracks()[0], { + direction: "sendonly", + streams: [stream], + sendEncodings: [ + // for firefox order matters... first high resolution, then scaled resolutions... + { + rid: "f", + }, + { + rid: "h", + scaleResolutionDownBy: 2.0, + }, + { + rid: "q", + scaleResolutionDownBy: 4.0, + }, + ], + }); + pc.addTransceiver("video"); + pc.addTransceiver("video"); + pc.addTransceiver("video"); + }); + +window.startSession = () => { + const sd = document.getElementById("remoteSessionDescription").value; + if (sd === "") { + return alert("Session Description must not be empty"); + } + + try { + console.log("answer", JSON.parse(atob(sd))); + pc.setRemoteDescription(new RTCSessionDescription(JSON.parse(atob(sd)))); + } catch (e) { + alert(e); + } +}; diff --git a/examples/simulcast/main.go b/examples/simulcast/main.go new file mode 100644 index 00000000000..1217ba8e6ef --- /dev/null +++ b/examples/simulcast/main.go @@ -0,0 +1,173 @@ +package main + +import ( + "fmt" + "io" + "log" + "math/rand" + "net/url" + "time" + + "github.com/pion/rtcp" + "github.com/pion/sdp/v2" + "github.com/pion/webrtc/v3" + "github.com/pion/webrtc/v3/examples/internal/signal" +) + +func main() { + // Everything below is the Pion WebRTC API! Thanks for using it ❤️. + + // Wait for the offer to be pasted + offer := webrtc.SessionDescription{} + signal.Decode(signal.MustReadStdin(), &offer) + + // We make our own mediaEngine so we can place the sender's codecs in it. Since we are echoing their RTP packet + // back to them we are actually codec agnostic - we can accept all their codecs. This also ensures that we use the + // dynamic media type from the sender in our answer. + mediaEngine := webrtc.MediaEngine{} + + // Add codecs to the mediaEngine. Note that even though we are only going to echo back the sender's video we also + // add audio codecs. This is because createAnswer will create an audioTransceiver and associated SDP and we currently + // cannot tell it not to. The audio SDP must match the sender's codecs too... + err := mediaEngine.PopulateFromSDP(offer) + if err != nil { + panic(err) + } + + videoCodecs := mediaEngine.GetCodecsByKind(webrtc.RTPCodecTypeVideo) + if len(videoCodecs) == 0 { + panic("Offer contained no video codecs") + } + + //Configure required extensions + + sdes, _ := url.Parse(sdp.SDESRTPStreamIDURI) + sdedMid, _ := url.Parse(sdp.SDESMidURI) + exts := []sdp.ExtMap{ + { + URI: sdes, + }, + { + URI: sdedMid, + }, + } + + se := webrtc.SettingEngine{} + se.AddSDPExtensions(webrtc.SDPSectionVideo, exts) + + api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine), webrtc.WithSettingEngine((se))) + + // Prepare the configuration + config := webrtc.Configuration{ + ICEServers: []webrtc.ICEServer{ + { + URLs: []string{"stun:stun.l.google.com:19302"}, + }, + }, + } + // Create a new RTCPeerConnection + peerConnection, err := api.NewPeerConnection(config) + if err != nil { + panic(err) + } + + outputTracks := map[string]*webrtc.Track{} + + // Create Track that we send video back to browser on + outputTrack, err := peerConnection.NewTrack(videoCodecs[0].PayloadType, rand.Uint32(), "video_q", "pion_q") + if err != nil { + panic(err) + } + outputTracks["q"] = outputTrack + + outputTrack, err = peerConnection.NewTrack(videoCodecs[0].PayloadType, rand.Uint32(), "video_h", "pion_h") + if err != nil { + panic(err) + } + outputTracks["h"] = outputTrack + + outputTrack, err = peerConnection.NewTrack(videoCodecs[0].PayloadType, rand.Uint32(), "video_f", "pion_f") + if err != nil { + panic(err) + } + outputTracks["f"] = outputTrack + + // Add this newly created track to the PeerConnection + if _, err = peerConnection.AddTrack(outputTracks["q"]); err != nil { + panic(err) + } + if _, err = peerConnection.AddTrack(outputTracks["h"]); err != nil { + panic(err) + } + if _, err = peerConnection.AddTrack(outputTracks["f"]); err != nil { + panic(err) + } + + // fmt.Printf("offer: %s\n", offer.SDP) + // Set the remote SessionDescription + err = peerConnection.SetRemoteDescription(offer) + if err != nil { + panic(err) + } + + // Set a handler for when a new remote track starts + peerConnection.OnTrack(func(track *webrtc.Track, receiver *webrtc.RTPReceiver) { + fmt.Printf("Track has started\n") + log.Println("Track has started", track) + + // Start reading from all the streams and sending them to the related output track + rid := track.RID() + go func() { + ticker := time.NewTicker(3 * time.Second) + for range ticker.C { + fmt.Printf("Sending pli for stream with rid: %q, ssrc: %d\n", track.RID(), track.SSRC()) + if writeErr := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: track.SSRC()}}); writeErr != nil { + fmt.Println(writeErr) + } + // Send a remb message with a very high bandwidth to trigger chrome to send also the high bitrate stream + fmt.Printf("Sending remb for stream with rid: %q, ssrc: %d\n", track.RID(), track.SSRC()) + if writeErr := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.ReceiverEstimatedMaximumBitrate{Bitrate: 10000000, SenderSSRC: track.SSRC()}}); writeErr != nil { + fmt.Println(writeErr) + } + } + }() + for { + var readErr error + // Read RTP packets being sent to Pion + packet, readErr := track.ReadRTP() + if err != nil { + panic(readErr) + } + + packet.SSRC = outputTracks[rid].SSRC() + + if writeErr := outputTracks[rid].WriteRTP(packet); writeErr != nil && writeErr != io.ErrClosedPipe { + panic(writeErr) + } + } + }) + // Set the handler for ICE connection state and update chan if connected + peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) { + fmt.Printf("Connection State has changed %s \n", connectionState.String()) + }) + + // Create an answer + answer, err := peerConnection.CreateAnswer(nil) + if err != nil { + panic(err) + } + + fmt.Printf("answer: %s\n", answer.SDP) + + // Sets the LocalDescription, and starts our UDP listeners + err = peerConnection.SetLocalDescription(answer) + if err != nil { + panic(err) + } + + // Output the answer in base64 so we can paste it in browser + fmt.Printf("Paste below base64 in browser:\n%v\n", signal.Encode(answer)) + + // Block forever + select {} +}