feat: Add support for Spatial audio rendering via Omnitone (#181)

videojs · Sep 9, 2019 · 2af9aa3 · 2af9aa3
1 parent ebe0727
commit 2af9aa3
Show file tree

Hide file tree

Showing 7 changed files with 2,811 additions and 2,180 deletions.
diff --git a/README.md b/README.md
@@ -47,6 +47,8 @@ Maintenance Status: Stable
     - [`'EAC_LR'`](#eac_lr)
   - [`player.mediainfo.projection`](#playermediainfoprojection)
   - [`debug`](#debug)
+  - [`omnitone`](#omnitone)
+  - [`omnitoneOptions`](#omnitoneoptions)
 - [Credits](#credits)
 - [Support](#support)
 
@@ -243,13 +245,28 @@ See [`projection`](#projection) above for information of values. Note that `AUTO
 
 Enable debug logging for this plugin
 
+### `omnitone`
+
+> type: `Omnitone library object`
+
+Use this property to pass the Omnitone library object to the plugin.
+Please be aware of, the Omnitone library is not included in the build files.
+
+### `omnitoneOptions`
+
+> type: `object`, default: `{}`
+
+Default options for the Omnitone library. Please check available options on https://github.com/GoogleChrome/omnitone
+
+
 ## Credits ##
 
 This project is a conglomeration of a few amazing open source libraries.
 
 * [VideoJS](http://www.videojs.com)
 * [Three.js](http://threejs.org)
 * [webvr-polyfill](https://github.com/borismus/webvr-polyfill)
+* [Omnitone](https://googlechrome.github.io/omnitone)
 
 ## Support ##
 This work is sponsored by [Brightcove](https://www.brightcove.com), [HapYak](http://corp.hapyak.com/) and [StreamShark](https://streamshark.io)
diff --git a/examples/spatial.html b/examples/spatial.html
@@ -0,0 +1,30 @@
+<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>videojs-vr Demo</title>
+   <link href="../node_modules/video.js/dist/video-js.css" rel="stylesheet">
+   <link href="../dist/videojs-vr.css" rel="stylesheet">
+</head>
+<body>
+  <video width="640" height="300" id="videojs-vr-player" class="video-js vjs-default-skin" crossorigin="anonymous" controls playsinline>
+    <source src="https://storage.googleapis.com/omnitone-demo.google.com.a.appspot.com/fuerza-imprevista-1080p-h264-aac.mp4" type="video/mp4">
+  </video>
+  <ul>
+    <li><a href="../">return to main example</a></li>
+  </ul>
+  <script src="../node_modules/video.js/dist/video.js"></script>
+  <script src="../node_modules/omnitone/build/omnitone.js"></script>
+  <script src="../dist/videojs-vr.js"></script>
+  <script>
+    (function(window, videojs) {
+      var player = window.player = videojs('videojs-vr-player');
+      player.mediainfo = player.mediainfo || {};
+      player.mediainfo.projection = '360';
+
+      // AUTO is the default and looks at mediainfo
+      var vr = window.vr = player.vr({projection: 'AUTO', debug: true, forceCardboard: false, omnitone: Omnitone, omnitoneOptions: {}});
+    }(window, window.videojs));
+  </script>
+</body>
+</html>
diff --git a/index.html b/index.html
@@ -16,6 +16,7 @@
     <li><a href="examples/eac.html">EAC video example</a></li>
     <li><a href="examples/eac-lr.html">EAC LR video example</a></li>
     <li><a href="examples/fluid.html">"Fluid" video size example</a></li>
+    <li><a href="examples/spatial.html">Spatial audio example</a></li>
   </ul>
 </body>
 </html>
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -88,6 +88,7 @@
     "@videojs/generator-helpers": "~1.2.0",
     "jsdoc": "BrandonOCasey/jsdoc#feat/plugin-from-cli",
     "karma": "^4.0.0",
+    "omnitone": "^1.3.0",
     "rollup": "^1.12.0",
     "sinon": "^7.2.2",
     "videojs-generate-karma-config": "~5.3.1",

diff --git a/src/omnitone-controller.js b/src/omnitone-controller.js
@@ -0,0 +1,64 @@
+import videojs from 'video.js';
+
+/**
+ * This class manages ambisonic decoding and binaural rendering via Omnitone library.
+ */
+class OmnitoneController extends videojs.EventTarget {
+  /**
+   * Omnitone controller class.
+   *
+   * @class
+   * @param {AudioContext} audioContext - associated AudioContext.
+   * @param {Omnitone library} omnitone - Omnitone library element.
+   * @param {HTMLVideoElement} video - vidoe tag element.
+   * @param {Object} options - omnitone options.
+   */
+  constructor(audioContext, omnitone, video, options) {
+    super();
+
+    const settings = videojs.mergeOptions({
+      // Safari uses the different AAC decoder than FFMPEG. The channel order is
+      // The default 4ch AAC channel layout for FFMPEG AAC channel ordering.
+      channelMap: videojs.browser.IS_SAFARI ? [2, 0, 1, 3] : [0, 1, 2, 3],
+      ambisonicOrder: 1
+    }, options);
+
+    this.videoElementSource = audioContext.createMediaElementSource(video);
+    this.foaRenderer = omnitone.createFOARenderer(audioContext, settings);
+
+    this.foaRenderer.initialize().then(() => {
+      if (audioContext.state === 'suspended') {
+        this.trigger({type: 'audiocontext-suspended'});
+      }
+      this.videoElementSource.connect(this.foaRenderer.input);
+      this.foaRenderer.output.connect(audioContext.destination);
+      this.initialized = true;
+      this.trigger({type: 'omnitone-ready'});
+    }, (error) => {
+      videojs.log.warn(`videojs-vr: Omnitone initializes failed with the following error: ${error})`);
+    });
+  }
+
+  /**
+   * Updates the rotation of the Omnitone decoder based on three.js camera matrix.
+   *
+   * @param {Camera} camera Three.js camera object
+   */
+  update(camera) {
+    if (!this.initialized) {
+      return;
+    }
+    this.foaRenderer.setRotationMatrixFromCamera(camera.matrix);
+  }
+
+  /**
+   * Destroys the controller and does any necessary cleanup.
+   */
+  dispose() {
+    this.initialized = false;
+    this.foaRenderer.setRenderingMode('bypass');
+    this.foaRenderer = null;
+  }
+}
+
+export default OmnitoneController;
diff --git a/src/plugin.js b/src/plugin.js
@@ -9,16 +9,19 @@ import VREffect from 'three/examples/js/effects/VREffect.js';
 import OrbitOrientationContols from './orbit-orientation-controls.js';
 import * as utils from './utils';
 import CanvasPlayerControls from './canvas-player-controls';
+import OmnitoneController from './omnitone-controller';
 
 // import controls so they get regisetered with videojs
 import './cardboard-button';
 import './big-vr-play-button';
 
 // Default options for the plugin.
 const defaults = {
-  projection: 'AUTO',
+  debug: false,
+  omnitone: false,
   forceCardboard: false,
-  debug: false
+  omnitoneOptions: {},
+  projection: 'AUTO'
 };
 
 const errors = {
@@ -530,6 +533,9 @@ void main() {
     }
 
     this.controls3d.update();
+    if (this.omniController) {
+      this.omniController.update(this.camera);
+    }
     this.effect.render(this.scene, this.camera);
 
     if (window.navigator.getGamepads) {
@@ -711,6 +717,21 @@ void main() {
       this.triggerError_({code: 'web-vr-not-supported', dismiss: false});
     }
 
+    if (this.options_.omnitone) {
+      const audiocontext = THREE.AudioContext.getContext();
+
+      this.omniController = new OmnitoneController(
+        audiocontext,
+        this.options_.omnitone, this.getVideoEl_(), this.options_.omnitoneOptions
+      );
+      this.omniController.one('audiocontext-suspended', () => {
+        this.player.pause();
+        this.player.one('playing', () => {
+          audiocontext.resume();
+        });
+      });
+    }
+
     this.on(this.player_, 'fullscreenchange', this.handleResize_);
     window.addEventListener('vrdisplaypresentchange', this.handleResize_, true);
     window.addEventListener('resize', this.handleResize_, true);
@@ -736,6 +757,12 @@ void main() {
       return;
     }
 
+    if (this.omniController) {
+      this.omniController.off('audiocontext-suspended');
+      this.omniController.dispose();
+      this.omniController = undefined;
+    }
+
     if (this.controls3d) {
       this.controls3d.dispose();
       this.controls3d = null;