make accommodations necessary for iOS 10 HLS (#2597)

* make accommodations necessary for iOS 10 HLS

* add candidate ios10hls shader

* use ios10hls shader automatically if needed

* minor edits per discussion on PR

* move isHLS into utils/material

* fix isHLS issue

src/shaders/index.js

@@ -2,3 +2,4 @@ require('./flat');
 require('./standard');
 require('./sdf');
 require('./msdf');
+require('./ios10hls');

src/shaders/ios10hls.js

@@ -0,0 +1,33 @@
+var registerShader = require('../core/shader').registerShader;
+
+/**
+ * Custom shader for iOS 10 HTTP Live Streaming (HLS).
+ * For more information on HLS, see https://datatracker.ietf.org/doc/draft-pantos-http-live-streaming/
+ */
+module.exports.Shader = registerShader('ios10hls', {
+  schema: {
+    src: {type: 'map', is: 'uniform'},
+    opacity: {type: 'number', is: 'uniform', default: 1}
+  },
+
+  vertexShader: [
+    'varying vec2 vUV;',
+    'void main(void) {',
+    '  gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);',
+    '  vUV = uv;',
+    '}'
+  ].join('\n'),
+
+  fragmentShader: [
+    'uniform sampler2D src;',
+    'uniform float opacity;',
+    'varying vec2 vUV;',
+    'void main() {',
+    '  vec2 offset = vec2(0, 0);',
+    '  vec2 repeat = vec2(1, 1);',
+    '  vec4 color = texture2D(src, vec2(vUV.x / repeat.x + offset.x, (1.0 - vUV.y) / repeat.y + offset.y)).bgra;',
+    '  gl_FragColor = vec4(color.rgb, opacity);',
+    '}'
+  ].join('\n')
+});
+

src/systems/material.js

@@ -1,6 +1,7 @@
 var registerSystem = require('../core/system').registerSystem;
 var THREE = require('../lib/three');
 var utils = require('../utils/');
+var isHLS = require('../utils/material').isHLS;
 
 var debug = utils.debug;
 var error = debug('components:texture:error');
@@ -144,6 +145,16 @@ module.exports.System = registerSystem('material', {
     texture.minFilter = THREE.LinearFilter;
     setTextureProperties(texture, data);
 
+    // if we're on iOS, and the video is HLS, we currently need to do some hacks
+    if (utils.device.isIOS() && isHLS(videoEl)) {
+      // really it's BGRA, so this needs correction in shader
+      texture.format = THREE.RGBAFormat;
+      texture.needsCorrectionBGRA = true;
+      // apparently this is needed for HLS, so this needs correction in shader
+      texture.flipY = false;
+      texture.needsCorrectionFlipY = true;
+    }
+
     // Cache as promise to be consistent with image texture caching.
     videoTextureResult = {texture: texture, videoEl: videoEl};
     textureCache[hash] = Promise.resolve(videoTextureResult);

src/utils/material.js

@@ -119,6 +119,10 @@ function handleTextureEvents (el, texture) {
   // Video events.
   if (!texture.image || texture.image.tagName !== 'VIDEO') { return; }
   texture.image.addEventListener('loadeddata', function emitVideoTextureLoadedDataAll () {
+    // Check to see if we need to use iOS 10 HLS shader.
+    if (texture.needsCorrectionBGRA && texture.needsCorrectionFlipY) {
+      el.setAttribute('material', 'shader', 'ios10hls');
+    }
     el.emit('materialvideoloadeddata', {src: texture.image, texture: texture});
   });
   texture.image.addEventListener('ended', function emitVideoTextureEndedAll () {
@@ -127,3 +131,9 @@ function handleTextureEvents (el, texture) {
   });
 }
 module.exports.handleTextureEvents = handleTextureEvents;
+
+module.exports.isHLS = function (videoEl) {
+  if (videoEl.type && videoEl.type.toLowerCase() in ['application/x-mpegurl', 'application/vnd.apple.mpegurl']) { return true; }
+  if (videoEl.src && videoEl.src.toLowerCase().indexOf('.m3u8') > 0) { return true; }
+  return false;
+};