WebGPU: Improve copy video to texture

packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts

@@ -2,6 +2,7 @@ import { ThinEngine } from "../../Engines/thinEngine";
 import type { InternalTexture } from "../../Materials/Textures/internalTexture";
 import type { Nullable } from "../../types";
 import { Constants } from "../constants";
+import type { ExternalTexture } from "../../Materials/Textures/externalTexture";
 
 declare module "../../Engines/thinEngine" {
     export interface ThinEngine {
@@ -11,7 +12,7 @@ declare module "../../Engines/thinEngine" {
          * @param video defines the video element to use
          * @param invertY defines if data must be stored with Y axis inverted
          */
-        updateVideoTexture(texture: Nullable<InternalTexture>, video: HTMLVideoElement, invertY: boolean): void;
+        updateVideoTexture(texture: Nullable<InternalTexture>, video: HTMLVideoElement | Nullable<ExternalTexture>, invertY: boolean): void;
     }
 }
 

packages/dev/core/src/Engines/WebGPU/Extensions/engine.videoTexture.ts

@@ -2,8 +2,13 @@ import type { InternalTexture } from "../../../Materials/Textures/internalTextur
 import type { Nullable } from "../../../types";
 import { WebGPUEngine } from "../../webgpuEngine";
 import type { WebGPUHardwareTexture } from "../webgpuHardwareTexture";
+import type { ExternalTexture } from "../../../Materials/Textures/externalTexture";
 
-WebGPUEngine.prototype.updateVideoTexture = function (texture: Nullable<InternalTexture>, video: HTMLVideoElement, invertY: boolean): void {
+function IsExternalTexture(texture: Nullable<ExternalTexture> | HTMLVideoElement): texture is ExternalTexture {
+    return texture && (texture as ExternalTexture).underlyingResource !== undefined ? true : false;
+}
+
+WebGPUEngine.prototype.updateVideoTexture = function (texture: Nullable<InternalTexture>, video: HTMLVideoElement | Nullable<ExternalTexture>, invertY: boolean): void {
     if (!texture || texture._isDisabled) {
         return;
     }
@@ -18,18 +23,26 @@ WebGPUEngine.prototype.updateVideoTexture = function (texture: Nullable<Internal
         gpuTextureWrapper = this._textureHelper.createGPUTextureForInternalTexture(texture);
     }
 
-    this.createImageBitmap(video)
-        .then((bitmap) => {
-            this._textureHelper.updateTexture(bitmap, texture, texture.width, texture.height, texture.depth, gpuTextureWrapper.format, 0, 0, !invertY, false, 0, 0);
-            if (texture.generateMipMaps) {
-                this._generateMipmaps(texture, this._uploadEncoder);
-            }
+    if (IsExternalTexture(video)) {
+        this._textureHelper.copyVideoToTexture(video, texture, gpuTextureWrapper.format, !invertY);
+        if (texture.generateMipMaps) {
+            this._generateMipmaps(texture, this._uploadEncoder);
+        }
+        texture.isReady = true;
+    } else if (video) {
+        this.createImageBitmap(video)
+            .then((bitmap) => {
+                this._textureHelper.updateTexture(bitmap, texture, texture.width, texture.height, texture.depth, gpuTextureWrapper.format, 0, 0, !invertY, false, 0, 0);
+                if (texture.generateMipMaps) {
+                    this._generateMipmaps(texture, this._uploadEncoder);
+                }
 
-            texture.isReady = true;
-        })
-        .catch(() => {
-            // Sometimes createImageBitmap(video) fails with "Failed to execute 'createImageBitmap' on 'Window': The provided element's player has no current data."
-            // Just keep going on
-            texture.isReady = true;
-        });
+                texture.isReady = true;
+            })
+            .catch(() => {
+                // Sometimes createImageBitmap(video) fails with "Failed to execute 'createImageBitmap' on 'Window': The provided element's player has no current data."
+                // Just keep going on
+                texture.isReady = true;
+            });
+    }
 };

packages/dev/core/src/Engines/WebGPU/webgpuTextureHelper.ts

@@ -31,6 +31,7 @@ import type { HardwareTextureWrapper } from "../../Materials/Textures/hardwareTe
 import type { BaseTexture } from "../../Materials/Textures/baseTexture";
 import { WebGPUHardwareTexture } from "./webgpuHardwareTexture";
 import type { WebGPUTintWASM } from "./webgpuTintWASM";
+import type { ExternalTexture } from "../../Materials/Textures/externalTexture";
 
 // TODO WEBGPU improve mipmap generation by using compute shaders
 
@@ -159,13 +160,74 @@ const clearFragmentSource = `
     }
     `;
 
+const copyVideoToTextureVertexSource = `
+    struct VertexOutput {
+        @builtin(position) Position : vec4<f32>,
+        @location(0) fragUV : vec2<f32>
+    }
+
+    @vertex
+    fn main(
+        @builtin(vertex_index) VertexIndex : u32
+    ) -> VertexOutput {
+        var pos = array<vec2<f32>, 4>(
+            vec2(-1.0,  1.0),
+            vec2( 1.0,  1.0),
+            vec2(-1.0, -1.0),
+            vec2( 1.0, -1.0)
+        );
+        var tex = array<vec2<f32>, 4>(
+            vec2(0.0, 0.0),
+            vec2(1.0, 0.0),
+            vec2(0.0, 1.0),
+            vec2(1.0, 1.0)
+        );
+
+        var output: VertexOutput;
+
+        output.Position = vec4<f32>(pos[VertexIndex], 0.0, 1.0);
+        output.fragUV = tex[VertexIndex];
+
+        return output;
+    }
+    `;
+
+const copyVideoToTextureFragmentSource = `
+    @group(0) @binding(0) var videoSampler: sampler;
+    @group(0) @binding(1) var videoTexture: texture_external;
+
+    @fragment
+    fn main(
+        @location(0) fragUV: vec2<f32>
+    ) -> @location(0) vec4<f32> {
+        return textureSampleBaseClampToEdge(videoTexture, videoSampler, fragUV);
+    }
+    `;
+
+const copyVideoToTextureInvertYFragmentSource = `
+    @group(0) @binding(0) var videoSampler: sampler;
+    @group(0) @binding(1) var videoTexture: texture_external;
+
+    @fragment
+    fn main(
+        @location(0) fragUV: vec2<f32>
+    ) -> @location(0) vec4<f32> {
+        return textureSampleBaseClampToEdge(videoTexture, videoSampler, vec2<f32>(fragUV.x, 1.0 - fragUV.y));
+    }
+    `;
+
 enum PipelineType {
     MipMap = 0,
     InvertYPremultiplyAlpha = 1,
     Clear = 2,
     InvertYPremultiplyAlphaWithOfst = 3,
 }
 
+enum VideoPipelineType {
+    DontInvertY = 0,
+    InvertY = 1,
+}
+
 interface IPipelineParameters {
     invertY?: boolean;
     premultiplyAlpha?: boolean;
@@ -238,9 +300,12 @@ export class WebGPUTextureHelper {
     private _tintWASM: Nullable<WebGPUTintWASM>;
     private _bufferManager: WebGPUBufferManager;
     private _mipmapSampler: GPUSampler;
+    private _videoSampler: GPUSampler;
     private _ubCopyWithOfst: GPUBuffer;
     private _pipelines: { [format: string]: Array<[GPURenderPipeline, GPUBindGroupLayout]> } = {};
     private _compiledShaders: GPUShaderModule[][] = [];
+    private _videoPipelines: { [format: string]: Array<[GPURenderPipeline, GPUBindGroupLayout]> } = {};
+    private _videoCompiledShaders: GPUShaderModule[][] = [];
     private _deferredReleaseTextures: Array<[Nullable<HardwareTextureWrapper | GPUTexture>, Nullable<BaseTexture>]> = [];
     private _commandEncoderForCreation: GPUCommandEncoder;
 
@@ -259,9 +324,11 @@ export class WebGPUTextureHelper {
         this._bufferManager = bufferManager;
 
         this._mipmapSampler = device.createSampler({ minFilter: WebGPUConstants.FilterMode.Linear });
+        this._videoSampler = device.createSampler({ minFilter: WebGPUConstants.FilterMode.Linear });
         this._ubCopyWithOfst = this._bufferManager.createBuffer(4 * 4, WebGPUConstants.BufferUsage.Uniform | WebGPUConstants.BufferUsage.CopyDst).underlyingResource;
 
         this._getPipeline(WebGPUConstants.TextureFormat.RGBA8Unorm);
+        this._getVideoPipeline(WebGPUConstants.TextureFormat.RGBA8Unorm);
     }
 
     private _getPipeline(format: GPUTextureFormat, type: PipelineType = PipelineType.MipMap, params?: IPipelineParameters): [GPURenderPipeline, GPUBindGroupLayout] {
@@ -338,6 +405,54 @@ export class WebGPUTextureHelper {
         return pipelineAndBGL;
     }
 
+    private _getVideoPipeline(format: GPUTextureFormat, type: VideoPipelineType = VideoPipelineType.DontInvertY): [GPURenderPipeline, GPUBindGroupLayout] {
+        const index = type === VideoPipelineType.InvertY ? 1 << 0 : 0;
+
+        if (!this._videoPipelines[format]) {
+            this._videoPipelines[format] = [];
+        }
+
+        let pipelineAndBGL = this._videoPipelines[format][index];
+        if (!pipelineAndBGL) {
+            let modules = this._videoCompiledShaders[index];
+            if (!modules) {
+                const vertexModule = this._device.createShaderModule({
+                    code: copyVideoToTextureVertexSource,
+                });
+                const fragmentModule = this._device.createShaderModule({
+                    code: index === 0 ? copyVideoToTextureFragmentSource : copyVideoToTextureInvertYFragmentSource,
+                });
+                modules = this._videoCompiledShaders[index] = [vertexModule, fragmentModule];
+            }
+
+            const pipeline = this._device.createRenderPipeline({
+                label: `CopyVideoToTexture_${format}_${index === 0 ? "DontInvertY" : "InvertY"}`,
+                layout: WebGPUConstants.AutoLayoutMode.Auto,
+                vertex: {
+                    module: modules[0],
+                    entryPoint: "main",
+                },
+                fragment: {
+                    module: modules[1],
+                    entryPoint: "main",
+                    targets: [
+                        {
+                            format,
+                        },
+                    ],
+                },
+                primitive: {
+                    topology: WebGPUConstants.PrimitiveTopology.TriangleStrip,
+                    stripIndexFormat: WebGPUConstants.IndexFormat.Uint16,
+                },
+            });
+
+            pipelineAndBGL = this._videoPipelines[format][index] = [pipeline, pipeline.getBindGroupLayout(0)];
+        }
+
+        return pipelineAndBGL;
+    }
+
     private static _GetTextureTypeFromFormat(format: GPUTextureFormat): number {
         switch (format) {
             // One Component = 8 bits
@@ -1010,6 +1125,68 @@ export class WebGPUTextureHelper {
         return false;
     }
 
+    public copyVideoToTexture(video: ExternalTexture, texture: InternalTexture, format: GPUTextureFormat, invertY = false, commandEncoder?: GPUCommandEncoder): void {
+        const useOwnCommandEncoder = commandEncoder === undefined;
+        const [pipeline, bindGroupLayout] = this._getVideoPipeline(format, invertY ? VideoPipelineType.InvertY : VideoPipelineType.DontInvertY);
+
+        if (useOwnCommandEncoder) {
+            commandEncoder = this._device.createCommandEncoder({});
+        }
+
+        commandEncoder!.pushDebugGroup?.(`copy video to texture - invertY=${invertY}`);
+
+        const webgpuHardwareTexture = texture._hardwareTexture as WebGPUHardwareTexture;
+
+        const renderPassDescriptor: GPURenderPassDescriptor = {
+            colorAttachments: [
+                {
+                    view: webgpuHardwareTexture.underlyingResource!.createView({
+                        format,
+                        dimension: WebGPUConstants.TextureViewDimension.E2d,
+                        mipLevelCount: 1,
+                        baseArrayLayer: 0,
+                        baseMipLevel: 0,
+                        arrayLayerCount: 1,
+                        aspect: WebGPUConstants.TextureAspect.All,
+                    }),
+                    loadOp: WebGPUConstants.LoadOp.Load,
+                    storeOp: WebGPUConstants.StoreOp.Store,
+                },
+            ],
+        };
+        const passEncoder = commandEncoder!.beginRenderPass(renderPassDescriptor);
+
+        const descriptor: GPUBindGroupDescriptor = {
+            layout: bindGroupLayout,
+            entries: [
+                {
+                    binding: 0,
+                    resource: this._videoSampler,
+                },
+                {
+                    binding: 1,
+                    resource: this._device.importExternalTexture({
+                        source: video.underlyingResource,
+                    }),
+                },
+            ],
+        };
+
+        const bindGroup = this._device.createBindGroup(descriptor);
+
+        passEncoder.setPipeline(pipeline);
+        passEncoder.setBindGroup(0, bindGroup);
+        passEncoder.draw(4, 1, 0, 0);
+        passEncoder.end();
+
+        commandEncoder!.popDebugGroup?.();
+
+        if (useOwnCommandEncoder) {
+            this._device.queue.submit([commandEncoder!.finish()]);
+            commandEncoder = null as any;
+        }
+    }
+
     public invertYPreMultiplyAlpha(
         gpuOrHdwTexture: GPUTexture | WebGPUHardwareTexture,
         width: number,

packages/dev/core/src/Materials/Textures/htmlElementTexture.ts

@@ -3,6 +3,7 @@ import { BaseTexture } from "../../Materials/Textures/baseTexture";
 import { Constants } from "../../Engines/constants";
 import { Matrix } from "../../Maths/math.vector";
 import { Observable } from "../../Misc/observable";
+import type { ExternalTexture } from "./externalTexture";
 
 import "../../Engines/Extensions/engine.dynamicTexture";
 import "../../Engines/Extensions/engine.videoTexture";
@@ -69,6 +70,7 @@ export class HtmlElementTexture extends BaseTexture {
     private _isVideo: boolean;
     private _generateMipMaps: boolean;
     private _samplingMode: number;
+    private _externalTexture: Nullable<ExternalTexture>;
 
     /**
      * Instantiates a HtmlElementTexture from the following parameters.
@@ -97,6 +99,7 @@ export class HtmlElementTexture extends BaseTexture {
         this.name = name;
         this.element = element;
         this._isVideo = element instanceof HTMLVideoElement;
+        this._externalTexture = this._isVideo ? this._engine?.createExternalTexture(element as HTMLVideoElement) ?? null : null;
 
         this.anisotropicFilteringLevel = 1;
 
@@ -147,7 +150,7 @@ export class HtmlElementTexture extends BaseTexture {
                 return;
             }
 
-            engine.updateVideoTexture(this._texture, videoElement, invertY === null ? true : invertY);
+            engine.updateVideoTexture(this._texture, this._externalTexture ? this._externalTexture : videoElement, invertY === null ? true : invertY);
         } else {
             const canvasElement = this.element as HTMLCanvasElement;
             engine.updateDynamicTexture(this._texture, canvasElement, invertY === null ? true : invertY, false, this._format);

packages/dev/core/src/Materials/Textures/videoTexture.ts

@@ -5,6 +5,7 @@ import type { Nullable } from "../../types";
 import type { Scene } from "../../scene";
 import { Texture } from "../../Materials/Textures/texture";
 import { Constants } from "../../Engines/constants";
+import type { ExternalTexture } from "./externalTexture";
 
 import "../../Engines/Extensions/engine.videoTexture";
 import "../../Engines/Extensions/engine.dynamicTexture";
@@ -76,6 +77,7 @@ export class VideoTexture extends Texture {
      */
     public readonly video: HTMLVideoElement;
 
+    private _externalTexture: Nullable<ExternalTexture>;
     private _onUserActionRequestedObservable: Nullable<Observable<Texture>> = null;
 
     /**
@@ -174,6 +176,7 @@ export class VideoTexture extends Texture {
         this._currentSrc = src;
         this.name = name || this._getName(src);
         this.video = this._getVideo(src);
+        this._externalTexture = this._engine?.createExternalTexture(this.video) ?? null;
 
         if (this._settings.poster) {
             this.video.poster = this._settings.poster;
@@ -367,7 +370,7 @@ export class VideoTexture extends Texture {
 
         this._frameId = frameId;
 
-        this._getEngine()!.updateVideoTexture(this._texture, this.video, this._invertY);
+        this._getEngine()!.updateVideoTexture(this._texture, this._externalTexture ? this._externalTexture : this.video, this._invertY);
     };
 
     /**
@@ -405,6 +408,8 @@ export class VideoTexture extends Texture {
         this.video.removeEventListener("seeked", this._updateInternalTexture);
         this.video.removeEventListener("emptied", this._reset);
         this.video.pause();
+
+        this._externalTexture?.dispose();
     }
 
     /**