Skip to content

Commit

Permalink
Merge pull request #13254 from Popov72/webgpu-improve-updatevideotexture
Browse files Browse the repository at this point in the history 
WebGPU: Improve copy video to texture
Former-commit-id: d1751530243fc1b75cb1b76afda72847ff59e2fb
  • Loading branch information
@sebavan
sebavan authored Nov 16, 2022
2 parents e0dad6b + a19c1ae commit 48f3492
Show file tree
Hide file tree
Showing 5 changed files with 216 additions and 17 deletions.
3 changes: 2 additions & 1 deletion packages/dev/core/src/Engines/Extensions/engine.videoTexture.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { ThinEngine } from "../../Engines/thinEngine";
import type { InternalTexture } from "../../Materials/Textures/internalTexture";
import type { Nullable } from "../../types";
import { Constants } from "../constants";
import type { ExternalTexture } from "../../Materials/Textures/externalTexture";

declare module "../../Engines/thinEngine" {
export interface ThinEngine {
Expand All @@ -11,7 +12,7 @@ declare module "../../Engines/thinEngine" {
* @param video defines the video element to use
* @param invertY defines if data must be stored with Y axis inverted
*/
updateVideoTexture(texture: Nullable<InternalTexture>, video: HTMLVideoElement, invertY: boolean): void;
updateVideoTexture(texture: Nullable<InternalTexture>, video: HTMLVideoElement | Nullable<ExternalTexture>, invertY: boolean): void;
}
}

Expand Down
41 changes: 27 additions & 14 deletions packages/dev/core/src/Engines/WebGPU/Extensions/engine.videoTexture.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,13 @@ import type { InternalTexture } from "../../../Materials/Textures/internalTextur
import type { Nullable } from "../../../types";
import { WebGPUEngine } from "../../webgpuEngine";
import type { WebGPUHardwareTexture } from "../webgpuHardwareTexture";
import type { ExternalTexture } from "../../../Materials/Textures/externalTexture";

WebGPUEngine.prototype.updateVideoTexture = function (texture: Nullable<InternalTexture>, video: HTMLVideoElement, invertY: boolean): void {
function IsExternalTexture(texture: Nullable<ExternalTexture> | HTMLVideoElement): texture is ExternalTexture {
return texture && (texture as ExternalTexture).underlyingResource !== undefined ? true : false;
}

WebGPUEngine.prototype.updateVideoTexture = function (texture: Nullable<InternalTexture>, video: HTMLVideoElement | Nullable<ExternalTexture>, invertY: boolean): void {
if (!texture || texture._isDisabled) {
return;
}
Expand All @@ -18,18 +23,26 @@ WebGPUEngine.prototype.updateVideoTexture = function (texture: Nullable<Internal
gpuTextureWrapper = this._textureHelper.createGPUTextureForInternalTexture(texture);
}

this.createImageBitmap(video)
.then((bitmap) => {
this._textureHelper.updateTexture(bitmap, texture, texture.width, texture.height, texture.depth, gpuTextureWrapper.format, 0, 0, !invertY, false, 0, 0);
if (texture.generateMipMaps) {
this._generateMipmaps(texture, this._uploadEncoder);
}
if (IsExternalTexture(video)) {
this._textureHelper.copyVideoToTexture(video, texture, gpuTextureWrapper.format, !invertY);
if (texture.generateMipMaps) {
this._generateMipmaps(texture, this._uploadEncoder);
}
texture.isReady = true;
} else if (video) {
this.createImageBitmap(video)
.then((bitmap) => {
this._textureHelper.updateTexture(bitmap, texture, texture.width, texture.height, texture.depth, gpuTextureWrapper.format, 0, 0, !invertY, false, 0, 0);
if (texture.generateMipMaps) {
this._generateMipmaps(texture, this._uploadEncoder);
}

texture.isReady = true;
})
.catch(() => {
// Sometimes createImageBitmap(video) fails with "Failed to execute 'createImageBitmap' on 'Window': The provided element's player has no current data."
// Just keep going on
texture.isReady = true;
});
texture.isReady = true;
})
.catch(() => {
// Sometimes createImageBitmap(video) fails with "Failed to execute 'createImageBitmap' on 'Window': The provided element's player has no current data."
// Just keep going on
texture.isReady = true;
});
}
};
177 changes: 177 additions & 0 deletions packages/dev/core/src/Engines/WebGPU/webgpuTextureHelper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import type { HardwareTextureWrapper } from "../../Materials/Textures/hardwareTe
import type { BaseTexture } from "../../Materials/Textures/baseTexture";
import { WebGPUHardwareTexture } from "./webgpuHardwareTexture";
import type { WebGPUTintWASM } from "./webgpuTintWASM";
import type { ExternalTexture } from "../../Materials/Textures/externalTexture";

// TODO WEBGPU improve mipmap generation by using compute shaders

Expand Down Expand Up @@ -159,13 +160,74 @@ const clearFragmentSource = `
}
`;

const copyVideoToTextureVertexSource = `
struct VertexOutput {
@builtin(position) Position : vec4<f32>,
@location(0) fragUV : vec2<f32>
}
@vertex
fn main(
@builtin(vertex_index) VertexIndex : u32
) -> VertexOutput {
var pos = array<vec2<f32>, 4>(
vec2(-1.0, 1.0),
vec2( 1.0, 1.0),
vec2(-1.0, -1.0),
vec2( 1.0, -1.0)
);
var tex = array<vec2<f32>, 4>(
vec2(0.0, 0.0),
vec2(1.0, 0.0),
vec2(0.0, 1.0),
vec2(1.0, 1.0)
);
var output: VertexOutput;
output.Position = vec4<f32>(pos[VertexIndex], 0.0, 1.0);
output.fragUV = tex[VertexIndex];
return output;
}
`;

const copyVideoToTextureFragmentSource = `
@group(0) @binding(0) var videoSampler: sampler;
@group(0) @binding(1) var videoTexture: texture_external;
@fragment
fn main(
@location(0) fragUV: vec2<f32>
) -> @location(0) vec4<f32> {
return textureSampleBaseClampToEdge(videoTexture, videoSampler, fragUV);
}
`;

const copyVideoToTextureInvertYFragmentSource = `
@group(0) @binding(0) var videoSampler: sampler;
@group(0) @binding(1) var videoTexture: texture_external;
@fragment
fn main(
@location(0) fragUV: vec2<f32>
) -> @location(0) vec4<f32> {
return textureSampleBaseClampToEdge(videoTexture, videoSampler, vec2<f32>(fragUV.x, 1.0 - fragUV.y));
}
`;

enum PipelineType {
MipMap = 0,
InvertYPremultiplyAlpha = 1,
Clear = 2,
InvertYPremultiplyAlphaWithOfst = 3,
}

enum VideoPipelineType {
DontInvertY = 0,
InvertY = 1,
}

interface IPipelineParameters {
invertY?: boolean;
premultiplyAlpha?: boolean;
Expand Down Expand Up @@ -238,9 +300,12 @@ export class WebGPUTextureHelper {
private _tintWASM: Nullable<WebGPUTintWASM>;
private _bufferManager: WebGPUBufferManager;
private _mipmapSampler: GPUSampler;
private _videoSampler: GPUSampler;
private _ubCopyWithOfst: GPUBuffer;
private _pipelines: { [format: string]: Array<[GPURenderPipeline, GPUBindGroupLayout]> } = {};
private _compiledShaders: GPUShaderModule[][] = [];
private _videoPipelines: { [format: string]: Array<[GPURenderPipeline, GPUBindGroupLayout]> } = {};
private _videoCompiledShaders: GPUShaderModule[][] = [];
private _deferredReleaseTextures: Array<[Nullable<HardwareTextureWrapper | GPUTexture>, Nullable<BaseTexture>]> = [];
private _commandEncoderForCreation: GPUCommandEncoder;

Expand All @@ -259,9 +324,11 @@ export class WebGPUTextureHelper {
this._bufferManager = bufferManager;

this._mipmapSampler = device.createSampler({ minFilter: WebGPUConstants.FilterMode.Linear });
this._videoSampler = device.createSampler({ minFilter: WebGPUConstants.FilterMode.Linear });
this._ubCopyWithOfst = this._bufferManager.createBuffer(4 * 4, WebGPUConstants.BufferUsage.Uniform | WebGPUConstants.BufferUsage.CopyDst).underlyingResource;

this._getPipeline(WebGPUConstants.TextureFormat.RGBA8Unorm);
this._getVideoPipeline(WebGPUConstants.TextureFormat.RGBA8Unorm);
}

private _getPipeline(format: GPUTextureFormat, type: PipelineType = PipelineType.MipMap, params?: IPipelineParameters): [GPURenderPipeline, GPUBindGroupLayout] {
Expand Down Expand Up @@ -338,6 +405,54 @@ export class WebGPUTextureHelper {
return pipelineAndBGL;
}

private _getVideoPipeline(format: GPUTextureFormat, type: VideoPipelineType = VideoPipelineType.DontInvertY): [GPURenderPipeline, GPUBindGroupLayout] {
const index = type === VideoPipelineType.InvertY ? 1 << 0 : 0;

if (!this._videoPipelines[format]) {
this._videoPipelines[format] = [];
}

let pipelineAndBGL = this._videoPipelines[format][index];
if (!pipelineAndBGL) {
let modules = this._videoCompiledShaders[index];
if (!modules) {
const vertexModule = this._device.createShaderModule({
code: copyVideoToTextureVertexSource,
});
const fragmentModule = this._device.createShaderModule({
code: index === 0 ? copyVideoToTextureFragmentSource : copyVideoToTextureInvertYFragmentSource,
});
modules = this._videoCompiledShaders[index] = [vertexModule, fragmentModule];
}

const pipeline = this._device.createRenderPipeline({
label: `CopyVideoToTexture_${format}_${index === 0 ? "DontInvertY" : "InvertY"}`,
layout: WebGPUConstants.AutoLayoutMode.Auto,
vertex: {
module: modules[0],
entryPoint: "main",
},
fragment: {
module: modules[1],
entryPoint: "main",
targets: [
{
format,
},
],
},
primitive: {
topology: WebGPUConstants.PrimitiveTopology.TriangleStrip,
stripIndexFormat: WebGPUConstants.IndexFormat.Uint16,
},
});

pipelineAndBGL = this._videoPipelines[format][index] = [pipeline, pipeline.getBindGroupLayout(0)];
}

return pipelineAndBGL;
}

private static _GetTextureTypeFromFormat(format: GPUTextureFormat): number {
switch (format) {
// One Component = 8 bits
Expand Down Expand Up @@ -1010,6 +1125,68 @@ export class WebGPUTextureHelper {
return false;
}

public copyVideoToTexture(video: ExternalTexture, texture: InternalTexture, format: GPUTextureFormat, invertY = false, commandEncoder?: GPUCommandEncoder): void {
const useOwnCommandEncoder = commandEncoder === undefined;
const [pipeline, bindGroupLayout] = this._getVideoPipeline(format, invertY ? VideoPipelineType.InvertY : VideoPipelineType.DontInvertY);

if (useOwnCommandEncoder) {
commandEncoder = this._device.createCommandEncoder({});
}

commandEncoder!.pushDebugGroup?.(`copy video to texture - invertY=${invertY}`);

const webgpuHardwareTexture = texture._hardwareTexture as WebGPUHardwareTexture;

const renderPassDescriptor: GPURenderPassDescriptor = {
colorAttachments: [
{
view: webgpuHardwareTexture.underlyingResource!.createView({
format,
dimension: WebGPUConstants.TextureViewDimension.E2d,
mipLevelCount: 1,
baseArrayLayer: 0,
baseMipLevel: 0,
arrayLayerCount: 1,
aspect: WebGPUConstants.TextureAspect.All,
}),
loadOp: WebGPUConstants.LoadOp.Load,
storeOp: WebGPUConstants.StoreOp.Store,
},
],
};
const passEncoder = commandEncoder!.beginRenderPass(renderPassDescriptor);

const descriptor: GPUBindGroupDescriptor = {
layout: bindGroupLayout,
entries: [
{
binding: 0,
resource: this._videoSampler,
},
{
binding: 1,
resource: this._device.importExternalTexture({
source: video.underlyingResource,
}),
},
],
};

const bindGroup = this._device.createBindGroup(descriptor);

passEncoder.setPipeline(pipeline);
passEncoder.setBindGroup(0, bindGroup);
passEncoder.draw(4, 1, 0, 0);
passEncoder.end();

commandEncoder!.popDebugGroup?.();

if (useOwnCommandEncoder) {
this._device.queue.submit([commandEncoder!.finish()]);
commandEncoder = null as any;
}
}

public invertYPreMultiplyAlpha(
gpuOrHdwTexture: GPUTexture | WebGPUHardwareTexture,
width: number,
Expand Down
5 changes: 4 additions & 1 deletion packages/dev/core/src/Materials/Textures/htmlElementTexture.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { BaseTexture } from "../../Materials/Textures/baseTexture";
import { Constants } from "../../Engines/constants";
import { Matrix } from "../../Maths/math.vector";
import { Observable } from "../../Misc/observable";
import type { ExternalTexture } from "./externalTexture";

import "../../Engines/Extensions/engine.dynamicTexture";
import "../../Engines/Extensions/engine.videoTexture";
Expand Down Expand Up @@ -69,6 +70,7 @@ export class HtmlElementTexture extends BaseTexture {
private _isVideo: boolean;
private _generateMipMaps: boolean;
private _samplingMode: number;
private _externalTexture: Nullable<ExternalTexture>;

/**
* Instantiates a HtmlElementTexture from the following parameters.
Expand Down Expand Up @@ -97,6 +99,7 @@ export class HtmlElementTexture extends BaseTexture {
this.name = name;
this.element = element;
this._isVideo = element instanceof HTMLVideoElement;
this._externalTexture = this._isVideo ? this._engine?.createExternalTexture(element as HTMLVideoElement) ?? null : null;

this.anisotropicFilteringLevel = 1;

Expand Down Expand Up @@ -147,7 +150,7 @@ export class HtmlElementTexture extends BaseTexture {
return;
}

engine.updateVideoTexture(this._texture, videoElement, invertY === null ? true : invertY);
engine.updateVideoTexture(this._texture, this._externalTexture ? this._externalTexture : videoElement, invertY === null ? true : invertY);
} else {
const canvasElement = this.element as HTMLCanvasElement;
engine.updateDynamicTexture(this._texture, canvasElement, invertY === null ? true : invertY, false, this._format);
Expand Down
7 changes: 6 additions & 1 deletion packages/dev/core/src/Materials/Textures/videoTexture.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type { Nullable } from "../../types";
import type { Scene } from "../../scene";
import { Texture } from "../../Materials/Textures/texture";
import { Constants } from "../../Engines/constants";
import type { ExternalTexture } from "./externalTexture";

import "../../Engines/Extensions/engine.videoTexture";
import "../../Engines/Extensions/engine.dynamicTexture";
Expand Down Expand Up @@ -76,6 +77,7 @@ export class VideoTexture extends Texture {
*/
public readonly video: HTMLVideoElement;

private _externalTexture: Nullable<ExternalTexture>;
private _onUserActionRequestedObservable: Nullable<Observable<Texture>> = null;

/**
Expand Down Expand Up @@ -174,6 +176,7 @@ export class VideoTexture extends Texture {
this._currentSrc = src;
this.name = name || this._getName(src);
this.video = this._getVideo(src);
this._externalTexture = this._engine?.createExternalTexture(this.video) ?? null;

if (this._settings.poster) {
this.video.poster = this._settings.poster;
Expand Down Expand Up @@ -367,7 +370,7 @@ export class VideoTexture extends Texture {

this._frameId = frameId;

this._getEngine()!.updateVideoTexture(this._texture, this.video, this._invertY);
this._getEngine()!.updateVideoTexture(this._texture, this._externalTexture ? this._externalTexture : this.video, this._invertY);
};

/**
Expand Down Expand Up @@ -405,6 +408,8 @@ export class VideoTexture extends Texture {
this.video.removeEventListener("seeked", this._updateInternalTexture);
this.video.removeEventListener("emptied", this._reset);
this.video.pause();

this._externalTexture?.dispose();
}

/**
Expand Down

0 comments on commit 48f3492

Please sign in to comment.