-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[js/webgpu] support customop FastGelu (#19392)
### Description Support WebGPU custom operator FastGelu.
- Loading branch information
Showing
10 changed files
with
353 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| import {DataType} from '../../../wasm-common'; | ||
| import {TensorView} from '../../tensor-view'; | ||
| import {ShapeUtil} from '../../util'; | ||
| import {ComputeContext, ProgramInfo} from '../types'; | ||
|
|
||
| import {inputVariable, outputVariable, ShaderHelper, tensorTypeToWsglValueType, UniformsArrayType, WORKGROUP_SIZE} from './common'; | ||
| import * as unary from './unary-op'; | ||
|
|
||
| // GELU is defined as Y=0.5*X*(1+tanh(0.797885*X+0.035677*X*X*X)), where X may pre-add a bias. | ||
|
|
||
| const createFastGeluProgramInfo = (inputTensors: readonly TensorView[]): ProgramInfo => { | ||
| const dataType = inputTensors[0].dataType; | ||
| const outputSize = ShapeUtil.size(inputTensors[0].dims); | ||
| const biasLength = ShapeUtil.size(inputTensors[1].dims); | ||
| // can only use vec4 when bias length is multiple of 4 | ||
| const useVec4 = biasLength % 4 === 0; | ||
| const getShaderSource = (shaderHelper: ShaderHelper): string => { | ||
| const x = inputVariable('x', dataType, [1], 4); | ||
| const bias = inputVariable('bias', dataType, [1], 4); | ||
| const y = outputVariable('y', dataType, [1], 4); | ||
|
|
||
| const uniforms: UniformsArrayType = [{name: 'output_vec_size', type: 'u32'}, {name: 'bias_size', type: 'u32'}]; | ||
|
|
||
| const singleElementBias = (i: 0|1|2|3) => ` | ||
| let bias${i}_offset: u32 = (global_idx * 4 + ${i}) % uniforms.bias_size; | ||
| let bias${i} = ${bias.getByOffset(`bias${i}_offset / 4`)}[bias${i}_offset % 4];`; | ||
| const biasGetExpression = useVec4 ? | ||
| ` | ||
| let bias = ${bias.getByOffset('global_idx % (uniforms.bias_size / 4)')};` : | ||
| `${singleElementBias(0)}${singleElementBias(1)}${singleElementBias(2)}${singleElementBias(3)} | ||
| let bias = ${x.type.value}(bias0, bias1, bias2, bias3);`; | ||
|
|
||
| return `${shaderHelper.registerUniforms(uniforms).declareVariables(x, bias, y)} | ||
| ${unary.fastGeluImpl(tensorTypeToWsglValueType(dataType))} | ||
| ${shaderHelper.mainStart(WORKGROUP_SIZE)} | ||
| ${shaderHelper.guardAgainstOutOfBoundsWorkgroupSizes('uniforms.output_vec_size')} | ||
| let x = ${x.getByOffset('global_idx')}; | ||
| ${biasGetExpression} | ||
| let x_in = x + bias; | ||
| ${y.setByOffset('global_idx', unary.fastGeluExpression('x_in'))} | ||
| }`; | ||
| }; | ||
|
|
||
| return { | ||
| name: 'FastGeluWithBias', | ||
| shaderCache: {hint: `${useVec4}`, inputDependencies: ['type', 'type']}, | ||
| getShaderSource, | ||
| getRunData: (inputs) => ({ | ||
| outputs: [{dims: inputs[0].dims, dataType: inputs[0].dataType}], | ||
| programUniforms: | ||
| [{type: DataType.uint32, data: Math.ceil(outputSize / 4)}, {type: DataType.uint32, data: biasLength}], | ||
| dispatchGroup: {x: Math.ceil(outputSize / WORKGROUP_SIZE / 4)} | ||
| }) | ||
| }; | ||
| }; | ||
|
|
||
| export const fastGelu = (context: ComputeContext): void => { | ||
| if (context.inputs.length < 2 || ShapeUtil.size(context.inputs[1].dims) === 0) { | ||
| unary.fastGelu(context); | ||
| } else { | ||
| context.compute(createFastGeluProgramInfo(context.inputs)); | ||
| } | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,211 @@ | ||
| [ | ||
| { | ||
| "name": "FastGelu test without bias", | ||
| "operator": "FastGelu", | ||
| "opset": { "domain": "com.microsoft", "version": 1 }, | ||
| "cases": [ | ||
| { | ||
| "name": "scalar", | ||
| "inputs": [ | ||
| { | ||
| "data": [1], | ||
| "dims": [], | ||
| "type": "float32" | ||
| } | ||
| ], | ||
| "outputs": [ | ||
| { | ||
| "data": [0.841192], | ||
| "dims": [], | ||
| "type": "float32" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "[2x4]", | ||
| "inputs": [ | ||
| { | ||
| "data": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8], | ||
| "dims": [2, 4], | ||
| "type": "float32" | ||
| } | ||
| ], | ||
| "outputs": [ | ||
| { | ||
| "data": [0.0539828, 0.115851, 0.185371, 0.262161, 0.345714, 0.435415, 0.53057, 0.630432], | ||
| "dims": [2, 4], | ||
| "type": "float32" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "[3x5]", | ||
| "inputs": [ | ||
| { | ||
| "data": [0.1, 0.2, 0.3, 0.4, 0.5, 1, 2, 3, 4, 5, 1.1, 1.2, 1.3, 1.4, 1.5], | ||
| "dims": [3, 5], | ||
| "type": "float32" | ||
| } | ||
| ], | ||
| "outputs": [ | ||
| { | ||
| "data": [ | ||
| 0.0539828, 0.115851, 0.185371, 0.262161, 0.345714, 0.841192, 1.9546, 2.99636, 3.99993, 5, 0.950581, | ||
| 1.0617, 1.17393, 1.28671, 1.39957 | ||
| ], | ||
| "dims": [3, 5], | ||
| "type": "float32" | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "FastGelu test with bias", | ||
| "operator": "FastGelu", | ||
| "opset": { "domain": "com.microsoft", "version": 1 }, | ||
| "cases": [ | ||
| { | ||
| "name": "scalar", | ||
| "inputs": [ | ||
| { | ||
| "data": [1], | ||
| "dims": [], | ||
| "type": "float32" | ||
| }, | ||
| { | ||
| "data": [0.5], | ||
| "dims": [], | ||
| "type": "float32" | ||
| } | ||
| ], | ||
| "outputs": [ | ||
| { | ||
| "data": [1.39957], | ||
| "dims": [], | ||
| "type": "float32" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "[2x4], [4]", | ||
| "inputs": [ | ||
| { | ||
| "data": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8], | ||
| "dims": [2, 4], | ||
| "type": "float32" | ||
| }, | ||
| { | ||
| "data": [1, 2, 3, 4], | ||
| "dims": [4], | ||
| "type": "float32" | ||
| } | ||
| ], | ||
| "outputs": [ | ||
| { | ||
| "data": [0.950581, 2.16968, 3.29869, 4.39999, 1.39957, 2.58835, 3.69973, 4.8], | ||
| "dims": [2, 4], | ||
| "type": "float32" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "[2x4], [3]", | ||
| "inputs": [ | ||
| { | ||
| "data": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8], | ||
| "dims": [2, 4], | ||
| "type": "float32" | ||
| }, | ||
| { | ||
| "data": [1, 2, 3], | ||
| "dims": [3], | ||
| "type": "float32" | ||
| } | ||
| ], | ||
| "outputs": [ | ||
| { | ||
| "data": [0.950581, 2.16968, 3.29869, 1.28671, 2.48492, 3.59959, 1.62411, 2.79331], | ||
| "dims": [2, 4], | ||
| "type": "float32" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "[3x5], [2]", | ||
| "inputs": [ | ||
| { | ||
| "data": [0.1, 0.2, 0.3, 0.4, 0.5, 1, 2, 3, 4, 5, 1.1, 1.2, 1.3, 1.4, 1.5], | ||
| "dims": [3, 5], | ||
| "type": "float32" | ||
| }, | ||
| { | ||
| "data": [2, 3], | ||
| "dims": [2], | ||
| "type": "float32" | ||
| } | ||
| ], | ||
| "outputs": [ | ||
| { | ||
| "data": [ | ||
| 2.06267, 3.19813, 2.27567, 3.39909, 2.48492, 3.99993, 3.99993, 6, 6, 8, 3.09737, 4.19997, 3.29869, | ||
| 4.39999, 3.49938 | ||
| ], | ||
| "dims": [3, 5], | ||
| "type": "float32" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "[3x5], [7]", | ||
| "inputs": [ | ||
| { | ||
| "data": [0.1, 0.2, 0.3, 0.4, 0.5, 1, 2, 3, 4, 5, 1.1, 1.2, 1.3, 1.4, 1.5], | ||
| "dims": [3, 5], | ||
| "type": "float32" | ||
| }, | ||
| { | ||
| "data": [2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7], | ||
| "dims": [7], | ||
| "type": "float32" | ||
| } | ||
| ], | ||
| "outputs": [ | ||
| { | ||
| "data": [ | ||
| 2.16968, 2.38072, 2.58835, 2.79331, 2.99636, 3.59959, 4.7, 5.1, 6.2, 7.3, 3.49938, 3.69973, 3.89989, | ||
| 4.09996, 3.59959 | ||
| ], | ||
| "dims": [3, 5], | ||
| "type": "float32" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "name": "[4x4], [8]", | ||
| "inputs": [ | ||
| { | ||
| "data": [0.8, -0.5, 0.0, 1, 1.3, 2.1, -0.2, 1.1, 0.5, 0.2, 0.3, -0.6, 3.1, 2.2, -1.1, 0.0], | ||
| "dims": [4, 4], | ||
| "type": "float32" | ||
| }, | ||
| { | ||
| "data": [-0.5, 0.6, 1.2, 2.1, 1.3, -1, 0, 3.1], | ||
| "dims": [8], | ||
| "type": "float32" | ||
| } | ||
| ], | ||
| "outputs": [ | ||
| { | ||
| "data": [ | ||
| 0.185371, 0.0539828, 1.0617, 3.09737, 2.58835, 0.950581, -0.0841486, 4.19997, 0, 0.630432, 1.39957, | ||
| 1.39957, 4.39999, 1.0617, -0.149419, 3.09737 | ||
| ], | ||
| "dims": [4, 4], | ||
| "type": "float32" | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| } | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.