From 61c2b13cc61054485d320f125e3a7da105fb1599 Mon Sep 17 00:00:00 2001 From: bd_ Date: Mon, 18 Sep 2023 21:30:38 +0900 Subject: [PATCH] chore: improve performance of blendshape manipulation This improves performance of blendshape manipulation by keeping blendshape data in flat arrays as much as possible, and using Burst where reasonable. --- Editor/Processors/MergeBoneProcessor.cs | 21 +- .../SkinnedMeshes/BlendshapeInfo.cs | 804 ++++++++++++++++++ .../SkinnedMeshes/BlendshapeInfo.cs.meta | 3 + .../FreezeBlendShapeProcessor.cs | 6 +- ...utoFreezeMeaninglessBlendShapeProcessor.cs | 8 +- .../MergeSkinnedMeshProcessor.cs | 7 +- Editor/Processors/SkinnedMeshes/MeshInfo2.cs | 177 +--- .../RemoveMeshByBlendShapeProcessor.cs | 8 +- ...m.anatawa12.avatar-optimizer.editor.asmdef | 3 +- 9 files changed, 866 insertions(+), 171 deletions(-) create mode 100644 Editor/Processors/SkinnedMeshes/BlendshapeInfo.cs create mode 100644 Editor/Processors/SkinnedMeshes/BlendshapeInfo.cs.meta diff --git a/Editor/Processors/MergeBoneProcessor.cs b/Editor/Processors/MergeBoneProcessor.cs index fe6a12c44..4f36b1c90 100644 --- a/Editor/Processors/MergeBoneProcessor.cs +++ b/Editor/Processors/MergeBoneProcessor.cs @@ -3,6 +3,7 @@ using System.Linq; using Anatawa12.AvatarOptimizer.ErrorReporting; using Anatawa12.AvatarOptimizer.Processors.SkinnedMeshes; +using Unity.Collections; using UnityEditor; using UnityEngine; using Object = UnityEngine.Object; @@ -136,7 +137,10 @@ private void DoBoneMap2(MeshInfo2 meshInfo2, Dictionary me if (!boneReplaced) return; + NativeArray blendshapeTransforms = new NativeArray(meshInfo2.InitialVertexCount, Allocator.TempJob); + // Optimization 1: if vertex is affected by only one bone, we can merge to one weight + int vIndex = 0; foreach (var vertex in meshInfo2.Vertices) { var singleBoneTransform = vertex.BoneWeights.Select(x => x.bone.Transform) @@ -160,30 +164,21 @@ private void DoBoneMap2(MeshInfo2 meshInfo2, Dictionary me foreach (var (bone, weight) in vertex.BoneWeights) mergedOldBindPose += bone.Bindpose * weight; var transBindPose = finalBone.Bindpose.inverse * mergedOldBindPose; + blendshapeTransforms[vertex.Index] = transBindPose; vertex.Position = transBindPose.MultiplyPoint3x4(vertex.Position); vertex.Normal = transBindPose.MultiplyPoint3x3(vertex.Normal); var tangentVec3 = transBindPose.MultiplyPoint3x3(vertex.Tangent); vertex.Tangent = new Vector4(tangentVec3.x, tangentVec3.y, tangentVec3.z, vertex.Tangent.w); - foreach (var frames in vertex.BlendShapes.Values) - { - for (var i = 0; i < frames.Count; i++) - { - var frame = frames[i]; - frames[i] = new Vertex.BlendShapeFrame( - weight: frame.Weight, - position: transBindPose.MultiplyPoint3x4(frame.Position), - normal: transBindPose.MultiplyPoint3x3(frame.Normal), - tangent: transBindPose.MultiplyPoint3x3(frame.Tangent) - ); - } - } var weightSum = vertex.BoneWeights.Select(x => x.weight).Sum(); // I want weightSum to be 1.0 but it may not. vertex.BoneWeights.Clear(); vertex.BoneWeights.Add((finalBone, weightSum)); } + + // This call takes ownership of and frees the native blendshapeTransforms array + meshInfo2.BlendShapeData.TransformVertexSpaces(blendshapeTransforms); // Optimization2: If there are same (BindPose, Transform) pair, merge // This is optimization for RestPose bone merging diff --git a/Editor/Processors/SkinnedMeshes/BlendshapeInfo.cs b/Editor/Processors/SkinnedMeshes/BlendshapeInfo.cs new file mode 100644 index 000000000..e898bcd25 --- /dev/null +++ b/Editor/Processors/SkinnedMeshes/BlendshapeInfo.cs @@ -0,0 +1,804 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Anatawa12.AvatarOptimizer.ErrorReporting; +using Unity.Burst; +using Unity.Collections; +using Unity.Jobs; +using UnityEditor; +using UnityEngine; + +namespace Anatawa12.AvatarOptimizer.Processors.SkinnedMeshes.Blendshape +{ + /// + /// Holds information about where in the native storage arrays a particular blend shape maps to + /// + internal struct BlendshapeDescriptor + { + public int StartFrame, NumFrames; + /// + /// True if any delta value for any frame is nonzero + /// + public bool IsMeaningful; + } + + /// + /// Holds the native arrays used to store all blendshape data + /// + internal class BlendshapeNativeStorage : IDisposable + { + public delegate JobHandle LaunchJobDelegate(); + + private bool IsValid = true; + private List ActiveJobs = new List(); + + public int NumVertices { get; } + + // These arrays are organized by shape, frame, then vertex. These are also dense; zero-delta vertices are + // represented in these arrays. + private NativeArray _positions, _normals, _tangents; + private NativeArray _frameWeights; + private NativeArray _shapeDescriptors; + + public NativeArray DeltaPositions { + get + { + if (!IsValid) throw new ObjectDisposedException(nameof(BlendshapeNativeStorage)); + return _positions; + } + } + + public NativeArray DeltaNormals { + get + { + if (!IsValid) throw new ObjectDisposedException(nameof(BlendshapeNativeStorage)); + return _normals; + } + } + + public NativeArray DeltaTangents { + get + { + if (!IsValid) throw new ObjectDisposedException(nameof(BlendshapeNativeStorage)); + return _tangents; + } + } + + public NativeArray FrameWeights { + get + { + if (!IsValid) throw new ObjectDisposedException(nameof(BlendshapeNativeStorage)); + return _frameWeights; + } + } + + public NativeArray ShapeDescriptors { + get + { + if (!IsValid) throw new ObjectDisposedException(nameof(BlendshapeNativeStorage)); + return _shapeDescriptors; + } + } + + public BlendshapeNativeStorage(Mesh m) + { + NumVertices = m.vertexCount; + + int frames = 0; + int shapes = m.blendShapeCount; + for (int s = 0; s < shapes; s++) + { + frames += m.GetBlendShapeFrameCount(s); + } + + _positions = new NativeArray(m.vertexCount * frames, Allocator.TempJob); + _normals = new NativeArray(m.vertexCount * frames, Allocator.TempJob); + _tangents = new NativeArray(m.vertexCount * frames, Allocator.TempJob); + _frameWeights = new NativeArray(frames, Allocator.TempJob); + _shapeDescriptors = new NativeArray(shapes, Allocator.TempJob); + + EditorApplication.delayCall += Dispose; + } + + public BlendshapeNativeStorage(int verticesCount, BlendshapeDescriptor[] shapeDescriptors, float[] weights) + { + NumVertices = verticesCount; + + _shapeDescriptors = new NativeArray(shapeDescriptors, Allocator.TempJob); + _frameWeights = new NativeArray(weights, Allocator.TempJob); + + _positions = new NativeArray(verticesCount * weights.Length, Allocator.TempJob); + _normals = new NativeArray(verticesCount * weights.Length, Allocator.TempJob); + _tangents = new NativeArray(verticesCount * weights.Length, Allocator.TempJob); + + EditorApplication.delayCall += Dispose; + } + + /// + /// Records that a job is in progress, so that disposal can wait for it + /// + /// + /// + public JobHandle LaunchJob(JobHandle h) + { + ActiveJobs.Add(h); + return h; + } + + public void WaitForAllJobs() + { + foreach (var job in ActiveJobs) + { + job.Complete(); + } + ActiveJobs.Clear(); + } + + ~BlendshapeNativeStorage() + { + Dispose(); + } + + public void Dispose() + { + if (!IsValid) return; + + IsValid = false; + WaitForAllJobs(); + + _positions.Dispose(); + _normals.Dispose(); + _tangents.Dispose(); + _frameWeights.Dispose(); + _shapeDescriptors.Dispose(); + + GC.SuppressFinalize(this); + } + } + + /// + /// This class tracks blendshape data from a mesh in a compact and burst-compatible format. + /// + internal class BlendshapeInfo + { + private BlendshapeNativeStorage Storage; + + private Dictionary _shapeNameToIndex = new Dictionary(); + private List _shapeNames = new List(); + + public int InitialVertexCount => Storage.NumVertices; + + struct MergeRange + { + public BlendshapeInfo sourceInfo; + public int startSrcVertex, startDstVertex, vertexCount; + } + + private BlendshapeInfo(List vertices) + { + // Assign shape indexes + AssignMergeShapeIndexes(vertices); + + // Determine vertex index mappings (and update vertex fields) + Dictionary> mappingRanges = AssignMergeIndices(vertices); + + // Now determine what frames exist + // This is a mapping of dest shape index -> [weight => frame index] + // We also generate shape descriptors in this step + MergeFrames(mappingRanges, out var shapeDescriptors, out var weights); + + // Allocate sufficient space for the merged data + Storage = new BlendshapeNativeStorage(vertices.Count, shapeDescriptors, weights); + + JobHandle.CombineDependencies( + new ZeroFill(Storage.DeltaPositions).Schedule(Storage.DeltaPositions.Length, 1024), + new ZeroFill(Storage.DeltaNormals).Schedule(Storage.DeltaNormals.Length, 1024), + new ZeroFill(Storage.DeltaTangents).Schedule(Storage.DeltaTangents.Length, 1024) + ).Complete(); + + CopyFrames(mappingRanges); + } + + private void CopyFrames(Dictionary> mappingRanges) + { + for (int s = 0; s < _shapeNames.Count; s++) + { + var desc = Storage.ShapeDescriptors[s]; + for (int f = 0; f < desc.NumFrames; f++) + { + CopySingleFrame(s, f, mappingRanges); + } + } + } + + private void CopySingleFrame(int shapeIndex, int frame, Dictionary> mappingRanges) + { + var shapeName = _shapeNames[shapeIndex]; + var desc = Storage.ShapeDescriptors[shapeIndex]; + + int dstOffset = Storage.NumVertices * (desc.StartFrame + frame); + + foreach (var kvp in mappingRanges) + { + var srcShapes = kvp.Key; + var ranges = kvp.Value; + + if (!srcShapes._shapeNameToIndex.TryGetValue(shapeName, out var srcIndex)) + { + continue; + } + + var srcDesc = srcShapes.Storage.ShapeDescriptors[srcIndex]; + int srcOffset = srcShapes.Storage.NumVertices * (srcDesc.StartFrame + frame); + + foreach (var range in ranges) + { + int srcStart = srcOffset + range.startSrcVertex; + int dstStart = dstOffset + range.startDstVertex; + int count = range.vertexCount; + + Storage.DeltaPositions.Slice(dstStart, count).CopyFrom(srcShapes.Storage.DeltaPositions.Slice(srcStart, count)); + Storage.DeltaNormals.Slice(dstStart, count).CopyFrom(srcShapes.Storage.DeltaNormals.Slice(srcStart, count)); + Storage.DeltaTangents.Slice(dstStart, count).CopyFrom(srcShapes.Storage.DeltaTangents.Slice(srcStart, count)); + } + } + } + + private Dictionary> AssignMergeIndices(List vertices) + { + MergeRange currentRange = new MergeRange(); + + Dictionary> rangeBuffer = + new Dictionary>(); + + for (int v = 0; v < vertices.Count; v++) + { + var dstVertex = vertices[v]; + var srcIndex = dstVertex.Index; + var srcInfo = dstVertex.BlendshapeInfo; + + if (currentRange.sourceInfo != srcInfo && currentRange.sourceInfo != null) + { + if (!rangeBuffer.TryGetValue(currentRange.sourceInfo, out var ranges)) + { + ranges = new List(); + rangeBuffer[currentRange.sourceInfo] = ranges; + } + ranges.Add(currentRange); + currentRange = new MergeRange(); + } + + if (currentRange.sourceInfo == null) + { + currentRange.sourceInfo = srcInfo; + currentRange.startSrcVertex = srcIndex; + currentRange.startDstVertex = v; + } + + currentRange.vertexCount++; + + dstVertex.BlendshapeInfo = this; + dstVertex.Index = v; + } + + if (currentRange.sourceInfo != null) + { + if (!rangeBuffer.TryGetValue(currentRange.sourceInfo, out var ranges)) + { + ranges = new List(); + rangeBuffer[currentRange.sourceInfo] = ranges; + } + ranges.Add(currentRange); + } + + return rangeBuffer; + } + + private void AssignMergeShapeIndexes(List vertices) + { + HashSet visitedInfos = new HashSet(); + + foreach (var vertex in vertices) + { + if (!visitedInfos.Add(vertex.BlendshapeInfo)) + { + continue; + } + + foreach (var shapeName in vertex.BlendshapeInfo._shapeNames) + { + if (shapeName != null && !_shapeNameToIndex.ContainsKey(shapeName)) + { + var index = _shapeNames.Count; + _shapeNames.Add(shapeName); + _shapeNameToIndex[shapeName] = index; + } + } + } + } + + private void MergeFrames( + Dictionary> mappingRanges, + out BlendshapeDescriptor[] shapeDescriptors, + out float[] weights + ) + { + shapeDescriptors = new BlendshapeDescriptor[_shapeNames.Count]; + List weightList = new List(); + + int nextFrameIndex = 0; + for (int s = 0; s < _shapeNames.Count; s++) + { + BlendshapeDescriptor descriptor = new BlendshapeDescriptor() + { + IsMeaningful = false, + StartFrame = nextFrameIndex, + NumFrames = 0 + }; + + bool isFirst = true; + foreach (var info in mappingRanges.Keys) + { + if (!info._shapeNameToIndex.TryGetValue(_shapeNames[s], out var index)) + { + continue; + } + + var srcDesc = info.Storage.ShapeDescriptors[index]; + + if (isFirst) + { + descriptor.NumFrames = srcDesc.NumFrames; + nextFrameIndex += srcDesc.NumFrames; + + shapeDescriptors[s] = descriptor; + + // Copy weights + for (int i = 0; i < srcDesc.NumFrames; i++) + { + weightList.Add(info.Storage.FrameWeights[srcDesc.StartFrame + i]); + } + + isFirst = false; + } + else + { + // Verify weights are consistent + if (descriptor.NumFrames != srcDesc.NumFrames) + { + BuildReport.LogWarning("MergeSkinnedMesh:warning:blendShapeWeightMismatch", _shapeNames[s]); + continue; + } + + for (int f = 0; f < srcDesc.NumFrames; f++) + { + if (weightList[descriptor.StartFrame + f] != info.Storage.FrameWeights[srcDesc.StartFrame + f]) + { + BuildReport.LogWarning("MergeSkinnedMesh:warning:blendShapeWeightMismatch", _shapeNames[s]); + break; + } + } + } + } + } + + weights = weightList.ToArray(); + } + + + /// + /// Constructs a BlendshapeInfo by merging blendshape data from vertices originally from multiple meshes. + /// The input vertices will be updated to reference this new BlendshapeInfo. + /// + /// + public static BlendshapeInfo MergeVertices(List vertices) + { + return new BlendshapeInfo(vertices); + } + + public BlendshapeInfo(Mesh mesh) + { + Storage = new BlendshapeNativeStorage(mesh); + + Vector3[] positions = new Vector3[mesh.vertexCount]; + Vector3[] normals = new Vector3[mesh.vertexCount]; + Vector3[] tangents = new Vector3[mesh.vertexCount]; + + int nVerts = mesh.vertexCount; + int nShapes = mesh.blendShapeCount; + int offset = 0; + for (int i = 0; i < nShapes; i++) + { + var name = mesh.GetBlendShapeName(i); + _shapeNameToIndex[name] = i; + _shapeNames.Add(name); + + BlendshapeDescriptor descriptor = new BlendshapeDescriptor(); + descriptor.StartFrame = offset; + descriptor.NumFrames = mesh.GetBlendShapeFrameCount(i); + descriptor.IsMeaningful = false; + + int nFrames = mesh.GetBlendShapeFrameCount(i); + for (int f = 0; f < nFrames; f++) + { + float weight = mesh.GetBlendShapeFrameWeight(i, f); + mesh.GetBlendShapeFrameVertices(i, f, positions, normals, tangents); + + Storage.DeltaPositions.Slice(offset * nVerts, nVerts).CopyFrom(positions); + Storage.DeltaNormals.Slice(offset * nVerts, nVerts).CopyFrom(normals); + Storage.DeltaTangents.Slice(offset * nVerts, nVerts).CopyFrom(tangents); + + var frameWeights = Storage.FrameWeights; + frameWeights[offset] = weight; + + offset++; + } + + var descriptors = Storage.ShapeDescriptors; + descriptors[i] = descriptor; + } + } + + /// + /// Returns an enumerable of blendshape names which do not influence any bones. + /// + /// + public IEnumerable GetZeroInfluenceBlendshapes() + { + Storage.WaitForAllJobs(); + + var descriptors = Storage.ShapeDescriptors; + for (int i = 0; i < _shapeNames.Count; i++) + { + var desc = descriptors[i]; + desc.IsMeaningful = false; + descriptors[i] = desc; + } + + Storage.LaunchJob(new FindMeaningfulShapes() + { + nVertices = Storage.NumVertices, + DeltaPositions = Storage.DeltaPositions, + DeltaNormals = Storage.DeltaNormals, + DeltaTangents = Storage.DeltaTangents, + ShapeDescriptors = Storage.ShapeDescriptors + }.Schedule(Storage.ShapeDescriptors.Length, 1)).Complete(); + + for (int i = 0; i < _shapeNames.Count; i++) + { + var name = _shapeNames[i]; + var descriptor = Storage.ShapeDescriptors[i]; + + if (name != null && !descriptor.IsMeaningful) + { + yield return name; + } + } + } + + /// + /// Returns an enumerable of vertices affected by a specific blend shape. + /// + /// + /// + /// + public IEnumerable VerticesAffectedByShape(string name, double sqrTolerance) + { + int shapeIndex = _shapeNameToIndex[name]; + var descriptor = Storage.ShapeDescriptors[shapeIndex]; + + for (int v = 0; v < Storage.NumVertices; v++) + { + for (int f = descriptor.StartFrame; f < descriptor.StartFrame + descriptor.NumFrames; f++) + { + int index = f * Storage.NumVertices + v; + + if (Storage.DeltaPositions[index].sqrMagnitude > sqrTolerance) + { + yield return v; + break; // skip remaining frames + } + } + } + } + + /// + /// Tries to find the interpolated blend shape delta for a given shape name, frame weight, and vertex. + /// + /// + /// + /// + /// + /// + /// + /// + public bool TryGetBlendshape( + string name, + float weight, + int vertexIndex, + out Vector3 position, + out Vector3 normal, + out Vector3 tangent + ) + { + if (!_shapeNameToIndex.TryGetValue(name, out var shapeIndex)) + { + position = default; + normal = default; + tangent = default; + + return false; + } + + var descriptor = Storage.ShapeDescriptors[shapeIndex]; + + int nverts = Storage.NumVertices; + int nframes = descriptor.NumFrames; + int startFrame = descriptor.StartFrame; + + if (Mathf.Abs(weight) <= 0.0001f && ZeroForWeightZero()) + { + position = Vector3.zero; + normal = Vector3.zero; + tangent = Vector3.zero; + return true; + } + + bool ZeroForWeightZero() + { + if (nframes == 1) return true; + var first = Frame(0); + var end = Frame(nframes - 1); + + // both weight are same sign, zero for 0 weight + if (first.Weight <= 0 && end.Weight <= 0) return true; + if (first.Weight >= 0 && end.Weight >= 0) return true; + + return false; + } + + if (nframes == 1) + { + // simplest and likely + var frame = Frame(0); + var ratio = weight / frame.Weight; + position = frame.Position * ratio; + normal = frame.Normal * ratio; + tangent = frame.Tangent * ratio; + return true; + } + else + { + // multi frame + var (lessFrame, greaterFrame) = FindFrame(); + var ratio = InverseLerpUnclamped(lessFrame.Weight, greaterFrame.Weight, weight); + + position = Vector3.LerpUnclamped(lessFrame.Position, greaterFrame.Position, ratio); + normal = Vector3.LerpUnclamped(lessFrame.Normal, greaterFrame.Normal, ratio); + tangent = Vector3.LerpUnclamped(lessFrame.Tangent, greaterFrame.Tangent, ratio); + return true; + } + + (BlendshapeVertexFrame, BlendshapeVertexFrame) FindFrame() + { + var firstFrame = Frame(0); + var lastFrame = Frame(nframes - 1); + + if (firstFrame.Weight > 0 && weight < firstFrame.Weight) + { + // if all weights are positive and the weight is less than first weight: lerp 0..first + return (default, firstFrame); + } + + if (lastFrame.Weight < 0 && weight > lastFrame.Weight) + { + // if all weights are negative and the weight is more than last weight: lerp last..0 + return (lastFrame, default); + } + + // otherwise, lerp between two surrounding frames OR nearest two frames + + for (var i = 1; i < nframes; i++) + { + if (weight <= Frame(i).Weight) + return (Frame(i-1), Frame(i)); + } + + return (Frame(nframes - 2), Frame(nframes - 1)); + } + + BlendshapeVertexFrame Frame(int f) + { + int index = (startFrame + f) * nverts + vertexIndex; + + return new BlendshapeVertexFrame( + Storage.DeltaPositions[index], + Storage.DeltaNormals[index], + Storage.DeltaTangents[index], + Storage.FrameWeights[startFrame + f] + ); + } + + float InverseLerpUnclamped(float a, float b, float value) => (value - a) / (b - a); + } + + /// + /// Transforms all blendshape data based on an array of per-vertex affine transformations. + /// + /// The array of affine transformations to apply. Ownership of this array is + /// transferred to this function; this array will be deallocated asynchronously. + public void TransformVertexSpaces(NativeArray perVertexTransforms) + { + if (perVertexTransforms.Length != InitialVertexCount) + { + throw new ArgumentException("Wrong vertex count for TransformVertexSpaces"); + } + + Storage.LaunchJob(new TransformVertices() + { + PerVertexTransforms = perVertexTransforms, + DeltaPositions = Storage.DeltaPositions, + DeltaNormals = Storage.DeltaNormals, + DeltaTangents = Storage.DeltaTangents + }.Schedule(Storage.DeltaPositions.Length, 32)); + } + + /// + /// Marks a blendshape for deletion. + /// + /// + public void DeleteBlendshape(string name) + { + if (_shapeNameToIndex.TryGetValue(name, out var index)) + { + _shapeNameToIndex.Remove(name); + _shapeNames[index] = null; + } + } + + /// + /// Saves all blendshape data to a mesh. + /// + /// + /// + public void SaveToMesh(Mesh destMesh, List vertices) + { + Storage.WaitForAllJobs(); + destMesh.ClearBlendShapes(); + + int nShapes = _shapeNameToIndex.Count; + List shapeIndices = _shapeNameToIndex.Values.OrderBy(i => i).ToList(); + + // Vertices may have been deleted during processing, so create a lookup table to map between the different + // indices + int[] dstToSrcVertexIndex = new int[vertices.Count]; + int idx = 0; + foreach (var v in vertices) + { + dstToSrcVertexIndex[idx++] = v.Index; + } + + Vector3[] positions = new Vector3[dstToSrcVertexIndex.Length]; + Vector3[] normals = new Vector3[dstToSrcVertexIndex.Length]; + Vector3[] tangents = new Vector3[dstToSrcVertexIndex.Length]; + + foreach (var nativeIndex in shapeIndices) + { + var descriptor = Storage.ShapeDescriptors[nativeIndex]; + for (int f = descriptor.StartFrame; f < descriptor.StartFrame + descriptor.NumFrames; f++) + { + /*Storage.DeltaPositions.Slice(f * Mesh.vertexCount, Mesh.vertexCount).CopyTo(positions); + Storage.DeltaNormals.Slice(f * Mesh.vertexCount, Mesh.vertexCount).CopyTo(normals); + Storage.DeltaTangents.Slice(f * Mesh.vertexCount, Mesh.vertexCount).CopyTo(tangents);*/ + + // TODO: should this move to Burst? [might be best to wait until post-2021 when we can access the + // blendshape data directly from Burst] + for (int v = 0; v < dstToSrcVertexIndex.Length; v++) + { + var srcVertex = dstToSrcVertexIndex[v]; + positions[v] = Storage.DeltaPositions[f * Storage.NumVertices + srcVertex]; + normals[v] = Storage.DeltaNormals[f * Storage.NumVertices + srcVertex]; + tangents[v] = Storage.DeltaTangents[f * Storage.NumVertices + srcVertex]; + } + + destMesh.AddBlendShapeFrame(_shapeNames[nativeIndex], Storage.FrameWeights[f], + positions, normals, tangents); + } + } + } + + [BurstCompile] + private struct TransformVertices : IJobParallelFor + { + [ReadOnly] [DeallocateOnJobCompletion] public NativeArray PerVertexTransforms; + + public NativeArray DeltaPositions; + public NativeArray DeltaNormals; + public NativeArray DeltaTangents; + + public void Execute(int index) + { + // index here is the index into each of the Delta arrays + int vertexIndex = index % PerVertexTransforms.Length; + var transformation = PerVertexTransforms[vertexIndex]; + + if (transformation.m33 < 0.1) + { + // Uninitialized element, skip it + return; + } + + var position = DeltaPositions[index]; + var normal = DeltaNormals[index]; + var tangent = DeltaTangents[index]; + + DeltaPositions[index] = transformation.MultiplyPoint3x4(position); + DeltaNormals[index] = transformation.MultiplyPoint3x3(normal); + DeltaTangents[index] = transformation.MultiplyPoint3x3(tangent); + } + } + + [BurstCompile] + private struct FindMeaningfulShapes : IJobParallelFor + { + public int nVertices; + + [ReadOnly] public NativeArray DeltaPositions; + [ReadOnly] public NativeArray DeltaNormals; + [ReadOnly] public NativeArray DeltaTangents; + + public NativeArray ShapeDescriptors; + + public void Execute(int index) + { + var descriptor = ShapeDescriptors[index]; + + int startIndex = descriptor.StartFrame * nVertices; + int endIndex = startIndex + descriptor.NumFrames * nVertices; + + for (int i = startIndex; i < endIndex; i++) + { + if (DeltaPositions[i] != Vector3.zero + || DeltaNormals[i] != Vector3.zero + || DeltaTangents[i] != Vector3.zero) + { + descriptor.IsMeaningful = true; + ShapeDescriptors[index] = descriptor; + break; + } + } + } + } + + [BurstCompile] + private struct ZeroFill : IJobParallelFor + { + [WriteOnly] + public NativeArray Array; + + public ZeroFill(NativeArray array) + { + Array = array; + } + + public void Execute(int index) + { + Array[index] = Vector3.zero; + } + } + } + + internal struct BlendshapeVertexFrame + { + public Vector3 Position, Normal, Tangent; + public float Weight; + + public BlendshapeVertexFrame(Vector3 deltaPosition, Vector3 deltaNormal, Vector3 deltaTangent, float weight) + { + this.Position = deltaPosition; + this.Normal = deltaNormal; + this.Tangent = deltaTangent; + this.Weight = weight; + } + } +} \ No newline at end of file diff --git a/Editor/Processors/SkinnedMeshes/BlendshapeInfo.cs.meta b/Editor/Processors/SkinnedMeshes/BlendshapeInfo.cs.meta new file mode 100644 index 000000000..952b322ec --- /dev/null +++ b/Editor/Processors/SkinnedMeshes/BlendshapeInfo.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: d15498c4a2864e8cb93212732700068d +timeCreated: 1695020865 \ No newline at end of file diff --git a/Editor/Processors/SkinnedMeshes/FreezeBlendShapeProcessor.cs b/Editor/Processors/SkinnedMeshes/FreezeBlendShapeProcessor.cs index 6642f68bb..21144ef31 100644 --- a/Editor/Processors/SkinnedMeshes/FreezeBlendShapeProcessor.cs +++ b/Editor/Processors/SkinnedMeshes/FreezeBlendShapeProcessor.cs @@ -41,10 +41,14 @@ HashSet freezeNames vertex.Normal += normal; tangent += (Vector3)vertex.Tangent; vertex.Tangent = new Vector4(tangent.x, tangent.y, tangent.z, vertex.Tangent.w); - vertex.BlendShapes.Remove(name); } } + foreach (var name in freezeNames) + { + target.BlendShapeData.DeleteBlendshape(name); + } + { int srcI = 0, dstI = 0; for (; srcI < target.BlendShapes.Count; srcI++) diff --git a/Editor/Processors/SkinnedMeshes/InternalAutoFreezeMeaninglessBlendShapeProcessor.cs b/Editor/Processors/SkinnedMeshes/InternalAutoFreezeMeaninglessBlendShapeProcessor.cs index 00f98b4ca..7d86e9963 100644 --- a/Editor/Processors/SkinnedMeshes/InternalAutoFreezeMeaninglessBlendShapeProcessor.cs +++ b/Editor/Processors/SkinnedMeshes/InternalAutoFreezeMeaninglessBlendShapeProcessor.cs @@ -14,18 +14,12 @@ public InternalAutoFreezeMeaninglessBlendShapeProcessor(InternalAutoFreezeMeanin public override void Process(OptimizerSession session, MeshInfo2 target) { - var meaningfulBlendShapes = new HashSet(); - - foreach (var vertex in target.Vertices) - foreach (var kvp in vertex.BlendShapes.Where(kvp => kvp.Value != default)) - meaningfulBlendShapes.Add(kvp.Key); - var freezeBlendShape = Target.GetComponent(); var serialized = new SerializedObject(freezeBlendShape); var editorUtil = PrefabSafeSet.EditorUtil.Create( serialized.FindProperty(nameof(FreezeBlendShape.shapeKeysSet)), 0, p => p.stringValue, (p, v) => p.stringValue = v); - foreach (var (meaningLess, _) in target.BlendShapes.Where(x => !meaningfulBlendShapes.Contains(x.name))) + foreach (var meaningLess in target.BlendShapeData.GetZeroInfluenceBlendshapes()) editorUtil.GetElementOf(meaningLess).EnsureAdded(); serialized.ApplyModifiedPropertiesWithoutUndo(); } diff --git a/Editor/Processors/SkinnedMeshes/MergeSkinnedMeshProcessor.cs b/Editor/Processors/SkinnedMeshes/MergeSkinnedMeshProcessor.cs index 915844096..88b2255b6 100644 --- a/Editor/Processors/SkinnedMeshes/MergeSkinnedMeshProcessor.cs +++ b/Editor/Processors/SkinnedMeshes/MergeSkinnedMeshProcessor.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using Anatawa12.AvatarOptimizer.ErrorReporting; +using Anatawa12.AvatarOptimizer.Processors.SkinnedMeshes.Blendshape; using UnityEngine; using Object = UnityEngine.Object; @@ -48,7 +49,7 @@ TexCoordStatus TexCoordStatusMax(TexCoordStatus x, TexCoordStatus y) => var mappings = new List<(string, string)>(); var weightMismatchBlendShapes = new HashSet(); - + for (var i = 0; i < meshInfos.Length; i++) { var meshInfo = meshInfos[i]; @@ -69,7 +70,6 @@ TexCoordStatus TexCoordStatusMax(TexCoordStatus x, TexCoordStatus y) => $"m_Materials.Array.data[{targetSubMeshIndex}]")); } - // add blend shape if not defined by name for (var sourceI = 0; sourceI < meshInfo.BlendShapes.Count; sourceI++) { @@ -104,7 +104,6 @@ TexCoordStatus TexCoordStatusMax(TexCoordStatus x, TexCoordStatus y) => newBoundMax.y = Mathf.Max(vector3.y, newBoundMax.y); newBoundMax.z = Mathf.Max(vector3.z, newBoundMax.z); } - } session.MappingBuilder.RecordMoveProperties(meshInfo.SourceRenderer, mappings.ToArray()); @@ -117,6 +116,8 @@ TexCoordStatus TexCoordStatusMax(TexCoordStatus x, TexCoordStatus y) => target.AssertInvariantContract($"processing meshInfo {Target.gameObject.name}"); } + + target.BlendShapeData = BlendshapeInfo.MergeVertices(target.Vertices); foreach (var weightMismatchBlendShape in weightMismatchBlendShapes) BuildReport.LogWarning("MergeSkinnedMesh:warning:blendShapeWeightMismatch", weightMismatchBlendShape); diff --git a/Editor/Processors/SkinnedMeshes/MeshInfo2.cs b/Editor/Processors/SkinnedMeshes/MeshInfo2.cs index 3d611e710..981ed952f 100644 --- a/Editor/Processors/SkinnedMeshes/MeshInfo2.cs +++ b/Editor/Processors/SkinnedMeshes/MeshInfo2.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Linq; using Anatawa12.AvatarOptimizer.ErrorReporting; +using Anatawa12.AvatarOptimizer.Processors.SkinnedMeshes.Blendshape; using JetBrains.Annotations; using Unity.Collections; using UnityEngine; @@ -15,10 +16,12 @@ namespace Anatawa12.AvatarOptimizer.Processors.SkinnedMeshes { internal class MeshInfo2 { + public int InitialVertexCount { get; private set; } public readonly Renderer SourceRenderer; public Transform RootBone; public Bounds Bounds; public readonly List Vertices = new List(0); + public BlendshapeInfo BlendShapeData; // TexCoordStatus which is 3 bits x 8 = 24 bits private ushort _texCoordStatus; @@ -41,7 +44,7 @@ public MeshInfo2(SkinnedMeshRenderer renderer) BuildReport.LogFatal("The Mesh is not readable. Please Check Read/Write")?.WithContext(mesh); return; } - + BuildReport.ReportingObject(renderer, true, () => { if (mesh) @@ -150,33 +153,22 @@ public void ReadSkinnedMesh([NotNull] Mesh mesh) } BlendShapes.Clear(); - var deltaVertices = new Vector3[Vertices.Count]; - var deltaNormals = new Vector3[Vertices.Count]; - var deltaTangents = new Vector3[Vertices.Count]; for (var i = 0; i < mesh.blendShapeCount; i++) { var shapeName = mesh.GetBlendShapeName(i); BlendShapes.Add((shapeName, 0.0f)); - - var frames = Vertices.Select(v => v.BlendShapes[shapeName] = new List()).ToArray(); - - for (int frame = 0; frame < mesh.GetBlendShapeFrameCount(i); frame++) - { - mesh.GetBlendShapeFrameVertices(i, frame, deltaVertices, deltaNormals, deltaTangents); - var weight = mesh.GetBlendShapeFrameWeight(i, frame); - - for (var vertex = 0; vertex < deltaNormals.Length; vertex++) - frames[vertex].Add(new Vertex.BlendShapeFrame(weight, deltaVertices[vertex], deltaNormals[vertex], deltaTangents[vertex])); - } } } public void ReadStaticMesh([NotNull] Mesh mesh) { + InitialVertexCount = mesh.vertexCount; + BlendShapeData = new BlendshapeInfo(mesh); + Vertices.Capacity = Math.Max(Vertices.Capacity, mesh.vertexCount); Vertices.Clear(); - for (var i = 0; i < mesh.vertexCount; i++) Vertices.Add(new Vertex()); + for (var i = 0; i < mesh.vertexCount; i++) Vertices.Add(new Vertex(BlendShapeData, i)); CopyVertexAttr(mesh.vertices, (x, v) => x.Position = v); CopyVertexAttr(mesh.normals, (x, v) => x.Normal = v); @@ -263,6 +255,7 @@ public void Clear() Bones.Clear(); HasColor = false; HasTangent = false; + BlendShapeData = null; } public void Optimize() @@ -413,46 +406,9 @@ public void WriteToMesh(Mesh destMesh) } // BlendShapes - if (BlendShapes.Count != 0) + if (BlendShapes.Count != 0 && BlendShapeData != null) { - for (var i = 0; i < BlendShapes.Count; i++) - { - Debug.Assert(destMesh.blendShapeCount == i, "Unexpected state: blend shape count"); - var (shapeName, _) = BlendShapes[i]; - var weightsSet = new HashSet(); - - foreach (var vertex in Vertices) - if (vertex.BlendShapes.TryGetValue(shapeName, out var frames)) - foreach (var frame in frames) - weightsSet.Add(frame.Weight); - - // blendShape with no weights is not allowed. - if (weightsSet.Count == 0) - weightsSet.Add(100); - - var weights = weightsSet.ToArray(); - Array.Sort(weights); - - var positions = new Vector3[Vertices.Count]; - var normals = new Vector3[Vertices.Count]; - var tangents = new Vector3[Vertices.Count]; - - foreach (var weight in weights) - { - for (var vertexI = 0; vertexI < Vertices.Count; vertexI++) - { - var vertex = Vertices[vertexI]; - - vertex.TryGetBlendShape(shapeName, weight, out var position, out var normal, - out var tangent); - positions[vertexI] = position; - normals[vertexI] = normal; - tangents[vertexI] = tangent; - } - - destMesh.AddBlendShapeFrame(shapeName, weight, positions, normals, tangents); - } - } + BlendShapeData.SaveToMesh(destMesh, Vertices); } } @@ -467,7 +423,21 @@ public void WriteToSkinnedMeshRenderer(SkinnedMeshRenderer targetRenderer, Optim WriteToMesh(mesh); targetRenderer.sharedMesh = mesh; for (var i = 0; i < BlendShapes.Count; i++) - targetRenderer.SetBlendShapeWeight(i, BlendShapes[i].weight); + { + try + { + if (i > targetRenderer.sharedMesh.blendShapeCount) + { + UnityEngine.Debug.Log($"Blendshape {BlendShapes[i].name} is out of range on {targetRenderer.gameObject.name}"); + } + targetRenderer.SetBlendShapeWeight(i, BlendShapes[i].weight); + } + catch (Exception e) + { + UnityEngine.Debug.LogException(e); + } + } + targetRenderer.sharedMaterials = SubMeshes.Select(x => x.SharedMaterial).ToArray(); targetRenderer.bones = Bones.Select(x => x.Transform).ToArray(); @@ -505,6 +475,7 @@ public SubMesh(List vertices, ReadOnlySpan triangles, SubMeshDescri internal class Vertex { + public int Index { get; set; } public Vector3 Position { get; set; } public Vector3 Normal { get; set; } public Vector4 Tangent { get; set; } = new Vector4(1, 0, 0, 1); @@ -522,9 +493,7 @@ internal class Vertex // SkinnedMesh related public List<(Bone bone, float weight)> BoneWeights = new List<(Bone, float)>(); - // Each frame must sorted increasingly - public readonly Dictionary> BlendShapes = - new Dictionary>(); + public BlendshapeInfo BlendshapeInfo; public readonly struct BlendShapeFrame { @@ -588,94 +557,18 @@ public void SetTexCoord(int index, Vector4 value) public bool TryGetBlendShape(string name, float weight, out Vector3 position, out Vector3 normal, out Vector3 tangent) { - if (!BlendShapes.TryGetValue(name, out var frames)) - { - position = default; - normal = default; - tangent = default; - return false; - } - - if (Mathf.Abs(weight) <= 0.0001f && ZeroForWeightZero()) - { - position = Vector3.zero; - normal = Vector3.zero; - tangent = Vector3.zero; - return true; - } - - bool ZeroForWeightZero() - { - if (frames.Count == 1) return true; - var first = frames.First(); - var end = frames.Last(); - - // both weight are same sign, zero for 0 weight - if (first.Weight <= 0 && end.Weight <= 0) return true; - if (first.Weight >= 0 && end.Weight >= 0) return true; - - return false; - } - - if (frames.Count == 1) - { - // simplest and likely - var frame = frames[0]; - var ratio = weight / frame.Weight; - position = frame.Position * ratio; - normal = frame.Normal * ratio; - tangent = frame.Tangent * ratio; - return true; - } - else - { - // multi frame - var (lessFrame, greaterFrame) = FindFrame(); - var ratio = InverseLerpUnclamped(lessFrame.Weight, greaterFrame.Weight, weight); - - position = Vector3.LerpUnclamped(lessFrame.Position, greaterFrame.Position, ratio); - normal = Vector3.LerpUnclamped(lessFrame.Normal, greaterFrame.Normal, ratio); - tangent = Vector3.LerpUnclamped(lessFrame.Tangent, greaterFrame.Tangent, ratio); - return true; - } - - (BlendShapeFrame, BlendShapeFrame) FindFrame() - { - var firstFrame = frames[0]; - var lastFrame = frames.Last(); - - if (firstFrame.Weight > 0 && weight < firstFrame.Weight) - { - // if all weights are positive and the weight is less than first weight: lerp 0..first - return (default, firstFrame); - } - - if (lastFrame.Weight < 0 && weight > lastFrame.Weight) - { - // if all weights are negative and the weight is more than last weight: lerp last..0 - return (lastFrame, default); - } - - // otherwise, lerp between two surrounding frames OR nearest two frames - - for (var i = 1; i < frames.Count; i++) - { - if (weight <= frames[i].Weight) - return (frames[i - 1], frames[i]); - } - - return (frames[frames.Count - 2], frames[frames.Count - 1]); - } - - float InverseLerpUnclamped(float a, float b, float value) => (value - a) / (b - a); + return BlendshapeInfo.TryGetBlendshape(name, weight, Index, out position, out normal, out tangent); } - public Vertex() + public Vertex(BlendshapeInfo blendshapeInfo, int index) { + this.BlendshapeInfo = blendshapeInfo; + Index = index; } - private Vertex(Vertex vertex) + internal Vertex(Vertex vertex) { + Index = vertex.Index; Position = vertex.Position; Normal = vertex.Normal; Tangent = vertex.Tangent; @@ -689,7 +582,7 @@ private Vertex(Vertex vertex) TexCoord7 = vertex.TexCoord7; Color = vertex.Color; BoneWeights = vertex.BoneWeights.ToList(); - BlendShapes = new Dictionary>(vertex.BlendShapes); + BlendshapeInfo = vertex.BlendshapeInfo; } public Vertex Clone() => new Vertex(this); diff --git a/Editor/Processors/SkinnedMeshes/RemoveMeshByBlendShapeProcessor.cs b/Editor/Processors/SkinnedMeshes/RemoveMeshByBlendShapeProcessor.cs index aef705724..1de50bbc7 100644 --- a/Editor/Processors/SkinnedMeshes/RemoveMeshByBlendShapeProcessor.cs +++ b/Editor/Processors/SkinnedMeshes/RemoveMeshByBlendShapeProcessor.cs @@ -17,12 +17,12 @@ public override void Process(OptimizerSession session, MeshInfo2 target) var byBlendShapeVertices = new HashSet(); var sqrTolerance = Component.tolerance * Component.tolerance; - foreach (var vertex in target.Vertices) foreach (var shapeName in Component.RemovingShapeKeys) { - if (!vertex.BlendShapes.TryGetValue(shapeName, out var value)) continue; - if (value.Any(f => f.Position.sqrMagnitude > sqrTolerance)) - byBlendShapeVertices.Add(vertex); + foreach (var vertIndex in target.BlendShapeData.VerticesAffectedByShape(shapeName, sqrTolerance)) + { + byBlendShapeVertices.Add(target.Vertices[vertIndex]); + } } foreach (var subMesh in target.SubMeshes) diff --git a/Editor/com.anatawa12.avatar-optimizer.editor.asmdef b/Editor/com.anatawa12.avatar-optimizer.editor.asmdef index 41396b371..004330a75 100644 --- a/Editor/com.anatawa12.avatar-optimizer.editor.asmdef +++ b/Editor/com.anatawa12.avatar-optimizer.editor.asmdef @@ -9,7 +9,8 @@ "GUID:f69eeb3e25674f4a9bd20e6d7e69e0e6", "GUID:2633ab9fa94544a69517fc9a1bc143c9", "GUID:b9880ca0b6584453a2627bd3c038759f", - "GUID:8542dbf824204440a818dbc2377cb4d6" + "GUID:8542dbf824204440a818dbc2377cb4d6", + "GUID:2665a8d13d1b3f18800f46e256720795" ], "includePlatforms": [ "Editor"