From 8212db6a3efd831188a4accc1f57661e2183c7de Mon Sep 17 00:00:00 2001 From: bd_ Date: Thu, 28 Nov 2024 15:24:06 -0800 Subject: [PATCH] Improve asset saving performance (#479) * feat: new, more efficient serialization APIs * fix: unpacking generated assets breaks references Also implement support for subassets. * chore: remove dead code and fix tests --- CHANGELOG.md | 7 +- Editor/API/BuildContext.cs | 83 +++------ Editor/API/Serialization.meta | 3 + Editor/API/Serialization/AssetSaver.cs | 164 ++++++++++++++++++ Editor/API/Serialization/AssetSaver.cs.meta | 3 + Editor/API/Serialization/IAssetSaver.cs | 64 +++++++ Editor/API/Serialization/IAssetSaver.cs.meta | 3 + Editor/API/Serialization/NullAssetSaver.cs | 31 ++++ .../API/Serialization/NullAssetSaver.cs.meta | 3 + .../API/Serialization/SerializationScope.cs | 47 +++++ .../Serialization/SerializationScope.cs.meta | 3 + Editor/API/Serialization/SingleAssetSaver.cs | 39 +++++ .../Serialization/SingleAssetSaver.cs.meta | 3 + Editor/UI/GeneratedAssetsEditor.cs | 126 ++++++++++---- Runtime/GeneratedAssets.cs | 4 +- Runtime/SubAssetContainer.cs | 12 ++ Runtime/SubAssetContainer.cs.meta | 3 + UnitTests~/AvatarNameFilterTests.cs | 26 +-- UnitTests~/SerializationSweepTest.cs | 10 +- 19 files changed, 522 insertions(+), 112 deletions(-) create mode 100644 Editor/API/Serialization.meta create mode 100644 Editor/API/Serialization/AssetSaver.cs create mode 100644 Editor/API/Serialization/AssetSaver.cs.meta create mode 100644 Editor/API/Serialization/IAssetSaver.cs create mode 100644 Editor/API/Serialization/IAssetSaver.cs.meta create mode 100644 Editor/API/Serialization/NullAssetSaver.cs create mode 100644 Editor/API/Serialization/NullAssetSaver.cs.meta create mode 100644 Editor/API/Serialization/SerializationScope.cs create mode 100644 Editor/API/Serialization/SerializationScope.cs.meta create mode 100644 Editor/API/Serialization/SingleAssetSaver.cs create mode 100644 Editor/API/Serialization/SingleAssetSaver.cs.meta create mode 100644 Runtime/SubAssetContainer.cs create mode 100644 Runtime/SubAssetContainer.cs.meta diff --git a/CHANGELOG.md b/CHANGELOG.md index e93754b..92c8fe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `Mesh.RecalculateUVDistributionMetrics` on generated meshes. - [#478] Added `ProfilerScope` API - [#480] Added `IExtensionContext.Owner` API. Setting this property will allow errors to be correctly attributed to the - plugin that contains an extension context. + plugin that contains an extension context. +- [#479] Added `IAssetSaver` and `SerializationScope` APIs. + - NDMF plugins are encouraged to use `IAssetSaver.SaveAsset` instead of directly accessing `AssetContainer`. This will + split saved assets across multiple files, to avoid performance degradation as the number of assets in a container + grows. ### Fixed +- [#479] Unpacking generated asset containers can break inter-asset references ### Changed diff --git a/Editor/API/BuildContext.cs b/Editor/API/BuildContext.cs index 69bc055..3aacdc1 100644 --- a/Editor/API/BuildContext.cs +++ b/Editor/API/BuildContext.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; +using JetBrains.Annotations; using nadena.dev.ndmf.reporting; using nadena.dev.ndmf.runtime; using nadena.dev.ndmf.ui; @@ -43,6 +44,7 @@ public void Dispose() /// The BuildContext is passed to all plugins during the build process. It provides access to the avatar being /// built, as well as various other context information. /// + [PublicAPI] public sealed partial class BuildContext { private readonly GameObject _avatarRootObject; @@ -70,8 +72,10 @@ public sealed partial class BuildContext /// referenced by the avatar to this container when the build completes, but in some cases it can be necessary /// to manually save assets (e.g. when using AnimatorController builtins). /// - public UnityObject AssetContainer { get; private set; } + public UnityObject AssetContainer => AssetSaver.CurrentContainer; + public IAssetSaver AssetSaver { get; } + public bool Successful => !_report.Errors.Any(e => e.TheError.Severity >= ErrorSeverity.Error); private Dictionary _state = new Dictionary(); @@ -122,22 +126,17 @@ public BuildContext(GameObject obj, string assetRootPath, bool isClone = true) #endif var avatarName = _avatarRootObject.name; - - AssetContainer = ScriptableObject.CreateInstance(); + if (assetRootPath != null) { // Ensure the target directory exists Directory.CreateDirectory(assetRootPath); - var pathAvatarName = FilterAvatarName(avatarName); - - var avatarPath = Path.Combine(assetRootPath, pathAvatarName) + ".asset"; - AssetDatabase.GenerateUniqueAssetPath(avatarPath); - AssetDatabase.CreateAsset(AssetContainer, avatarPath); - if (string.IsNullOrEmpty(AssetDatabase.GetAssetPath(AssetContainer))) - { - throw new Exception("Failed to persist asset container"); - } + AssetSaver = new AssetSaver(assetRootPath, avatarName); + } + else + { + AssetSaver = new NullAssetSaver(); } // Ensure that no prefab instances remain somehow @@ -166,44 +165,14 @@ public BuildContext(GameObject obj, string assetRootPath, bool isClone = true) } } - private static readonly Regex WindowsReservedFileNames = new Regex( - "(CON|PRN|AUX|NUL|COM[0-9]|LPT[0-9])([.].*)?", - RegexOptions.IgnoreCase - ); - - private static readonly Regex WindowsReservedFileCharacters = new Regex( - "[<>:\"/\\\\|?*\x00-\x1f]", - RegexOptions.IgnoreCase - ); - - private static readonly Regex StripLeadingTrailingWhitespace = new Regex( - "^[\\s]*((?=\\S).*\\S)[\\s]*$" - ); - - internal static string FilterAvatarName(string avatarName) + public SerializationScope OpenSerializationScope() { - avatarName = WindowsReservedFileCharacters.Replace(avatarName, "_"); - - if (WindowsReservedFileNames.IsMatch(avatarName)) - { - avatarName = "_" + avatarName; - } - - var match = StripLeadingTrailingWhitespace.Match(avatarName); - if (match.Success) - { - avatarName = match.Groups[1].Value; - } else { - avatarName = Guid.NewGuid().ToString(); - } - - return avatarName; + return new SerializationScope(AssetSaver); } public bool IsTemporaryAsset(UnityObject obj) { - return !EditorUtility.IsPersistent(obj) - || AssetDatabase.GetAssetPath(obj) == AssetDatabase.GetAssetPath(AssetContainer); + return AssetSaver.IsTemporaryAsset(obj); } public void Serialize() @@ -217,15 +186,12 @@ public void Serialize() try { - AssetDatabase.StartAssetEditing(); - - HashSet _savedObjects = - new HashSet( - AssetDatabase.LoadAllAssetsAtPath(AssetDatabase.GetAssetPath(AssetContainer))); + HashSet _savedObjects = new HashSet(AssetSaver.GetPersistedAssets()); _savedObjects.Remove(AssetContainer); int index = 0; + List assetsToSave = new List(); foreach (var asset in _avatarRootObject.ReferencedAssets(traverseSaved: true, includeScene: false)) { if (asset is MonoScript) @@ -252,7 +218,7 @@ public void Serialize() { try { - AssetDatabase.AddObjectToAsset(asset, AssetContainer); + assetsToSave.Add(asset); } catch (UnityException ex) { @@ -263,17 +229,17 @@ public void Serialize() } } } - - foreach (var assetToHide in AssetDatabase.LoadAllAssetsAtPath( - AssetDatabase.GetAssetPath(AssetContainer))) + + foreach (var assetToHide in AssetSaver.GetPersistedAssets().Concat(assetsToSave)) { - if (assetToHide != AssetContainer && - GeneratedAssetBundleExtractor.IsAssetTypeHidden(assetToHide.GetType())) + if (GeneratedAssetBundleExtractor.IsAssetTypeHidden(assetToHide.GetType())) { assetToHide.hideFlags = HideFlags.HideInHierarchy; } } + AssetSaver.SaveAssets(assetsToSave); + // Remove obsolete temporary assets foreach (var asset in _savedObjects) { @@ -288,10 +254,7 @@ public void Serialize() } finally { - AssetDatabase.StopAssetEditing(); - - // SaveAssets to make sub-assets visible on the Project window - AssetDatabase.SaveAssets(); + AssetSaver.Dispose(); } Profiler.EndSample(); diff --git a/Editor/API/Serialization.meta b/Editor/API/Serialization.meta new file mode 100644 index 0000000..3a788b2 --- /dev/null +++ b/Editor/API/Serialization.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 8db6996355ad4fada3d51972a4257577 +timeCreated: 1732827274 \ No newline at end of file diff --git a/Editor/API/Serialization/AssetSaver.cs b/Editor/API/Serialization/AssetSaver.cs new file mode 100644 index 0000000..abacc7e --- /dev/null +++ b/Editor/API/Serialization/AssetSaver.cs @@ -0,0 +1,164 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using nadena.dev.ndmf.runtime; +using UnityEditor; +using UnityEditor.VersionControl; +using UnityEngine; +using Object = UnityEngine.Object; + +namespace nadena.dev.ndmf +{ + internal class AssetSaver : IAssetSaver + { + private readonly string subAssetPath, rootAssetPath; + private readonly int assetsPerContainer; + + private GeneratedAssets _rootAsset; + private SubAssetContainer _currentSubContainer; + private int _assetCount; + private HashSet _temporaryAssets = new HashSet(); + private List _containers = new(); + + public Object CurrentContainer => _currentSubContainer; + + internal AssetSaver(string generatedAssetsRoot, string avatarName, int assetsPerContainer = 256) + { + this.assetsPerContainer = assetsPerContainer; + + avatarName = FilterAssetName(avatarName, "avatar"); + var rootPath = generatedAssetsRoot + "/" + avatarName; + subAssetPath = rootPath + "/_assets"; + + this.rootAssetPath = rootPath + "/"; + + if (AssetDatabase.IsValidFolder(subAssetPath)) + { + AssetDatabase.DeleteAsset(rootPath); + } + + // Ensure directory exists recursively + if (!AssetDatabase.IsValidFolder(subAssetPath)) + { + Directory.CreateDirectory(subAssetPath); + AssetDatabase.Refresh(); + } + + var rootAssetPath = AssetDatabase.GenerateUniqueAssetPath(rootPath + "/" + avatarName + ".asset"); + _rootAsset = ScriptableObject.CreateInstance(); + AssetDatabase.CreateAsset(_rootAsset, rootAssetPath); + _currentSubContainer = CreateAssetContainer(); + + _assetCount = 0; + } + + public void SaveAsset(UnityEngine.Object? obj) + { + if (obj == null || EditorUtility.IsPersistent(obj)) return; + + _temporaryAssets.Add(obj); + + if (obj is Texture) + { + // Textures can be quite large, so push them off to their own files. + // However, be sure to create them as a subasset, as this appears to be much faster, for some reason... + var texName = FilterAssetName(obj.name, "texture"); + var container = CreateAssetContainer(texName); + + AssetDatabase.AddObjectToAsset(obj, container); + return; + } + + if (_assetCount >= assetsPerContainer) + { + _currentSubContainer = CreateAssetContainer(); + _assetCount = 0; + } + + AssetDatabase.AddObjectToAsset(obj, _currentSubContainer); + _assetCount++; + } + + private SubAssetContainer CreateAssetContainer(string name = "assets") + { + var subContainerPath = AssetDatabase.GenerateUniqueAssetPath(subAssetPath + "/" + name + ".asset"); + var subContainer = ScriptableObject.CreateInstance(); + AssetDatabase.CreateAsset(subContainer, subContainerPath); + + _containers.Add(subContainer); + + return subContainer; + } + + public bool IsTemporaryAsset(UnityEngine.Object? obj) + { + if (obj == null) return true; + + if (_temporaryAssets.Contains(obj) || !EditorUtility.IsPersistent(obj)) + { + return true; + } + + var path = AssetDatabase.GetAssetPath(obj); + if (path.StartsWith(rootAssetPath)) + { + _temporaryAssets.Add(obj); + return true; + } + + return false; + } + + + private static readonly Regex WindowsReservedFileNames = new Regex( + "(CON|PRN|AUX|NUL|COM[0-9]|LPT[0-9])([.].*)?", + RegexOptions.IgnoreCase + ); + + private static readonly Regex WindowsReservedFileCharacters = new Regex( + "[<>:\"/\\\\|?*\x00-\x1f]", + RegexOptions.IgnoreCase + ); + + private static readonly Regex StripLeadingTrailingWhitespace = new Regex( + "^[\\s]*((?=\\S).*\\S)[\\s]*$" + ); + + internal static string FilterAssetName(string assetName, string? fallbackName = null) + { + assetName = WindowsReservedFileCharacters.Replace(assetName, "_"); + + if (WindowsReservedFileNames.IsMatch(assetName)) + { + assetName = "_" + assetName; + } + + var match = StripLeadingTrailingWhitespace.Match(assetName); + if (match.Success) + { + assetName = match.Groups[1].Value; + } else { + assetName = fallbackName ?? Guid.NewGuid().ToString(); + } + + return assetName; + } + + + public IEnumerable GetPersistedAssets() + { + return _temporaryAssets.Where(o => o != null); + } + + public void Dispose() + { + _rootAsset.SubAssets = _containers; + EditorUtility.SetDirty(_rootAsset); + AssetDatabase.SaveAssets(); + } + } +} \ No newline at end of file diff --git a/Editor/API/Serialization/AssetSaver.cs.meta b/Editor/API/Serialization/AssetSaver.cs.meta new file mode 100644 index 0000000..2845892 --- /dev/null +++ b/Editor/API/Serialization/AssetSaver.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: c1095b05ff5545b6b19f8f08e783eb6b +timeCreated: 1732827288 \ No newline at end of file diff --git a/Editor/API/Serialization/IAssetSaver.cs b/Editor/API/Serialization/IAssetSaver.cs new file mode 100644 index 0000000..55d00cc --- /dev/null +++ b/Editor/API/Serialization/IAssetSaver.cs @@ -0,0 +1,64 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using UnityEditor; + +namespace nadena.dev.ndmf +{ + /// + /// This interface allows you to explicitly save temporary assets. This can be useful when writing textures, or when + /// you need to save assets (e.g. animator objects) prior to the automatic serialization pass at the end of processing. + /// + /// The asset saver must be disposed at the end of the avatar build. + /// + public interface IAssetSaver: IDisposable + { + /// + /// Saves an asset immediately. If the asset is already persistent or is null, this function does nothing. + /// + /// The asset to save. + void SaveAsset(UnityEngine.Object? asset); + /// + /// Determines if an asset is temporary and safe to overwrite. Returns true for null. + /// + /// The asset to check. + /// true if the object is non-persistent, or was saved as part of this avatar's processing + bool IsTemporaryAsset(UnityEngine.Object? asset); + /// + /// Returns the current unity object which is being used as a container for assets. May return null if asset + /// saving is disabled. + /// + /// Normally, it's better to use SaveAsset; saving too many assets to the same asset container can be slow. + /// However this property can be used for compatibility with legacy NDMF APIs. + /// + UnityEngine.Object? CurrentContainer { get; } + + /// + /// Returns all assets persisted using this IAssetSaver. + /// + /// an enumerable of assets + IEnumerable GetPersistedAssets(); + + /// + /// Saves a list of assets in batch. This can be more efficient than saving assets one by one. + /// + /// + void SaveAssets(IEnumerable assets) + { + try + { + AssetDatabase.StartAssetEditing(); + + foreach (var asset in assets) + { + SaveAsset(asset); + } + } + finally + { + AssetDatabase.StopAssetEditing(); + } + } + } +} \ No newline at end of file diff --git a/Editor/API/Serialization/IAssetSaver.cs.meta b/Editor/API/Serialization/IAssetSaver.cs.meta new file mode 100644 index 0000000..03e3b45 --- /dev/null +++ b/Editor/API/Serialization/IAssetSaver.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 00f2da8bdac84ef59e0239974ff8e732 +timeCreated: 1732827816 \ No newline at end of file diff --git a/Editor/API/Serialization/NullAssetSaver.cs b/Editor/API/Serialization/NullAssetSaver.cs new file mode 100644 index 0000000..c959633 --- /dev/null +++ b/Editor/API/Serialization/NullAssetSaver.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using UnityEditor; +using UnityEngine; + +namespace nadena.dev.ndmf +{ + public class NullAssetSaver : IAssetSaver + { + public void SaveAsset(Object asset) + { + // no-op + } + + public bool IsTemporaryAsset(Object asset) + { + return !AssetDatabase.Contains(asset); + } + + public Object CurrentContainer => null; + + public IEnumerable GetPersistedAssets() + { + yield break; + } + + public void Dispose() + { + // no-op + } + } +} \ No newline at end of file diff --git a/Editor/API/Serialization/NullAssetSaver.cs.meta b/Editor/API/Serialization/NullAssetSaver.cs.meta new file mode 100644 index 0000000..ecd512a --- /dev/null +++ b/Editor/API/Serialization/NullAssetSaver.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 8c82a17ee691455db39412aa78ddfe71 +timeCreated: 1732827829 \ No newline at end of file diff --git a/Editor/API/Serialization/SerializationScope.cs b/Editor/API/Serialization/SerializationScope.cs new file mode 100644 index 0000000..3b276d2 --- /dev/null +++ b/Editor/API/Serialization/SerializationScope.cs @@ -0,0 +1,47 @@ +using System; +using UnityEditor; + +namespace nadena.dev.ndmf +{ + /// + /// This helper invokes Unity's StartAssetEditing API, and will StopAssetEditing on disposal. + /// + /// It also provides an API to serialize an asset to an asset container; NDMF internally maintains multiple asset + /// container files to balance between the overhead of creating a new asset container and the overhead of adding + /// new assets to existing containers. + /// + public sealed class SerializationScope : IDisposable + { + private static bool _assetEditing; + + private readonly IAssetSaver _assetSaver; + + private bool _wasEditing; + + internal SerializationScope(IAssetSaver saver) + { + _assetSaver = saver; + _wasEditing = _assetEditing; + + if (!_assetEditing) + { + _assetEditing = true; + AssetDatabase.StartAssetEditing(); + } + } + + public void SaveAsset(UnityEngine.Object asset) + { + _assetSaver.SaveAsset(asset); + } + + public void Dispose() + { + if (!_wasEditing) + { + _assetEditing = false; + AssetDatabase.StopAssetEditing(); + } + } + } +} \ No newline at end of file diff --git a/Editor/API/Serialization/SerializationScope.cs.meta b/Editor/API/Serialization/SerializationScope.cs.meta new file mode 100644 index 0000000..4905c8a --- /dev/null +++ b/Editor/API/Serialization/SerializationScope.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 2bf710b093ce43fa8f2d23c8dc7d38e4 +timeCreated: 1732827173 \ No newline at end of file diff --git a/Editor/API/Serialization/SingleAssetSaver.cs b/Editor/API/Serialization/SingleAssetSaver.cs new file mode 100644 index 0000000..94c095b --- /dev/null +++ b/Editor/API/Serialization/SingleAssetSaver.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using UnityEditor; +using UnityEngine; + +namespace nadena.dev.ndmf +{ + public class SingleAssetSaver : IAssetSaver + { + private readonly UnityEngine.Object _container; + + public SingleAssetSaver(UnityEngine.Object container) + { + _container = container; + } + + public void SaveAsset(UnityEngine.Object obj) + { + if (AssetDatabase.Contains(obj)) return; + AssetDatabase.AddObjectToAsset(obj, _container); + } + + public bool IsTemporaryAsset(Object asset) + { + return AssetDatabase.GetAssetPath(asset) == AssetDatabase.GetAssetPath(_container); + } + + public Object CurrentContainer => _container; + + public IEnumerable GetPersistedAssets() + { + return AssetDatabase.LoadAllAssetsAtPath(AssetDatabase.GetAssetPath(_container)); + } + + public void Dispose() + { + AssetDatabase.SaveAssets(); + } + } +} \ No newline at end of file diff --git a/Editor/API/Serialization/SingleAssetSaver.cs.meta b/Editor/API/Serialization/SingleAssetSaver.cs.meta new file mode 100644 index 0000000..ee3afaf --- /dev/null +++ b/Editor/API/Serialization/SingleAssetSaver.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 9aeeaa903be545e0ba67e8c9306ab8a8 +timeCreated: 1732827847 \ No newline at end of file diff --git a/Editor/UI/GeneratedAssetsEditor.cs b/Editor/UI/GeneratedAssetsEditor.cs index 239cb3f..665ebc6 100644 --- a/Editor/UI/GeneratedAssetsEditor.cs +++ b/Editor/UI/GeneratedAssetsEditor.cs @@ -99,7 +99,7 @@ class AssetInfo { public readonly UnityEngine.Object Asset; public readonly HashSet IncomingReferences = new HashSet(); - public readonly HashSet OutgoingReferences = new HashSet(); + public readonly Dictionary> OutgoingReferences = new(); public AssetInfo Root; @@ -134,7 +134,12 @@ public void PopulateReferences(Dictionary assets) var value = prop.objectReferenceValue; if (value != null && assets.TryGetValue(value, out var target)) { - OutgoingReferences.Add(target); + if (!OutgoingReferences.TryGetValue(target, out var fixups)) + { + fixups = new(); + OutgoingReferences[target] = fixups; + } + fixups.Add(prop.propertyPath); target.IncomingReferences.Add(this); } } @@ -145,6 +150,30 @@ public void PopulateReferences(Dictionary assets) } } + public void ApplyFixups() + { + if (OutgoingReferences.Count == 0) return; + + var so = new SerializedObject(Asset); + + foreach (var (target, fixups) in OutgoingReferences) + { + foreach (var fixup in fixups) + { + var prop = so.FindProperty(fixup); + if (prop == null) + { + Debug.LogWarning($"Failed to find property {fixup} on {Asset.name}"); + continue; + } + + prop.objectReferenceValue = target.Asset; + } + } + + so.ApplyModifiedPropertiesWithoutUndo(); + } + public void ForceAssignRoot() { // First, see if we're reachable only from one root. @@ -184,15 +213,7 @@ public void ForceAssignRoot() public static void Unpack(GeneratedAssets bundle) { - try - { - AssetDatabase.StartAssetEditing(); - new GeneratedAssetBundleExtractor(bundle).Extract(); - } - finally - { - AssetDatabase.StopAssetEditing(); - } + new GeneratedAssetBundleExtractor(bundle).Extract(); } @@ -223,43 +244,69 @@ private bool TryAssignRoot(AssetInfo info) internal void Extract() { string path = AssetDatabase.GetAssetPath(Bundle); + var directory = System.IO.Path.GetDirectoryName(path); _unassigned = new HashSet(_assets.Keys); - foreach (var info in _assets.Values) + try { - info.PopulateReferences(_assets); - } + AssetDatabase.StartAssetEditing(); - var queue = new Queue(); - while (_unassigned.Count > 0) - { - // Bootstrap - if (queue.Count == 0) + foreach (var info in _assets.Values) { - _unassigned.Where(o => TryAssignRoot(_assets[o])).ToList().ForEach(o => { queue.Enqueue(o); }); + info.PopulateReferences(_assets); + } + var queue = new Queue(); + while (_unassigned.Count > 0) + { + // Bootstrap if (queue.Count == 0) { - _assets[_unassigned.First()].ForceAssignRoot(); - queue.Enqueue(_unassigned.First()); - } - } + _unassigned.Where(o => TryAssignRoot(_assets[o])).ToList().ForEach(o => { queue.Enqueue(o); }); - while (queue.Count > 0) - { - var next = queue.Dequeue(); - ProcessSingleAsset(directory, next); - _unassigned.Remove(next); + if (queue.Count == 0) + { + _assets[_unassigned.First()].ForceAssignRoot(); + queue.Enqueue(_unassigned.First()); + } + } - foreach (var outgoingReference in _assets[next].OutgoingReferences) + while (queue.Count > 0) { - if (_unassigned.Contains(outgoingReference.Asset) && TryAssignRoot(outgoingReference)) + var next = queue.Dequeue(); + ProcessSingleAsset(directory, next); + _unassigned.Remove(next); + + foreach (var outgoingReference in _assets[next].OutgoingReferences.Keys) { - queue.Enqueue(outgoingReference.Asset); + if (_unassigned.Contains(outgoingReference.Asset) && TryAssignRoot(outgoingReference)) + { + queue.Enqueue(outgoingReference.Asset); + } } } } + + // The above movements can break some inter-asset references. Fix them now before we save assets. + foreach (var asset in _assets.Values) + { + asset.ApplyFixups(); + } + } + finally + { + AssetDatabase.StopAssetEditing(); + AssetDatabase.SaveAssets(); + } + + foreach (var subcontainer in Bundle.SubAssets) + { + var remaining = AssetDatabase.LoadAllAssetsAtPath(AssetDatabase.GetAssetPath(subcontainer)) + .Where(asset => asset != subcontainer).ToList(); + if (remaining.Count > 0) Debug.Log($"Failed to extract: " + string.Join(", ", remaining.Select(o => o.name + " " + o.GetType().Name))); + + AssetDatabase.DeleteAsset(AssetDatabase.GetAssetPath(subcontainer)); } AssetDatabase.DeleteAsset(path); @@ -318,17 +365,22 @@ private void ProcessSingleAsset(string directory, Object next) private static Dictionary GetContainedAssets(GeneratedAssets bundle) { string path = AssetDatabase.GetAssetPath(bundle); - var rawAssets = AssetDatabase.LoadAllAssetsAtPath(path); - Dictionary infos = new Dictionary(rawAssets.Length); + var rawAssets = new List(AssetDatabase.LoadAllAssetsAtPath(path)); + + foreach (var subcontainer in bundle.SubAssets) + { + rawAssets.AddRange(AssetDatabase.LoadAllAssetsAtPath(AssetDatabase.GetAssetPath(subcontainer))); + } + + Dictionary infos = new Dictionary(rawAssets.Count); foreach (var asset in rawAssets) { - if (!(asset is GeneratedAssets)) + if (!(asset is GeneratedAssets or SubAssetContainer)) { infos.Add(asset, new AssetInfo(asset)); } } - - + return infos; } } diff --git a/Runtime/GeneratedAssets.cs b/Runtime/GeneratedAssets.cs index a4c5136..82736b4 100644 --- a/Runtime/GeneratedAssets.cs +++ b/Runtime/GeneratedAssets.cs @@ -5,10 +5,12 @@ namespace nadena.dev.ndmf.runtime { /// - /// This ScriptableObject is used as the root asset when storing generated assets. + /// This ScriptableObject is used as a "main asset", allowing users to repack the assets into a more sensible + /// structure. It contains references to all subassets. /// [PreferBinarySerialization] public class GeneratedAssets : ScriptableObject { + public List SubAssets = new(); } } \ No newline at end of file diff --git a/Runtime/SubAssetContainer.cs b/Runtime/SubAssetContainer.cs new file mode 100644 index 0000000..7cf22c6 --- /dev/null +++ b/Runtime/SubAssetContainer.cs @@ -0,0 +1,12 @@ +using UnityEngine; + +namespace nadena.dev.ndmf.runtime +{ + /// + /// This ScriptableObject is used as the root asset when storing generated assets. + /// + [PreferBinarySerialization] + public class SubAssetContainer : ScriptableObject + { + } +} \ No newline at end of file diff --git a/Runtime/SubAssetContainer.cs.meta b/Runtime/SubAssetContainer.cs.meta new file mode 100644 index 0000000..28c0397 --- /dev/null +++ b/Runtime/SubAssetContainer.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 4ee4713ac71e4f8a911a54de59b4829d +timeCreated: 1732832163 \ No newline at end of file diff --git a/UnitTests~/AvatarNameFilterTests.cs b/UnitTests~/AvatarNameFilterTests.cs index 17465aa..2424b29 100644 --- a/UnitTests~/AvatarNameFilterTests.cs +++ b/UnitTests~/AvatarNameFilterTests.cs @@ -1,6 +1,7 @@ using System; using nadena.dev.ndmf; using NUnit.Framework; +using UnityEditor; namespace UnitTests { @@ -11,51 +12,56 @@ public void TestAvatarNameFilter() { Assert.AreEqual( "foo", - BuildContext.FilterAvatarName("foo") + AssetSaver.FilterAssetName("foo") ); Assert.AreEqual( "_con", - BuildContext.FilterAvatarName("con") + AssetSaver.FilterAssetName("con") ); Assert.AreEqual( "_LPT4", - BuildContext.FilterAvatarName("LPT4") + AssetSaver.FilterAssetName("LPT4") ); Assert.AreEqual( "_AUX.avatar", - BuildContext.FilterAvatarName("AUX.avatar") + AssetSaver.FilterAssetName("AUX.avatar") ); Assert.AreEqual( "foo_bar", - BuildContext.FilterAvatarName("foo/bar") + AssetSaver.FilterAssetName("foo/bar") ); Assert.AreEqual( "foo_bar_baz_quux", - BuildContext.FilterAvatarName("foo\\bar?baz*quux") + AssetSaver.FilterAssetName("foo\\bar?baz*quux") ); Assert.AreEqual( "foo", - BuildContext.FilterAvatarName(" foo") + AssetSaver.FilterAssetName(" foo") ); Assert.AreEqual( "foo", - BuildContext.FilterAvatarName("foo ") + AssetSaver.FilterAssetName("foo ") ); Assert.AreEqual( "f", - BuildContext.FilterAvatarName(" f ") + AssetSaver.FilterAssetName(" f ") ); Assert.AreEqual( Guid.NewGuid().ToString().Length, - BuildContext.FilterAvatarName(" ").Length + AssetSaver.FilterAssetName(" ").Length + ); + + Assert.AreEqual( + "fallback", + AssetSaver.FilterAssetName(" ", "fallback") ); } } diff --git a/UnitTests~/SerializationSweepTest.cs b/UnitTests~/SerializationSweepTest.cs index 8b3a170..4e8b38b 100644 --- a/UnitTests~/SerializationSweepTest.cs +++ b/UnitTests~/SerializationSweepTest.cs @@ -42,10 +42,14 @@ public void testSerialization() bc.Serialize(); var path = AssetDatabase.GetAssetPath(testScriptable1); + var pathParts = path.Split('/'); + Assert.AreEqual("_assets", pathParts[^2]); + var avatarRootPath = string.Join("/", pathParts[..^2]); + Assert.IsFalse(string.IsNullOrEmpty(path)); - Assert.AreEqual(path, AssetDatabase.GetAssetPath(testScriptable2)); - Assert.AreEqual(path, AssetDatabase.GetAssetPath(testScriptable3)); - Assert.AreEqual(path, AssetDatabase.GetAssetPath(testScriptable4)); + Assert.IsTrue(AssetDatabase.GetAssetPath(testScriptable2).StartsWith(avatarRootPath)); + Assert.IsTrue(AssetDatabase.GetAssetPath(testScriptable3).StartsWith(avatarRootPath)); + Assert.IsTrue(AssetDatabase.GetAssetPath(testScriptable4).StartsWith(avatarRootPath)); Assert.IsTrue(string.IsNullOrEmpty(AssetDatabase.GetAssetPath(testComponent))); Assert.IsTrue(string.IsNullOrEmpty(AssetDatabase.GetAssetPath(root)));