From 8e0da2f71f40ec022f22fcc4c07b2b428b7b5287 Mon Sep 17 00:00:00 2001 From: Andreas Herbig Date: Sat, 5 Aug 2023 01:51:10 +0200 Subject: [PATCH] Disable file-backed caching on webGL platform (#37) Hey @StephenHodgson, as mentioned on discord the current implementation of the DownloadCache was running into trouble on the webGL platform. With the changes I implemented here I tried to separate the caching so that the DownloadCache can be replaced depending on the platforms needs. I originally wanted the DownloadCache to already return the assets instead of just returning the URLs to load, so that on webGL there could be a runtime-cache for the textures and audioClips. I decided against that for now, because I would have needed to change more of your existing code which I don't wanna do without your consent. Would be glad if these changes will go live promptly. --------- Co-authored-by: Andreas Herbig Co-authored-by: Stephen Hodgson --- .../Runtime/DiskDownloadCache.cs | 122 +++++++++++++++ .../Runtime/DiskDownloadCache.cs.meta | 3 + .../Runtime/Interfaces/IDownloadCache.cs | 22 +++ .../Runtime/Interfaces/IDownloadCache.cs.meta | 3 + .../Runtime/NoOpDownloadCache.cs | 27 ++++ .../Runtime/NoOpDownloadCache.cs.meta | 3 + .../com.utilities.rest/Runtime/Rest.cs | 144 +++++------------- 7 files changed, 215 insertions(+), 109 deletions(-) create mode 100644 Utilities.Rest/Packages/com.utilities.rest/Runtime/DiskDownloadCache.cs create mode 100644 Utilities.Rest/Packages/com.utilities.rest/Runtime/DiskDownloadCache.cs.meta create mode 100644 Utilities.Rest/Packages/com.utilities.rest/Runtime/Interfaces/IDownloadCache.cs create mode 100644 Utilities.Rest/Packages/com.utilities.rest/Runtime/Interfaces/IDownloadCache.cs.meta create mode 100644 Utilities.Rest/Packages/com.utilities.rest/Runtime/NoOpDownloadCache.cs create mode 100644 Utilities.Rest/Packages/com.utilities.rest/Runtime/NoOpDownloadCache.cs.meta diff --git a/Utilities.Rest/Packages/com.utilities.rest/Runtime/DiskDownloadCache.cs b/Utilities.Rest/Packages/com.utilities.rest/Runtime/DiskDownloadCache.cs new file mode 100644 index 0000000..9dbab90 --- /dev/null +++ b/Utilities.Rest/Packages/com.utilities.rest/Runtime/DiskDownloadCache.cs @@ -0,0 +1,122 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using UnityEngine; +using Utilities.Async; +using Utilities.WebRequestRest.Interfaces; + +namespace Utilities.WebRequestRest +{ + internal class DiskDownloadCache : IDownloadCache + { + /// + /// Generates a based on the string. + /// + /// The string to generate the . + /// A new that represents the string. + private static Guid GenerateGuid(string @string) + { + using MD5 md5 = MD5.Create(); + return new Guid(md5.ComputeHash(Encoding.Default.GetBytes(@string))); + } + + public void ValidateCacheDirectory() + { + if (!Directory.Exists(Rest.DownloadCacheDirectory)) + { + Directory.CreateDirectory(Rest.DownloadCacheDirectory); + } + } + + public async Task ValidateCacheDirectoryAsync() + { + await Awaiters.UnityMainThread; + ValidateCacheDirectory(); + } + + public bool TryGetDownloadCacheItem(string uri, out string filePath) + { + ValidateCacheDirectory(); + bool exists; + + if (uri.Contains(Rest.FileUriPrefix)) + { + filePath = uri; + return File.Exists(uri.Replace(Rest.FileUriPrefix, string.Empty)); + } + + if (Rest.TryGetFileNameFromUrl(uri, out var fileName)) + { + filePath = Path.Combine(Rest.DownloadCacheDirectory, fileName); + exists = File.Exists(filePath); + } + else + { + filePath = Path.Combine(Rest.DownloadCacheDirectory, GenerateGuid(uri).ToString()); + exists = File.Exists(filePath); + } + + if (exists) + { + filePath = $"{Rest.FileUriPrefix}{Path.GetFullPath(filePath)}"; + } + + return exists; + } + + public bool TryDeleteCacheItem(string uri) + { + if (!TryGetDownloadCacheItem(uri, out var filePath)) + { + return false; + } + + try + { + File.Delete(filePath); + } + catch (Exception e) + { + Debug.LogError(e); + } + + return !File.Exists(filePath); + } + + public void DeleteDownloadCache() + { + if (Directory.Exists(Rest.DownloadCacheDirectory)) + { + Directory.Delete(Rest.DownloadCacheDirectory, true); + } + } + + public async Task WriteCacheItemAsync(byte[] data, string cachePath, CancellationToken cancellationToken) + { + if (File.Exists(cachePath)) + { + return; + } + + var fileStream = File.OpenWrite(cachePath); + + try + { + await fileStream.WriteAsync(data, 0, data.Length, cancellationToken); + } + catch (Exception e) + { + Debug.LogError($"Failed to write asset to disk! {e}"); + } + finally + { + await fileStream.DisposeAsync(); + } + } + } +} diff --git a/Utilities.Rest/Packages/com.utilities.rest/Runtime/DiskDownloadCache.cs.meta b/Utilities.Rest/Packages/com.utilities.rest/Runtime/DiskDownloadCache.cs.meta new file mode 100644 index 0000000..e32f71a --- /dev/null +++ b/Utilities.Rest/Packages/com.utilities.rest/Runtime/DiskDownloadCache.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: c79cf124fa5d45a1b39caf8d6c057d14 +timeCreated: 1691048782 \ No newline at end of file diff --git a/Utilities.Rest/Packages/com.utilities.rest/Runtime/Interfaces/IDownloadCache.cs b/Utilities.Rest/Packages/com.utilities.rest/Runtime/Interfaces/IDownloadCache.cs new file mode 100644 index 0000000..f7226b0 --- /dev/null +++ b/Utilities.Rest/Packages/com.utilities.rest/Runtime/Interfaces/IDownloadCache.cs @@ -0,0 +1,22 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; + +namespace Utilities.WebRequestRest.Interfaces +{ + internal interface IDownloadCache + { + void ValidateCacheDirectory(); + + Task ValidateCacheDirectoryAsync(); + + bool TryGetDownloadCacheItem(string uri, out string filePath); + + bool TryDeleteCacheItem(string uri); + + void DeleteDownloadCache(); + + Task WriteCacheItemAsync(byte[] data, string cachePath, CancellationToken cancellationToken); + } +} diff --git a/Utilities.Rest/Packages/com.utilities.rest/Runtime/Interfaces/IDownloadCache.cs.meta b/Utilities.Rest/Packages/com.utilities.rest/Runtime/Interfaces/IDownloadCache.cs.meta new file mode 100644 index 0000000..f9cbc64 --- /dev/null +++ b/Utilities.Rest/Packages/com.utilities.rest/Runtime/Interfaces/IDownloadCache.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 5b070bc88bbb4390921d907c8012d0a5 +timeCreated: 1691048260 \ No newline at end of file diff --git a/Utilities.Rest/Packages/com.utilities.rest/Runtime/NoOpDownloadCache.cs b/Utilities.Rest/Packages/com.utilities.rest/Runtime/NoOpDownloadCache.cs new file mode 100644 index 0000000..a0ed74e --- /dev/null +++ b/Utilities.Rest/Packages/com.utilities.rest/Runtime/NoOpDownloadCache.cs @@ -0,0 +1,27 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; +using Utilities.WebRequestRest.Interfaces; + +namespace Utilities.Rest +{ + internal class NoOpDownloadCache : IDownloadCache + { + public void ValidateCacheDirectory() { } + + public Task ValidateCacheDirectoryAsync() => Task.CompletedTask; + + public bool TryGetDownloadCacheItem(string uri, out string filePath) + { + filePath = uri; + return false; + } + + public bool TryDeleteCacheItem(string uri) => true; + + public void DeleteDownloadCache() { } + + public Task WriteCacheItemAsync(byte[] data, string cachePath, CancellationToken cancellationToken) => Task.CompletedTask; + } +} diff --git a/Utilities.Rest/Packages/com.utilities.rest/Runtime/NoOpDownloadCache.cs.meta b/Utilities.Rest/Packages/com.utilities.rest/Runtime/NoOpDownloadCache.cs.meta new file mode 100644 index 0000000..d881bd3 --- /dev/null +++ b/Utilities.Rest/Packages/com.utilities.rest/Runtime/NoOpDownloadCache.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: d3d49896318c45d6853ad1e32ce8b550 +timeCreated: 1691055806 \ No newline at end of file diff --git a/Utilities.Rest/Packages/com.utilities.rest/Runtime/Rest.cs b/Utilities.Rest/Packages/com.utilities.rest/Runtime/Rest.cs index d999289..b188989 100644 --- a/Utilities.Rest/Packages/com.utilities.rest/Runtime/Rest.cs +++ b/Utilities.Rest/Packages/com.utilities.rest/Runtime/Rest.cs @@ -5,13 +5,14 @@ using System.IO; using System.Linq; using System.Runtime.CompilerServices; -using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; using UnityEngine; using UnityEngine.Networking; using Utilities.Async; +using Utilities.Rest; +using Utilities.WebRequestRest.Interfaces; using Debug = UnityEngine.Debug; namespace Utilities.WebRequestRest @@ -21,8 +22,8 @@ namespace Utilities.WebRequestRest /// public static class Rest { + internal const string FileUriPrefix = "file://"; private const string kHttpVerbPATCH = "PATCH"; - private const string fileUriPrefix = "file://"; private const string eventDelimiter = "data: "; private const string stopEventDelimiter = "[DONE]"; @@ -349,15 +350,25 @@ public static async Task DeleteAsync( private const string DOWNLOAD_CACHE = "download_cache"; - /// - /// Generates a based on the string. - /// - /// The string to generate the . - /// A new that represents the string. - private static Guid GenerateGuid(string @string) + private static IDownloadCache cache; + + private static IDownloadCache Cache { - using MD5 md5 = MD5.Create(); - return new Guid(md5.ComputeHash(Encoding.Default.GetBytes(@string))); + get + { + if (cache != null) + { + return cache; + } + + cache = Application.platform switch + { + RuntimePlatform.WebGLPlayer => new NoOpDownloadCache(), + _ => new DiskDownloadCache() + }; + + return cache; + } } /// @@ -370,21 +381,13 @@ public static string DownloadCacheDirectory /// Creates the if it doesn't exist. /// public static void ValidateCacheDirectory() - { - if (!Directory.Exists(DownloadCacheDirectory)) - { - Directory.CreateDirectory(DownloadCacheDirectory); - } - } + => Cache.ValidateCacheDirectory(); /// /// Creates the if it doesn't exist. /// - public static async Task ValidateCacheDirectoryAsync() - { - await Awaiters.UnityMainThread; - ValidateCacheDirectory(); - } + public static Task ValidateCacheDirectoryAsync() + => Cache.ValidateCacheDirectoryAsync(); /// /// Try to get a file out of the download cache by uri reference. @@ -393,34 +396,7 @@ public static async Task ValidateCacheDirectoryAsync() /// The file path to the cached item. /// True, if the item was in cache, otherwise false. public static bool TryGetDownloadCacheItem(string uri, out string filePath) - { - ValidateCacheDirectory(); - bool exists; - - if (uri.Contains(fileUriPrefix)) - { - filePath = uri; - return File.Exists(uri.Replace(fileUriPrefix, string.Empty)); - } - - if (TryGetFileNameFromUrl(uri, out var fileName)) - { - filePath = Path.Combine(DownloadCacheDirectory, fileName); - exists = File.Exists(filePath); - } - else - { - filePath = Path.Combine(DownloadCacheDirectory, GenerateGuid(uri).ToString()); - exists = File.Exists(filePath); - } - - if (exists) - { - filePath = $"{fileUriPrefix}{Path.GetFullPath(filePath)}"; - } - - return exists; - } + => Cache.TryGetDownloadCacheItem(uri, out filePath); /// /// Try to delete the cached item at the uri. @@ -428,34 +404,13 @@ public static bool TryGetDownloadCacheItem(string uri, out string filePath) /// The uri key of the item. /// True, if the cached item was successfully deleted. public static bool TryDeleteCacheItem(string uri) - { - if (!TryGetDownloadCacheItem(uri, out var filePath)) - { - return false; - } - - try - { - File.Delete(filePath); - } - catch (Exception e) - { - Debug.LogError(e); - } - - return !File.Exists(filePath); - } + => Cache.TryDeleteCacheItem(uri); /// /// Deletes all the files in the download cache. /// public static void DeleteDownloadCache() - { - if (Directory.Exists(DownloadCacheDirectory)) - { - Directory.Delete(DownloadCacheDirectory, true); - } - } + => Cache.DeleteDownloadCache(); /// /// We will try go guess the name based on the url. @@ -466,7 +421,7 @@ public static void DeleteDownloadCache() public static bool TryGetFileNameFromUrl(string url, out string fileName) { var baseUrl = UnityWebRequest.UnEscapeURL(url); - var rootUrl = baseUrl.Split("?")[0]; + var rootUrl = baseUrl.Split('?')[0]; var index = rootUrl.LastIndexOf('/') + 1; fileName = rootUrl.Substring(index, rootUrl.Length - index); return Path.HasExtension(fileName); @@ -498,7 +453,7 @@ public static async Task DownloadTextureAsync( bool isCached; string cachePath; - if (url.Contains(fileUriPrefix)) + if (url.Contains(FileUriPrefix)) { isCached = true; cachePath = url; @@ -526,23 +481,9 @@ public static async Task DownloadTextureAsync( var downloadHandler = (DownloadHandlerTexture)webRequest.downloadHandler; - if (!isCached && - !File.Exists(cachePath)) + if (!isCached) { - var fileStream = File.OpenWrite(cachePath); - - try - { - await fileStream.WriteAsync(downloadHandler.data, 0, downloadHandler.data.Length, cancellationToken); - } - catch (Exception e) - { - Debug.LogError($"Failed to write texture to disk!\n{e}"); - } - finally - { - await fileStream.DisposeAsync(); - } + await Cache.WriteCacheItemAsync(downloadHandler.data, cachePath, cancellationToken); } await Awaiters.UnityMainThread; @@ -578,7 +519,7 @@ public static async Task DownloadAudioClipAsync( bool isCached; string cachePath; - if (url.Contains(fileUriPrefix)) + if (url.Contains(FileUriPrefix)) { isCached = true; cachePath = url; @@ -605,23 +546,9 @@ public static async Task DownloadAudioClipAsync( var downloadHandler = (DownloadHandlerAudioClip)webRequest.downloadHandler; - if (!isCached && - !File.Exists(cachePath)) + if (!isCached) { - var fileStream = File.OpenWrite(cachePath); - - try - { - await fileStream.WriteAsync(downloadHandler.data, 0, downloadHandler.data.Length, cancellationToken); - } - catch (Exception e) - { - Debug.LogError($"Failed to write audio asset to disk! {e}"); - } - finally - { - await fileStream.DisposeAsync(); - } + await Cache.WriteCacheItemAsync(downloadHandler.data, cachePath, cancellationToken); } await Awaiters.UnityMainThread; @@ -664,7 +591,7 @@ public static async Task StreamAudioAsync( TryGetFileNameFromUrl(url, out fileName); } - if (url.Contains(fileUriPrefix)) + if (url.Contains(FileUriPrefix)) { // override the httpMethod httpMethod = UnityWebRequest.kHttpVerbGET; @@ -898,7 +825,6 @@ public static async Task DownloadFileAsync( if (!response.Successful) { Debug.LogError($"Failed to download file from \"{url}\"!\n[{response.Code}] {response.Body}"); - return null; }