-
Notifications
You must be signed in to change notification settings - Fork 1
/
assets-sync.js
1632 lines (1441 loc) · 65.6 KB
/
assets-sync.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* Copyright (C) 2021 - The Forge VTT Inc.
* Author: Evan Clarke
* Youness Alaoui <[email protected]>
* Arcanist <[email protected]>
* This file is part of The Forge VTT.
*
* All Rights Reserved
*
* NOTICE: All information contained herein is, and remains
* the property of The Forge VTT. The intellectual and technical concepts
* contained herein are proprietary of its author and may be covered by
* U.S. and Foreign Patents, and are protected by trade secret or copyright law.
* Dissemination of this information or reproduction of this material
* is strictly forbidden unless prior written permission is obtained
* from the author.
*/
/**
* @class ForgeAssetSync
* Worker-class to reconcile Forge assets with local (ie. Foundry server) assets and download any missing files.
* A worker should be instantiated anytime a unique reconciliation is needed.
*/
class ForgeAssetSync {
/**
* Sync Status enum
* @returns {Object} STATUSES
*/
static SYNC_STATUSES = {
READY: `Ready to Sync`,
PREPARING: `Preparing Assets for Sync`,
NOKEY: `Missing API Key. Please enter key in Module Settings then try again`,
SYNCING: `Syncing Assets...`,
POSTSYNC: `Cleaning up Sync Data`,
DBREWRITE: `Updating game to use local assets...`,
COMPLETE: `Sync Completed Successfully!`,
WITHERRORS: `Sync Completed with Errors. See below for folders and files which could not be synced. Check console for more details.`,
FAILED: `Failed to Sync. Check console for more details.`,
UNAUTHORIZED: `Unauthorized. Please check your API Key and try again.`,
CANCELLED: `Sync process Cancelled`,
};
constructor(app=null, {forceLocalRehash=false, overwriteLocalMismatches=false, updateFoundryDb=false}={}) {
// Number of retries to perform for error-prone operations
this.retries = 2;
// Array of matched local files
this.localFiles = null;
// Object containing local files and dirs
this.localInventory = null;
// Set containing missing local dirs
this.missingDirs = null;
// Root path of the API key, to prefix synced assets with
this.apiKeyPath = null;
// Map of Name=>ForgeAssetSyncMapping
this.assetMap = null;
// Map of Hash=>etag[]
this.etagMap = null;
// Dictates whether all local files should be rehashed
this.forceLocalRehash = forceLocalRehash;
// Dictates if local files with mismatched hashes should be overwrittent
this.overwriteLocalMismatches = overwriteLocalMismatches;
// Dictates if the local game database needs to be rewritten to use local assets
this.updateFoundryDb = updateFoundryDb;
// Holds the current syncing status
this.status = ForgeAssetSync.SYNC_STATUSES.READY;
// Array of Assets that successfully synced
this.syncedAssets = [];
// Array of Assets that failed to sync
this.failedAssets = [];
// Array of Folders that weren't able to be created
this.failedFolders = [];
// Reference the current Sync App instance
this.app = app;
}
/**
* Sets the sync status and updates the sync app
* @param {String} status
*/
async setStatus(status) {
this.status = status;
if (this.app) return this.app.updateStatus(status);
}
/**
* Sync orchestrator method
*/
async sync() {
/* ------------------------------- Preparation ------------------------------ */
if (this.status !== ForgeAssetSync.SYNC_STATUSES.READY) throw new Error(`Sync already started`);
await this.setStatus(ForgeAssetSync.SYNC_STATUSES.PREPARING);
// 1. Does user have an API token set?
const apiKey = game.settings.get("forge-vtt", "apiKey");
if (!apiKey || !apiKey?.length || !ForgeAPI.isValidAPIKey(apiKey)) {
await this.setStatus(ForgeAssetSync.SYNC_STATUSES.NOKEY);
throw Error("Forge VTT | Asset Sync: please set a valid API Key in Settings before attempting to sync!");
}
// Get the root path of the API key
const apiKeyInfo = ForgeAPI._tokenToInfo(apiKey);
const rootDir = apiKeyInfo?.keyOptions?.assets?.rootDir ?? null;
this.apiKeyPath = rootDir && rootDir !== "/" ? rootDir : null;
if (this.apiKeyPath) {
console.log(`Forge VTT | Asset Sync: API key references root folder ${this.apiKeyPath}`);
}
// logging/notification
console.log("Forge VTT | Asset Sync: starting sync");
// 2. get the existing mapping
const {etagMap, assetMap} = await ForgeAssetSync.fetchAssetMap({retries: this.retries});
this.assetMap = assetMap;
this.etagMap = etagMap;
if (this.status === ForgeAssetSync.SYNC_STATUSES.CANCELLED) return;
/* ----------------------------- Reconciliation ----------------------------- */
// 1. get Forge inventory
const {forgeDirMap, forgeFileMap} = await this.buildForgeInventory();
const forgeDirSet = new Set(forgeDirMap.keys());
const forgeFileSet = new Set(forgeFileMap.keys());
if (this.status === ForgeAssetSync.SYNC_STATUSES.CANCELLED) return;
// 2. Build Local inventory
this.localInventory = await this.buildLocalInventory(forgeDirSet);
const {localDirSet, localFileSet} = this.localInventory;
if (this.status === ForgeAssetSync.SYNC_STATUSES.CANCELLED) return;
// look for missing dirs
const missingDirs = ForgeAssetSync.reconcileSets(forgeDirSet, localDirSet);
let createdDirCount = 0;
this.app.updateProgress({current: 0, name: "", total: missingDirs.size, step: "Creating missing folders", type: "Folder"});
for (const dir of missingDirs) {
const createdDir = await this.createDirectory(dir, {retries: this.retries});
if (this.status === ForgeAssetSync.SYNC_STATUSES.CANCELLED) {
return this.updateMapFile();
}
if (createdDir) createdDirCount += createdDir;
this.app.updateProgress({current: createdDirCount, name: dir});
}
// Update local inventory and re-reconcile
const {localDirSet: updatedLocalDirSet, localFileSet: _updatedLocalFileSet} = await this.buildLocalInventory(forgeDirSet);
this.missingDirs = ForgeAssetSync.reconcileSets(forgeDirSet, updatedLocalDirSet);
this.failedFolders = Array.from(this.missingDirs);
if (this.status === ForgeAssetSync.SYNC_STATUSES.CANCELLED) {
return this.updateMapFile();
}
this.setStatus(ForgeAssetSync.SYNC_STATUSES.SYNCING);
const {synced, failed} = await this.assetSyncProcessor(forgeFileMap, localFileSet);
// logging/notification
console.log(`Forge VTT | Asset Sync complete. ${synced.length + failed.length} assets processed.${failed?.length ? ` ${failed.length} failed to sync` : ``}`);
if (this.status === ForgeAssetSync.SYNC_STATUSES.CANCELLED) {
return this.updateMapFile();
}
this.setStatus(ForgeAssetSync.SYNC_STATUSES.POSTSYNC);
// update map
await this.updateMapFile();
if (this.status === ForgeAssetSync.SYNC_STATUSES.CANCELLED) return;
// Update Foundry World & Compendiums to use local assets
let rewriteErrors = false;
if (this.updateFoundryDb) {
this.setStatus(ForgeAssetSync.SYNC_STATUSES.DBREWRITE);
const migration = new WorldMigration(this.app, forgeFileMap);
const success = await migration.migrateWorld();
if (!success) {
rewriteErrors = true;
new Dialog({
title: "World database conversion",
content: migration.errorMessage,
buttons: {ok: {label: "OK"}}
}, {width: 700}).render(true);
}
}
if (synced.length) {
if (failed.length || rewriteErrors) {
return this.setStatus(ForgeAssetSync.SYNC_STATUSES.WITHERRORS);
}
} else if (failed.length || rewriteErrors) {
return this.setStatus(ForgeAssetSync.SYNC_STATUSES.FAILED);
}
this.setStatus(ForgeAssetSync.SYNC_STATUSES.COMPLETE);
}
async cancelSync() {
this.setStatus(ForgeAssetSync.SYNC_STATUSES.CANCELLED);
}
async updateMapFile() {
const mapFileData = this.buildAssetMapFileData();
return ForgeAssetSync.uploadAssetMapFile(mapFileData);
}
/**
* Processes syncing tasks for a given Forge assets
* @param {Map} assets
* @param {Set} localFiles
* @returns {Promise<Object>} Object containing synced and failed assets
*/
async assetSyncProcessor(assets, localFiles) {
if (!assets || !localFiles) {
throw Error(`Forge VTT | Asset Sync could not process assets`);
}
let assetIndex = 1;
this.app.updateProgress({current: 0, name: "", total: assets.size, step: "Synchronizing assets", type: "Asset"});
for (const [_key, asset] of assets) {
if (this.status === ForgeAssetSync.SYNC_STATUSES.CANCELLED) break;
try {
// Check if there is a local file match for this asset
const localFileExists = localFiles.has(asset.name);
const targetDir = localFileExists || asset.name.split("/").slice(0, -1).join("/") + "/";
const localDirMissing = localFileExists || this.missingDirs.has(targetDir);
// console.log(`Attempting to sync \"${asset.name}\" to \"${targetDir}\", which ${localFileExists ? "exists" : "doesn't exist"} locally.`);
let result;
// If there is, jump to the reconcile method
if (localFileExists) {
result = await this.reconcileLocalMatch(asset);
} else if (localDirMissing) {
// If the local directory couldn't be created, treat it as a failed sync
result = false;
} else {
// If not, the asset needs to be fully synced
result = await this.syncAsset(asset);
}
this.app.updateProgress({current: assetIndex, name: asset.name});
// If all is good, mark the asset as synced
if (!!result)
this.syncedAssets.push(asset);
else
this.failedAssets.push(asset);
} catch (error) {
console.warn(error);
// If any errors occured mark the asset as failed and move on
this.failedAssets.push(asset);
}
assetIndex++;
}
return {synced: this.syncedAssets, failed: this.failedAssets};
}
_updateAssetMapping(asset, etag) {
const assetMapping = this.assetMap.get(asset.name);
const newMapRow = ForgeAssetSync.buildMappingRow(asset, {etag: etag, hash: asset.hash});
if (assetMapping) newMapRow.firstSyncDate = assetMapping.firstSyncDate;
this.assetMap.set(asset.name, newMapRow);
}
_updateEtagMapping(hash, etag) {
const etagValues = this.etagMap.get(hash) || new Set();
etagValues.add(etag);
this.etagMap.set(hash, etagValues);
}
/**
* Verifies that the local file corresponding to an asset matches the expected hash from the remote asset
*
* @returns true if they match
*/
async _verifyLocalFileMatchesRemote(asset, expectedEtag, etag) {
const etagValues = this.etagMap.get(asset.hash) || new Set();
// Verify if the local file has one of the expected etags for this asset
if (expectedEtag === etag || etagValues.has(etag)) {
return true;
} else if (!expectedEtag) {
// We don't know the exact etag to expect, so let's hash the file instead
const localHash = await ForgeAssetSync.fetchLocalHash(asset.name);
// Save the etag for this hash
this._updateEtagMapping(localHash, etag);
// Verify if the hashes match
if (localHash === asset.hash) {
return true;
}
}
return false;
}
/**
* Reconcile a Forge Asset with a Local File
* If local file exists, then it's either from a previous sync and needs to be ignored/updated,
* or it's not from a previous sync and needs to be ignored/overwritten
* @param {*} asset
* @returns {Promise<Boolean>} Asset Reconciled
*/
async reconcileLocalMatch(asset) {
// Get mapped data for this asset
const assetMapping = this.assetMap.get(asset.name);
const expectedEtag = assetMapping && assetMapping.localHash === asset.hash ? assetMapping.localEtag : null;
// If an asset mapping exists, then the file comes from a previous sync
if (!this.forceLocalRehash && assetMapping) {
// Asset has not been modified remotely, don't do anything
if (assetMapping.forgeHash === asset.hash) return true;
// Asset has been modified remotely, compare with local hash
const etag = await ForgeAssetSync.fetchLocalEtag(asset.name);
// If the local file is the same as the remote, then no need to sync
const matches = await this._verifyLocalFileMatchesRemote(asset, expectedEtag, etag);
if (matches) {
this._updateAssetMapping(asset, etag);
return true;
}
if (!this.overwriteLocalMismatches) {
// If local etag changed, file was modified locally, do not overwrite
if (etag !== assetMapping.localEtag) {
console.log(`Conflict detected: ${asset.name} has been modified both locally and remotely`);
return false;
}
}
return this.syncAsset(asset);
} else {
// File doesn't come from a previous sync (or we force a local rehash)
// If we're not overwriting local files and we're not re-hashing existing ones, then we're done
if (!this.forceLocalRehash && !this.overwriteLocalMismatches) return false;
const etag = await ForgeAssetSync.fetchLocalEtag(asset.name);
// If the local file is the same as the remote, then consider it 'synced' and keep track of it for the future
const matches = await this._verifyLocalFileMatchesRemote(asset, expectedEtag, etag);
if (matches) {
this._updateAssetMapping(asset, etag);
return true;
}
// If the local file is different from the remote, then overwrite or mark as conflict
if (this.overwriteLocalMismatches || (assetMapping && etag === assetMapping.localEtag)) {
return this.syncAsset(asset);
} else {
// If local etag changed, file was modified locally, do not overwrite
console.log(`Conflict detected: ${asset.name} exists locally and is different from the expected remote asset`);
return false;
}
}
}
/**
* Fully sync a Forge asset
* 1. Download the Asset blob
* 2. Upload the Asset to Foundry server
* 3. Fetch the new file's etag
* 4. Update asset map
* 5. Update etag map
* @param {*} asset
*/
async syncAsset(asset) {
if (asset.name && this.failedFolders.some(f => asset.name.startsWith(f)))
throw new Error(`Forge VTT | Could not upload ${asset.name} because the path contains invalid characters.`);
const assetMap = this.assetMap;
const etagMap = this.etagMap;
// Fetch the Forge asset blob
const blob = await ForgeAssetSync.fetchBlobFromUrl(asset.url, {retries: this.retries});
// Upload to Foundry
const upload = await ForgeAssetSync.uploadAssetToFoundry(asset, blob);
// Catch issues where upload is not valid, or it's an empty object
if (!upload || (typeof upload === "object" && Object.keys(upload).length === 0))
return false;
// Fetch the etag of the uploaded file
const etag = await ForgeAssetSync.fetchLocalEtag(asset.name);
// Insert/Update Asset Mapping row
const localFileData = {etag: etag, hash: asset.hash};
const assetMapRow = assetMap.get(asset.name);
const newAssetMapRow = ForgeAssetSync.buildMappingRow(asset, localFileData);
if (assetMapRow) newAssetMapRow.firstSyncDate = assetMapRow.firstSyncDate;
assetMap.set(newAssetMapRow.forgeName, newAssetMapRow);
// Insert/Update a Etag mapping Row
const etagValues = etagMap.get(asset.hash) || new Set();
etagValues.add(etag);
etagMap.set(asset.hash, etagValues);
return true;
}
/**
* Creates a new ForgeAssetSyncMapping instance with the provided data
* @todo error handling
* @param {*} asset
*/
static buildMappingRow(assetData={}, localFileData={}) {
return {
forgeName: assetData?.name,
forgeHash: assetData?.hash,
localEtag: localFileData?.etag,
localHash: localFileData?.hash,
firstSyncDate: new Date(),
lastSyncDate: new Date()
};
}
// compare inventories
/**
* Reconcile a source inventory with the target
* Returns any missing assets
* @param {Set} source A Set of keys from an Asset Inventory
* @param {Set} target A Set of keys from an Asset Inventory
* @returns {Set} a Set of missing assets
*/
static reconcileSets(source, target) {
if (!source || !target) {
return;
}
const missing = new Set();
// iterate through the source
for (const item of source) {
if (!target.has(item)) {
missing.add(item);
}
}
return missing;
}
/**
* Call the Forge API and retrieve asset list
* @returns {Array} Array of Forge Assets
*/
static async getForgeAssets() {
const assetsResponse = await ForgeAPI.call("assets");
if (!assetsResponse || assetsResponse.error) {
const error = assetsResponse?.error || `Forge VTT | Asset Sync: Unknown error occurred communicating with the Forge API`;
ui.notifications.error(error);
throw new Error(error);
}
if (assetsResponse.assets.length === 0) {
ui.notifications.warn(`You have no assets in your Forge Assets Library`);
}
return assetsResponse.assets;
}
/**
* Build Forge file inventory, keyed on Name (Asset Path)
* @returns {Promise<Object>} Object containing two Maps of Name=>ForgeAsset -- one for dirs and one for files
*/
async buildForgeInventory() {
const forgeAssets = await ForgeAssetSync.getForgeAssets();
if (!forgeAssets) throw new Error("Could not error Forge VTT Assets Library content");;
const forgeDirMap = new Map();
const forgeFileMap = new Map();
for (const asset of forgeAssets) {
if (!asset.name) continue;
asset.name = `${this.apiKeyPath ? this.apiKeyPath : ""}${asset.name}`;
// Remove leading and multiple slashes, if they exist
asset.name = asset.name.replace(/^\/+/g, "").replace(/\/+/g, "/");
asset.name = ForgeAssetSync.sanitizePath(asset.name);
if (asset.name.endsWith("/")) forgeDirMap.set(asset.name, asset);
else forgeFileMap.set(asset.name, asset);
}
return {forgeDirMap, forgeFileMap};
}
/**
* Build local inventory of dirs and files based on a set of reference assets (eg. Forge assets)
* @returns {Promise<Object>} LocalInventory -- contains a list of dirs and a list of file paths
*/
async buildLocalInventory(referenceDirs) {
referenceDirs = referenceDirs || new Set();
// Add the sanitized root dir to the reference list
// apiKeyPath indicates that a specific folder was shared, and acts as root dir
const rootDir = this.apiKeyPath ? this.apiKeyPath : "/";
const rootDirSanitized = ForgeAssetSync.sanitizePath(rootDir);
referenceDirs.add(rootDirSanitized);
const localFileSet = new Set();
const localDirSet = new Set();
let dirIndex = 1;
this.app.updateProgress({current: 0, name: "", total: referenceDirs.size, step: "Listing local files", type: "Folder"});
// use filepicker.browse to check in the paths provided in referenceassets
for (const dir of referenceDirs) {
try {
const fp = await FilePicker.browse("data", encodeURIComponent(dir));
this.app.updateProgress({current: dirIndex, name: dir});
dirIndex++;
if (!fp || decodeURIComponent(fp.target) !== dir) continue;
localDirSet.add(dir);
fp.files.forEach(f => localFileSet.add(decodeURIComponent(f)));
} catch (error) {
const errorMessage = error.message || error;
if (errorMessage?.match("does not exist")) continue;
else throw Error(error);
}
}
return { localDirSet, localFileSet };
}
/**
* Use Fetch API to get the etag header from a local file
* @param {*} path
*/
static async fetchLocalEtag(path) {
const headers = new Headers();
let etag;
try {
const request = await fetch(`/${encodeURL(path)}`, {
method: "HEAD",
headers
});
etag = request?.headers?.get("etag");
} catch (error) {
console.warn(error);
return;
}
return etag;
}
/**
* Get the hash for a local file
* @param {String} path
*/
static async fetchLocalHash(path) {
try {
const request = await fetch(`/${path}`, {
method: "GET",
headers: new Headers()
});
if (!request.ok) {
if (request.status === 404) {
throw new Error(`Asset ${path} not found`);
}
throw new Error(`An error occurred fetching this asset: ${path}`);
}
const blob = await request?.blob();
if (blob) return await ForgeVTT_FilePicker.etagFromFile(blob);
} catch(error) {
console.warn(error);
}
}
// build etag inventory
// check/update mapping
/**
* Fetch the Asset Map file from the Foundry server
* @todo maybe implement a proper custom exception method to throw and catch?
*/
static async fetchAssetMap({retries=0}={}) {
let errorType = null;
let assetMap = new Map();
let etagMap = new Map();
try {
const request = await fetch("/forge-assets.json");
if (!request.ok) {
switch (request.status) {
case 404:
errorType = `notfound`;
throw new Error(`Forge VTT | Asset Mapping file not found, but will be created with a successful sync.`);
default:
// @todo error handling -- maybe retry?
errorType = `unknown`;
throw new Error(`Forge VTT | Server error retrieving Forge Asset map!${retries ? ` Retrying...` : ``}`);
}
}
const mapJson = await request.json().catch(err => null);
if (!mapJson || (!mapJson.etags && !mapJson.assets)) {
errorType = `empty`;
throw new Error("Forge VTT | Asset Mapping file is empty.");
}
for (const row of mapJson.etags) {
try {
etagMap.set(row.hash, new Set(row.etags));
} catch (err) {}
}
for (const asset of mapJson.assets) {
assetMap.set(asset.forgeName, asset);
}
} catch(error) {
switch (errorType) {
case `notfound`:
case `empty`:
console.log(error);
break;
case `unknown`:
console.warn(error);
if (retries > 0) {
return ForgeAssetSync.fetchAssetMap({retries: retries - 1});
}
default:
throw new Error(error);
}
}
return {etagMap, assetMap};
}
/**
* Constructs an object for casting into a JSON and uploading
* @returns {Object} Mapping File Data
*/
buildAssetMapFileData() {
// Coerce Asset/etag Maps into arrays for JSON-ifying
const assetMapArray = this.assetMap instanceof Map ? [...this.assetMap.values()] : (this.assetMap instanceof Array ? this.assetMap : []);
const etagMapArray = [];
for (const [key, value] of this.etagMap) {
try {
if (key) etagMapArray.push({hash: key, etags: Array.from(value)});
} catch (err) {}
}
return {assets: assetMapArray, etags: etagMapArray}
}
/**
* Upload Asset Mapping file from the provided map.
*/
static async uploadAssetMapFile(fileData) {
if (!fileData) {
return false;
}
const fileName = "forge-assets.json";
const fileType = "application/json";
const file = new File([JSON.stringify(fileData, null, 2)], fileName, {type: fileType});
try {
const result = FilePicker.upload("data", "/", file, {}, {notify: false});
console.log(`Forge VTT | Asset mapping file upload succeeded.`)
return result;
} catch (error) {
console.warn(`Forge VTT | Asset mapping file upload failed. Please try sync again.`)
return false;
}
}
/**
* Download a single asset blob from its URL
* @param {URL} url - the URL of the Asset to fetch a Blob of
* @returns {Promise<Blob>} A Promise resolving to the Asset's Blob
*/
static async fetchBlobFromUrl(url, {retries=0}={}) {
if (!url) throw new Error(`Forge VTT | Asset Sync: no URL provided for Blob download`);
try {
const imageExtensions = isNewerVersion(ForgeVTT.foundryVersion, "9.0") ? Object.keys(CONST.IMAGE_FILE_EXTENSIONS) : CONST.IMAGE_FILE_EXTENSIONS;
const isImage = imageExtensions.some(e => url.endsWith(e));
const queryParams = isImage ? `?optimizer=disabled` : ``;
const request = await fetch(`${url}${queryParams}`, { mode: "cors" });
if (!request.ok) {
throw new Error(`Forge VTT | Failed to download asset file from The Forge`);
}
return await request.blob();
} catch(error) {
console.warn(error);
if (retries > 0) return ForgeAssetSync.fetchBlobFromUrl(url, {retries: retries - 1});
}
}
/**
* Upload an Asset to Foundry
* @param {ForgeAsset} asset
* @param {Blob} blob
*/
static async uploadAssetToFoundry(asset, blob) {
if (!asset) throw new Error(`Forge VTT | No Asset provided for uploading to Foundry.`);
if (!asset.name) throw new Error(`Forge VTT | Asset with URL ${asset.url} has no name and cannot be uploaded.`);
if (asset.name.endsWith("/")) throw new Error(`Forge VTT | Asset with URL ${asset.url} appears to be a folder.`);
if (!blob) throw new Error(`Forge VTT | No Blob data provided for ${asset.name} and therefore it cannot be uploaded to Foundry.`);
try {
const nameParts = asset.name.split("/");
const fileName = nameParts.pop();
const path = `/${nameParts.join("/")}`;
const file = new File([blob], fileName, {type: blob.type});
const upload = await FilePicker.upload("data", path, file, {}, {notify: false});
return upload;
} catch (error) {
console.warn(error);
}
}
/**
* For a given path, create the recursive directory tree necessary to reach the path
* @param {String} path
*/
async createDirectory(path, {retries=0}={}) {
path = path.replace(/\/+$|^\/+/g, "").replace(/\/+/g, "/");
const pathParts = path.split("/");
let created = 0;
for (let i = 0; i < pathParts.length; i++) {
const subPath = pathParts.slice(0, i + 1).join("/") + "/";
if (this.failedFolders.includes(subPath)) {
return false;
}
const pathExists = this.localPathExists(subPath);
if (!pathExists) {
try {
await FilePicker.createDirectory("data", encodeURIComponent(subPath));
this.localInventory.localDirSet.add(subPath);
created++;
continue; // Don't return yet, we may still need to check the rest of the path
} catch (error) {
const message = error.message ?? error;
if (message.includes("EEXIST:")) {
// There might be a case where the folder already exists, especially in the case of Windows
// where the case sensitivity could cause folder `music` to be created and `Music` to fail because
// it already exists.
this.localInventory.localDirSet.add(subPath);
continue; // Don't return yet, we may still need to check the rest of the path
} else if(message.includes("EINVAL:")) {
// If there's an invalid character in the directory to be created, then ignore this directory
// since the OS can't create the directory. And attempting to alter the character to something
// else could lead to a whole host of issues
this.failedFolders.push(subPath);
return created;
}
console.warn(error);
if (retries > 0) return created + this.createDirectory(path, {retries: retries - 1});
else return created;
}
}
}
return created;
}
/**
* Checks for the existence of a local path using the provided comparison path
* @param {String} path
* @returns {Boolean} pathExists
*/
localPathExists(path) {
return !!(this.localInventory?.localDirSet?.has(path) || this.localInventory?.localFileSet?.has(path));
}
/**
* Sanitises a given path, removing extraneous slashes and other problematic characters for Windows OS
* @see https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
* @param {*} path
* @returns
*/
static sanitizePath(path) {
path = path.replace(/\:+/g, "_58_")
.replace(/\<+/g, "_60_")
.replace(/\>+/g, "_62_")
.replace(/\"+/g, "_34_")
.replace(/\|+/g, "_124_")
.replace(/\?+/g, "_63_")
.replace(/\*+/g, "_42_")
.replace(/[\u0000-\u001F\u007F\uFFFE\uFFFF\t]/g, "�")
// Slashes should be handled elsewhere
// .replace(/)
// .replace(\)
return path;
}
}
/**
* @class ForgeAssetSyncApp
* Forge Asset Sync Application
* This app spawns an instance of ForgeAssetSync and calls the `sync` method when the Sync button is clicked.
* This class must derive from FormApplication so it can be registered as a settings menu
*/
class ForgeAssetSyncApp extends FormApplication {
constructor(data, options) {
super(data,options);
/**
* A boolean to capture if there is a sync in progress
*/
this.isSyncing = false;
/**
* The current status of the sync. Values pulled from ForgeAssetSync.SYNC_STATUSES.
*/
this.currentStatus = ForgeAssetSync.SYNC_STATUSES.READY;
/**
* The currently processed Asset
*/
this.currentAsset = ``;
this.currentAssetIndex = 0;
this.currentSyncStep = "";
this.currentAssetType = "Asset";
this.totalAssetCount = 1;
/**
* The general status of the sync to be used for determining which icon/animation to display
*/
this.syncStatusIcon = `ready`
/**
* Last timestamp where the UI was refresh for progress bar purposes.
* This will limit the refreshes to 1 per second instead of as much as the CPU can take. Without throttling it, the UI becomes unusable.
*/
this._lastRefresh = 0;
/**
* Options For Sync
*/
this.syncOptions = [
{
name: "overwriteLocalMismatches",
htmlName: "overwrite-local-mismatches",
checked: false,
label: "Overwrite Mismatched Local Files",
disabled: false
},
{
name: "forceLocalRehash",
htmlName: "force-local-rehash",
checked: false,
label: "Force resync (ignores assets cache)",
disabled: false
},
{
name: "updateFoundryDb",
htmlName: "update-foundry-db",
checked: false,
label: "Update Foundry World & Compendiums to use Local Assets",
disabled: false
}
]
}
/**
* Get the default options for the Application, merged with the super's
*/
static get defaultOptions() {
return mergeObject(super.defaultOptions, {
id: `forgevtt-asset-sync`,
title: `Forge VTT - Asset Sync`,
template: `modules/forge-vtt/templates/asset-sync-form.hbs`,
resizable: true,
width: 500,
height: "auto"
});
}
/**
* Builds the syncProgress property
*/
get syncProgress() {
return {
value: ((this.currentAssetIndex / this.totalAssetCount) || 0).toFixed(2) * 100,
countValue: this.currentAssetIndex || 0,
countMax: this.totalAssetCount || 0,
step: this.currentSyncStep || "",
type: this.currentAssetType || "Asset"
}
}
/**
* Get data for template rendering
* @returns
*/
getData() {
const apiKey = game.settings.get("forge-vtt", "apiKey");
if (!apiKey) {
this.currentStatus = ForgeAssetSync.SYNC_STATUSES.NOKEY;
ui.notifications.error(ForgeAssetSync.SYNC_STATUSES.NOKEY);
this.syncStatusIcon = `failed`;
}
let iconClass = null;
if (this.syncStatusIcon === "ready")
iconClass = "fas fa-clipboard-check";
if (this.syncStatusIcon === "complete")
iconClass = "fas fa-check";
if (this.syncStatusIcon === "failed")
iconClass = "fas fa-times";
if (this.syncStatusIcon === "witherrors")
iconClass = "fas fa-exclamation-triangle";
const syncButtonText = this.isSyncing ? "Cancel" : "Sync";
const syncButtonIcon = this.isSyncing ? "fas fa-ban" : "fas fa-sync";
return {
canSync: true,
isSyncing: this.isSyncing,
currentStatus: this.currentStatus,
currentAsset: this.currentAsset,
syncButtonIcon: syncButtonIcon,
syncButtonText: syncButtonText,
syncOptions: this.syncOptions,
syncProgress: this.syncProgress,
syncStatusIcon: this.syncStatusIcon,
syncStatusIconClass: iconClass,
failedFolders: this.syncWorker?.failedFolders ?? [],
failedAssets: (this.syncWorker?.failedAssets ?? []).map(a => a.name),
}
}
async updateProgress({current, name, total, step, type}={}) {
if (total !== undefined) {
this.totalAssetCount = total;
}
if (name !== undefined) {
this.currentAsset = name;
}
if (current !== undefined) {
this.currentAssetIndex = current;
}
if (step !== undefined) {
this.currentSyncStep = step;
}
if (type !== undefined) {
this.currentAssetType = type;
}
if (this._lastRefresh < Date.now() - 1000) {
await this.render();
this._lastRefresh = Date.now();
}
}
/**
* Update the Sync status with the provided status data
* @param {*} status
*/
async updateStatus(status) {
this.currentStatus = status;
switch (status) {
case ForgeAssetSync.SYNC_STATUSES.SYNCING:
case ForgeAssetSync.SYNC_STATUSES.PREPARING:
case ForgeAssetSync.SYNC_STATUSES.POSTSYNC:
case ForgeAssetSync.SYNC_STATUSES.DBREWRITE:
this.syncStatusIcon = `syncing`;
this.isSyncing = true;
break;
case ForgeAssetSync.SYNC_STATUSES.COMPLETE:
this.syncStatusIcon = `complete`;
this.isSyncing = false;