From 88628d10ae4b12feb5b57a1191771079737906a7 Mon Sep 17 00:00:00 2001 From: Nigel Breslaw Date: Sun, 5 May 2024 23:16:43 +0300 Subject: [PATCH] feat: Speed up minifier and add icon sequences (#1532) --- minifier/bun.lockb | Bin 3142 -> 3142 bytes minifier/index.ts | 138 +++++-- minifier/next_gen.ts | 858 ++++++++++++++++++++++++++++++++++++++++++ minifier/package.json | 4 +- 4 files changed, 964 insertions(+), 36 deletions(-) create mode 100644 minifier/next_gen.ts diff --git a/minifier/bun.lockb b/minifier/bun.lockb index f5c05f447d07b9828ee98d45af04ba6d7912572f..668fef9c115c888227de991655d2205ac26a7222 100755 GIT binary patch delta 246 zcmX>maZF-@o{Yzq`cl)@7MyNi5Gs0j;h|w`uvgwjKP$$`EsPqI zH!w;tvQ7M`zWD>=SH}9O?t9m4R%|es+IeE;HipMflvQuNd*`?QgngXtL^b!n8oNX@ znYwzCW-z$cKfC41AwBzus(jmLf3Hrr>k~w7&68kcU|?kU$NvvVGczCni_Y3v8PgqE zO;z}n>n2Kc%4{qT$vRMYlSOXZ!M!^tPdqNv^2aK5WufGe71un33g)+}ba_OUZFl&* r?PgT^l_r;vq|LP~iY$zVljn07Gn#F_%yEPf$eF?A1mxW2%4P%rH~V1b delta 245 zcmX>maZF-@o`Prh2bCXtw?2JZ7oW60D{RB|J-Wn1=t*BqZ%X2gjeb^)lRelp zCU0Pr0MVO2Fn(pMcSzcD(oebFx$6qw;Zwp-oE|NfWoUY4(wSYJ8PTXH_y4u+w_j(r z-(}iv@$acfim-5z#nO$-FUA#LvwoX#a9{OJMg|5(hJXD3fHX4$0N0^^{%W3<1^@#Zg3AUV@xAs-4^>38?-s!Mea^J+%O?(%ghg}ahIuLZeHKH> = { + Descriptions: new Map(), + DisplaySources: new Map(), + ExpirationTooltip: new Map(), + ItemTypeDisplayName: new Map(), + ExpiredInActivityMessage: new Map(), + IconWaterMark: new Map(), + TraitIds: new Map(), + UiItemDisplayStyle: new Map(), + PlugCategoryIdentifier: new Map(), + UiPlugLabel: new Map(), + InsertionMaterialRequirementHash: new Map(), + StackUniqueLabel: new Map(), + BucketTypeHash: new Map(), + Versions: new Map(), + StatHash: new Map(), + StatGroupHash: new Map(), + DamageTypeHashes: new Map(), + ItemValue: new Map(), + TooltipNotifications: new Map(), + ReusablePlugSetHash: new Map(), + SingleInitialItemHash: new Map(), + SocketCategoryHash: new Map(), + SocketIndexes: new Map(), + SocketCategories: new Map(), + PlugCategoryHash: new Map(), + SocketEntries: new Map(), + SocketTypeHash: new Map(), + TalentGridHash: new Map(), + }; + // Send a repeat string and get a index value back - function getRepeatStringIndex(name: RepeatStringsName, s: string): number { - const index = repeatStrings[name].indexOf(s); - if (index === -1) { + // function getRepeatStringIndexMap(name: RepeatStringsName, s: string): number { + // const index = repeatStrings[name].indexOf(s); + // if (index === -1) { + // repeatStrings[name].push(s); + // return getRepeatStringIndexMap(name, s); + // } + + // return index; + // } + + function getRepeatStringIndexMap(name: RepeatStringsName, s: string): number { + if (!repeatStringsMap[name].has(s)) { + repeatStringsMap[name].set(s, repeatStringsMap[name].size); repeatStrings[name].push(s); - return getRepeatStringIndex(name, s); } - return index; + return repeatStringsMap[name].get(s)!; } const processedData: ProcessedData = { helpers: {}, items: {}, version: 3, id: uniqueKey }; @@ -189,7 +259,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const description = displayProperties.description; if (description) { - item.d = getRepeatStringIndex(RepeatStringsName.Descriptions, description); + item.d = getRepeatStringIndexMap(RepeatStringsName.Descriptions, description); } const icon = displayProperties.icon; @@ -239,7 +309,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const displaySource = jsonData[key].displaySource; if (displaySource) { - item.ds = getRepeatStringIndex(RepeatStringsName.DisplaySources, displaySource); + item.ds = getRepeatStringIndexMap(RepeatStringsName.DisplaySources, displaySource); } const itemType = jsonData[key].itemType; @@ -259,7 +329,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const itemTypeDisplayName = jsonData[key].itemTypeDisplayName; if (itemTypeDisplayName) { - item.itd = getRepeatStringIndex(RepeatStringsName.ItemTypeDisplayName, itemTypeDisplayName); + item.itd = getRepeatStringIndexMap(RepeatStringsName.ItemTypeDisplayName, itemTypeDisplayName); } /// Values @@ -276,7 +346,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD if (itemHash === 0) { continue; } - val.ih = getRepeatStringIndex(RepeatStringsName.ItemValue, itemHash); + val.ih = getRepeatStringIndexMap(RepeatStringsName.ItemValue, itemHash); if (itemValue.quantity > 0) { val.q = itemValue.quantity; } @@ -301,22 +371,22 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const bucketTypeHash = inventory?.bucketTypeHash; if (bucketTypeHash) { - item.b = getRepeatStringIndex(RepeatStringsName.BucketTypeHash, bucketTypeHash); + item.b = getRepeatStringIndexMap(RepeatStringsName.BucketTypeHash, bucketTypeHash); } const stackUniqueLabel = inventory?.stackUniqueLabel; if (stackUniqueLabel) { - item.su = getRepeatStringIndex(RepeatStringsName.StackUniqueLabel, stackUniqueLabel); + item.su = getRepeatStringIndexMap(RepeatStringsName.StackUniqueLabel, stackUniqueLabel); } const expirationTooltip = inventory?.expirationTooltip; if (expirationTooltip) { - item.et = getRepeatStringIndex(RepeatStringsName.ExpirationTooltip, expirationTooltip); + item.et = getRepeatStringIndexMap(RepeatStringsName.ExpirationTooltip, expirationTooltip); } const expiredInActivityMessage = inventory?.expiredInActivityMessage; if (expiredInActivityMessage) { - item.em = getRepeatStringIndex(RepeatStringsName.ExpiredInActivityMessage, expiredInActivityMessage); + item.em = getRepeatStringIndexMap(RepeatStringsName.ExpiredInActivityMessage, expiredInActivityMessage); } const maxStackSize = inventory?.maxStackSize; @@ -349,7 +419,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const dt: any[] = []; for (const damageHash of damageTypeHashes) { - dt.push(getRepeatStringIndex(RepeatStringsName.DamageTypeHashes, damageHash)); + dt.push(getRepeatStringIndexMap(RepeatStringsName.DamageTypeHashes, damageHash)); } if (dt.length > 0) { @@ -371,17 +441,17 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD /// Is this needed any more? const talentGridHash = jsonData[key].talentGrid?.talentGridHash; if (talentGridHash && talentGridHash !== 0) { - item.th = getRepeatStringIndex(RepeatStringsName.TalentGridHash, talentGridHash); + item.th = getRepeatStringIndexMap(RepeatStringsName.TalentGridHash, talentGridHash); } const uiItemDisplayStyle = jsonData[key].uiItemDisplayStyle; if (uiItemDisplayStyle) { - item.ids = getRepeatStringIndex(RepeatStringsName.UiItemDisplayStyle, uiItemDisplayStyle); + item.ids = getRepeatStringIndexMap(RepeatStringsName.UiItemDisplayStyle, uiItemDisplayStyle); } const iconWatermark = jsonData[key].iconWatermark; if (iconWatermark) { - item.iw = getRepeatStringIndex(RepeatStringsName.IconWaterMark, stripImageUrl(iconWatermark)); + item.iw = getRepeatStringIndexMap(RepeatStringsName.IconWaterMark, stripImageUrl(iconWatermark)); } // Quality @@ -392,7 +462,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD if (versions) { const qv: any[] = []; for (const version of versions) { - qv.push(getRepeatStringIndex(RepeatStringsName.Versions, version.powerCapHash)); + qv.push(getRepeatStringIndexMap(RepeatStringsName.Versions, version.powerCapHash)); } if (qv.length > 0) { item.qv = qv; @@ -407,7 +477,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD if (!watermark) { continue; } - dvwi.push(getRepeatStringIndex(RepeatStringsName.IconWaterMark, stripImageUrl(watermark))); + dvwi.push(getRepeatStringIndexMap(RepeatStringsName.IconWaterMark, stripImageUrl(watermark))); } if (dvwi.length > 0) { @@ -426,7 +496,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const s: any = {}; const sortedStatKeys = Object.keys(itemStats).sort((a, b) => parseFloat(a) - parseFloat(b)); for (const key of sortedStatKeys) { - s[getRepeatStringIndex(RepeatStringsName.StatHash, key)] = itemStats[key].value; + s[getRepeatStringIndexMap(RepeatStringsName.StatHash, key)] = itemStats[key].value; } if (Object.keys(s).length > 0) { @@ -435,7 +505,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD var statGroupHash = stats.statGroupHash; if (statGroupHash) { - st.sgs = getRepeatStringIndex(RepeatStringsName.StatGroupHash, statGroupHash); + st.sgs = getRepeatStringIndexMap(RepeatStringsName.StatGroupHash, statGroupHash); } if (Object.keys(st).length > 0) { @@ -475,7 +545,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const ttString = tt.displayString; if (ttString) { - ttn.push(getRepeatStringIndex(RepeatStringsName.TooltipNotifications, ttString)); + ttn.push(getRepeatStringIndexMap(RepeatStringsName.TooltipNotifications, ttString)); } /// NOTE!!! Ishtar only uses the first tooltip so no need to keep the others? @@ -520,7 +590,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const plugCategoryHash = plug?.plugCategoryHash; if (plugCategoryHash) { - p.p = getRepeatStringIndex(RepeatStringsName.PlugCategoryHash, plugCategoryHash); + p.p = getRepeatStringIndexMap(RepeatStringsName.PlugCategoryHash, plugCategoryHash); } /// NOTE: This change breaks the existing app. All it needs to do to get the correct @@ -529,17 +599,17 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const plugCategoryIdentifier = plug.plugCategoryIdentifier; if (plugCategoryIdentifier) { /// Intentionally call the function but don't save the result here. The p.p index will be the same. - getRepeatStringIndex(RepeatStringsName.PlugCategoryIdentifier, plugCategoryIdentifier); + getRepeatStringIndexMap(RepeatStringsName.PlugCategoryIdentifier, plugCategoryIdentifier); } var uiPlugLabel = plug.uiPlugLabel; if (uiPlugLabel) { - p.pl = getRepeatStringIndex(RepeatStringsName.UiPlugLabel, uiPlugLabel); + p.pl = getRepeatStringIndexMap(RepeatStringsName.UiPlugLabel, uiPlugLabel); } const insertionMaterialRequirementHash = plug?.insertionMaterialRequirementHash; if (insertionMaterialRequirementHash && insertionMaterialRequirementHash !== 0) { - p.im = getRepeatStringIndex( + p.im = getRepeatStringIndexMap( RepeatStringsName.InsertionMaterialRequirementHash, insertionMaterialRequirementHash, ); @@ -555,7 +625,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const ti: any[] = []; for (const traitId of traitIds) { - ti.push(getRepeatStringIndex(RepeatStringsName.TraitIds, traitId)); + ti.push(getRepeatStringIndexMap(RepeatStringsName.TraitIds, traitId)); } if (ti.length > 0) { @@ -581,17 +651,17 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD const st = socketEntry?.socketTypeHash; if (st) { - socEntry.st = getRepeatStringIndex(RepeatStringsName.SocketTypeHash, st); + socEntry.st = getRepeatStringIndexMap(RepeatStringsName.SocketTypeHash, st); } const rp = socketEntry.reusablePlugSetHash; if (rp) { - socEntry.r = getRepeatStringIndex(RepeatStringsName.ReusablePlugSetHash, rp); + socEntry.r = getRepeatStringIndexMap(RepeatStringsName.ReusablePlugSetHash, rp); } const s = socketEntry.singleInitialItemHash; if (s && s !== 0) { - socEntry.s = getRepeatStringIndex(RepeatStringsName.SingleInitialItemHash, s); + socEntry.s = getRepeatStringIndexMap(RepeatStringsName.SingleInitialItemHash, s); } if (socEntry) { @@ -600,7 +670,7 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD } if (se.length > 0) { - sk.se = getRepeatStringIndex(RepeatStringsName.SocketEntries, JSON.stringify(se)); + sk.se = getRepeatStringIndexMap(RepeatStringsName.SocketEntries, JSON.stringify(se)); } const scJson: any[] = []; @@ -609,19 +679,19 @@ function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedD var h = socketCategory?.socketCategoryHash; if (h) { - socCatEntry.h = getRepeatStringIndex(RepeatStringsName.SocketCategoryHash, h); + socCatEntry.h = getRepeatStringIndexMap(RepeatStringsName.SocketCategoryHash, h); } /// NOTE: In ishtar you want to Json.parse the string you get to turn it into a json array. var socketIndexes = socketCategory?.socketIndexes; if (socketIndexes) { - socCatEntry.i = getRepeatStringIndex(RepeatStringsName.SocketIndexes, JSON.stringify(socketIndexes)); + socCatEntry.i = getRepeatStringIndexMap(RepeatStringsName.SocketIndexes, JSON.stringify(socketIndexes)); scJson.push(socCatEntry); } } if (scJson.length > 0) { /// NOTE: In ishtar you want to Json.parse the string you get to turn it into a json array. - sk.sc = getRepeatStringIndex(RepeatStringsName.SocketCategories, JSON.stringify(scJson)); + sk.sc = getRepeatStringIndexMap(RepeatStringsName.SocketCategories, JSON.stringify(scJson)); } if (Object.keys(sk).length > 0) { diff --git a/minifier/next_gen.ts b/minifier/next_gen.ts new file mode 100644 index 000000000..83979bb37 --- /dev/null +++ b/minifier/next_gen.ts @@ -0,0 +1,858 @@ +import * as fs from "fs"; +import path from "path"; + +// Enum for all the repeatedStrings +enum RepeatStringsName { + Descriptions = "Descriptions", + DisplaySources = "DisplaySources", + ExpirationTooltip = "ExpirationTooltip", + ItemTypeDisplayName = "ItemTypeDisplayName", + ExpiredInActivityMessage = "ExpiredInActivityMessage", + IconWaterMark = "IconWaterMark", + TraitIds = "TraitIds", + UiItemDisplayStyle = "UiItemDisplayStyle", + PlugCategoryIdentifier = "PlugCategoryIdentifier", + UiPlugLabel = "UiPlugLabel", + InsertionMaterialRequirementHash = "InsertionMaterialRequirementHash", + StackUniqueLabel = "StackUniqueLabel", + BucketTypeHash = "BucketTypeHash", + Versions = "Versions", + StatHash = "StatHash", + StatGroupHash = "StatGroupHash", + DamageTypeHashes = "DamageTypeHashes", + ItemValue = "ItemValue", + TooltipNotifications = "TooltipNotifications", + ReusablePlugSetHash = "ReusablePlugSetHash", + SingleInitialItemHash = "SingleInitialItemHash", + SocketCategoryHash = "SocketCategoryHash", + SocketIndexes = "SocketIndexes", + SocketCategories = "SocketCategories", + PlugCategoryHash = "PlugCategoryHash", + SocketEntries = "SocketEntries", + SocketTypeHash = "SocketTypeHash", + TalentGridHash = "TalentGridHash", + Icons = "Icons", +} + +// Interface (Schema) for the DestinyItemDefinition +interface JsonData { + [key: string]: { + sockets: any; + plug: any; + perks: any; + tooltipNotifications: any; + setData: any; + preview: any; + traitIds: any; + stats: any; + + quality?: { + versions?: any[]; + displayVersionWatermarkIcons: any; + }; + iconWatermark?: any; + uiItemDisplayStyle?: any; + equippingBlock?: any; + breakerType?: any; + talentGrid?: any; + damageTypeHashes?: any; + redacted?: any; + value?: { + itemValue?: [itemHash?: string, quantity?: any]; + }; + displayProperties?: { + iconSequences: any; + name?: any; + description?: any; + icon?: any; + }; + secondaryIcon?: any; + secondarySpecial?: any; + secondaryOverlay?: any; + screenshot?: any; + investmentStats?: [statTypeHash?: any, value?: any]; + nonTransferrable?: any; + allowActions?: any; + equippable?: any; + doesPostmasterPullHaveSideEffects?: any; + displaySource?: any; + itemType?: any; + itemSubType?: any; + classType?: any; + itemTypeDisplayName?: any; + inventory?: { + tierType?: any; + maxStackSize?: any; + bucketTypeHash?: any; + stackUniqueLabel?: any; + expirationTooltip?: any; + expiredInActivityMessage?: any; + }; + }; +} + +// Strip off the url so only the image name is left +// http:bungie.com/blah/blah/123.jpg -> 123.jpg +function stripImageUrl(url: string): string { + const index = url.lastIndexOf("/"); + let shortUrl = url.substring(index + 1); + + // Change the extension + // if (shortUrl.endsWith(".jpg")) { + // shortUrl = shortUrl.replace(".jpg", ".j"); + // } else if (shortUrl.endsWith(".png")) { + // shortUrl = shortUrl.replace(".png", ".p"); + // } + + return shortUrl; +} + +// TODO: Update to retry a couple of times instead of failing right away +async function downloadJsonFile(url: string): Promise { + try { + const response = await fetch(url); + + if (!response.ok) { + console.error(`Failed to fetch JSON: ${response.statusText}`); + process.exit(1); + } + + return response.json(); + } catch (error) { + console.error(`Failed to download JSON file: ${error}`); + process.exit(1); + } +} + +type ProcessedData = { + helpers: { [key: string]: any }; + items: { [key: string]: any }; + version: number; + id: string; +}; + +function createMiniDefinition(jsonData: JsonData, uniqueKey: string): ProcessedData { + // Dictionary of the repeat string arrays + const repeatStrings: Record = { + [RepeatStringsName.Descriptions]: [], + [RepeatStringsName.DisplaySources]: [], + [RepeatStringsName.ExpirationTooltip]: [], + [RepeatStringsName.ItemTypeDisplayName]: [], + [RepeatStringsName.ExpiredInActivityMessage]: [], + [RepeatStringsName.IconWaterMark]: [], + [RepeatStringsName.TraitIds]: [], + [RepeatStringsName.UiItemDisplayStyle]: [], + [RepeatStringsName.PlugCategoryIdentifier]: [], + [RepeatStringsName.UiPlugLabel]: [], + [RepeatStringsName.InsertionMaterialRequirementHash]: [], + [RepeatStringsName.StackUniqueLabel]: [], + [RepeatStringsName.BucketTypeHash]: [], + [RepeatStringsName.Versions]: [], + [RepeatStringsName.StatHash]: [], + [RepeatStringsName.StatGroupHash]: [], + [RepeatStringsName.DamageTypeHashes]: [], + [RepeatStringsName.ItemValue]: [], + [RepeatStringsName.TooltipNotifications]: [], + [RepeatStringsName.ReusablePlugSetHash]: [], + [RepeatStringsName.SingleInitialItemHash]: [], + [RepeatStringsName.SocketCategoryHash]: [], + [RepeatStringsName.SocketIndexes]: [], + [RepeatStringsName.SocketCategories]: [], + [RepeatStringsName.PlugCategoryHash]: [], + [RepeatStringsName.SocketEntries]: [], + [RepeatStringsName.SocketTypeHash]: [], + [RepeatStringsName.TalentGridHash]: [], + [RepeatStringsName.Icons]: [], + }; + + type RepeatStringsName = + | "Descriptions" + | "DisplaySources" + | "ExpirationTooltip" + | "ItemTypeDisplayName" + | "ExpiredInActivityMessage" + | "IconWaterMark" + | "TraitIds" + | "UiItemDisplayStyle" + | "PlugCategoryIdentifier" + | "UiPlugLabel" + | "InsertionMaterialRequirementHash" + | "StackUniqueLabel" + | "BucketTypeHash" + | "Versions" + | "StatHash" + | "StatGroupHash" + | "DamageTypeHashes" + | "ItemValue" + | "TooltipNotifications" + | "ReusablePlugSetHash" + | "SingleInitialItemHash" + | "SocketCategoryHash" + | "SocketIndexes" + | "SocketCategories" + | "PlugCategoryHash" + | "SocketEntries" + | "SocketTypeHash" + | "TalentGridHash" + | "Icons"; + + const repeatStringsMap: Record> = { + Descriptions: new Map(), + DisplaySources: new Map(), + ExpirationTooltip: new Map(), + ItemTypeDisplayName: new Map(), + ExpiredInActivityMessage: new Map(), + IconWaterMark: new Map(), + TraitIds: new Map(), + UiItemDisplayStyle: new Map(), + PlugCategoryIdentifier: new Map(), + UiPlugLabel: new Map(), + InsertionMaterialRequirementHash: new Map(), + StackUniqueLabel: new Map(), + BucketTypeHash: new Map(), + Versions: new Map(), + StatHash: new Map(), + StatGroupHash: new Map(), + DamageTypeHashes: new Map(), + ItemValue: new Map(), + TooltipNotifications: new Map(), + ReusablePlugSetHash: new Map(), + SingleInitialItemHash: new Map(), + SocketCategoryHash: new Map(), + SocketIndexes: new Map(), + SocketCategories: new Map(), + PlugCategoryHash: new Map(), + SocketEntries: new Map(), + SocketTypeHash: new Map(), + TalentGridHash: new Map(), + Icons: new Map(), + }; + + // Send a repeat string and get a index value back + // function getRepeatStringIndexMap(name: RepeatStringsName, s: string): number { + // const index = repeatStrings[name].indexOf(s); + // if (index === -1) { + // repeatStrings[name].push(s); + // return getRepeatStringIndexMap(name, s); + // } + + // return index; + // } + + function getRepeatStringIndexMap(name: RepeatStringsName, s: string): number { + if (!repeatStringsMap[name].has(s)) { + repeatStringsMap[name].set(s, repeatStringsMap[name].size); + repeatStrings[name].push(s); + } + + return repeatStringsMap[name].get(s)!; + } + + const processedData: ProcessedData = { helpers: {}, items: {}, version: 3, id: uniqueKey }; + + const sortedDataKeys = Object.keys(jsonData).sort((a, b) => parseFloat(a) - parseFloat(b)); + + for (const key of sortedDataKeys) { + const item: any = {}; + + if (jsonData.hasOwnProperty(key)) { + const redacted = jsonData[key].redacted; + if (redacted) { + item.r = 1; + } + + const displayProperties = jsonData[key].displayProperties; + if (displayProperties) { + const name = displayProperties.name; + if (name) { + item.n = name; + } else if (!redacted) { + continue; + } + + const description = displayProperties.description; + if (description) { + item.d = getRepeatStringIndexMap(RepeatStringsName.Descriptions, description); + } + + const icon = displayProperties.icon; + if (icon) { + item.i = getRepeatStringIndexMap(RepeatStringsName.Icons, stripImageUrl(icon)); + } + + const iconSequences = displayProperties.iconSequences; + if (iconSequences) { + const is: any[] = []; + for (const i of iconSequences) { + const f: any[] = []; + for (const frame of i.frames) { + const icon = getRepeatStringIndexMap(RepeatStringsName.Icons, stripImageUrl(frame)); + f.push(icon); + } + is.push(f); + } + + item.if = is; + } + } + + const secondaryIcon = jsonData[key].secondaryIcon; + if (secondaryIcon) { + item.si = stripImageUrl(secondaryIcon); + } + + const secondaryOverlay = jsonData[key].secondaryOverlay; + if (secondaryOverlay) { + item.so = stripImageUrl(secondaryOverlay); + } + const secondarySpecial = jsonData[key].secondarySpecial; + if (secondarySpecial) { + item.ss = stripImageUrl(secondarySpecial); + } + + const screenshot = jsonData[key].screenshot; + if (screenshot) { + item.s = stripImageUrl(screenshot); + } + + const allowActions = jsonData[key].allowActions; + if (!allowActions) { + item.a = 0; + } + + const nonTransferrable = jsonData[key].nonTransferrable; + if (nonTransferrable) { + item.nt = 1; + } + + const equippable = jsonData[key].equippable; + if (equippable) { + item.e = 1; + } + + const doesPostmasterPullHaveSideEffects = jsonData[key].doesPostmasterPullHaveSideEffects; + if (doesPostmasterPullHaveSideEffects) { + item.pm = 1; + } + + const displaySource = jsonData[key].displaySource; + if (displaySource) { + item.ds = getRepeatStringIndexMap(RepeatStringsName.DisplaySources, displaySource); + } + + const itemType = jsonData[key].itemType; + if (itemType) { + item.it = itemType; + } + + const itemSubType = jsonData[key].itemSubType; + if (itemSubType) { + item.is = itemSubType; + } + + const classType = jsonData[key].classType; + if (classType) { + item.c = classType; + } + + const itemTypeDisplayName = jsonData[key].itemTypeDisplayName; + if (itemTypeDisplayName) { + item.itd = getRepeatStringIndexMap(RepeatStringsName.ItemTypeDisplayName, itemTypeDisplayName); + } + + /// Values + const itemValues = jsonData[key].value?.itemValue; + + if (itemValues) { + const v: any[] = []; + + for (const itemValue of itemValues) { + const val: any = {}; + + const itemHash = itemValue.itemHash; + + if (itemHash === 0) { + continue; + } + val.ih = getRepeatStringIndexMap(RepeatStringsName.ItemValue, itemHash); + if (itemValue.quantity > 0) { + val.q = itemValue.quantity; + } + + if (Object.keys(val).length > 0) { + v.push(val); + } + + if (Object.keys(v).length > 0) { + item.v = v; + } + } + } + + /// inventory + const inventory = jsonData[key].inventory; + if (inventory) { + const tierType = inventory?.tierType; + if (tierType) { + item.t = tierType; + } + + const bucketTypeHash = inventory?.bucketTypeHash; + if (bucketTypeHash) { + item.b = getRepeatStringIndexMap(RepeatStringsName.BucketTypeHash, bucketTypeHash); + } + + const stackUniqueLabel = inventory?.stackUniqueLabel; + if (stackUniqueLabel) { + item.su = getRepeatStringIndexMap(RepeatStringsName.StackUniqueLabel, stackUniqueLabel); + } + + const expirationTooltip = inventory?.expirationTooltip; + if (expirationTooltip) { + item.et = getRepeatStringIndexMap(RepeatStringsName.ExpirationTooltip, expirationTooltip); + } + + const expiredInActivityMessage = inventory?.expiredInActivityMessage; + if (expiredInActivityMessage) { + item.em = getRepeatStringIndexMap(RepeatStringsName.ExpiredInActivityMessage, expiredInActivityMessage); + } + + const maxStackSize = inventory?.maxStackSize; + if (maxStackSize) { + item.m = maxStackSize; + } + } + + const investmentStats = jsonData[key].investmentStats; + + if (investmentStats) { + const iv: any = {}; + + for (const stat of investmentStats) { + const value = stat.value; + + if (value > 0) { + const statTypeHash = stat.statTypeHash; + iv[statTypeHash] = value; + } + } + + if (Object.keys(iv).length > 0) { + item.iv = iv; + } + } + + const damageTypeHashes = jsonData[key].damageTypeHashes; + if (damageTypeHashes) { + const dt: any[] = []; + + for (const damageHash of damageTypeHashes) { + dt.push(getRepeatStringIndexMap(RepeatStringsName.DamageTypeHashes, damageHash)); + } + + if (dt.length > 0) { + item.dt = dt; + } + } + + ///equipping block? + const ammoType = jsonData[key].equippingBlock?.ammoType; + if (ammoType) { + item.at = ammoType; + } + + const breakerType = jsonData[key].breakerType; + if (breakerType) { + item.bt = breakerType; + } + + /// Is this needed any more? + const talentGridHash = jsonData[key].talentGrid?.talentGridHash; + if (talentGridHash && talentGridHash !== 0) { + item.th = getRepeatStringIndexMap(RepeatStringsName.TalentGridHash, talentGridHash); + } + + const uiItemDisplayStyle = jsonData[key].uiItemDisplayStyle; + if (uiItemDisplayStyle) { + item.ids = getRepeatStringIndexMap(RepeatStringsName.UiItemDisplayStyle, uiItemDisplayStyle); + } + + const iconWatermark = jsonData[key].iconWatermark; + if (iconWatermark) { + item.iw = getRepeatStringIndexMap(RepeatStringsName.IconWaterMark, stripImageUrl(iconWatermark)); + } + + // Quality + const quality = jsonData[key].quality; + if (quality) { + const versions = quality.versions; + + if (versions) { + const qv: any[] = []; + for (const version of versions) { + qv.push(getRepeatStringIndexMap(RepeatStringsName.Versions, version.powerCapHash)); + } + if (qv.length > 0) { + item.qv = qv; + } + } + + const displayVersionWatermarkIcons = quality.displayVersionWatermarkIcons; + if (displayVersionWatermarkIcons) { + const dvwi: any[] = []; + + for (const watermark of displayVersionWatermarkIcons) { + if (!watermark) { + continue; + } + dvwi.push(getRepeatStringIndexMap(RepeatStringsName.IconWaterMark, stripImageUrl(watermark))); + } + + if (dvwi.length > 0) { + item.dvwi = dvwi; + } + } + } + + /// Stats + var stats = jsonData[key].stats; + if (stats) { + const st: any = {}; + + const itemStats = stats.stats; + + const s: any = {}; + const sortedStatKeys = Object.keys(itemStats).sort((a, b) => parseFloat(a) - parseFloat(b)); + for (const key of sortedStatKeys) { + s[getRepeatStringIndexMap(RepeatStringsName.StatHash, key)] = itemStats[key].value; + } + + if (Object.keys(s).length > 0) { + st.s = s; + } + + var statGroupHash = stats.statGroupHash; + if (statGroupHash) { + st.sgs = getRepeatStringIndexMap(RepeatStringsName.StatGroupHash, statGroupHash); + } + + if (Object.keys(st).length > 0) { + item.st = st; + } + } + + var previewVendorHash = jsonData[key].preview?.previewVendorHash; + + if (previewVendorHash) { + const p = previewVendorHash; + if (p) { + item.pv = p; + } + } + + /// setData + const setData = jsonData[key].setData; + if (setData) { + const sD: any = {}; + + const questLineName = setData.questLineName; + if (questLineName) { + sD.qN = questLineName; + } + + if (sD) { + item.sD = sD; + } + } + + const tooltipNotifications = jsonData[key].tooltipNotifications; + if (tooltipNotifications) { + const ttn: any[] = []; + + for (const tt of tooltipNotifications) { + const ttString = tt.displayString; + + if (ttString) { + ttn.push(getRepeatStringIndexMap(RepeatStringsName.TooltipNotifications, ttString)); + } + + /// NOTE!!! Ishtar only uses the first tooltip so no need to keep the others? + /// Hmmm probably was used by some items in the detail view? + break; + } + + if (ttn.length > 0) { + item.ttn = ttn; + } + } + + /// Perks + const perks = jsonData[key].perks; + if (perks) { + const ph: any[] = []; + + for (const perk of perks) { + const jPerk: any = {}; + const perkHash = perk.perkHash; + if (perkHash) { + jPerk.ph = perkHash; + } + var perkVisibility = perk.perkVisibility; + if (perkVisibility) { + jPerk.pv = perkVisibility; + } + + if (Object.keys(jPerk).length > 0) { + ph.push(jPerk); + } + } + + if (ph.length > 0) { + item.ph = ph; + } + } + + var plug = jsonData[key].plug; + if (plug) { + const p: any = {}; + + const plugCategoryHash = plug?.plugCategoryHash; + if (plugCategoryHash) { + p.p = getRepeatStringIndexMap(RepeatStringsName.PlugCategoryHash, plugCategoryHash); + } + + /// NOTE: This change breaks the existing app. All it needs to do to get the correct + /// PlugCategoryIdentifier is use the p.p index number to get the name from the + /// PlugCategoryIdentifier array in the jsonDef + const plugCategoryIdentifier = plug.plugCategoryIdentifier; + if (plugCategoryIdentifier) { + /// Intentionally call the function but don't save the result here. The p.p index will be the same. + getRepeatStringIndexMap(RepeatStringsName.PlugCategoryIdentifier, plugCategoryIdentifier); + } + + var uiPlugLabel = plug.uiPlugLabel; + if (uiPlugLabel) { + p.pl = getRepeatStringIndexMap(RepeatStringsName.UiPlugLabel, uiPlugLabel); + } + + const insertionMaterialRequirementHash = plug?.insertionMaterialRequirementHash; + if (insertionMaterialRequirementHash && insertionMaterialRequirementHash !== 0) { + p.im = getRepeatStringIndexMap( + RepeatStringsName.InsertionMaterialRequirementHash, + insertionMaterialRequirementHash, + ); + } + + if (Object.keys(p).length > 0) { + item.p = p; + } + } + + var traitIds = jsonData[key].traitIds; + if (traitIds) { + const ti: any[] = []; + + for (const traitId of traitIds) { + ti.push(getRepeatStringIndexMap(RepeatStringsName.TraitIds, traitId)); + } + + if (ti.length > 0) { + item.tI = ti; + } + } + + /// NOTE:!!!! This changes the names of many socket properties and will break current Ishtar + const sockets = jsonData[key].sockets; + if (sockets) { + const sk: any = {}; + + const socketEntries = sockets?.socketEntries; + + const se: any[] = []; + for (const socketEntry of socketEntries) { + const socEntry: any = {}; + const p = socketEntry?.plugSources; + + if (p) { + socEntry.p = p; + } + + const st = socketEntry?.socketTypeHash; + if (st) { + socEntry.st = getRepeatStringIndexMap(RepeatStringsName.SocketTypeHash, st); + } + + const rp = socketEntry.reusablePlugSetHash; + if (rp) { + socEntry.r = getRepeatStringIndexMap(RepeatStringsName.ReusablePlugSetHash, rp); + } + + const s = socketEntry.singleInitialItemHash; + if (s && s !== 0) { + socEntry.s = getRepeatStringIndexMap(RepeatStringsName.SingleInitialItemHash, s); + } + + if (socEntry) { + se.push(socEntry); + } + } + + if (se.length > 0) { + sk.se = getRepeatStringIndexMap(RepeatStringsName.SocketEntries, JSON.stringify(se)); + } + + const scJson: any[] = []; + for (const socketCategory of sockets.socketCategories) { + const socCatEntry: any = {}; + + var h = socketCategory?.socketCategoryHash; + if (h) { + socCatEntry.h = getRepeatStringIndexMap(RepeatStringsName.SocketCategoryHash, h); + } + + /// NOTE: In ishtar you want to Json.parse the string you get to turn it into a json array. + var socketIndexes = socketCategory?.socketIndexes; + if (socketIndexes) { + socCatEntry.i = getRepeatStringIndexMap(RepeatStringsName.SocketIndexes, JSON.stringify(socketIndexes)); + scJson.push(socCatEntry); + } + } + if (scJson.length > 0) { + /// NOTE: In ishtar you want to Json.parse the string you get to turn it into a json array. + sk.sc = getRepeatStringIndexMap(RepeatStringsName.SocketCategories, JSON.stringify(scJson)); + } + + if (Object.keys(sk).length > 0) { + item.sk = sk; + } + } + } + // Only add items with data + if (Object.keys(item).length > 0) { + processedData.items[key] = item; // Assign 'item' directly to the key + } + } + + // Add the repeatStrings to the output JSON + // Create an array of enum names by filtering out invalid enum values + const enumNames = Object.keys(RepeatStringsName).filter((key) => + isNaN(Number(key)), + ) as (keyof typeof RepeatStringsName)[]; + + // Iterate over the enum names + for (const enumName of enumNames) { + switch (enumName) { + case RepeatStringsName.SocketIndexes: { + const socketIndexesDefinition = repeatStrings[RepeatStringsName[enumName]]; + const si: number[][] = []; + for (const socketIndex of socketIndexesDefinition) { + si.push(JSON.parse(socketIndex) as number[]); + } + processedData.helpers[enumName] = si; + + break; + } + case RepeatStringsName.SocketCategories: + case RepeatStringsName.SocketEntries: + const entries = repeatStrings[RepeatStringsName[enumName]]; + const si: JSON[][] = []; + for (const entry of entries) { + si.push(JSON.parse(entry) as JSON[]); + } + processedData.helpers[enumName] = si; + break; + default: { + const stringArray = repeatStrings[RepeatStringsName[enumName]]; + processedData.helpers[enumName] = stringArray; + } + } + } + return processedData; +} + +async function saveToJsonFile(data: any, filePath: string): Promise { + try { + const jsonString = JSON.stringify(data, null, 0); + await Bun.write(filePath, jsonString); + console.log(`Data saved to ${filePath}`); + } catch (error) { + throw new Error(`Failed to save data to file: ${error}`); + } +} + +async function useContentPaths(jsonWorldComponentContentPaths: any, uniqueKey: string): Promise { + const promises: Promise[] = []; + + for (const key in jsonWorldComponentContentPaths) { + const definitionUrl = "https://bungie.com" + jsonWorldComponentContentPaths[key].DestinyInventoryItemDefinition; + + promises.push(downloadAndMinifyDefinition(definitionUrl, key, uniqueKey)); + } + + // Wait for all promises to resolve in parallel + await Promise.all(promises); +} + +async function downloadAndMinifyDefinition(definitionUrl: string, key: string, uniqueKey: string): Promise { + console.time(`${key} download-json`); + const jsonData = await downloadJsonFile(definitionUrl); + console.timeEnd(`${key} download-json`); + + console.time(`${key} parse-took:`); + const processedData = createMiniDefinition(jsonData, uniqueKey); + console.timeEnd(`${key} parse-took:`); + + const outputFilePath = path.join(__dirname, `json/1/${key}.json`); + + console.time(`${key} save-took:`); + await saveToJsonFile(processedData, outputFilePath); + console.timeEnd(`${key} save-took:`); + + console.log(""); +} + +async function main() { + // make directory called json + const jsonPath = path.join(__dirname, `json`); + if (!fs.existsSync(jsonPath)) { + fs.mkdirSync(jsonPath); + } + const jsonPath1 = path.join(__dirname, `json/1`); + if (!fs.existsSync(jsonPath1)) { + fs.mkdirSync(jsonPath1); + } + try { + console.time("download-manifest"); + + const manifestUrl = "https://www.bungie.net/Platform/Destiny2/Manifest/"; + + const jsonManifest = await downloadJsonFile(manifestUrl); + const id = jsonManifest.Response.jsonWorldComponentContentPaths["en"].DestinyInventoryItemDefinition; + console.timeEnd("download-manifest"); + + const jsonWorldComponentContentPaths = jsonManifest.Response.jsonWorldComponentContentPaths; + console.time("total-json-parse"); + await useContentPaths(jsonWorldComponentContentPaths, id); + console.timeEnd("total-json-parse"); + + // const uniqueJsonManifest = { + // version: id, + // }; + + // const savePath = path.join(__dirname, `json/manifest.json`); + // await saveToJsonFile(uniqueJsonManifest, savePath); + + // To avoid committing a 3MB JSON blob into this repo download the demo.json file + // from the unintuitive.com site and save it to the json folder. This is only used + // for the apps demo mode. + // const demoJsonUrl = "https://unintuitive.com/demo.json"; + // const demoJson = await downloadJsonFile(demoJsonUrl); + // const saveDemoPath = path.join(__dirname, `json/demo.json`); + // await saveToJsonFile(demoJson, saveDemoPath); + } catch (error) { + console.error(error); + process.exit(1); + } +} + +main(); diff --git a/minifier/package.json b/minifier/package.json index 805b41abe..ac06bed66 100644 --- a/minifier/package.json +++ b/minifier/package.json @@ -3,10 +3,10 @@ "version": "0.0.0", "main": "index.ts", "scripts": { - "start": "bun run index.ts" + "start": "bun run index.ts && bun run next_gen.ts" }, "devDependencies": { - "@types/bun": "1.1.0", + "@types/bun": "1.1.1", "typescript": "5.4.5" } }