Feature: Caching (#13)

* caching stuff is fun Co-authored-by: Nawias <[email protected]> * asset caches kind of work? * 3ds asset file caching * more updates * we do some code cleanup?? * fix pr review comments * update some tests * update some automation, frontend code * finish up tests * not the logs! --------- Co-authored-by: Nawias <[email protected]>
lovebrew · Jun 14, 2024 · c1b1eb6 · c1b1eb6
1 parent 9e7c6a9
commit c1b1eb6
Show file tree

Hide file tree

Showing 22 changed files with 1,029 additions and 273 deletions.
diff --git a/Bundler.Client/package-lock.json b/Bundler.Client/package-lock.json
diff --git a/Bundler.Client/package.json b/Bundler.Client/package.json
@@ -12,7 +12,9 @@
   },
   "dependencies": {
     "debug": "^4.3.5",
+    "imurmurhash": "^0.1.4",
     "jszip": "^3.10.1",
+    "localforage": "^1.10.0",
     "mime": "^3.0.0",
     "mime-types": "^2.1.35",
     "react": "^18.2.0",
@@ -22,6 +24,7 @@
     "use-sound": "^4.0.1"
   },
   "devDependencies": {
+    "@types/imurmurhash": "^0.1.4",
     "@types/mime": "^3.0.3",
     "@types/node": "^20.8.9",
     "@types/react": "^18.2.15",

diff --git a/Bundler.Client/src/App.tsx b/Bundler.Client/src/App.tsx
@@ -13,8 +13,20 @@ import errorSfx from "@assets/sound/error.ogg";
 import useSound from "use-sound";
 import JSZip from "jszip";
 
-import { MediaFile } from "./services/MediaConverter";
-import { isZipFile, convertFiles, isValidFile, isFontFile, isImageFile } from "./services/utilities";
+import { MediaFile } from "./services/types";
+import MediaConverter from "./services/MediaConverter";
+
+import mime from "mime";
+
+const ZipTypes = ["application/x-zip-compressed", "application/zip"];
+
+const isZipFile = (file: File) => {
+  const mimeType = mime.getType(file.name);
+
+  if (mimeType === null) return false;
+
+  return ZipTypes.includes(mimeType);
+}
 
 const downloadBlob = (blob: Blob) => {
   const link = document.createElement("a");
@@ -57,24 +69,27 @@ function App() {
   const handleZipUpload = async (archive: File) => {
     const bundler = new Bundler(archive);
 
-    toast.promise(bundler.prepareContent(), {
+    toast.promise(bundler.bundleContent(), {
       loading: "Uploading..",
       success: handleUploadSuccess,
       error: handleUploadError,
     });
   };
 
-  const handleConversions = async (files: File[]) => {
-    toast.promise(convertFiles(files), {
+  const handleConversions = async (files: Array<File>) => {
+    toast.promise(MediaConverter.instance.convert(files), {
       loading: "Uploading..",
-      success: (files: MediaFile[]) => {
+      success: (files: Array<MediaFile>) => {
         playSuccess();
         const zip = new JSZip();
 
         for (const file of files) {
           zip.file(file.filepath, file.data);
         }
 
+        const log = MediaConverter.getConversionLog();
+        if (log && log.size > 0) zip.file("convert.log", log);
+
         zip
           .generateAsync({ type: "blob" })
           .then((blob: Blob) => downloadBlob(blob));
@@ -84,20 +99,19 @@ function App() {
     });
   };
 
-  const handleUpload = async (files: File[]) => {
+  const handleUpload = async (files: Array<File>) => {
     try {
       for (const file of files) {
         if (file.size === 0) throw Error("Invalid file.");
 
-        if (!isValidFile(file)) throw Error("Invalid file type.");
+        if (!MediaConverter.isValidFileType(file) && !isZipFile(file)) throw Error("Invalid file type.");
 
-        if (isZipFile(file)) handleZipUpload(file);
+        if (isZipFile(file)) return await handleZipUpload(file);
       }
 
-      if (files.length > 0 && files.every((file) => isFontFile(file) ||isImageFile(file))) {
-        handleConversions(files);
+      if (MediaConverter.areFilesValid(files)) {
+        await handleConversions(files);
       }
-
     } catch (exception) {
       toast.error(handleUploadError((exception as Error).message));
     }

diff --git a/Bundler.Client/src/dbutils.ts b/Bundler.Client/src/dbutils.ts
@@ -0,0 +1,48 @@
+import localforage from "localforage";
+import { BundlerCacheItem } from "./services/types";
+
+class Botanist {
+  private storage: LocalForage;
+  private isReady: boolean = false;
+
+  constructor(name: string, storeName: string) {
+    this.storage = localforage.createInstance({
+      name,
+      storeName,
+      driver: localforage.INDEXEDDB,
+    });
+  }
+
+  private async checkTimestamps(): Promise<void> {
+    if (this.isReady) return;
+
+    const currentTime = Date.now();
+
+    if ((await this.storage.length()) === 0) return;
+
+    const keys = await this.storage.keys();
+
+    for (const key of keys) {
+      const item: BundlerCacheItem | null = await this.storage.getItem(key);
+
+      if (item !== null && item.timestamp < currentTime) {
+        await this.storage.removeItem(key);
+      }
+    }
+
+    this.isReady = true;
+  }
+
+  // this doesn't like any but who cares
+  public async getItem(key: string): Promise<any | null> {
+    if (!this.isReady) await this.checkTimestamps();
+    return await this.storage.getItem(key);
+  }
+
+  public async setItem(key: string, value: BundlerCacheItem): Promise<void> {
+    await this.storage.setItem(key, value);
+  }
+}
+
+export const binariesCache = new Botanist("bundler", "binaryCache");
+export const assetsCache = new Botanist("bundler", "assetCache");
diff --git a/Bundler.Client/src/main.tsx b/Bundler.Client/src/main.tsx
@@ -6,5 +6,5 @@ import './index.css'
 ReactDOM.createRoot(document.getElementById('root')!).render(
   <React.StrictMode>
     <App />
-  </React.StrictMode>,
+  </React.StrictMode>
 )
diff --git a/Bundler.Client/src/services/Bundle.ts b/Bundler.Client/src/services/Bundle.ts
@@ -1,11 +1,7 @@
 import JSZip, { JSZipObject } from "jszip";
 import Config, { ConfigMetadata, parseConfig } from "./Config";
 
-export type BundleIcons = {
-  ctr?: Blob;
-  cafe?: Blob;
-  hac?: Blob;
-};
+import { BundleIcons, BundleType } from "./types";
 
 /*
  ** Bundler class
@@ -103,6 +99,43 @@ export default class Bundle {
     return files;
   }
 
+  private async blobToBase64(blob: Blob): Promise<string> {
+    // const value = reader.result as string;
+    // const base64data = value[value.indexOf(", ") + 1];
+
+    return new Promise((resolve, reject) => {
+      const reader = new FileReader();
+      reader.onloadend = () => {
+        const res = reader.result;
+        if (typeof res === "string") {
+          console.log(res);
+          resolve(res);
+        } else {
+          reject(res);
+        }
+      };
+      reader.readAsDataURL(blob);
+    });
+  }
+
+  public async getHashData(): Promise<string> {
+    if (this.config === undefined) {
+      throw Error("Configuration file not loaded.");
+    }
+
+    const icons = await this.findDefinedIcons();
+
+    let iconData = "";
+    let key: BundleType;
+
+    for (key in icons) {
+      if (icons[key] === undefined) continue;
+      iconData += await this.blobToBase64(icons[key] as Blob);
+    }
+
+    return this.config.source + iconData;
+  }
+
   public getMetadata(): ConfigMetadata {
     if (this.config === undefined) {
       throw Error("Configuration file not loaded.");
@@ -111,12 +144,12 @@ export default class Bundle {
     return this.config.metadata;
   }
 
-  public getTargets(): Array<string> {
+  public getTargets(): Array<BundleType> {
     if (this.config === undefined) {
       throw Error("Configuration file not loaded.");
     }
 
-    return this.config.build.targets;
+    return this.config.getTargets();
   }
 
   public isPackaged(): boolean {