Merge pull request #836 from vcync/next

Merge next into main

.github/FUNDING.yml

@@ -1,3 +1,4 @@
 github: 2xAA
+ko_fi: twoxAA
 patreon: vcync
 custom: ["https://www.paypal.me/2xAA"]

package.json

@@ -37,11 +37,10 @@
     "golden-layout": "^1.5.9",
     "grandiose": "github:vcync/grandiose#feat/workerCompatibility",
     "interactive-shader-format": "github:vcync/interactive-shader-format-js#78b62b6f4c787b870852df4c8e7b1131e331d8a6",
-    "lfo-for-modv": "0.0.1",
     "lodash.get": "^4.4.2",
     "lodash.set": "^4.3.2",
     "mathjs": "^7.5.1",
-    "meyda": "5.6.0",
+    "meyda": "^5.6.0",
     "mkdirp": "^0.5.1",
     "npm": "6.14.6",
     "nwjs-menu-browser": "^1.0.0",
@@ -54,15 +53,15 @@
     "stream-to-blob": "^2.0.0",
     "tap-tempo": "^0.1.1",
     "three": "^0.131.3",
-    "uuid": "^3.3.3",
-    "vue": "^2.6.10",
+    "uuid": "^9.0.0",
+    "vue": "^2.7.14",
     "vue-class-component": "^7.2.3",
     "vue-color": "^2.7.1",
     "vue-fragment": "^1.5.1",
     "vue-golden-layout": "^2.1.0",
     "vue-property-decorator": "^8.3.0",
     "vue-smooth-dnd": "^0.8.1",
-    "vuex": "^3.1.2",
+    "vuex": "^3.6.2",
     "vuex-persistedstate": "^4.0.0-beta.3",
     "webpack": "^4.43.0",
     "webpack-2": "npm:[email protected]"
@@ -78,7 +77,7 @@
     "@vue/eslint-config-prettier": "^4.0.1",
     "babel-eslint": "^10.0.3",
     "core-js": "^3.19.1",
-    "electron": "^18.3.7",
+    "electron": "^23.1.2",
     "electron-builder": "^22.9.1",
     "electron-notarize": "^1.2.2",
     "eslint": "^5.16.0",
@@ -89,7 +88,8 @@
     "sass-loader": "^7.3.1",
     "text-loader": "0.0.1",
     "vue-cli-plugin-electron-builder": "^2.0.0",
-    "vue-template-compiler": "^2.6.10",
+    "vue-template-babel-compiler": "^2.0.0",
+    "vue-template-compiler": "^2.7.14",
     "vuex-localstorage": "^1.0.0",
     "worker-loader": "^2.0.0"
   },

src/application/createWebcodecVideo.js

@@ -0,0 +1,42 @@
+export function createWebcodecVideo({ id, url, textureDefinition }) {
+  return new Promise(async (resolve, reject) => {
+    const video = document.createElement("video");
+    video.setAttribute("crossorigin", "anonymous");
+    video.setAttribute("loop", true);
+    video.onerror = reject;
+    video.muted = true;
+
+    video.onloadedmetadata = async () => {
+      const stream = video.captureStream();
+      const [track] = stream.getVideoTracks();
+
+      // eslint-disable-next-line
+      const processor = new MediaStreamTrackProcessor(track);
+      const frameStream = processor.readable;
+
+      // Transfer the readable stream to the worker.
+      // NOTE: transferring frameStream and reading it in the worker is more
+      // efficient than reading frameStream here and transferring VideoFrames individually.
+      this.store.dispatch(
+        "videos/assignVideoStream",
+        {
+          id,
+          stream: frameStream,
+          width: video.videoWidth || 256,
+          height: video.videoHeight || 256
+        },
+        [frameStream]
+      );
+
+      resolve({ id, video, stream });
+    };
+
+    video.setAttribute("src", url);
+    video.playbackRate = textureDefinition?.options?.playbackrate;
+    if (textureDefinition?.options?.paused) {
+      video.pause();
+    } else {
+      video.play();
+    }
+  });
+}

src/application/index.js

@@ -16,6 +16,7 @@ import PromiseWorker from "promise-worker-transferable";
 import Vue from "vue";
 import { ipcRenderer } from "electron";
 import { app } from "@electron/remote";
+import { createWebcodecVideo } from "./createWebcodecVideo";
 import { GROUP_ENABLED } from "./constants";
 
 let imageBitmap;
@@ -29,6 +30,7 @@ export default class ModV {
   setupBeatDetektor = setupBeatDetektor;
   setupMidi = setupMidi;
   windowHandler = windowHandler;
+  createWebcodecVideo = createWebcodecVideo;
   use = use;
   debug = false;
   features = Vue.observable({
@@ -45,6 +47,7 @@ export default class ModV {
     perceptualSpread: 0,
     perceptualSharpness: 0
   });
+  videos = {};
 
   _store = store;
   store = {
@@ -64,7 +67,7 @@ export default class ModV {
       payload: app.getAppPath()
     });
 
-    this.$worker.addEventListener("message", e => {
+    this.$worker.addEventListener("message", async e => {
       const message = e.data;
       const { type } = message;
 
@@ -78,6 +81,19 @@ export default class ModV {
       //   console.log(`⚙️%c ${type}`, "color: red");
       // }
 
+      if (type === "createWebcodecVideo") {
+        const videoContext = await this.createWebcodecVideo(message);
+        this.videos[videoContext.id] = videoContext;
+      }
+
+      if (type === "removeWebcodecVideo") {
+        const { video, stream } = this.videos[message.id];
+        video.src = "";
+        // eslint-disable-next-line no-for-each/no-for-each
+        stream.getTracks().forEach(track => track.stop());
+        delete this.videos[message.id];
+      }
+
       if (e.data.type === "tick" && this.ready) {
         this.tick(e.data.payload);
         return;

src/application/sample-modules/Texture2d.js

@@ -32,24 +32,75 @@ export default {
       min: -100,
       max: 100,
       step: 1
+    },
+    constrain: {
+      label: "Constrain",
+      type: "enum",
+      default: "none",
+      enum: [
+        { label: "None (Scale)", value: "none" },
+        { label: "Contain", value: "contain" },
+        { label: "Cover", value: "cover" }
+      ]
     }
   },
   draw({ canvas: { width, height }, context, props }) {
-    const { scale, offsetX, offsetY } = props;
+    const { constrain, offsetX, offsetY, scale, texture } = props;
+
+    if (texture.value) {
+      let { width: imageWidth, height: imageHeight } = texture.value;
+
+      let x;
+      let y;
+
+      if (constrain === "contain") {
+        imageHeight = (imageHeight / imageWidth) * width;
+        imageWidth = width;
+
+        y = (height - imageHeight) / 2;
+        x = 0;
+
+        if (imageHeight > height) {
+          imageWidth = (imageWidth / imageHeight) * height;
+          imageHeight = height;
+
+          y = 0;
+          x = (width - imageWidth) / 2;
+        }
+      } else if (constrain === "cover") {
+        const imageRatio = imageHeight / imageWidth;
+        const canvasRatio = height / width;
+
+        if (imageRatio < canvasRatio) {
+          imageWidth = (width * canvasRatio) / imageRatio;
+          imageHeight = height;
+
+          x = (width - imageWidth) / 2;
+          y = 0;
+        } else {
+          imageHeight = width * imageRatio;
+          imageWidth = width;
+
+          x = 0;
+          y = (height - imageHeight * scale) / 2;
+        }
+      } else {
+        imageWidth = imageWidth * scale;
+        imageHeight = imageHeight * scale;
+
+        x = (width - imageWidth) / 2;
+        y = (height - imageHeight) / 2;
+      }
 
-    if (props.texture.value) {
-      const { width: imageWidth, height: imageHeight } = props.texture.value;
-      const x = (width - imageWidth * scale) / 2;
-      const y = (height - imageHeight * scale) / 2;
       const calculatedOffsetX = (width / 100) * offsetX;
       const calculatedOffsetY = (height / 100) * offsetY;
 
       context.drawImage(
-        props.texture.value,
+        texture.value,
         x + calculatedOffsetX,
         y + calculatedOffsetY,
-        imageWidth * scale,
-        imageHeight * scale
+        imageWidth,
+        imageHeight
       );
     }
   }

src/application/utils/conform-file-path.js

@@ -0,0 +1,6 @@
+import path from "path";
+
+export const conformFilePath = (filePath = "") =>
+  filePath
+    .split(process.platform === "win32" ? path.posix.sep : path.win32.sep)
+    .join(path.sep);

src/application/utils/get-prop-default.js

@@ -22,7 +22,8 @@ export default async function getPropDefault(
     if (store.state.dataTypes[type].create) {
       return await store.state.dataTypes[type].create(
         propData,
-        module.meta.isGallery
+        module.meta.isGallery,
+        useExistingData
       );
     }
   }

src/application/worker/store/modules/dataTypes.js

@@ -28,10 +28,10 @@ const state = {
     inputs: () => ({ r: 0, g: 0, b: 0, a: 0 })
   },
   texture: {
-    async create(textureDefinition = {}, isGallery) {
+    async create(textureDefinition = {}, isGallery, useExistingData = false) {
       const { type, options } = textureDefinition;
-      textureDefinition.location = "";
-      textureDefinition.id = "";
+      textureDefinition.location = textureDefinition.location ?? "";
+      // textureDefinition.id = textureDefinition.id ?? "";
 
       if (type === "image") {
         const { path } = options;
@@ -48,7 +48,22 @@ const state = {
         textureDefinition.id = id;
       }
 
-      if (type === "canvas" || type == "group") {
+      if (type === "video" && (useExistingData || !textureDefinition.id)) {
+        let id;
+        try {
+          ({ id } = await store.dispatch(
+            "videos/createVideoFromPath",
+            textureDefinition
+          ));
+        } catch (e) {
+          console.error(e);
+        }
+
+        textureDefinition.location = "videos/video";
+        textureDefinition.id = id;
+      }
+
+      if (type === "canvas" || type === "group") {
         const { id } = options;
         textureDefinition.location = "outputs/auxillaryCanvas";
         textureDefinition.id = id;
@@ -63,6 +78,15 @@ const state = {
         }
       });
     },
+    async destroy(textureDefinition) {
+      const { type, id } = textureDefinition;
+
+      if (type === "video") {
+        await store.dispatch("videos/removeVideoById", {
+          id
+        });
+      }
+    },
     get: textureDefinition => {
       if (!textureDefinition.location.length) {
         return false;

src/application/worker/store/modules/errors.js

@@ -1,5 +1,5 @@
 import Vue from "vue";
-import uuidv4 from "uuid/v4";
+import { v4 as uuidv4 } from "uuid";
 
 const state = {
   messages: {}

src/application/worker/store/modules/expressions.js

@@ -1,4 +1,4 @@
-import uuidv4 from "uuid/v4";
+import { v4 as uuidv4 } from "uuid";
 const math = require("mathjs");
 
 const state = {

src/application/worker/store/modules/groups.js

@@ -1,7 +1,7 @@
 import SWAP from "./common/swap";
 import store from "../";
 import constants, { GROUP_DISABLED } from "../../../constants";
-import uuidv4 from "uuid/v4";
+import { v4 as uuidv4 } from "uuid";
 import { applyExpression } from "../../../utils/apply-expression";
 
 /**

src/application/worker/store/modules/images.js

@@ -3,9 +3,10 @@ import fs from "fs";
 import path from "path";
 
 import Vue from "vue";
-import uuidv4 from "uuid/v4";
+import { v4 as uuidv4 } from "uuid";
 
 import store from "../";
+import { conformFilePath } from "../../../utils/conform-file-path";
 
 const state = {};
 
@@ -19,7 +20,10 @@ const actions = {
     let joinedFilePath;
 
     try {
-      joinedFilePath = path.join(store.state.media.path, filePath);
+      joinedFilePath = path.join(
+        store.state.media.path,
+        conformFilePath(filePath)
+      );
     } catch (e) {
       console.log(e);
     }

src/application/worker/store/modules/inputs.js

@@ -1,5 +1,5 @@
 import Vue from "vue";
-import uuidv4 from "uuid/v4";
+import { v4 as uuidv4 } from "uuid";
 import SWAP from "./common/swap";
 
 /**

src/application/worker/store/modules/meyda.js

@@ -6,7 +6,7 @@ import {
   MAX_SMOOTHING,
   SMOOTHING_STEP
 } from "../../audio-features";
-import uuidv4 from "uuid/v4";
+import { v4 as uuidv4 } from "uuid";
 
 const state = {
   features: [