feat(video): add support for webcodec video, re #695

src/application/createWebcodecVideo.js

@@ -1,10 +1,9 @@
-export function createWebcodecVideo(id, modV) {
-  return Promise(async (resolve, reject) => {
-    const url = modV.store.state.videos[id];
+export function createWebcodecVideo(id, url) {
+  return new Promise(async (resolve, reject) => {
     const video = document.createElement("video");
     video.setAttribute("crossorigin", "anonymous");
     video.setAttribute("loop", true);
-    video.onerror(reject);
+    video.onerror = reject;
     video.muted = true;
 
     video.onloadedmetadata = async () => {
@@ -18,7 +17,7 @@ export function createWebcodecVideo(id, modV) {
       // Transfer the readable stream to the worker.
       // NOTE: transferring frameStream and reading it in the worker is more
       // efficient than reading frameStream here and transferring VideoFrames individually.
-      this.$modV.store.dispatch(
+      this.store.dispatch(
         "videos/assignVideoStream",
         {
           id,
@@ -29,7 +28,7 @@ export function createWebcodecVideo(id, modV) {
         [frameStream]
       );
 
-      resolve();
+      resolve({ id, video, stream });
     };
 
     video.setAttribute("src", url);

src/application/index.js

@@ -45,6 +45,7 @@ export default class ModV {
     perceptualSpread: 0,
     perceptualSharpness: 0
   });
+  videos = {};
 
   _store = store;
   store = {
@@ -64,7 +65,7 @@ export default class ModV {
       payload: app.getAppPath()
     });
 
-    this.$worker.addEventListener("message", e => {
+    this.$worker.addEventListener("message", async e => {
       const message = e.data;
       const { type } = message;
 
@@ -79,7 +80,19 @@ export default class ModV {
       // }
 
       if (type === "createWebcodecVideo") {
-        this.createWebcodecVideo();
+        const videoContext = await this.createWebcodecVideo(
+          message.id,
+          message.path
+        );
+        this.videos[videoContext.id] = videoContext;
+      }
+
+      if (type === "removeWebcodecVideo") {
+        const { video, stream } = this.videos[message.id];
+        video.src = "";
+        // eslint-disable-next-line no-for-each/no-for-each
+        stream.getTracks().forEach(track => track.stop());
+        delete this.videos[message.id];
       }
 
       if (e.data.type === "tick" && this.ready) {

src/application/worker/store/modules/dataTypes.js

@@ -52,7 +52,7 @@ const state = {
         const { path } = options;
         let id;
         try {
-          ({ id } = await store.dispatch("images/createVideoFromPath", {
+          ({ id } = await store.dispatch("videos/createVideoFromPath", {
             path
           }));
         } catch (e) {
@@ -78,6 +78,15 @@ const state = {
         }
       });
     },
+    async destroy(textureDefinition) {
+      const { type, id } = textureDefinition;
+
+      if (type === "video") {
+        await store.dispatch("videos/removeVideoById", {
+          id
+        });
+      }
+    },
     get: textureDefinition => {
       if (!textureDefinition.location.length) {
         return false;

src/application/worker/store/modules/modules.js

@@ -564,12 +564,24 @@ const actions = {
       meta.compositeOperationInputId,
       meta.enabledInputId
     ];
-    const moduleProperties = Object.values(module.$props).map(prop => ({
-      id: prop.id,
-      type: prop.type
-    }));
+    const moduleProperties = Object.entries(module.$props).map(
+      ([key, prop]) => ({
+        key,
+        id: prop.id,
+        type: prop.type
+      })
+    );
     const inputIds = [...moduleProperties, ...metaInputIds.map(id => ({ id }))];
 
+    for (let i = 0, len = moduleProperties.length; i < len; i++) {
+      const { key, type: propType } = moduleProperties[i];
+
+      // destroy anything created by datatypes we don't need anymore
+      if (store.state.dataTypes[propType].destroy) {
+        store.state.dataTypes[propType].destroy(module.props[key]);
+      }
+    }
+
     for (let i = 0, len = inputIds.length; i < len; i++) {
       const { id: inputId, type: propType } = inputIds[i];
 

src/application/worker/store/modules/videos.js

@@ -1,16 +1,17 @@
 import Vue from "vue";
 import uuidv4 from "uuid/v4";
+import store from "../";
 
 const state = {};
 
 const getters = {
-  video: state => id => state[id]
+  video: state => id => state[id]?.outputContext?.context.canvas
 };
 
 const actions = {
-  createVideoFromPath({ commit }, { path: filePath }) {
+  createVideoFromPath({ rootState, commit }, { path: filePath }) {
     const id = uuidv4();
-    const path = `modv://${filePath}`;
+    const path = `modv://${rootState.media.path}${filePath}`;
 
     if (typeof window !== "undefined") {
       self.postMessage({
@@ -24,19 +25,85 @@ const actions = {
     return { id };
   },
 
-  assignVideoStream({ commit }, { id, stream, width, height }) {
-    commit("UPDATE_VIDEO", { id, stream, width, height });
+  async assignVideoStream({ commit }, { id, stream, width, height }) {
+    const frameReader = stream.getReader();
+    const outputContext = await store.dispatch("outputs/getAuxillaryOutput", {
+      name: state[id].path,
+      options: {
+        desynchronized: true
+      },
+      group: "videos",
+      reactToResize: false,
+      width,
+      height
+    });
+
+    frameReader.read().then(function processFrame({ done, value: frame }) {
+      const { stream, needsRemoval } = state[id];
+      if (done) {
+        return;
+      }
+
+      // NOTE: all paths below must call frame.close(). Otherwise, the GC won't
+      // be fast enough to recollect VideoFrames, and decoding can stall.
+
+      if (needsRemoval) {
+        // TODO: There might be a more elegant way of closing a stream, or other
+        // events to listen for - do we need to use frameReader.cancel(); somehow?
+        frameReader.releaseLock();
+        stream.cancel();
+
+        frame.close();
+
+        commit("REMOVE_VIDEO", { id });
+
+        if (typeof window !== "undefined") {
+          self.postMessage({
+            type: "removeWebcodecVideo",
+            id
+          });
+        }
+        return;
+      }
+
+      // Processing on 'frame' goes here!
+      // E.g. this is where encoding via a VideoEncoder could be set up, or
+      // rendering to an OffscreenCanvas.
+
+      outputContext.context.drawImage(frame, 0, 0);
+      frame.close();
+
+      frameReader.read().then(processFrame);
+    });
+
+    commit("UPDATE_VIDEO", {
+      id,
+      stream,
+      width,
+      height,
+      frameReader,
+      outputContext,
+      needsRemoval: false
+    });
+  },
+
+  async removeVideoById({ commit }, { id }) {
+    commit("UPDATE_VIDEO", { id, needsRemoval: true });
   }
 };
 
 const mutations = {
   CREATE_VIDEO(state, { id, path }) {
-    Vue.set(state, id, path);
+    Vue.set(state, id, { path });
   },
 
   UPDATE_VIDEO(state, video) {
     const { id } = video;
     state[id] = { ...state[id], ...video };
+  },
+
+  REMOVE_VIDEO(state, { id }) {
+    delete state[id];
   }
 };
 

src/background/background.js

@@ -1,6 +1,6 @@
 import { app, protocol } from "electron";
 import { APP_SCHEME } from "./background-constants";
-// import { getMediaManager } from "./media-manager";
+import { getMediaManager } from "./media-manager";
 import { openFile } from "./open-file";
 import { createWindow } from "./windows";
 
@@ -42,10 +42,10 @@ app.on("activate", async () => {
 
 // https://stackoverflow.com/a/66673831
 function fileHandler(req, callback) {
-  // const { mediaDirectoryPath } = getMediaManager();
-  const requestedPath = req.url;
+  const { mediaDirectoryPath } = getMediaManager();
+  const requestedPath = req.url.substr(7);
   // Write some code to resolve path, calculate absolute path etc
-  const check = true; // requestedPath.indexOf(mediaDirectoryPath) > -1;
+  const check = requestedPath.indexOf(mediaDirectoryPath) > -1;
 
   if (!check) {
     callback({

src/components/Controls/TextureControl.vue

@@ -36,7 +36,12 @@
     </div>
 
     <div v-if="type === 'image'">
-      <select v-model="modelImagePath" @change="setTexture('image')">
+      <select
+        v-model="modelImagePath"
+        @change="setTexture('image')"
+        :disabled="!images"
+      >
+        <option selected value="">No image</option>
         <option
           v-for="(image, index) in images"
           :key="index"
@@ -45,20 +50,47 @@
         >
       </select>
     </div>
+
+    <div v-if="type === 'video'">
+      <select
+        v-model="modelVideoPath"
+        @change="setTexture('video')"
+        :disabled="!videos"
+      >
+        <option selected value="">No video</option>
+        <option
+          v-for="output in videos"
+          :key="output.path"
+          :value="output.path"
+          >{{ output.name }}</option
+        >
+      </select>
+
+      <VideoControl
+        v-if="type === 'video' && modelVideoPath && value.id"
+        :video-id="value.id"
+      />
+    </div>
   </div>
 </template>
 
 <script>
 import constants from "../../application/constants";
+import { VideoControl } from "./VideoControl.vue";
 
 export default {
   props: ["value"],
 
+  components: {
+    VideoControl
+  },
+
   data() {
     return {
-      textureTypes: ["inherit", "group", "canvas", "image"],
+      textureTypes: ["inherit", "group", "canvas", "image", "video"],
       type: "",
       modelImagePath: "",
+      modelVideoPath: "",
       modelCanvasId: ""
     };
   },
@@ -68,8 +100,8 @@ export default {
       this.value.type && this.value.type.length
         ? this.value.type
         : this.textureTypes[0];
-    this.modelImagePath = this.value.options.path || "";
-    this.modelCanvasId = this.value.options.id || "";
+    this.modelImagePath = this.value.options?.path || "";
+    this.modelCanvasId = this.value.options?.id || "";
   },
 
   computed: {
@@ -79,8 +111,9 @@ export default {
 
     groupOutputs() {
       return Object.values(this.auxillaries).filter(
-        group =>
-          group.group === "group" && group.name !== constants.GALLERY_GROUP_NAME
+        auxillary =>
+          auxillary.group === "group" &&
+          auxillary.name !== constants.GALLERY_GROUP_NAME
       );
     },
 
@@ -101,9 +134,19 @@ export default {
     },
 
     images() {
-      return this.$modV.store.state.media.media[
-        this.$modV.store.state.projects.currentProject
-      ].image;
+      return (
+        this.$modV.store.state.media.media[
+          this.$modV.store.state.projects.currentProject
+        ].image ?? false
+      );
+    },
+
+    videos() {
+      return (
+        this.$modV.store.state.media.media[
+          this.$modV.store.state.projects.currentProject
+        ].video ?? false
+      );
     }
   },
 
@@ -118,6 +161,14 @@ export default {
         textureDefinition.options.path = this.modelImagePath;
       }
 
+      if (type === "video") {
+        if (!this.modelVideoPath) {
+          return;
+        }
+
+        textureDefinition.options.path = this.modelVideoPath;
+      }
+
       if (type === "canvas" || type === "group") {
         if (!this.modelCanvasId) {
           return;