diff --git a/src/application/createWebcodecVideo.js b/src/application/createWebcodecVideo.js index e888b4abf..7d1e5cb1a 100644 --- a/src/application/createWebcodecVideo.js +++ b/src/application/createWebcodecVideo.js @@ -1,10 +1,9 @@ -export function createWebcodecVideo(id, modV) { - return Promise(async (resolve, reject) => { - const url = modV.store.state.videos[id]; +export function createWebcodecVideo(id, url) { + return new Promise(async (resolve, reject) => { const video = document.createElement("video"); video.setAttribute("crossorigin", "anonymous"); video.setAttribute("loop", true); - video.onerror(reject); + video.onerror = reject; video.muted = true; video.onloadedmetadata = async () => { @@ -18,7 +17,7 @@ export function createWebcodecVideo(id, modV) { // Transfer the readable stream to the worker. // NOTE: transferring frameStream and reading it in the worker is more // efficient than reading frameStream here and transferring VideoFrames individually. - this.$modV.store.dispatch( + this.store.dispatch( "videos/assignVideoStream", { id, @@ -29,7 +28,7 @@ export function createWebcodecVideo(id, modV) { [frameStream] ); - resolve(); + resolve({ id, video, stream }); }; video.setAttribute("src", url); diff --git a/src/application/index.js b/src/application/index.js index bb2439a11..6ba60fe1f 100644 --- a/src/application/index.js +++ b/src/application/index.js @@ -45,6 +45,7 @@ export default class ModV { perceptualSpread: 0, perceptualSharpness: 0 }); + videos = {}; _store = store; store = { @@ -64,7 +65,7 @@ export default class ModV { payload: app.getAppPath() }); - this.$worker.addEventListener("message", e => { + this.$worker.addEventListener("message", async e => { const message = e.data; const { type } = message; @@ -79,7 +80,19 @@ export default class ModV { // } if (type === "createWebcodecVideo") { - this.createWebcodecVideo(); + const videoContext = await this.createWebcodecVideo( + message.id, + message.path + ); + this.videos[videoContext.id] = videoContext; + } + + if (type === "removeWebcodecVideo") { + const { video, stream } = this.videos[message.id]; + video.src = ""; + // eslint-disable-next-line no-for-each/no-for-each + stream.getTracks().forEach(track => track.stop()); + delete this.videos[message.id]; } if (e.data.type === "tick" && this.ready) { diff --git a/src/application/worker/store/modules/dataTypes.js b/src/application/worker/store/modules/dataTypes.js index 1b48494d3..3d9bca3da 100644 --- a/src/application/worker/store/modules/dataTypes.js +++ b/src/application/worker/store/modules/dataTypes.js @@ -52,7 +52,7 @@ const state = { const { path } = options; let id; try { - ({ id } = await store.dispatch("images/createVideoFromPath", { + ({ id } = await store.dispatch("videos/createVideoFromPath", { path })); } catch (e) { @@ -78,6 +78,15 @@ const state = { } }); }, + async destroy(textureDefinition) { + const { type, id } = textureDefinition; + + if (type === "video") { + await store.dispatch("videos/removeVideoById", { + id + }); + } + }, get: textureDefinition => { if (!textureDefinition.location.length) { return false; diff --git a/src/application/worker/store/modules/modules.js b/src/application/worker/store/modules/modules.js index 7eba092ed..ef3c9ccfa 100644 --- a/src/application/worker/store/modules/modules.js +++ b/src/application/worker/store/modules/modules.js @@ -564,12 +564,24 @@ const actions = { meta.compositeOperationInputId, meta.enabledInputId ]; - const moduleProperties = Object.values(module.$props).map(prop => ({ - id: prop.id, - type: prop.type - })); + const moduleProperties = Object.entries(module.$props).map( + ([key, prop]) => ({ + key, + id: prop.id, + type: prop.type + }) + ); const inputIds = [...moduleProperties, ...metaInputIds.map(id => ({ id }))]; + for (let i = 0, len = moduleProperties.length; i < len; i++) { + const { key, type: propType } = moduleProperties[i]; + + // destroy anything created by datatypes we don't need anymore + if (store.state.dataTypes[propType].destroy) { + store.state.dataTypes[propType].destroy(module.props[key]); + } + } + for (let i = 0, len = inputIds.length; i < len; i++) { const { id: inputId, type: propType } = inputIds[i]; diff --git a/src/application/worker/store/modules/videos.js b/src/application/worker/store/modules/videos.js index f6c76219b..5ccbe51af 100644 --- a/src/application/worker/store/modules/videos.js +++ b/src/application/worker/store/modules/videos.js @@ -1,16 +1,17 @@ import Vue from "vue"; import uuidv4 from "uuid/v4"; +import store from "../"; const state = {}; const getters = { - video: state => id => state[id] + video: state => id => state[id]?.outputContext?.context.canvas }; const actions = { - createVideoFromPath({ commit }, { path: filePath }) { + createVideoFromPath({ rootState, commit }, { path: filePath }) { const id = uuidv4(); - const path = `modv://${filePath}`; + const path = `modv://${rootState.media.path}${filePath}`; if (typeof window !== "undefined") { self.postMessage({ @@ -24,19 +25,85 @@ const actions = { return { id }; }, - assignVideoStream({ commit }, { id, stream, width, height }) { - commit("UPDATE_VIDEO", { id, stream, width, height }); + async assignVideoStream({ commit }, { id, stream, width, height }) { + const frameReader = stream.getReader(); + const outputContext = await store.dispatch("outputs/getAuxillaryOutput", { + name: state[id].path, + options: { + desynchronized: true + }, + group: "videos", + reactToResize: false, + width, + height + }); + + frameReader.read().then(function processFrame({ done, value: frame }) { + const { stream, needsRemoval } = state[id]; + if (done) { + return; + } + + // NOTE: all paths below must call frame.close(). Otherwise, the GC won't + // be fast enough to recollect VideoFrames, and decoding can stall. + + if (needsRemoval) { + // TODO: There might be a more elegant way of closing a stream, or other + // events to listen for - do we need to use frameReader.cancel(); somehow? + frameReader.releaseLock(); + stream.cancel(); + + frame.close(); + + commit("REMOVE_VIDEO", { id }); + + if (typeof window !== "undefined") { + self.postMessage({ + type: "removeWebcodecVideo", + id + }); + } + return; + } + + // Processing on 'frame' goes here! + // E.g. this is where encoding via a VideoEncoder could be set up, or + // rendering to an OffscreenCanvas. + + outputContext.context.drawImage(frame, 0, 0); + frame.close(); + + frameReader.read().then(processFrame); + }); + + commit("UPDATE_VIDEO", { + id, + stream, + width, + height, + frameReader, + outputContext, + needsRemoval: false + }); + }, + + async removeVideoById({ commit }, { id }) { + commit("UPDATE_VIDEO", { id, needsRemoval: true }); } }; const mutations = { CREATE_VIDEO(state, { id, path }) { - Vue.set(state, id, path); + Vue.set(state, id, { path }); }, UPDATE_VIDEO(state, video) { const { id } = video; state[id] = { ...state[id], ...video }; + }, + + REMOVE_VIDEO(state, { id }) { + delete state[id]; } }; diff --git a/src/background/background.js b/src/background/background.js index 0569ac21f..789c7c6d8 100644 --- a/src/background/background.js +++ b/src/background/background.js @@ -1,6 +1,6 @@ import { app, protocol } from "electron"; import { APP_SCHEME } from "./background-constants"; -// import { getMediaManager } from "./media-manager"; +import { getMediaManager } from "./media-manager"; import { openFile } from "./open-file"; import { createWindow } from "./windows"; @@ -42,10 +42,10 @@ app.on("activate", async () => { // https://stackoverflow.com/a/66673831 function fileHandler(req, callback) { - // const { mediaDirectoryPath } = getMediaManager(); - const requestedPath = req.url; + const { mediaDirectoryPath } = getMediaManager(); + const requestedPath = req.url.substr(7); // Write some code to resolve path, calculate absolute path etc - const check = true; // requestedPath.indexOf(mediaDirectoryPath) > -1; + const check = requestedPath.indexOf(mediaDirectoryPath) > -1; if (!check) { callback({ diff --git a/src/components/Controls/TextureControl.vue b/src/components/Controls/TextureControl.vue index 5def84c22..bb0e3c1f3 100644 --- a/src/components/Controls/TextureControl.vue +++ b/src/components/Controls/TextureControl.vue @@ -36,7 +36,12 @@