From 6985cf6337430d6da3379a5872d1073430043b3d Mon Sep 17 00:00:00 2001 From: Sam Wray Date: Tue, 23 Aug 2022 19:20:51 +0100 Subject: [PATCH] feat(video): add preliminary changes for video support, re #695 --- src/application/createWebcodecVideo.js | 38 ++++++++++++++ src/application/index.js | 6 +++ .../worker/store/modules/dataTypes.js | 17 ++++++- .../worker/store/modules/videos.js | 49 +++++++++++++++++++ src/background/background.js | 24 +++++++++ 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 src/application/createWebcodecVideo.js create mode 100644 src/application/worker/store/modules/videos.js diff --git a/src/application/createWebcodecVideo.js b/src/application/createWebcodecVideo.js new file mode 100644 index 000000000..e888b4abf --- /dev/null +++ b/src/application/createWebcodecVideo.js @@ -0,0 +1,38 @@ +export function createWebcodecVideo(id, modV) { + return Promise(async (resolve, reject) => { + const url = modV.store.state.videos[id]; + const video = document.createElement("video"); + video.setAttribute("crossorigin", "anonymous"); + video.setAttribute("loop", true); + video.onerror(reject); + video.muted = true; + + video.onloadedmetadata = async () => { + const stream = video.captureStream(); + const [track] = stream.getVideoTracks(); + + // eslint-disable-next-line + const processor = new MediaStreamTrackProcessor(track); + const frameStream = processor.readable; + + // Transfer the readable stream to the worker. + // NOTE: transferring frameStream and reading it in the worker is more + // efficient than reading frameStream here and transferring VideoFrames individually. + this.$modV.store.dispatch( + "videos/assignVideoStream", + { + id, + stream: frameStream, + width: video.videoWidth || 256, + height: video.videoHeight || 256 + }, + [frameStream] + ); + + resolve(); + }; + + video.setAttribute("src", url); + await video.play(); + }); +} diff --git a/src/application/index.js b/src/application/index.js index f76f85f15..bb2439a11 100644 --- a/src/application/index.js +++ b/src/application/index.js @@ -15,6 +15,7 @@ import PromiseWorker from "promise-worker-transferable"; import Vue from "vue"; import { ipcRenderer } from "electron"; import { app } from "@electron/remote"; +import { createWebcodecVideo } from "./createWebcodecVideo"; let imageBitmap; const imageBitmapQueue = []; @@ -27,6 +28,7 @@ export default class ModV { setupBeatDetektor = setupBeatDetektor; setupMidi = setupMidi; windowHandler = windowHandler; + createWebcodecVideo = createWebcodecVideo; use = use; debug = false; features = Vue.observable({ @@ -76,6 +78,10 @@ export default class ModV { // console.log(`⚙️%c ${type}`, "color: red"); // } + if (type === "createWebcodecVideo") { + this.createWebcodecVideo(); + } + if (e.data.type === "tick" && this.ready) { this.tick(e.data.payload); return; diff --git a/src/application/worker/store/modules/dataTypes.js b/src/application/worker/store/modules/dataTypes.js index 28561bfdf..1b48494d3 100644 --- a/src/application/worker/store/modules/dataTypes.js +++ b/src/application/worker/store/modules/dataTypes.js @@ -48,7 +48,22 @@ const state = { textureDefinition.id = id; } - if (type === "canvas" || type == "group") { + if (type === "video") { + const { path } = options; + let id; + try { + ({ id } = await store.dispatch("images/createVideoFromPath", { + path + })); + } catch (e) { + console.error(e); + } + + textureDefinition.location = "videos/video"; + textureDefinition.id = id; + } + + if (type === "canvas" || type === "group") { const { id } = options; textureDefinition.location = "outputs/auxillaryCanvas"; textureDefinition.id = id; diff --git a/src/application/worker/store/modules/videos.js b/src/application/worker/store/modules/videos.js new file mode 100644 index 000000000..f6c76219b --- /dev/null +++ b/src/application/worker/store/modules/videos.js @@ -0,0 +1,49 @@ +import Vue from "vue"; +import uuidv4 from "uuid/v4"; + +const state = {}; + +const getters = { + video: state => id => state[id] +}; + +const actions = { + createVideoFromPath({ commit }, { path: filePath }) { + const id = uuidv4(); + const path = `modv://${filePath}`; + + if (typeof window !== "undefined") { + self.postMessage({ + type: "createWebcodecVideo", + id, + path + }); + } + + commit("CREATE_VIDEO", { id, path }); + return { id }; + }, + + assignVideoStream({ commit }, { id, stream, width, height }) { + commit("UPDATE_VIDEO", { id, stream, width, height }); + } +}; + +const mutations = { + CREATE_VIDEO(state, { id, path }) { + Vue.set(state, id, path); + }, + + UPDATE_VIDEO(state, video) { + const { id } = video; + state[id] = { ...state[id], ...video }; + } +}; + +export default { + namespaced: true, + state, + getters, + actions, + mutations +}; diff --git a/src/background/background.js b/src/background/background.js index deb720c65..0569ac21f 100644 --- a/src/background/background.js +++ b/src/background/background.js @@ -1,5 +1,6 @@ import { app, protocol } from "electron"; import { APP_SCHEME } from "./background-constants"; +// import { getMediaManager } from "./media-manager"; import { openFile } from "./open-file"; import { createWindow } from "./windows"; @@ -39,10 +40,33 @@ app.on("activate", async () => { createWindow("mainWindow"); }); +// https://stackoverflow.com/a/66673831 +function fileHandler(req, callback) { + // const { mediaDirectoryPath } = getMediaManager(); + const requestedPath = req.url; + // Write some code to resolve path, calculate absolute path etc + const check = true; // requestedPath.indexOf(mediaDirectoryPath) > -1; + + if (!check) { + callback({ + // -6 is FILE_NOT_FOUND + // https://source.chromium.org/chromium/chromium/src/+/master:net/base/net_error_list.h + error: -6 + }); + return; + } + + callback({ + path: requestedPath + }); +} + // This method will be called when Electron has finished // initialization and is ready to create browser windows. // Some APIs can only be used after this event occurs. app.on("ready", async () => { + protocol.registerFileProtocol("modv", fileHandler); + app.commandLine.appendSwitch( "disable-backgrounding-occluded-windows", "true"