diff --git a/app/api/routes/channels/index.js b/app/api/routes/channels/index.js index 312b38d..89ff9a7 100644 --- a/app/api/routes/channels/index.js +++ b/app/api/routes/channels/index.js @@ -1,6 +1,7 @@ const channels = require('express').Router(); channels.use('/zoom', require('./zoom')); +channels.use('/youtube', require('./youtube')); const iconPrefix = '/static/channel-icons'; const configPatePathPrefix = '/captioner/settings/channels/new?type='; @@ -55,12 +56,32 @@ const channelsList = [ iconPath: `${iconPrefix}/youtube.png`, limit: 1, configPagePath: `${configPatePathPrefix}youtube`, + requiredExperiment: 'youtube', }, ]; channels.get('/', async (req, res, next) => { - // Sort alphabetically by name - res.send(channelsList.sort((a, b) => a.name.localeCompare(b.name))); + let { experiments } = req.query; + experiments = experiments ? String(experiments).split(',') : []; + + let channelsToReturn = [ + ...channelsList.sort((a, b) => + // Sort alphabetically by name + a.name.localeCompare(b.name) + ), + ].map((c) => ({ ...c })); // clone + + // Filter out any channels that require experiments + // to be enabled, unless those experiments are enabled + channelsToReturn = channelsToReturn.filter( + (channel) => + !channel.requiredExperiment || + experiments.includes(channel.requiredExperiment) + ); + + channelsToReturn.forEach((channel) => delete channel.requiredExperiment); + + res.send(channelsToReturn); }); channels.get('/:id', async (req, res, next) => { diff --git a/app/api/routes/channels/youtube/index.js b/app/api/routes/channels/youtube/index.js new file mode 100644 index 0000000..64a0d6e --- /dev/null +++ b/app/api/routes/channels/youtube/index.js @@ -0,0 +1,76 @@ +const youtube = require('express').Router(); +const axios = require('axios'); +const rateLimit = require('express-rate-limit'); + +const rateLimitWindowMinutes = 5; +const requestsAllowedPerSecond = 1; // Frontend limits to one request per second +const rateLimitLeeway = 10; +const rateLimiter = rateLimit({ + windowMs: rateLimitWindowMinutes * 60 * 1000, + max: rateLimitWindowMinutes * requestsAllowedPerSecond * 60 + rateLimitLeeway, +}); + +youtube.use('/', rateLimiter); + +youtube.post('/', async (req, res) => { + const { apiPath, transcript } = req.body; + if (!apiPath || !transcript) { + return res.sendStatus(400); + } + + try { + // Verify this is actually a URL and a YouTube closed caption URL + const url = new URL(apiPath); + if ( + url.origin !== 'http://upload.youtube.com' || + url.pathname !== '/closedcaption' + ) { + throw new Error(); + } + } catch (e) { + return res.sendStatus(400); + } + + // At one point I thought doing this might improve caption delay + // in YouTube but I don't think so + const offsetTimestampSeconds = 0; + + const body = `${ + new Date(new Date().getTime() - 1000 * offsetTimestampSeconds) + .toISOString() + .split('Z')[0] + }\n${transcript}\n`; + + axios + .post(apiPath, body, { + headers: { + 'Content-Type': 'text/plain', + }, + }) + .then(() => { + // We got a successful response + res.sendStatus(200); + }) + .catch((e) => { + if (e.code === 'ENOTFOUND') { + return res.sendStatus(404); + } else { + const errorCode = + e && e.response && e.response.status ? e.response.status : undefined; + switch (errorCode) { + case 400: + return res + .status(400) + .send( + `Error: The Zoom meeting has not started yet or it has already ended.` + ); + default: + return res + .status(520) + .send(`Something went wrong. (${errorCode || 'Unknown error'})`); + } + } + }); +}); + +module.exports = youtube; diff --git a/app/api/routes/channels/zoom/index.js b/app/api/routes/channels/zoom/index.js index 0c4b0a8..3bdd09b 100644 --- a/app/api/routes/channels/zoom/index.js +++ b/app/api/routes/channels/zoom/index.js @@ -12,7 +12,7 @@ const zoomRateLimiter = rateLimit({ zoom.use('/', zoomRateLimiter); -zoom.post('/api', async (req, res) => { +zoom.post('/', async (req, res) => { const { apiPath, transcript } = req.body; if (!apiPath || !transcript) { return res.sendStatus(400); diff --git a/app/components/channels/ChannelsPopup.vue b/app/components/channels/ChannelsPopup.vue index f17763a..4d632b0 100644 --- a/app/components/channels/ChannelsPopup.vue +++ b/app/components/channels/ChannelsPopup.vue @@ -72,8 +72,8 @@ export default { channels: [], }; }, - async mounted() { - this.channels = await this.$axios.$get('/api/channels'); + async created() { + this.channels = await this.$store.dispatch('channels/GET_CHANNELS'); }, methods: { toggleChannel(channelId, onOrOff) { diff --git a/app/components/channels/editors/youtube.vue b/app/components/channels/editors/youtube.vue new file mode 100644 index 0000000..09c90b8 --- /dev/null +++ b/app/components/channels/editors/youtube.vue @@ -0,0 +1,87 @@ + + + diff --git a/app/pages/captioner/settings/channels.vue b/app/pages/captioner/settings/channels.vue index 36668eb..741939f 100755 --- a/app/pages/captioner/settings/channels.vue +++ b/app/pages/captioner/settings/channels.vue @@ -141,11 +141,23 @@ export default { channels: [], }; }, - async asyncData({ $axios }) { - const channels = await $axios.$get('/api/channels'); - return { - channels, - }; + async created() { + // Wait until settings are loaded + if (!this.$store.state.settingsLoaded) { + await new Promise((resolve) => { + this.$store.watch( + (state) => { + return state.settingsLoaded; + }, + (loaded) => { + if (loaded) { + resolve(); + } + } + ); + }); + } + this.channels = await this.$store.dispatch('channels/GET_CHANNELS'); }, methods: { channelInfo(id) { diff --git a/app/pages/captioner/settings/channels/_channelId.vue b/app/pages/captioner/settings/channels/_channelId.vue index cfa0d21..4dc7744 100644 --- a/app/pages/captioner/settings/channels/_channelId.vue +++ b/app/pages/captioner/settings/channels/_channelId.vue @@ -67,7 +67,7 @@ export default { }, async beforeCreate() { try { - const channels = await this.$axios.$get('/api/channels'); + const channels = await this.$store.dispatch('channels/GET_CHANNELS'); await this.settingsLoaded(); diff --git a/app/pages/captioner/settings/channels/deletemezoom.vue b/app/pages/captioner/settings/channels/deletemezoom.vue deleted file mode 100644 index cb2d002..0000000 --- a/app/pages/captioner/settings/channels/deletemezoom.vue +++ /dev/null @@ -1,114 +0,0 @@ - - - diff --git a/app/pages/captioner/settings/experiments/index.vue b/app/pages/captioner/settings/experiments/index.vue index e7d2846..537ee97 100755 --- a/app/pages/captioner/settings/experiments/index.vue +++ b/app/pages/captioner/settings/experiments/index.vue @@ -222,6 +222,12 @@ export default { description: 'After speech is converted to text, convert the text back to speech using speech synthesis.', }, + { + id: 'youtube', + name: 'YouTube integration', + description: + 'Add an experimental YouTube live closed captions integration. Go to the Channels page to set it up.', + }, ], experimentIdToAdd: '', diff --git a/app/plugins/channels/index.js b/app/plugins/channels/index.js index 6507b14..d7590ba 100644 --- a/app/plugins/channels/index.js +++ b/app/plugins/channels/index.js @@ -1,8 +1,9 @@ -import zoom from './zoom'; import webhook from './webhook'; +import youtube from './youtube'; +import zoom from './zoom'; // Available channels -const channels = { zoom, webhook }; +const channels = { webhook, youtube, zoom }; // Every time a channel is registered, it returns a function // we need to run if we need to deregister it in the future diff --git a/app/plugins/channels/youtube.js b/app/plugins/channels/youtube.js new file mode 100644 index 0000000..10dd3ab --- /dev/null +++ b/app/plugins/channels/youtube.js @@ -0,0 +1,190 @@ +export default ({ $store, $axios, channelId, channelParameters }) => { + // Register + if (!channelParameters.url) { + $store.commit('UPDATE_CHANNEL_ERROR', { + channelId, + error: 'YouTube API path is missing.', + }); + + // Turn off the channel because it's not configured correctly + $store.commit('TOGGLE_CHANNEL_ON_OR_OFF', { channelId, onOrOff: false }); + // No need to unregister here because we haven't registered yet + return; + } + + try { + new URL(channelParameters.url); + } catch (e) { + $store.commit('UPDATE_CHANNEL_ERROR', { + channelId, + error: + 'This channel has been turned off because the YouTube closed captions ingestion URL is not a valid URL. Make sure the YouTube closed captions ingestion URL is correct and try again.', + }); + + // Turn off the channel because it's not configured correctly + $store.commit('TOGGLE_CHANNEL_ON_OR_OFF', { channelId, onOrOff: false }); + // No need to unregister here because we haven't registered yet + return; + } + + let transcriptBuffer = []; + let transcriptCurrentlyDisplayed = []; + const maxCharactersPerLine = 40; + let lastSequenceNumber = 0; + const youtubeSequenceNumberLocalStorageKey = + 'webcaptioner-channels-youtube-sequence-number'; + + const unsubscribeFn = $store.subscribe((mutation, state) => { + if ( + [ + 'captioner/APPEND_TRANSCRIPT_STABILIZED', + 'captioner/APPEND_TRANSCRIPT_FINAL', + 'captioner/CLEAR_TRANSCRIPT', + ].includes(mutation.type) + ) { + if (mutation.type === 'captioner/APPEND_TRANSCRIPT_STABILIZED') { + transcriptBuffer.push(mutation.payload.transcript); + } else if ( + (mutation.type === 'captioner/APPEND_TRANSCRIPT_FINAL' && + mutation.payload.clearLimitedSpaceReceivers) || + mutation.type === 'captioner/CLEAR_TRANSCRIPT' + ) { + // Clear the output (this doesn't work completely yet) + transcriptBuffer = ['\n', '\n']; + } + } + }); + + const errorDates = []; + let intervalsWithoutBufferClear = 0; + const youtubeSendInterval = setInterval(() => { + if (!transcriptBuffer.length) { + return; + } + + try { + let localStorageValues = JSON.parse( + localStorage.getItem(youtubeSequenceNumberLocalStorageKey) + ); + + if (localStorageValues.url === channelParameters.url) { + // The stored sequenceNumber is for the current API token and not + // a previous one. Restore the value. + lastSequenceNumber = Number(localStorageValues.lastSequenceNumber); + } + } catch (e) { + // No local storage value found. Assume we're starting over. + lastSequenceNumber = 0; + } + + // Consume the buffer ONLY WHEN IT IS FULL + + transcriptCurrentlyDisplayed.push(...transcriptBuffer); + transcriptBuffer = []; + + // Add line breaks if necessary + const firstWordAfterLastLineBreakIndex = + transcriptCurrentlyDisplayed.lastIndexOf('\n') + 1; // or this may be '0' if there are no line breaks yet + for ( + let i = firstWordAfterLastLineBreakIndex; + i < transcriptCurrentlyDisplayed.length; + i++ + ) { + // Check the length by adding one more word at a time + // up to but not including last + const someWordsAfterLastLineBreak = transcriptCurrentlyDisplayed.slice( + firstWordAfterLastLineBreakIndex, + i + 1 + ); + + if (someWordsAfterLastLineBreak.join(' ').length > maxCharactersPerLine) { + // Add a line break before the `i`th word + transcriptCurrentlyDisplayed.splice(i, 0, '\n'); + break; + } + } + + // Enforce two lines max by removing content before the + // first line break if we now have two line breaks + let transcriptRemovedFromBufferToSend = []; + if ( + transcriptCurrentlyDisplayed.filter((word) => word === '\n').length >= 2 + ) { + const firstLineBreakIndex = transcriptCurrentlyDisplayed.findIndex( + (word) => word === '\n' + ); + + transcriptRemovedFromBufferToSend = transcriptCurrentlyDisplayed.splice( + 0, + firstLineBreakIndex + 1 + ); + } + + if (intervalsWithoutBufferClear > 6) { + transcriptRemovedFromBufferToSend = transcriptCurrentlyDisplayed; + transcriptCurrentlyDisplayed = []; + } + + if (transcriptRemovedFromBufferToSend.length > 0) { + intervalsWithoutBufferClear = 0; + const transcript = transcriptRemovedFromBufferToSend + .join(' ') + .replace(' \n ', '\n') // remove spaces around line breaks + .trim(); + + if (!transcript) { + return; + } + + let apiPath = new URL(channelParameters.url); + apiPath.searchParams.append('seq', String(lastSequenceNumber)); + + $axios + .$post('/api/channels/youtube', { + apiPath, + transcript, + }) + .catch((e) => { + errorDates.push(new Date()); + + const errorPeriodSeconds = 30; + const maxErrorsInPeriod = 10; + const errorPeriodStartDate = new Date( + Date.now() - 1000 * errorPeriodSeconds + ); + + if ( + errorDates.filter((date) => date > errorPeriodStartDate).length > + maxErrorsInPeriod + ) { + $store.commit('UPDATE_CHANNEL_ERROR', { + channelId, + error: `This channel has been turned off because we received an error back from YouTube ${maxErrorsInPeriod} times in the last ${errorPeriodSeconds} seconds that this channel was on. Make sure your YouTube closed captions ingestion URL is correct. Note that you will need a new YouTube closed captions ingestion URL for every live stream.`, + }); + + // Turn off the channel because it's not configured correctly + $store.commit('TOGGLE_CHANNEL_ON_OR_OFF', { + channelId, + onOrOff: false, + }); + return; + } + }); + + lastSequenceNumber++; + localStorage.setItem( + youtubeSequenceNumberLocalStorageKey, + JSON.stringify({ + lastSequenceNumber, + url: channelParameters.url, + }) + ); + } + }, 1000); + + return () => { + // Unregister function + unsubscribeFn(); + clearInterval(youtubeSendInterval); + }; +}; diff --git a/app/plugins/channels/zoom.js b/app/plugins/channels/zoom.js index 0717c2d..b68a16c 100644 --- a/app/plugins/channels/zoom.js +++ b/app/plugins/channels/zoom.js @@ -131,7 +131,7 @@ export default ({ $store, $axios, channelId, channelParameters }) => { .trim(); $axios - .$post('/api/channels/zoom/api', { + .$post('/api/channels/zoom', { apiPath, transcript, }) diff --git a/app/store/channels.js b/app/store/channels.js new file mode 100644 index 0000000..1017665 --- /dev/null +++ b/app/store/channels.js @@ -0,0 +1,12 @@ +export const actions = { + async GET_CHANNELS({ state, rootState }) { + const experiments = rootState.settings.exp; + let channelsPath = '/api/channels'; + if (experiments.length) { + channelsPath += `?experiments=${experiments.join(',')}`; + } + + const channels = await this.$axios.$get(channelsPath); + return channels; + }, +};