From 9b9af361082eee5317002574b1642d4c694b0434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Wed, 4 Oct 2023 19:40:33 +0700 Subject: [PATCH 1/6] Stream audio/image input values into and out of the server --- assets/js/hooks/audio_input.js | 180 +++----------- assets/js/hooks/image_input.js | 102 +++++--- assets/js/hooks/js_view/channel.js | 38 +-- assets/js/lib/codec.js | 103 ++++++++ assets/package-lock.json | 6 +- lib/livebook/notebook/cell.ex | 6 - lib/livebook/session.ex | 28 +-- lib/livebook_web/channels/js_view_channel.ex | 18 +- .../components/form_components.ex | 69 ++++-- .../controllers/session_controller.ex | 137 +++++++++++ lib/livebook_web/helpers/codec.ex | 133 +++++++++++ .../live/annotated_tmp_file_writer.ex | 58 +++++ lib/livebook_web/live/output.ex | 88 ++++++- .../live/output/audio_input_component.ex | 218 +++++++++++------ .../live/output/file_input_component.ex | 42 ++-- .../live/output/image_input_component.ex | 226 +++++++++++------- .../live/output/input_component.ex | 8 + lib/livebook_web/live/session_helpers.ex | 38 +++ lib/livebook_web/live/session_live.ex | 11 + lib/livebook_web/router.ex | 2 + mix.exs | 5 +- mix.lock | 10 +- 22 files changed, 1053 insertions(+), 473 deletions(-) create mode 100644 assets/js/lib/codec.js create mode 100644 lib/livebook_web/helpers/codec.ex create mode 100644 lib/livebook_web/live/annotated_tmp_file_writer.ex diff --git a/assets/js/hooks/audio_input.js b/assets/js/hooks/audio_input.js index 18085863164..c9eb3efe388 100644 --- a/assets/js/hooks/audio_input.js +++ b/assets/js/hooks/audio_input.js @@ -1,5 +1,9 @@ -import { getAttributeOrThrow, parseInteger } from "../lib/attribute"; -import { base64ToBuffer, bufferToBase64 } from "../lib/utils"; +import { + getAttributeOrDefault, + getAttributeOrThrow, + parseInteger, +} from "../lib/attribute"; +import { encodeAnnotatedBuffer, encodePcmAsWav } from "../lib/codec"; const dropClasses = ["bg-yellow-100", "border-yellow-300"]; @@ -18,6 +22,8 @@ const dropClasses = ["bg-yellow-100", "border-yellow-300"]; * * * `data-endianness` - the server endianness, either `"little"` or `"big"` * + * * `data-audio-url` - the URL to audio file to use for the current preview + * */ const AudioInput = { mounted() { @@ -32,21 +38,8 @@ const AudioInput = { this.mediaRecorder = null; - // Render updated value - this.handleEvent( - `audio_input_change:${this.props.id}`, - ({ audio_info: audioInfo }) => { - if (audioInfo) { - this.updatePreview({ - data: this.decodeAudio(base64ToBuffer(audioInfo.data)), - numChannels: audioInfo.num_channels, - samplingRate: audioInfo.sampling_rate, - }); - } else { - this.clearPreview(); - } - } - ); + // Set the current value URL + this.audioEl.src = this.props.audioUrl; // File selection @@ -105,6 +98,8 @@ const AudioInput = { updated() { this.props = this.getProps(); + + this.audioEl.src = this.props.audioUrl; }, getProps() { @@ -118,6 +113,7 @@ const AudioInput = { ), endianness: getAttributeOrThrow(this.el, "data-endianness"), format: getAttributeOrThrow(this.el, "data-format"), + audioUrl: getAttributeOrDefault(this.el, "data-audio-url", null), }; }, @@ -176,6 +172,8 @@ const AudioInput = { }, loadEncodedAudio(buffer) { + this.pushEventTo(this.props.phxTarget, "decoding", {}); + const context = new AudioContext({ sampleRate: this.props.samplingRate }); context.decodeAudioData(buffer, (audioBuffer) => { @@ -184,66 +182,30 @@ const AudioInput = { }); }, - updatePreview(audioInfo) { - const oldUrl = this.audioEl.src; - const blob = audioInfoToWavBlob(audioInfo); - this.audioEl.src = URL.createObjectURL(blob); - oldUrl && URL.revokeObjectURL(oldUrl); - }, - - clearPreview() { - const oldUrl = this.audioEl.src; - this.audioEl.src = ""; - oldUrl && URL.revokeObjectURL(oldUrl); - }, - pushAudio(audioInfo) { - this.pushEventTo(this.props.phxTarget, "change", { - data: bufferToBase64(this.encodeAudio(audioInfo)), + const meta = { num_channels: audioInfo.numChannels, sampling_rate: audioInfo.samplingRate, - }); + }; + + const buffer = this.encodeAudio(audioInfo); + + this.uploadTo(this.props.phxTarget, "file", [ + new Blob([encodeAnnotatedBuffer(meta, buffer)]), + ]); }, encodeAudio(audioInfo) { if (this.props.format === "pcm_f32") { - return this.fixEndianness32(audioInfo.data); + return convertEndianness32(audioInfo.data, this.props.endianness); } else if (this.props.format === "wav") { - return encodeWavData( + return encodePcmAsWav( audioInfo.data, audioInfo.numChannels, audioInfo.samplingRate ); } }, - - decodeAudio(buffer) { - if (this.props.format === "pcm_f32") { - return this.fixEndianness32(buffer); - } else if (this.props.format === "wav") { - return decodeWavData(buffer); - } - }, - - fixEndianness32(buffer) { - if (getEndianness() === this.props.endianness) { - return buffer; - } - - // If the server uses different endianness, we swap bytes accordingly - for (let i = 0; i < buffer.byteLength / 4; i++) { - const b1 = buffer[i]; - const b2 = buffer[i + 1]; - const b3 = buffer[i + 2]; - const b4 = buffer[i + 3]; - buffer[i] = b4; - buffer[i + 1] = b3; - buffer[i + 2] = b2; - buffer[i + 3] = b1; - } - - return buffer; - }, }; function audioBufferToAudioInfo(audioBuffer) { @@ -267,88 +229,24 @@ function audioBufferToAudioInfo(audioBuffer) { return { data: pcmArray.buffer, numChannels, samplingRate }; } -function audioInfoToWavBlob({ data, numChannels, samplingRate }) { - const wavBytes = encodeWavData(data, numChannels, samplingRate); - return new Blob([wavBytes], { type: "audio/wav" }); -} - -// See http://soundfile.sapp.org/doc/WaveFormat -function encodeWavData(buffer, numChannels, samplingRate) { - const HEADER_SIZE = 44; - - const wavBuffer = new ArrayBuffer(HEADER_SIZE + buffer.byteLength); - const view = new DataView(wavBuffer); - - const numFrames = buffer.byteLength / 4; - const bytesPerSample = 4; - - const blockAlign = numChannels * bytesPerSample; - const byteRate = samplingRate * blockAlign; - const dataSize = numFrames * blockAlign; - - let offset = 0; - - function writeUint32Big(int) { - view.setUint32(offset, int, false); - offset += 4; - } - - function writeUint32(int) { - view.setUint32(offset, int, true); - offset += 4; - } - - function writeUint16(int) { - view.setUint16(offset, int, true); - offset += 2; - } - - function writeFloat32(int) { - view.setFloat32(offset, int, true); - offset += 4; - } - - writeUint32Big(0x52494646); - writeUint32(36 + dataSize); - writeUint32Big(0x57415645); - - writeUint32Big(0x666d7420); - writeUint32(16); - writeUint16(3); // 3 represents 32-bit float PCM - writeUint16(numChannels); - writeUint32(samplingRate); - writeUint32(byteRate); - writeUint16(blockAlign); - writeUint16(bytesPerSample * 8); - - writeUint32Big(0x64617461); - writeUint32(dataSize); - - const array = new Float32Array(buffer); - - for (let i = 0; i < array.length; i++) { - writeFloat32(array[i]); +function convertEndianness32(buffer, targetEndianness) { + if (getEndianness() === targetEndianness) { + return buffer; } - return wavBuffer; -} - -// We assume the exact same format as above, since we only need to -// decode data we encoded previously -function decodeWavData(buffer) { - const HEADER_SIZE = 44; - - const pcmBuffer = new ArrayBuffer(buffer.byteLength - HEADER_SIZE); - const pcmArray = new Float32Array(pcmBuffer); - - const view = new DataView(buffer); - - for (let i = 0; i < pcmArray.length; i++) { - const offset = HEADER_SIZE + i * 4; - pcmArray[i] = view.getFloat32(offset, true); + // If the server uses different endianness, we swap bytes accordingly + for (let i = 0; i < buffer.byteLength / 4; i++) { + const b1 = buffer[i]; + const b2 = buffer[i + 1]; + const b3 = buffer[i + 2]; + const b4 = buffer[i + 3]; + buffer[i] = b4; + buffer[i + 1] = b3; + buffer[i + 2] = b2; + buffer[i + 3] = b1; } - return pcmBuffer; + return buffer; } function getEndianness() { diff --git a/assets/js/hooks/image_input.js b/assets/js/hooks/image_input.js index b19f00d125f..e0b91fa5c65 100644 --- a/assets/js/hooks/image_input.js +++ b/assets/js/hooks/image_input.js @@ -3,7 +3,7 @@ import { getAttributeOrThrow, parseInteger, } from "../lib/attribute"; -import { base64ToBuffer, bufferToBase64 } from "../lib/utils"; +import { encodeAnnotatedBuffer } from "../lib/codec"; const dropClasses = ["bg-yellow-100", "border-yellow-300"]; @@ -24,6 +24,12 @@ const dropClasses = ["bg-yellow-100", "border-yellow-300"]; * * * `data-fit` - the fit strategy * + * * `data-image-url` - the URL to the image binary value + * + * * `data-value-height` - the height of the current image value + * + * * `data-value-width` - the width fo the current image value + * */ const ImageInput = { mounted() { @@ -50,18 +56,7 @@ const ImageInput = { this.cameraVideoEl = null; this.cameraStream = null; - // Render updated value - this.handleEvent( - `image_input_change:${this.props.id}`, - ({ image_info: imageInfo }) => { - if (imageInfo) { - const canvas = imageInfoToElement(imageInfo, this.props.format); - this.setPreview(canvas); - } else { - this.setPreview(this.initialPreviewContentEl); - } - } - ); + this.updateImagePreview(); // File selection @@ -139,6 +134,8 @@ const ImageInput = { updated() { this.props = this.getProps(); + + this.updateImagePreview(); }, getProps() { @@ -149,9 +146,37 @@ const ImageInput = { width: getAttributeOrDefault(this.el, "data-width", null, parseInteger), format: getAttributeOrThrow(this.el, "data-format"), fit: getAttributeOrThrow(this.el, "data-fit"), + imageUrl: getAttributeOrDefault(this.el, "data-image-url", null), + valueHeight: getAttributeOrDefault( + this.el, + "data-value-height", + null, + parseInteger + ), + valueWidth: getAttributeOrDefault( + this.el, + "data-value-width", + null, + parseInteger + ), }; }, + updateImagePreview() { + if (this.props.imageUrl) { + buildPreviewElement( + this.props.imageUrl, + this.props.valueHeight, + this.props.valueWidth, + this.props.format + ).then((element) => { + this.setPreview(element); + }); + } else { + this.setPreview(this.initialPreviewContentEl); + } + }, + loadFile(file) { const reader = new FileReader(); @@ -272,10 +297,15 @@ const ImageInput = { }, pushImage(canvas) { - this.pushEventTo(this.props.phxTarget, "change", { - data: canvasToBase64(canvas, this.props.format), + const meta = { height: canvas.height, width: canvas.width, + }; + + canvasToBuffer(canvas, this.props.format).then((buffer) => { + this.uploadTo(this.props.phxTarget, "file", [ + new Blob([encodeAnnotatedBuffer(meta, buffer)]), + ]); }); }, @@ -397,11 +427,15 @@ const ImageInput = { }, }; -function canvasToBase64(canvas, format) { +function canvasToBuffer(canvas, format) { if (format === "png" || format === "jpeg") { - const prefix = `data:image/${format};base64,`; - const dataUrl = canvas.toDataURL(`image/${format}`); - return dataUrl.slice(prefix.length); + return new Promise((resolve, reject) => { + canvas.toBlob((blob) => { + blob.arrayBuffer().then((buffer) => { + resolve(buffer); + }); + }, `image/${format}`); + }); } if (format === "rgb") { @@ -410,7 +444,7 @@ function canvasToBase64(canvas, format) { .getImageData(0, 0, canvas.width, canvas.height); const buffer = imageDataToRGBBuffer(imageData); - return bufferToBase64(buffer); + return Promise.resolve(buffer); } throw new Error(`Unexpected format: ${format}`); @@ -429,26 +463,24 @@ function imageDataToRGBBuffer(imageData) { return bytes.buffer; } -function imageInfoToElement(imageInfo, format) { +function buildPreviewElement(imageUrl, height, width, format) { if (format === "png" || format === "jpeg") { - const src = `data:image/${format};base64,${imageInfo.data}`; const img = document.createElement("img"); - img.src = src; - return img; + img.src = imageUrl; + return Promise.resolve(img); } if (format === "rgb") { - const canvas = document.createElement("canvas"); - canvas.height = imageInfo.height; - canvas.width = imageInfo.width; - const buffer = base64ToBuffer(imageInfo.data); - const imageData = imageDataFromRGBBuffer( - buffer, - imageInfo.width, - imageInfo.height - ); - canvas.getContext("2d").putImageData(imageData, 0, 0); - return canvas; + return fetch(imageUrl) + .then((response) => response.arrayBuffer()) + .then((buffer) => { + const canvas = document.createElement("canvas"); + canvas.height = height; + canvas.width = width; + const imageData = imageDataFromRGBBuffer(buffer, width, height); + canvas.getContext("2d").putImageData(imageData, 0, 0); + return canvas; + }); } throw new Error(`Unexpected format: ${format}`); diff --git a/assets/js/hooks/js_view/channel.js b/assets/js/hooks/js_view/channel.js index 3f5e80d70c9..e7e70c3f4b7 100644 --- a/assets/js/hooks/js_view/channel.js +++ b/assets/js/hooks/js_view/channel.js @@ -1,4 +1,5 @@ import { Socket } from "phoenix"; +import { decodeAnnotatedBuffer, encodeAnnotatedBuffer } from "../../lib/codec"; const csrfToken = document .querySelector("meta[name='csrf-token']") @@ -43,7 +44,7 @@ export function transportEncode(meta, payload) { payload[1].constructor === ArrayBuffer ) { const [info, buffer] = payload; - return encode([meta, info], buffer); + return encodeAnnotatedBuffer([meta, info], buffer); } else { return { root: [meta, payload] }; } @@ -51,7 +52,7 @@ export function transportEncode(meta, payload) { export function transportDecode(raw) { if (raw.constructor === ArrayBuffer) { - const [[meta, info], buffer] = decode(raw); + const [[meta, info], buffer] = decodeAnnotatedBuffer(raw); return [meta, [info, buffer]]; } else { const { @@ -60,36 +61,3 @@ export function transportDecode(raw) { return [meta, payload]; } } - -const HEADER_LENGTH = 4; - -function encode(meta, buffer) { - const encoder = new TextEncoder(); - const metaArray = encoder.encode(JSON.stringify(meta)); - - const raw = new ArrayBuffer( - HEADER_LENGTH + metaArray.byteLength + buffer.byteLength - ); - const view = new DataView(raw); - - view.setUint32(0, metaArray.byteLength); - new Uint8Array(raw, HEADER_LENGTH, metaArray.byteLength).set(metaArray); - new Uint8Array(raw, HEADER_LENGTH + metaArray.byteLength).set( - new Uint8Array(buffer) - ); - - return raw; -} - -function decode(raw) { - const view = new DataView(raw); - const metaArrayLength = view.getUint32(0); - - const metaArray = new Uint8Array(raw, HEADER_LENGTH, metaArrayLength); - const buffer = raw.slice(HEADER_LENGTH + metaArrayLength); - - const decoder = new TextDecoder(); - const meta = JSON.parse(decoder.decode(metaArray)); - - return [meta, buffer]; -} diff --git a/assets/js/lib/codec.js b/assets/js/lib/codec.js new file mode 100644 index 00000000000..90181836e43 --- /dev/null +++ b/assets/js/lib/codec.js @@ -0,0 +1,103 @@ +/** + * Encodes PCM float-32 in native endianness into a WAV binary. + */ +export function encodePcmAsWav(buffer, numChannels, samplingRate) { + // See http://soundfile.sapp.org/doc/WaveFormat + + const HEADER_SIZE = 44; + + const wavBuffer = new ArrayBuffer(HEADER_SIZE + buffer.byteLength); + const view = new DataView(wavBuffer); + + const numFrames = buffer.byteLength / 4; + const bytesPerSample = 4; + + const blockAlign = numChannels * bytesPerSample; + const byteRate = samplingRate * blockAlign; + const dataSize = numFrames * blockAlign; + + let offset = 0; + + function writeUint32Big(int) { + view.setUint32(offset, int, false); + offset += 4; + } + + function writeUint32(int) { + view.setUint32(offset, int, true); + offset += 4; + } + + function writeUint16(int) { + view.setUint16(offset, int, true); + offset += 2; + } + + function writeFloat32(int) { + view.setFloat32(offset, int, true); + offset += 4; + } + + writeUint32Big(0x52494646); + writeUint32(36 + dataSize); + writeUint32Big(0x57415645); + + writeUint32Big(0x666d7420); + writeUint32(16); + writeUint16(3); // 3 represents 32-bit float PCM + writeUint16(numChannels); + writeUint32(samplingRate); + writeUint32(byteRate); + writeUint16(blockAlign); + writeUint16(bytesPerSample * 8); + + writeUint32Big(0x64617461); + writeUint32(dataSize); + + const array = new Float32Array(buffer); + + for (let i = 0; i < array.length; i++) { + writeFloat32(array[i]); + } + + return wavBuffer; +} + +const HEADER_LENGTH = 4; + +/** + * Builds a single buffer with JSON-serialized `meta` and `buffer`. + */ +export function encodeAnnotatedBuffer(meta, buffer) { + const encoder = new TextEncoder(); + const metaArray = encoder.encode(JSON.stringify(meta)); + + const raw = new ArrayBuffer( + HEADER_LENGTH + metaArray.byteLength + buffer.byteLength + ); + const view = new DataView(raw); + + view.setUint32(0, metaArray.byteLength); + new Uint8Array(raw, HEADER_LENGTH, metaArray.byteLength).set(metaArray); + new Uint8Array(raw, HEADER_LENGTH + metaArray.byteLength).set( + new Uint8Array(buffer) + ); + + return raw; +} + +/** + * Decodes binary annotated with JSON-serialized metadata. + */ +export function decodeAnnotatedBuffer(raw) { + const view = new DataView(raw); + const metaArrayLength = view.getUint32(0); + + const metaArray = new Uint8Array(raw, HEADER_LENGTH, metaArrayLength); + const buffer = raw.slice(HEADER_LENGTH + metaArrayLength); + + const decoder = new TextDecoder(); + const meta = JSON.parse(decoder.decode(metaArray)); + + return [meta, buffer]; +} diff --git a/assets/package-lock.json b/assets/package-lock.json index c649a1abf18..444e29f051c 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -51,14 +51,14 @@ } }, "../deps/phoenix": { - "version": "1.7.5", + "version": "1.7.7", "license": "MIT" }, "../deps/phoenix_html": { - "version": "3.3.1" + "version": "3.3.2" }, "../deps/phoenix_live_view": { - "version": "0.19.5", + "version": "0.20.0", "license": "MIT" }, "node_modules/@alloc/quick-lru": { diff --git a/lib/livebook/notebook/cell.ex b/lib/livebook/notebook/cell.ex index 046f421afc3..b992a46d472 100644 --- a/lib/livebook/notebook/cell.ex +++ b/lib/livebook/notebook/cell.ex @@ -104,10 +104,4 @@ defmodule Livebook.Notebook.Cell do """ @spec setup_cell_id() :: id() def setup_cell_id(), do: @setup_cell_id - - @doc """ - Checks if the given term is a file input value (info map). - """ - defguard is_file_input_value(value) - when is_map_key(value, :file_ref) and is_map_key(value, :client_name) end diff --git a/lib/livebook/session.ex b/lib/livebook/session.ex index 4c0e14c3799..719bcf64ae3 100644 --- a/lib/livebook/session.ex +++ b/lib/livebook/session.ex @@ -83,8 +83,6 @@ defmodule Livebook.Session do use GenServer, restart: :temporary - import Livebook.Notebook.Cell, only: [is_file_input_value: 1] - alias Livebook.NotebookManager alias Livebook.Session.{Data, FileGuard} alias Livebook.{Utils, Notebook, Delta, Runtime, LiveMarkdown, FileSystem} @@ -1854,6 +1852,17 @@ defmodule Livebook.Session do end end + @doc """ + Returns a local path to a session-registered file with the given + reference. + """ + @spec registered_file_path(id(), Livebook.Runtime.file_ref()) :: String.t() + def registered_file_path(session_id, file_ref) do + {:file, file_id} = file_ref + %{path: session_dir} = session_tmp_dir(session_id) + Path.join([session_dir, "registered_files", file_id]) + end + defp encode_path_component(component) do String.replace(component, [".", "/", "\\", ":"], "_") end @@ -2283,14 +2292,8 @@ defmodule Livebook.Session do end defp handle_action(state, {:clean_up_input_values, input_infos}) do - for {_input_id, %{value: value}} <- input_infos do - case value do - value when is_file_input_value(value) -> - schedule_file_deletion(state, value.file_ref) - - _ -> - :ok - end + for {_input_id, %{value: %{file_ref: file_ref}}} <- input_infos do + schedule_file_deletion(state, file_ref) end state @@ -2527,11 +2530,6 @@ defmodule Livebook.Session do end end - defp registered_file_path(session_id, {:file, file_id}) do - %{path: session_dir} = session_tmp_dir(session_id) - Path.join([session_dir, "registered_files", file_id]) - end - defp schedule_file_deletion(state, file_ref) do Process.send_after( self(), diff --git a/lib/livebook_web/channels/js_view_channel.ex b/lib/livebook_web/channels/js_view_channel.ex index 7ca6b2d6c55..d969a74774d 100644 --- a/lib/livebook_web/channels/js_view_channel.ex +++ b/lib/livebook_web/channels/js_view_channel.ex @@ -1,6 +1,8 @@ defmodule LivebookWeb.JSViewChannel do use Phoenix.Channel + alias LivebookWeb.Helpers.Codec + @impl true def join("js_view", %{"session_token" => session_token}, socket) do case Phoenix.Token.verify(LivebookWeb.Endpoint, "session", session_token) do @@ -154,7 +156,7 @@ defmodule LivebookWeb.JSViewChannel do # payload accordingly defp transport_encode!(meta, {:binary, info, binary}) do - {:binary, encode!([meta, info], binary)} + {:binary, Codec.encode_annotated_binary!([meta, info], binary)} end defp transport_encode!(meta, payload) do @@ -162,7 +164,7 @@ defmodule LivebookWeb.JSViewChannel do end defp transport_decode!({:binary, raw}) do - {[meta, info], binary} = decode!(raw) + {[meta, info], binary} = Codec.decode_annotated_binary!(raw) {meta, {:binary, info, binary}} end @@ -170,16 +172,4 @@ defmodule LivebookWeb.JSViewChannel do %{"root" => [meta, payload]} = raw {meta, payload} end - - defp encode!(meta, binary) do - meta = Jason.encode!(meta) - meta_size = byte_size(meta) - <> - end - - defp decode!(raw) do - <> = raw - meta = Jason.decode!(meta) - {meta, binary} - end end diff --git a/lib/livebook_web/components/form_components.ex b/lib/livebook_web/components/form_components.ex index 010f4c95a01..11cae52bb59 100644 --- a/lib/livebook_web/components/form_components.ex +++ b/lib/livebook_web/components/form_components.ex @@ -603,30 +603,51 @@ defmodule LivebookWeb.FormComponents do <.live_file_input upload={@upload} class="hidden" /> <.label><%= @label %>
-
-
- <%= entry.client_name %> - - - - <%= entry.progress %>% - -
-
-
-
-
+ <.file_entry entry={entry} on_clear={@on_clear} /> +
+
+ """ + end + + @doc """ + Renders a file entry with progress. + + ## Examples + + <.file_entry + entry={entry} + on_clear={JS.push("clear_file", target: @myself)} + /> + + """ + attr :entry, Phoenix.LiveView.UploadEntry, required: true + attr :on_clear, Phoenix.LiveView.JS, required: true + attr :name, :string, default: nil + + def file_entry(assigns) do + ~H""" +
+
+ <%= @name || @entry.client_name %> + + + + <%= @entry.progress %>% + +
+
+
diff --git a/lib/livebook_web/controllers/session_controller.ex b/lib/livebook_web/controllers/session_controller.ex index fffa581be4e..4327f7d832b 100644 --- a/lib/livebook_web/controllers/session_controller.ex +++ b/lib/livebook_web/controllers/session_controller.ex @@ -2,6 +2,7 @@ defmodule LivebookWeb.SessionController do use LivebookWeb, :controller alias Livebook.{Sessions, Session, FileSystem} + alias LivebookWeb.Helpers.Codec def show_file(conn, %{"id" => id, "name" => name}) do with {:ok, session} <- Sessions.fetch_session(id), @@ -173,6 +174,91 @@ defmodule LivebookWeb.SessionController do end end + def show_input_audio(conn, %{"token" => token}) do + {live_view_pid, input_id} = LivebookWeb.SessionHelpers.verify_input_token!(token) + + case GenServer.call(live_view_pid, {:get_input_value, input_id}) do + {:ok, session_id, value} -> + path = Livebook.Session.registered_file_path(session_id, value.file_ref) + + conn = + conn + |> cache_permanently() + |> put_resp_header("accept-ranges", "bytes") + + case value.format do + :pcm_f32 -> + %{size: file_size} = File.stat!(path) + + total_size = Codec.pcm_as_wav_size(file_size) + + case parse_byte_range(conn, total_size) do + {range_start, range_end} when range_start > 0 or range_end < total_size - 1 -> + stream = + Codec.encode_pcm_as_wav_stream!( + path, + file_size, + value.num_channels, + value.sampling_rate, + range_start, + range_end - range_start + 1 + ) + + conn + |> put_content_range(range_start, range_end, total_size) + |> send_stream(206, stream) + + _ -> + stream = + Codec.encode_pcm_as_wav_stream!( + path, + file_size, + value.num_channels, + value.sampling_rate, + 0, + total_size + ) + + conn + |> put_resp_header("content-length", Integer.to_string(total_size)) + |> send_stream(200, stream) + end + + :wav -> + %{size: total_size} = File.stat!(path) + + case parse_byte_range(conn, total_size) do + {range_start, range_end} when range_start > 0 or range_end < total_size - 1 -> + conn + |> put_content_range(range_start, range_end, total_size) + |> send_file(206, path, range_start, range_end - range_start + 1) + + _ -> + send_file(conn, 200, path) + end + end + + :error -> + send_resp(conn, 404, "Not found") + end + end + + def show_input_image(conn, %{"token" => token}) do + {live_view_pid, input_id} = LivebookWeb.SessionHelpers.verify_input_token!(token) + + case GenServer.call(live_view_pid, {:get_input_value, input_id}) do + {:ok, session_id, value} -> + path = Livebook.Session.registered_file_path(session_id, value.file_ref) + + conn + |> cache_permanently() + |> send_file(200, path) + + :error -> + send_resp(conn, 404, "Not found") + end + end + defp accept_encoding?(conn, encoding) do encoding? = &String.contains?(&1, [encoding, "*"]) @@ -217,4 +303,55 @@ defmodule LivebookWeb.SessionController do content_type = MIME.from_path(path) put_resp_header(conn, "content-type", content_type) end + + defp parse_byte_range(conn, total_size) do + with [range] <- get_req_header(conn, "range"), + %{"bytes" => bytes} <- Plug.Conn.Utils.params(range), + {range_start, range_end} <- start_and_end(bytes, total_size) do + {range_start, range_end} + else + _ -> :error + end + end + + defp start_and_end("-" <> rest, total_size) do + case Integer.parse(rest) do + {last, ""} when last > 0 and last <= total_size -> {total_size - last, total_size - 1} + _ -> :error + end + end + + defp start_and_end(range, total_size) do + case Integer.parse(range) do + {first, "-"} when first >= 0 -> + {first, total_size - 1} + + {first, "-" <> rest} when first >= 0 -> + case Integer.parse(rest) do + {last, ""} when last >= first -> {first, min(last, total_size - 1)} + _ -> :error + end + + _ -> + :error + end + end + + defp put_content_range(conn, range_start, range_end, total_size) do + put_resp_header(conn, "content-range", "bytes #{range_start}-#{range_end}/#{total_size}") + end + + defp send_stream(conn, status, stream) do + conn = send_chunked(conn, status) + + Enum.reduce_while(stream, conn, fn chunk, conn -> + case Plug.Conn.chunk(conn, chunk) do + {:ok, conn} -> + {:cont, conn} + + {:error, :closed} -> + {:halt, conn} + end + end) + end end diff --git a/lib/livebook_web/helpers/codec.ex b/lib/livebook_web/helpers/codec.ex new file mode 100644 index 00000000000..93ead1e1b19 --- /dev/null +++ b/lib/livebook_web/helpers/codec.ex @@ -0,0 +1,133 @@ +defmodule LivebookWeb.Helpers.Codec do + @wav_header_size 44 + + @doc """ + Returns the size of a WAV binary that would wrap PCM data of the + given size. + + This size matches the result of `encode_pcm_as_wav_stream!/6`. + """ + @spec pcm_as_wav_size(pos_integer()) :: pos_integer() + def pcm_as_wav_size(pcm_size) do + @wav_header_size + pcm_size + end + + @doc """ + Encodes PCM float-32 in native endianness into a WAV binary. + + Accepts a range of the WAV binary that should be returned. Returns + a stream, where the PCM binary is streamed from the given file. + """ + @spec encode_pcm_as_wav_stream!( + Path.t(), + non_neg_integer(), + pos_integer(), + pos_integer(), + non_neg_integer(), + pos_integer() + ) :: Enumerable.t() + def encode_pcm_as_wav_stream!(path, file_size, num_channels, sampling_rate, offset, length) do + {header_enum, file_length} = + if offset < @wav_header_size do + header = encode_pcm_as_wav_header(file_size, num_channels, sampling_rate) + header_length = min(@wav_header_size - offset, length) + header_slice = binary_slice(header, offset, header_length) + {[header_slice], length - header_length} + else + {[], length} + end + + file_offset = max(offset - @wav_header_size, 0) + + file_stream = raw_file_range_stream!(path, file_offset, file_length) + + file_stream = + case System.endianness() do + :little -> + file_stream + + :big -> + Stream.map(file_stream, fn binary -> + for <>, reduce: <<>> do + acc -> <> + end + end) + end + + Stream.concat(header_enum, file_stream) + end + + defp encode_pcm_as_wav_header(pcm_size, num_channels, sampling_rate) do + # See http://soundfile.sapp.org/doc/WaveFormat + + num_frames = div(pcm_size, 4) + bytes_per_sample = 4 + + block_align = num_channels * bytes_per_sample + byte_rate = sampling_rate * block_align + data_size = num_frames * block_align + + << + # RIFF chunk + 0x52494646::32-unsigned-integer-big, + 36 + data_size::32-unsigned-integer-little, + 0x57415645::32-unsigned-integer-big, + # "fmt " sub-chunk + 0x666D7420::32-unsigned-integer-big, + 16::32-unsigned-integer-little, + # 3 indicates 32-bit float PCM + 3::16-unsigned-integer-little, + num_channels::16-unsigned-integer-little, + sampling_rate::32-unsigned-integer-little, + byte_rate::32-unsigned-integer-little, + block_align::16-unsigned-integer-little, + bytes_per_sample * 8::16-unsigned-integer-little, + # "data" sub-chunk + 0x64617461::32-unsigned-integer-big, + data_size::32-unsigned-integer-little + >> + end + + # We assume a local path and open a raw file for efficiency + defp raw_file_range_stream!(path, offset, length) do + chunk_size = 64_000 + + Stream.resource( + fn -> + {:ok, fd} = :file.open(path, [:raw, :binary, :read, :read_ahead]) + {:ok, _} = :file.position(fd, offset) + {fd, length} + end, + fn + {fd, 0} -> + {:halt, {fd, 0}} + + {fd, length} -> + size = min(chunk_size, length) + {:ok, chunk} = :file.read(fd, size) + {[chunk], {fd, length - size}} + end, + fn {fd, _} -> :file.close(fd) end + ) + end + + @doc """ + Builds a single binary with JSON-serialized `meta` and `binary`. + """ + @spec encode_annotated_binary!(term(), binary()) :: binary() + def encode_annotated_binary!(meta, binary) do + meta = Jason.encode!(meta) + meta_size = byte_size(meta) + <> + end + + @doc """ + Decodes binary annotated with JSON-serialized metadata. + """ + @spec decode_annotated_binary!(binary()) :: {term(), binary()} + def decode_annotated_binary!(raw) do + <> = raw + meta = Jason.decode!(meta) + {meta, binary} + end +end diff --git a/lib/livebook_web/live/annotated_tmp_file_writer.ex b/lib/livebook_web/live/annotated_tmp_file_writer.ex new file mode 100644 index 00000000000..0ea41b14b76 --- /dev/null +++ b/lib/livebook_web/live/annotated_tmp_file_writer.ex @@ -0,0 +1,58 @@ +defmodule LivebookWeb.AnnotatedTmpFileWriter do + # Custom writer for JSON-annotated binary. + # + # This corresponds to `LivebookWeb.Helpers.Codec.decode_annotated_binary!/1`. + # The entry metadata include `:meta` key with the annotation payload, + # while the actual binary is written to a temporary file, the same + # way the default `Phoenix.LiveView.UploadTmpFileWriter` would do. + + @behaviour Phoenix.LiveView.UploadWriter + + @impl true + def init(_opts) do + with {:ok, path} <- Plug.Upload.random_file("live_view_upload"), + {:ok, file} <- File.open(path, [:binary, :write]) do + {:ok, %{meta_size: nil, meta_binary: <<>>, path: path, file: file}} + end + end + + @impl true + def meta(state) do + meta = Jason.decode!(state.meta_binary) + %{meta: meta, path: state.path} + end + + @impl true + def write_chunk(data, %{meta_size: nil} = state) do + <> = data + write_chunk(rest, %{state | meta_size: meta_size}) + end + + def write_chunk(data, state) when byte_size(state.meta_binary) < state.meta_size do + data_size = byte_size(data) + pending_meta = state.meta_size - byte_size(state.meta_binary) + + if data_size > pending_meta do + left = binary_part(data, 0, pending_meta) + right = binary_slice(data, pending_meta..-1//1) + write_chunk(right, %{state | meta_binary: <>}) + else + {:ok, %{state | meta_binary: <>}} + end + end + + def write_chunk(data, state) do + case IO.binwrite(state.file, data) do + :ok -> {:ok, state} + {:error, reason} -> {:error, reason, state} + end + end + + @impl true + def close(state, _reason) do + case File.close(state.file) do + :ok -> {:ok, state} + {:error, reason} -> {:error, reason} + end + end +end diff --git a/lib/livebook_web/live/output.ex b/lib/livebook_web/live/output.ex index c3828340c7f..1fb9d9af7ea 100644 --- a/lib/livebook_web/live/output.ex +++ b/lib/livebook_web/live/output.ex @@ -45,17 +45,32 @@ defmodule LivebookWeb.Output do defp render_output(%{type: :terminal_text, text: text}, %{id: id}) do text = if(text == :__pruned__, do: nil, else: text) - live_component(Output.TerminalTextComponent, id: id, text: text) + + assigns = %{id: id, text: text} + + ~H""" + <.live_component module={Output.TerminalTextComponent} id={@id} text={@text} /> + """ end defp render_output(%{type: :plain_text, text: text}, %{id: id}) do text = if(text == :__pruned__, do: nil, else: text) - live_component(Output.PlainTextComponent, id: id, text: text) + + assigns = %{id: id, text: text} + + ~H""" + <.live_component module={Output.PlainTextComponent} id={@id} text={@text} /> + """ end defp render_output(%{type: :markdown, text: text}, %{id: id, session_id: session_id}) do text = if(text == :__pruned__, do: nil, else: text) - live_component(Output.MarkdownComponent, id: id, session_id: session_id, text: text) + + assigns = %{id: id, session_id: session_id, text: text} + + ~H""" + <.live_component module={Output.MarkdownComponent} id={@id} session_id={@session_id} text={@text} /> + """ end defp render_output(%{type: :image} = output, %{id: id}) do @@ -71,13 +86,23 @@ defmodule LivebookWeb.Output do session_id: session_id, client_id: client_id }) do - live_component(LivebookWeb.JSViewComponent, + assigns = %{ id: id, js_view: output.js_view, session_id: session_id, - client_id: client_id, - timeout_message: "Output data no longer available, please reevaluate this cell" - ) + client_id: client_id + } + + ~H""" + <.live_component + module={LivebookWeb.JSViewComponent} + id={@id} + js_view={@js_view} + session_id={@session_id} + client_id={@client_id} + timeout_message="Output data no longer available, please reevaluate this cell" + /> + """ end defp render_output(%{type: :frame} = output, %{ @@ -88,7 +113,7 @@ defmodule LivebookWeb.Output do client_id: client_id, cell_id: cell_id }) do - live_component(Output.FrameComponent, + assigns = %{ id: id, outputs: output.outputs, placeholder: output.placeholder, @@ -97,7 +122,21 @@ defmodule LivebookWeb.Output do input_views: input_views, client_id: client_id, cell_id: cell_id - ) + } + + ~H""" + <.live_component + module={Output.FrameComponent} + id={@id} + outputs={@outputs} + placeholder={@placeholder} + session_id={@session_id} + session_pid={@session_pid} + input_views={@input_views} + client_id={@client_id} + cell_id={@cell_id} + /> + """ end defp render_output(%{type: :tabs, outputs: outputs, labels: labels}, %{ @@ -226,13 +265,24 @@ defmodule LivebookWeb.Output do session_pid: session_pid, client_id: client_id }) do - live_component(Output.InputComponent, + assigns = %{ id: id, input: input, input_views: input_views, session_pid: session_pid, client_id: client_id - ) + } + + ~H""" + <.live_component + module={Output.InputComponent} + id={@id} + input={@input} + input_views={@input_views} + session_pid={@session_pid} + client_id={@client_id} + /> + """ end defp render_output(%{type: :control} = control, %{ @@ -242,14 +292,26 @@ defmodule LivebookWeb.Output do client_id: client_id, cell_id: cell_id }) do - live_component(Output.ControlComponent, + assigns = %{ id: id, control: control, input_views: input_views, session_pid: session_pid, client_id: client_id, cell_id: cell_id - ) + } + + ~H""" + <.live_component + module={Output.ControlComponent} + id={@id} + control={@control} + input_views={@input_views} + session_pid={@session_pid} + client_id={@client_id} + cell_id={@cell_id} + /> + """ end defp render_output( diff --git a/lib/livebook_web/live/output/audio_input_component.ex b/lib/livebook_web/live/output/audio_input_component.ex index 821b68ff6ca..042f61d518b 100644 --- a/lib/livebook_web/live/output/audio_input_component.ex +++ b/lib/livebook_web/live/output/audio_input_component.ex @@ -3,7 +3,22 @@ defmodule LivebookWeb.Output.AudioInputComponent do @impl true def mount(socket) do - {:ok, assign(socket, endianness: System.endianness(), value: nil)} + {:ok, + socket + |> assign( + endianness: System.endianness(), + value: nil, + audio_url: nil, + decoding?: false + ) + |> allow_upload(:file, + accept: :any, + max_entries: 1, + max_file_size: 100_000_000_000, + progress: &handle_progress/3, + auto_upload: true, + writer: fn _name, _entry, _socket -> {LivebookWeb.AnnotatedTmpFileWriter, []} end + )} end @impl true @@ -13,93 +28,150 @@ defmodule LivebookWeb.Output.AudioInputComponent do socket = assign(socket, assigns) socket = - if value == socket.assigns.value do - socket - else - audio_info = - if value do - %{ - data: Base.encode64(value.data), - num_channels: value.num_channels, - sampling_rate: value.sampling_rate - } - end - - push_event(socket, "audio_input_change:#{socket.assigns.id}", %{audio_info: audio_info}) + cond do + value == socket.assigns.value -> + socket + + value == nil -> + assign(socket, value: value, audio_url: nil) + + true -> + assign(socket, value: value, audio_url: audio_url(socket.assigns.input_id)) end - {:ok, assign(socket, value: value)} + {:ok, socket} + end + + defp audio_url(input_id) do + # For the client-side audio preview, we serve audio encoded as WAV + # from a separate endpoint. To do that, we encode information in a + # token and then the controller fetches input value from the LV. + # This is especially important for client-specific inputs in forms. + token = LivebookWeb.SessionHelpers.generate_input_token(self(), input_id) + ~p"/sessions/audio-input/#{token}" end @impl true def render(assigns) do ~H""" -
- - -
- - + - - + Stop recording + + + +
+
+ +
+ Decoding <.spinner /> +
+
+ <.file_entry name="Audio" entry={entry} on_clear={JS.push("clear_file", target: @myself)} />
""" end @impl true - def handle_event("change", params, socket) do - value = %{ - data: Base.decode64!(params["data"]), - num_channels: params["num_channels"], - sampling_rate: params["sampling_rate"], - format: socket.assigns.format - } - - send_update(LivebookWeb.Output.InputComponent, - id: socket.assigns.input_component_id, - event: :change, - value: value - ) + def handle_event("decoding", %{}, socket) do + {:noreply, assign(socket, decoding?: true)} + end + def handle_event("validate", %{}, socket) do {:noreply, socket} end + + def handle_event("clear_file", %{"ref" => ref}, socket) do + {:noreply, cancel_upload(socket, :file, ref)} + end + + defp handle_progress(:file, entry, socket) do + if entry.done? do + {meta, file_ref} = + consume_uploaded_entry(socket, entry, fn %{path: path, meta: meta} -> + {:ok, file_ref} = + LivebookWeb.SessionHelpers.register_input_file( + socket.assigns.session_pid, + path, + socket.assigns.input_id, + socket.assigns.local, + socket.assigns.client_id + ) + + {:ok, {meta, file_ref}} + end) + + %{"num_channels" => num_channels, "sampling_rate" => sampling_rate} = meta + + value = %{ + file_ref: file_ref, + num_channels: num_channels, + sampling_rate: sampling_rate, + format: socket.assigns.format + } + + send_update(LivebookWeb.Output.InputComponent, + id: socket.assigns.input_component_id, + event: :change, + value: value + ) + end + + {:noreply, assign(socket, decoding?: false)} + end end diff --git a/lib/livebook_web/live/output/file_input_component.ex b/lib/livebook_web/live/output/file_input_component.ex index 4373fbf4e76..197d2f9483f 100644 --- a/lib/livebook_web/live/output/file_input_component.ex +++ b/lib/livebook_web/live/output/file_input_component.ex @@ -71,35 +71,27 @@ defmodule LivebookWeb.Output.FileInputComponent do defp handle_progress(:file, entry, socket) do if entry.done? do - socket - |> consume_uploaded_entries(:file, fn %{path: path}, entry -> - {:ok, file_ref} = - if socket.assigns.local do - key = "#{socket.assigns.input_id}-#{socket.assigns.client_id}" - - Livebook.Session.register_file(socket.assigns.session_pid, path, key, - linked_client_id: socket.assigns.client_id + {file_ref, client_name} = + consume_uploaded_entry(socket, entry, fn %{path: path} -> + {:ok, file_ref} = + LivebookWeb.SessionHelpers.register_input_file( + socket.assigns.session_pid, + path, + socket.assigns.input_id, + socket.assigns.local, + socket.assigns.client_id ) - else - key = "#{socket.assigns.input_id}-global" - Livebook.Session.register_file(socket.assigns.session_pid, path, key) - end - {:ok, {file_ref, entry.client_name}} - end) - |> case do - [{file_ref, client_name}] -> - value = %{file_ref: file_ref, client_name: client_name} + {:ok, {file_ref, entry.client_name}} + end) - send_update(LivebookWeb.Output.InputComponent, - id: socket.assigns.input_component_id, - event: :change, - value: value - ) + value = %{file_ref: file_ref, client_name: client_name} - [] -> - :ok - end + send_update(LivebookWeb.Output.InputComponent, + id: socket.assigns.input_component_id, + event: :change, + value: value + ) end {:noreply, socket} diff --git a/lib/livebook_web/live/output/image_input_component.ex b/lib/livebook_web/live/output/image_input_component.ex index e36fa6972d0..04cae6b58f1 100644 --- a/lib/livebook_web/live/output/image_input_component.ex +++ b/lib/livebook_web/live/output/image_input_component.ex @@ -3,7 +3,17 @@ defmodule LivebookWeb.Output.ImageInputComponent do @impl true def mount(socket) do - {:ok, assign(socket, value: nil)} + {:ok, + socket + |> assign(value: nil, value: nil, image_url: nil) + |> allow_upload(:file, + accept: :any, + max_entries: 1, + max_file_size: 100_000_000_000, + progress: &handle_progress/3, + auto_upload: true, + writer: fn _name, _entry, _socket -> {LivebookWeb.AnnotatedTmpFileWriter, []} end + )} end @impl true @@ -13,101 +23,155 @@ defmodule LivebookWeb.Output.ImageInputComponent do socket = assign(socket, assigns) socket = - if value == socket.assigns.value do - socket - else - image_info = - if value do - %{data: Base.encode64(value.data), height: value.height, width: value.width} - end - - push_event(socket, "image_input_change:#{socket.assigns.id}", %{image_info: image_info}) + cond do + value == socket.assigns.value -> + socket + + value == nil -> + assign(socket, value: value, image_url: nil) + + true -> + assign(socket, value: value, image_url: image_url(socket.assigns.input_id)) end - {:ok, assign(socket, value: value)} + {:ok, socket} + end + + defp image_url(input_id) do + # For the client-side image preview, we serve the original binary + # value from a separate endpoint. To do that, we encode information + # in a token and then the controller fetches input value from the + # LV. This is especially important for client-specific inputs in + # forms. + token = LivebookWeb.SessionHelpers.generate_input_token(self(), input_id) + ~p"/sessions/image-input/#{token}" end @impl true def render(assigns) do ~H""" -
- -
-
- Drag an image file +
+
+ +
+
+ Drag an image file +
-
- -
- <.menu id={"#{@id}-camera-select-menu"} position={:bottom_left}> - <:toggle> - - -
- <.menu_item> - - -
- - - - + +
+ <.menu_item> + + +
+ + + + +
+
+ +
+ <.file_entry name="Audio" entry={entry} on_clear={JS.push("clear_file", target: @myself)} />
""" end @impl true - def handle_event("change", params, socket) do - value = %{ - data: Base.decode64!(params["data"]), - height: params["height"], - width: params["width"], - format: socket.assigns.format - } - - send_update(LivebookWeb.Output.InputComponent, - id: socket.assigns.input_component_id, - event: :change, - value: value - ) + def handle_event("validate", %{}, socket) do + {:noreply, socket} + end + + def handle_event("clear_file", %{"ref" => ref}, socket) do + {:noreply, cancel_upload(socket, :file, ref)} + end + + defp handle_progress(:file, entry, socket) do + if entry.done? do + {meta, file_ref} = + consume_uploaded_entry(socket, entry, fn %{path: path, meta: meta} -> + {:ok, file_ref} = + LivebookWeb.SessionHelpers.register_input_file( + socket.assigns.session_pid, + path, + socket.assigns.input_id, + socket.assigns.local, + socket.assigns.client_id + ) + + {:ok, {meta, file_ref}} + end) + + %{"height" => height, "width" => width} = meta + + value = %{ + file_ref: file_ref, + height: height, + width: width, + format: socket.assigns.format + } + + send_update(LivebookWeb.Output.InputComponent, + id: socket.assigns.input_component_id, + event: :change, + value: value + ) + end {:noreply, socket} end diff --git a/lib/livebook_web/live/output/input_component.ex b/lib/livebook_web/live/output/input_component.ex index f0763ca00d5..387ff7022b0 100644 --- a/lib/livebook_web/live/output/input_component.ex +++ b/lib/livebook_web/live/output/input_component.ex @@ -36,6 +36,10 @@ defmodule LivebookWeb.Output.InputComponent do width={@input.attrs.size && elem(@input.attrs.size, 1)} format={@input.attrs.format} fit={@input.attrs.fit} + input_id={@input.id} + session_pid={@session_pid} + client_id={@client_id} + local={@local} />
""" @@ -52,6 +56,10 @@ defmodule LivebookWeb.Output.InputComponent do value={@value} format={@input.attrs.format} sampling_rate={@input.attrs.sampling_rate} + input_id={@input.id} + session_pid={@session_pid} + client_id={@client_id} + local={@local} />
""" diff --git a/lib/livebook_web/live/session_helpers.ex b/lib/livebook_web/live/session_helpers.ex index 1eb9dbc2a91..ffde56f4cb0 100644 --- a/lib/livebook_web/live/session_helpers.ex +++ b/lib/livebook_web/live/session_helpers.ex @@ -272,4 +272,42 @@ defmodule LivebookWeb.SessionHelpers do end end end + + @doc """ + Generates a token for the given input. + """ + @spec generate_input_token(pid(), String.t()) :: String.t() + def generate_input_token(live_view_pid, input_id) do + Phoenix.Token.sign(LivebookWeb.Endpoint, "session-input", %{ + live_view_pid: live_view_pid, + input_id: input_id + }) + end + + @doc """ + Verifies token from `generate_input_token/2` and extracts the encoded + data. + """ + @spec verify_input_token!(String.t()) :: {pid(), String.t()} + def verify_input_token!(token) do + {:ok, %{live_view_pid: live_view_pid, input_id: input_id}} = + Phoenix.Token.verify(LivebookWeb.Endpoint, "session-input", token) + + {live_view_pid, input_id} + end + + @doc """ + Registers an uploaded input file in session. + """ + @spec register_input_file(pid(), String.t(), String.t(), boolean(), String.t()) :: + {:ok, Livebook.Runtime.file_ref()} + def register_input_file(session_pid, path, input_id, local, client_id) do + if local do + key = "#{input_id}-#{client_id}" + Livebook.Session.register_file(session_pid, path, key, linked_client_id: client_id) + else + key = "#{input_id}-global" + Livebook.Session.register_file(session_pid, path, key) + end + end end diff --git a/lib/livebook_web/live/session_live.ex b/lib/livebook_web/live/session_live.ex index 18a051652e9..8e6a1839c21 100644 --- a/lib/livebook_web/live/session_live.ex +++ b/lib/livebook_web/live/session_live.ex @@ -1730,6 +1730,17 @@ defmodule LivebookWeb.SessionLive do {:noreply, socket} end + @impl true + def handle_call({:get_input_value, input_id}, _from, socket) do + reply = + case socket.private.data.input_infos do + %{^input_id => %{value: value}} -> {:ok, socket.assigns.session.id, value} + %{} -> :error + end + + {:reply, reply, socket} + end + @impl true def handle_info({:operation, operation}, socket) do {:noreply, handle_operation(socket, operation)} diff --git a/lib/livebook_web/router.ex b/lib/livebook_web/router.ex index ad9c7b7cd83..f5c077581b0 100644 --- a/lib/livebook_web/router.ex +++ b/lib/livebook_web/router.ex @@ -100,6 +100,8 @@ defmodule LivebookWeb.Router do get "/sessions/:id/files/:name", SessionController, :show_file get "/sessions/:id/images/:name", SessionController, :show_image get "/sessions/:id/download/files/:name", SessionController, :download_file + get "/sessions/audio-input/:token", SessionController, :show_input_audio + get "/sessions/image-input/:token", SessionController, :show_input_image live "/sessions/:id/settings/custom-view", SessionLive, :custom_view_settings live "/sessions/:id/*path_parts", SessionLive, :catch_all end diff --git a/mix.exs b/mix.exs index 42565f0ca8c..730f9f72e91 100644 --- a/mix.exs +++ b/mix.exs @@ -94,10 +94,9 @@ defmodule Livebook.MixProject do # defp deps do [ - {:phoenix, "~> 1.7.0"}, + {:phoenix, "~> 1.7.7"}, {:phoenix_html, "~> 3.0"}, - # {:phoenix_live_view, "~> 0.19.0"}, - {:phoenix_live_view, github: "phoenixframework/phoenix_live_view", override: true}, + {:phoenix_live_view, "~> 0.20.0"}, {:phoenix_live_dashboard, "~> 0.8.0"}, {:telemetry_metrics, "~> 0.4"}, {:telemetry_poller, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index 95f2d82d9f1..cb5e5beab49 100644 --- a/mix.lock +++ b/mix.lock @@ -25,12 +25,12 @@ "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, - "phoenix": {:hex, :phoenix, "1.7.5", "3234bc87185e6a2103a15a3b1399f19775b093a6923c4064436e49cdab8ce5d2", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.1", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "5abad1789f06a3572ee5e5d5151993ed35b9e2711537904cc457a40229587979"}, + "phoenix": {:hex, :phoenix, "1.7.7", "4cc501d4d823015007ba3cdd9c41ecaaf2ffb619d6fb283199fa8ddba89191e0", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "8966e15c395e5e37591b6ed0bd2ae7f48e961f0f60ac4c733f9566b519453085"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.2", "b21bd01fdeffcfe2fab49e4942aa938b6d3e89e93a480d4aee58085560a0bc0d", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "70242edd4601d50b69273b057ecf7b684644c19ee750989fd555625ae4ce8f5d"}, - "phoenix_html": {:hex, :phoenix_html, "3.3.1", "4788757e804a30baac6b3fc9695bf5562465dd3f1da8eb8460ad5b404d9a2178", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "bed1906edd4906a15fd7b412b85b05e521e1f67c9a85418c55999277e553d0d3"}, - "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.0", "0b3158b5b198aa444473c91d23d79f52fb077e807ffad80dacf88ce078fa8df2", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "87785a54474fed91a67a1227a741097eb1a42c2e49d3c0d098b588af65cd410d"}, + "phoenix_html": {:hex, :phoenix_html, "3.3.2", "d6ce982c6d8247d2fc0defe625255c721fb8d5f1942c5ac051f6177bffa5973f", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "44adaf8e667c1c20fb9d284b6b0fa8dc7946ce29e81ce621860aa7e96de9a11d"}, + "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.2", "b9e33c950d1ed98494bfbde1c34c6e51c8a4214f3bea3f07ca9a510643ee1387", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "67a598441b5f583d301a77e0298719f9654887d3d8bf14e80ff0b6acf887ef90"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.4.1", "2aff698f5e47369decde4357ba91fc9c37c6487a512b41732818f2204a8ef1d3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "9bffb834e7ddf08467fe54ae58b5785507aaba6255568ae22b4d46e2bb3615ab"}, - "phoenix_live_view": {:git, "https://github.com/phoenixframework/phoenix_live_view.git", "64e22999c2900e2f9266a030ca7a135a042f0645", []}, + "phoenix_live_view": {:hex, :phoenix_live_view, "0.20.0", "3f3531c835e46a3b45b4c3ca4a09cef7ba1d0f0d0035eef751c7084b8adb1299", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "29875f8a58fb031f2dc8f3be025c92ed78d342b46f9bbf6dfe579549d7c81050"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, "phoenix_template": {:hex, :phoenix_template, "1.0.1", "85f79e3ad1b0180abb43f9725973e3b8c2c3354a87245f91431eec60553ed3ef", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "157dc078f6226334c91cb32c1865bf3911686f8bcd6bcff86736f6253e6993ee"}, "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, @@ -44,5 +44,5 @@ "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, "thousand_island": {:hex, :thousand_island, "0.6.7", "3a91a7e362ca407036c6691e8a4f6e01ac8e901db3598875863a149279ac8571", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "541a5cb26b88adf8d8180b6b96a90f09566b4aad7a6b3608dcac969648cf6765"}, "websock": {:hex, :websock, "0.5.1", "c496036ce95bc26d08ba086b2a827b212c67e7cabaa1c06473cd26b40ed8cf10", [:mix], [], "hexpm", "b9f785108b81cd457b06e5f5dabe5f65453d86a99118b2c0a515e1e296dc2d2c"}, - "websock_adapter": {:hex, :websock_adapter, "0.5.1", "292e6c56724e3457e808e525af0e9bcfa088cc7b9c798218e78658c7f9b85066", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "8e2e1544bfde5f9d0442f9cec2f5235398b224f75c9e06b60557debf64248ec1"}, + "websock_adapter": {:hex, :websock_adapter, "0.5.4", "7af8408e7ed9d56578539594d1ee7d8461e2dd5c3f57b0f2a5352d610ddde757", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d2c238c79c52cbe223fcdae22ca0bb5007a735b9e933870e241fce66afb4f4ab"}, } From 5bfdc37cd2985cf5a77ba413673b2d005281b9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 5 Oct 2023 02:33:04 +0700 Subject: [PATCH 2/6] Up --- mix.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 730f9f72e91..f3059b14471 100644 --- a/mix.exs +++ b/mix.exs @@ -96,7 +96,8 @@ defmodule Livebook.MixProject do [ {:phoenix, "~> 1.7.7"}, {:phoenix_html, "~> 3.0"}, - {:phoenix_live_view, "~> 0.20.0"}, + # {:phoenix_live_view, "~> 0.20.0"}, + {:phoenix_live_view, github: "phoenixframework/phoenix_live_view", override: true}, {:phoenix_live_dashboard, "~> 0.8.0"}, {:telemetry_metrics, "~> 0.4"}, {:telemetry_poller, "~> 1.0"}, From bfce3507dc2c6c9dee3a89ae2385aff4e215d4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 5 Oct 2023 03:04:17 +0700 Subject: [PATCH 3/6] Use client_meta on LV upload --- assets/js/hooks/audio_input.js | 7 ++- assets/js/hooks/image_input.js | 17 +++--- .../live/annotated_tmp_file_writer.ex | 58 ------------------- .../live/output/audio_input_component.ex | 11 ++-- .../live/output/image_input_component.ex | 11 ++-- mix.lock | 2 +- 6 files changed, 24 insertions(+), 82 deletions(-) delete mode 100644 lib/livebook_web/live/annotated_tmp_file_writer.ex diff --git a/assets/js/hooks/audio_input.js b/assets/js/hooks/audio_input.js index c9eb3efe388..6a3991cbd85 100644 --- a/assets/js/hooks/audio_input.js +++ b/assets/js/hooks/audio_input.js @@ -190,9 +190,10 @@ const AudioInput = { const buffer = this.encodeAudio(audioInfo); - this.uploadTo(this.props.phxTarget, "file", [ - new Blob([encodeAnnotatedBuffer(meta, buffer)]), - ]); + const blob = new Blob([buffer]); + blob.meta = () => meta; + + this.uploadTo(this.props.phxTarget, "file", [blob]); }, encodeAudio(audioInfo) { diff --git a/assets/js/hooks/image_input.js b/assets/js/hooks/image_input.js index e0b91fa5c65..ce42d31c85f 100644 --- a/assets/js/hooks/image_input.js +++ b/assets/js/hooks/image_input.js @@ -297,15 +297,16 @@ const ImageInput = { }, pushImage(canvas) { - const meta = { - height: canvas.height, - width: canvas.width, - }; - canvasToBuffer(canvas, this.props.format).then((buffer) => { - this.uploadTo(this.props.phxTarget, "file", [ - new Blob([encodeAnnotatedBuffer(meta, buffer)]), - ]); + const meta = { + height: canvas.height, + width: canvas.width, + }; + + const blob = new Blob([buffer]); + blob.meta = () => meta; + + this.uploadTo(this.props.phxTarget, "file", [blob]); }); }, diff --git a/lib/livebook_web/live/annotated_tmp_file_writer.ex b/lib/livebook_web/live/annotated_tmp_file_writer.ex deleted file mode 100644 index 0ea41b14b76..00000000000 --- a/lib/livebook_web/live/annotated_tmp_file_writer.ex +++ /dev/null @@ -1,58 +0,0 @@ -defmodule LivebookWeb.AnnotatedTmpFileWriter do - # Custom writer for JSON-annotated binary. - # - # This corresponds to `LivebookWeb.Helpers.Codec.decode_annotated_binary!/1`. - # The entry metadata include `:meta` key with the annotation payload, - # while the actual binary is written to a temporary file, the same - # way the default `Phoenix.LiveView.UploadTmpFileWriter` would do. - - @behaviour Phoenix.LiveView.UploadWriter - - @impl true - def init(_opts) do - with {:ok, path} <- Plug.Upload.random_file("live_view_upload"), - {:ok, file} <- File.open(path, [:binary, :write]) do - {:ok, %{meta_size: nil, meta_binary: <<>>, path: path, file: file}} - end - end - - @impl true - def meta(state) do - meta = Jason.decode!(state.meta_binary) - %{meta: meta, path: state.path} - end - - @impl true - def write_chunk(data, %{meta_size: nil} = state) do - <> = data - write_chunk(rest, %{state | meta_size: meta_size}) - end - - def write_chunk(data, state) when byte_size(state.meta_binary) < state.meta_size do - data_size = byte_size(data) - pending_meta = state.meta_size - byte_size(state.meta_binary) - - if data_size > pending_meta do - left = binary_part(data, 0, pending_meta) - right = binary_slice(data, pending_meta..-1//1) - write_chunk(right, %{state | meta_binary: <>}) - else - {:ok, %{state | meta_binary: <>}} - end - end - - def write_chunk(data, state) do - case IO.binwrite(state.file, data) do - :ok -> {:ok, state} - {:error, reason} -> {:error, reason, state} - end - end - - @impl true - def close(state, _reason) do - case File.close(state.file) do - :ok -> {:ok, state} - {:error, reason} -> {:error, reason} - end - end -end diff --git a/lib/livebook_web/live/output/audio_input_component.ex b/lib/livebook_web/live/output/audio_input_component.ex index 042f61d518b..4fe7a404224 100644 --- a/lib/livebook_web/live/output/audio_input_component.ex +++ b/lib/livebook_web/live/output/audio_input_component.ex @@ -16,8 +16,7 @@ defmodule LivebookWeb.Output.AudioInputComponent do max_entries: 1, max_file_size: 100_000_000_000, progress: &handle_progress/3, - auto_upload: true, - writer: fn _name, _entry, _socket -> {LivebookWeb.AnnotatedTmpFileWriter, []} end + auto_upload: true )} end @@ -142,8 +141,8 @@ defmodule LivebookWeb.Output.AudioInputComponent do defp handle_progress(:file, entry, socket) do if entry.done? do - {meta, file_ref} = - consume_uploaded_entry(socket, entry, fn %{path: path, meta: meta} -> + file_ref = + consume_uploaded_entry(socket, entry, fn %{path: path} -> {:ok, file_ref} = LivebookWeb.SessionHelpers.register_input_file( socket.assigns.session_pid, @@ -153,10 +152,10 @@ defmodule LivebookWeb.Output.AudioInputComponent do socket.assigns.client_id ) - {:ok, {meta, file_ref}} + {:ok, file_ref} end) - %{"num_channels" => num_channels, "sampling_rate" => sampling_rate} = meta + %{"num_channels" => num_channels, "sampling_rate" => sampling_rate} = entry.client_meta value = %{ file_ref: file_ref, diff --git a/lib/livebook_web/live/output/image_input_component.ex b/lib/livebook_web/live/output/image_input_component.ex index 04cae6b58f1..c69fdea1ed1 100644 --- a/lib/livebook_web/live/output/image_input_component.ex +++ b/lib/livebook_web/live/output/image_input_component.ex @@ -11,8 +11,7 @@ defmodule LivebookWeb.Output.ImageInputComponent do max_entries: 1, max_file_size: 100_000_000_000, progress: &handle_progress/3, - auto_upload: true, - writer: fn _name, _entry, _socket -> {LivebookWeb.AnnotatedTmpFileWriter, []} end + auto_upload: true )} end @@ -143,8 +142,8 @@ defmodule LivebookWeb.Output.ImageInputComponent do defp handle_progress(:file, entry, socket) do if entry.done? do - {meta, file_ref} = - consume_uploaded_entry(socket, entry, fn %{path: path, meta: meta} -> + file_ref = + consume_uploaded_entry(socket, entry, fn %{path: path} -> {:ok, file_ref} = LivebookWeb.SessionHelpers.register_input_file( socket.assigns.session_pid, @@ -154,10 +153,10 @@ defmodule LivebookWeb.Output.ImageInputComponent do socket.assigns.client_id ) - {:ok, {meta, file_ref}} + {:ok, file_ref} end) - %{"height" => height, "width" => width} = meta + %{"height" => height, "width" => width} = entry.client_meta value = %{ file_ref: file_ref, diff --git a/mix.lock b/mix.lock index cb5e5beab49..74d961d902e 100644 --- a/mix.lock +++ b/mix.lock @@ -30,7 +30,7 @@ "phoenix_html": {:hex, :phoenix_html, "3.3.2", "d6ce982c6d8247d2fc0defe625255c721fb8d5f1942c5ac051f6177bffa5973f", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "44adaf8e667c1c20fb9d284b6b0fa8dc7946ce29e81ce621860aa7e96de9a11d"}, "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.2", "b9e33c950d1ed98494bfbde1c34c6e51c8a4214f3bea3f07ca9a510643ee1387", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "67a598441b5f583d301a77e0298719f9654887d3d8bf14e80ff0b6acf887ef90"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.4.1", "2aff698f5e47369decde4357ba91fc9c37c6487a512b41732818f2204a8ef1d3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "9bffb834e7ddf08467fe54ae58b5785507aaba6255568ae22b4d46e2bb3615ab"}, - "phoenix_live_view": {:hex, :phoenix_live_view, "0.20.0", "3f3531c835e46a3b45b4c3ca4a09cef7ba1d0f0d0035eef751c7084b8adb1299", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "29875f8a58fb031f2dc8f3be025c92ed78d342b46f9bbf6dfe579549d7c81050"}, + "phoenix_live_view": {:git, "https://github.com/phoenixframework/phoenix_live_view.git", "9b267285cfce96072e9ecd8fbe065dc64a7ad004", []}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, "phoenix_template": {:hex, :phoenix_template, "1.0.1", "85f79e3ad1b0180abb43f9725973e3b8c2c3354a87245f91431eec60553ed3ef", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "157dc078f6226334c91cb32c1865bf3911686f8bcd6bcff86736f6253e6993ee"}, "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, From c0539367be99295e6e142fe51a8a9a758585f293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Wed, 4 Oct 2023 22:11:56 +0200 Subject: [PATCH 4/6] Update lib/livebook_web/helpers/codec.ex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Valim --- lib/livebook_web/helpers/codec.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/livebook_web/helpers/codec.ex b/lib/livebook_web/helpers/codec.ex index 93ead1e1b19..36592e1b9f8 100644 --- a/lib/livebook_web/helpers/codec.ex +++ b/lib/livebook_web/helpers/codec.ex @@ -48,7 +48,7 @@ defmodule LivebookWeb.Helpers.Codec do :big -> Stream.map(file_stream, fn binary -> - for <>, reduce: <<>> do + for <>, reduce: <<>> do acc -> <> end end) From bdfbeaba92e5e42d6d63fa1d9cba3762c130b17a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Wed, 4 Oct 2023 22:13:13 +0200 Subject: [PATCH 5/6] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Valim --- lib/livebook_web/helpers/codec.ex | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/livebook_web/helpers/codec.ex b/lib/livebook_web/helpers/codec.ex index 36592e1b9f8..1ea1dbf0010 100644 --- a/lib/livebook_web/helpers/codec.ex +++ b/lib/livebook_web/helpers/codec.ex @@ -68,12 +68,10 @@ defmodule LivebookWeb.Helpers.Codec do data_size = num_frames * block_align << - # RIFF chunk - 0x52494646::32-unsigned-integer-big, + "RIFF", 36 + data_size::32-unsigned-integer-little, - 0x57415645::32-unsigned-integer-big, - # "fmt " sub-chunk - 0x666D7420::32-unsigned-integer-big, + "WAVE", + "fmt ", 16::32-unsigned-integer-little, # 3 indicates 32-bit float PCM 3::16-unsigned-integer-little, @@ -82,8 +80,7 @@ defmodule LivebookWeb.Helpers.Codec do byte_rate::32-unsigned-integer-little, block_align::16-unsigned-integer-little, bytes_per_sample * 8::16-unsigned-integer-little, - # "data" sub-chunk - 0x64617461::32-unsigned-integer-big, + "data", data_size::32-unsigned-integer-little >> end From 29225d014f89504e7360713d2e79c9e532917f11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 5 Oct 2023 18:21:10 +0700 Subject: [PATCH 6/6] Test input endpoints --- .../controllers/session_controller_test.exs | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/test/livebook_web/controllers/session_controller_test.exs b/test/livebook_web/controllers/session_controller_test.exs index c60accb67f9..01b20fc9410 100644 --- a/test/livebook_web/controllers/session_controller_test.exs +++ b/test/livebook_web/controllers/session_controller_test.exs @@ -1,6 +1,8 @@ defmodule LivebookWeb.SessionControllerTest do use LivebookWeb.ConnCase, async: true + require Phoenix.LiveViewTest + alias Livebook.{Sessions, Session, Notebook, FileSystem} describe "show_file" do @@ -356,6 +358,100 @@ defmodule LivebookWeb.SessionControllerTest do end end + describe "show_audio_image" do + @tag :tmp_dir + test "given :wav input returns the audio binary", %{conn: conn, tmp_dir: tmp_dir} do + {session, input_id} = start_session_with_audio_input(:wav, "wav content", tmp_dir) + + {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}") + + token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id) + + conn = get(conn, ~p"/sessions/audio-input/#{token}") + + assert conn.status == 200 + assert conn.resp_body == "wav content" + assert get_resp_header(conn, "accept-ranges") == ["bytes"] + + Session.close(session.pid) + end + + @tag :tmp_dir + test "given :wav input supports range requests", %{conn: conn, tmp_dir: tmp_dir} do + {session, input_id} = start_session_with_audio_input(:wav, "wav content", tmp_dir) + + {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}") + + token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id) + + conn = + conn + |> put_req_header("range", "bytes=4-") + |> get(~p"/sessions/audio-input/#{token}") + + assert conn.status == 206 + assert conn.resp_body == "content" + assert get_resp_header(conn, "content-range") == ["bytes 4-10/11"] + + Session.close(session.pid) + end + + @tag :tmp_dir + test "given :pcm_f32 input returns a WAV binary", %{conn: conn, tmp_dir: tmp_dir} do + {session, input_id} = start_session_with_audio_input(:pcm_f32, "pcm content", tmp_dir) + + {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}") + + token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id) + + conn = get(conn, ~p"/sessions/audio-input/#{token}") + + assert conn.status == 200 + assert <<_header::44-binary, "pcm content">> = conn.resp_body + assert get_resp_header(conn, "accept-ranges") == ["bytes"] + + Session.close(session.pid) + end + + @tag :tmp_dir + test "given :pcm_f32 input supports range requests", %{conn: conn, tmp_dir: tmp_dir} do + {session, input_id} = start_session_with_audio_input(:pcm_f32, "pcm content", tmp_dir) + + {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}") + + token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id) + + conn = + conn + |> put_req_header("range", "bytes=48-") + |> get(~p"/sessions/audio-input/#{token}") + + assert conn.status == 206 + assert conn.resp_body == "content" + assert get_resp_header(conn, "content-range") == ["bytes 48-54/55"] + + Session.close(session.pid) + end + end + + describe "show_input_image" do + @tag :tmp_dir + test "returns the image binary", %{conn: conn, tmp_dir: tmp_dir} do + {session, input_id} = start_session_with_image_input(:rgb, "rgb content", tmp_dir) + + {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}") + + token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id) + + conn = get(conn, ~p"/sessions/image-input/#{token}") + + assert conn.status == 200 + assert conn.resp_body == "rgb content" + + Session.close(session.pid) + end + end + defp start_session_and_request_asset(conn, notebook, hash) do {:ok, session} = Sessions.create_session(notebook: notebook) # We need runtime in place to actually copy the archive @@ -387,4 +483,62 @@ defmodule LivebookWeb.SessionControllerTest do %{notebook: notebook, hash: hash} end + + defp start_session_with_audio_input(format, binary, tmp_dir) do + input = %{ + type: :input, + ref: "ref", + id: "input1", + destination: :noop, + attrs: %{type: :audio, default: nil, label: "Audio", format: format, sampling_rate: 16_000} + } + + cell = %{Notebook.Cell.new(:code) | outputs: [{1, input}]} + notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [cell]}]} + + {:ok, session} = Sessions.create_session(notebook: notebook) + + source_path = Path.join(tmp_dir, "audio.bin") + File.write!(source_path, binary) + + {:ok, file_ref} = Session.register_file(session.pid, source_path, "key") + + Session.set_input_value(session.pid, "input1", %{ + file_ref: file_ref, + sampling_rate: 16_000, + num_channels: 1, + format: format + }) + + {session, input.id} + end + + defp start_session_with_image_input(format, binary, tmp_dir) do + input = %{ + type: :input, + ref: "ref", + id: "input1", + destination: :noop, + attrs: %{type: :image, default: nil, label: "Image", format: :rgb, size: nil, fit: :contain} + } + + cell = %{Notebook.Cell.new(:code) | outputs: [{1, input}]} + notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [cell]}]} + + {:ok, session} = Sessions.create_session(notebook: notebook) + + source_path = Path.join(tmp_dir, "image.bin") + File.write!(source_path, binary) + + {:ok, file_ref} = Session.register_file(session.pid, source_path, "key") + + Session.set_input_value(session.pid, "input1", %{ + file_ref: file_ref, + height: 300, + width: 300, + format: format + }) + + {session, input.id} + end end