From e1bd6aabba6abc56e2800f7627934e062fa75e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 2 Jul 2024 11:14:51 +0200 Subject: [PATCH 1/2] Instructions for k8s agent --- .../live_editor/codemirror/languages.js | 8 + assets/package-lock.json | 59 +++++- assets/package.json | 1 + docs/deployment/clustering.md | 2 +- lib/livebook_web/components/app_components.ex | 2 +- .../components/core_components.ex | 41 +++++ .../live/app_session_live/source_component.ex | 27 +-- .../teams/deployment_group_agent_component.ex | 172 +++++++++++++++--- 8 files changed, 261 insertions(+), 51 deletions(-) diff --git a/assets/js/hooks/cell_editor/live_editor/codemirror/languages.js b/assets/js/hooks/cell_editor/live_editor/codemirror/languages.js index 43b2adedbc6..fa89d79f838 100644 --- a/assets/js/hooks/cell_editor/live_editor/codemirror/languages.js +++ b/assets/js/hooks/cell_editor/live_editor/codemirror/languages.js @@ -9,6 +9,7 @@ import { json } from "@codemirror/lang-json"; import { xml } from "@codemirror/lang-xml"; import { css } from "@codemirror/lang-css"; import { html } from "@codemirror/lang-html"; +import { yaml } from "@codemirror/lang-yaml"; import { javascript } from "@codemirror/lang-javascript"; import { erlang } from "@codemirror/legacy-modes/mode/erlang"; import { dockerFile } from "@codemirror/legacy-modes/mode/dockerfile"; @@ -29,6 +30,11 @@ const sqlDesc = LanguageDescription.of({ support: sql(), }); +const yamlDesc = LanguageDescription.of({ + name: "YAML", + support: yaml(), +}); + const jsonDesc = LanguageDescription.of({ name: "JSON", support: json(), @@ -67,6 +73,7 @@ const markdownDesc = LanguageDescription.of({ elixirDesc, erlangDesc, sqlDesc, + yamlDesc, jsonDesc, xmlDesc, cssDesc, @@ -81,6 +88,7 @@ export const languages = [ elixirDesc, erlangDesc, sqlDesc, + yamlDesc, jsonDesc, xmlDesc, cssDesc, diff --git a/assets/package-lock.json b/assets/package-lock.json index a783d32afb2..be0237ebe0e 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -14,6 +14,7 @@ "@codemirror/lang-markdown": "^6.2.3", "@codemirror/lang-sql": "^6.5.5", "@codemirror/lang-xml": "^6.0.2", + "@codemirror/lang-yaml": "^6.1.1", "@codemirror/language": "^6.10.0", "@codemirror/legacy-modes": "^6.3.3", "@codemirror/lint": "^6.4.2", @@ -1909,6 +1910,19 @@ "@lezer/xml": "^1.0.0" } }, + "node_modules/@codemirror/lang-yaml": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@codemirror/lang-yaml/-/lang-yaml-6.1.1.tgz", + "integrity": "sha512-HV2NzbK9bbVnjWxwObuZh5FuPCowx51mEfoFT9y3y+M37fA3+pbxx4I7uePuygFzDsAmCTwQSc/kXh/flab4uw==", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.2.0", + "@lezer/yaml": "^1.0.0" + } + }, "node_modules/@codemirror/language": { "version": "6.10.2", "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.2.tgz", @@ -3160,9 +3174,9 @@ } }, "node_modules/@lezer/lr": { - "version": "1.3.14", - "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.14.tgz", - "integrity": "sha512-z5mY4LStlA3yL7aHT/rqgG614cfcvklS+8oFRFBYrs4YaWLJyKKM4+nN6KopToX0o9Hj6zmH6M5kinOYuy06ug==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.1.tgz", + "integrity": "sha512-CHsKq8DMKBf9b3yXPDIU4DbH+ZJd/sJdYOW2llbW/HudP5u0VS6Bfq1hLYfgU7uAYGFIyGGQIsSOXGPEErZiJw==", "dependencies": { "@lezer/common": "^1.0.0" } @@ -3186,6 +3200,16 @@ "@lezer/lr": "^1.0.0" } }, + "node_modules/@lezer/yaml": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@lezer/yaml/-/yaml-1.0.3.tgz", + "integrity": "sha512-GuBLekbw9jDBDhGur82nuwkxKQ+a3W5H0GfaAthDXcAu+XdpS43VlnxA9E9hllkpSP5ellRDKjLLj7Lu9Wr6xA==", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.4.0" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -16181,6 +16205,19 @@ "@lezer/xml": "^1.0.0" } }, + "@codemirror/lang-yaml": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@codemirror/lang-yaml/-/lang-yaml-6.1.1.tgz", + "integrity": "sha512-HV2NzbK9bbVnjWxwObuZh5FuPCowx51mEfoFT9y3y+M37fA3+pbxx4I7uePuygFzDsAmCTwQSc/kXh/flab4uw==", + "requires": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.2.0", + "@lezer/yaml": "^1.0.0" + } + }, "@codemirror/language": { "version": "6.10.2", "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.2.tgz", @@ -17026,9 +17063,9 @@ } }, "@lezer/lr": { - "version": "1.3.14", - "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.14.tgz", - "integrity": "sha512-z5mY4LStlA3yL7aHT/rqgG614cfcvklS+8oFRFBYrs4YaWLJyKKM4+nN6KopToX0o9Hj6zmH6M5kinOYuy06ug==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.1.tgz", + "integrity": "sha512-CHsKq8DMKBf9b3yXPDIU4DbH+ZJd/sJdYOW2llbW/HudP5u0VS6Bfq1hLYfgU7uAYGFIyGGQIsSOXGPEErZiJw==", "requires": { "@lezer/common": "^1.0.0" } @@ -17052,6 +17089,16 @@ "@lezer/lr": "^1.0.0" } }, + "@lezer/yaml": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@lezer/yaml/-/yaml-1.0.3.tgz", + "integrity": "sha512-GuBLekbw9jDBDhGur82nuwkxKQ+a3W5H0GfaAthDXcAu+XdpS43VlnxA9E9hllkpSP5ellRDKjLLj7Lu9Wr6xA==", + "requires": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.4.0" + } + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", diff --git a/assets/package.json b/assets/package.json index e3058c60cc6..cf758d676d9 100644 --- a/assets/package.json +++ b/assets/package.json @@ -18,6 +18,7 @@ "@codemirror/lang-markdown": "^6.2.3", "@codemirror/lang-sql": "^6.5.5", "@codemirror/lang-xml": "^6.0.2", + "@codemirror/lang-yaml": "^6.1.1", "@codemirror/language": "^6.10.0", "@codemirror/legacy-modes": "^6.3.3", "@codemirror/lint": "^6.4.2", diff --git a/docs/deployment/clustering.md b/docs/deployment/clustering.md index 7f29cae6f26..ab5ff54017c 100644 --- a/docs/deployment/clustering.md +++ b/docs/deployment/clustering.md @@ -8,7 +8,7 @@ You may set `LIVEBOOK_CLUSTER` to one of the following values. ## `auto` -Detects the hosting platform and automatically sets up a cluster using DNS configuration. Currently the only supported platform is Fly.io. +Detects the hosting platform and automatically sets up a cluster using DNS configuration. Currently the only supported platform is Fly.io (and Kubernetes when using Livebook Teams). ## `dns:QUERY` diff --git a/lib/livebook_web/components/app_components.ex b/lib/livebook_web/components/app_components.ex index 2d647c9043f..34855738031 100644 --- a/lib/livebook_web/components/app_components.ex +++ b/lib/livebook_web/components/app_components.ex @@ -141,7 +141,7 @@ defmodule LivebookWeb.AppComponents do for more information.

- Automatic clustering is available when deploying to Fly.io. + Automatic clustering is available when deploying to Fly.io and Kubernetes.

diff --git a/lib/livebook_web/components/core_components.ex b/lib/livebook_web/components/core_components.ex index 6a086d643a2..661d1735b6c 100644 --- a/lib/livebook_web/components/core_components.ex +++ b/lib/livebook_web/components/core_components.ex @@ -521,6 +521,47 @@ defmodule LivebookWeb.CoreComponents do """ end + @doc """ + Renders a highlighted code snippet with a title and a copy button. + + ## Examples + + <.code_preview_with_title_and_copy + title=" + source_id="my-snippet" + language="elixir" + source="System.version()" /> + + """ + attr :title, :string, required: true + attr :source_id, :string, required: true + attr :language, :string, required: true + attr :source, :string, required: true + + def code_preview_with_title_and_copy(assigns) do + ~H""" +
+
+ <%= @title %> +
+ + <.icon_button + aria-label="copy source" + phx-click={JS.dispatch("lb:clipcopy", to: "##{@source_id}")} + > + <.remix_icon icon="clipboard-line" /> + + +
+
+ +
+ <.code_preview source_id={@source_id} language={@language} source={@source} /> +
+
+ """ + end + @doc """ Renders text with a tiny label. diff --git a/lib/livebook_web/live/app_session_live/source_component.ex b/lib/livebook_web/live/app_session_live/source_component.ex index ebd1647b135..902d44afef6 100644 --- a/lib/livebook_web/live/app_session_live/source_component.ex +++ b/lib/livebook_web/live/app_session_live/source_component.ex @@ -38,26 +38,13 @@ defmodule LivebookWeb.AppSessionLive.SourceComponent do

This app is built from the following notebook source:

-
-
- - <%= Session.file_name_for_download(@session) <> ".livemd" %> - -
- - <.icon_button - aria-label="copy source" - phx-click={JS.dispatch("lb:clipcopy", to: "#export-notebook-source")} - > - <.remix_icon icon="clipboard-line" /> - - -
-
-
- <.code_preview source_id="export-notebook-source" language="markdown" source={@source} /> -
-
+ + <.code_preview_with_title_and_copy + title={Session.file_name_for_download(@session) <> ".livemd"} + source_id="export-notebook-source" + language="markdown" + source={@source} + /> """ end diff --git a/lib/livebook_web/live/hub/teams/deployment_group_agent_component.ex b/lib/livebook_web/live/hub/teams/deployment_group_agent_component.ex index 213b1d5aae1..e37db9cb2e1 100644 --- a/lib/livebook_web/live/hub/teams/deployment_group_agent_component.ex +++ b/lib/livebook_web/live/hub/teams/deployment_group_agent_component.ex @@ -95,17 +95,13 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do to set the environment variables as secrets, if applicable. Below is an example calling Docker CLI directly, adapt it as necessary.

-
-
- CLI -
- - <.code_preview - source_id="agent-dockerfile-source" - source={@instructions.docker_instructions} - language="shell" - /> -
+ + <.code_preview_with_title_and_copy + title="CLI" + source_id="agent-dockerfile-source" + source={@instructions.docker_instructions} + language="shell" + /> <:tab id="fly_io" label="Fly.io"> @@ -113,17 +109,38 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do

Deploy an app server to Fly.io with a few commands.

-
-
- CLI -
- - <.code_preview - source_id="agent-dockerfile-source" - source={@instructions.fly_instructions} - language="shell" - /> -
+ + <.code_preview_with_title_and_copy + title="CLI" + source_id="agent-fly-source" + source={@instructions.fly_instructions} + language="shell" + /> + + + <:tab id="k8s" label="Kubernetes"> +
+

+ Deploy an app server to Kubernetes. First save the following Kubernetes resource file to disk: +

+ + <.code_preview_with_title_and_copy + title="livebook.yml" + source_id="agent-k8s-source" + source={@instructions.k8s_instructions} + language="yaml" + /> + +

+ Now run the following shell command: +

+ + <.code_preview_with_title_and_copy + title="CLI" + source_id="agent-k8s-cli" + source="kubectl apply -f livebook.yml" + language="shell" + />
@@ -178,7 +195,8 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do %{ docker_instructions: docker_instructions(image, env), - fly_instructions: fly_instructions(image, env, hub.hub_name, deployment_group.name) + fly_instructions: fly_instructions(image, env, hub.hub_name, deployment_group.name), + k8s_instructions: k8s_instructions(image, env) } end @@ -213,4 +231,112 @@ defmodule LivebookWeb.Hub.Teams.DeploymentGroupAgentComponent do fly deploy --ha=false """ end + + defp k8s_instructions(image, env) do + {secrets, envs} = + Map.split( + Map.new(env), + ~w(LIVEBOOK_TEAMS_KEY LIVEBOOK_TEAMS_AUTH LIVEBOOK_SECRET_KEY_BASE LIVEBOOK_COOKIE) + ) + + # We replace auto by the cluster setting. + {replicas, envs} = + case envs do + %{"LIVEBOOK_CLUSTER" => "auto"} -> {2, Map.delete(envs, "LIVEBOOK_CLUSTER")} + %{} -> {1, envs} + end + + envs = + Map.put_new( + envs, + "LIVEBOOK_CLUSTER", + "dns:livebook-headless.$(POD_NAMESPACE).svc.cluster.local" + ) + + k8s_instructions_template(image, envs, secrets, replicas) + end + + require EEx + + EEx.function_from_string( + :defp, + :k8s_instructions_template, + """ + apiVersion: v1 + kind: Service + metadata: + name: livebook-headless + spec: + clusterIP: None + selector: + app: livebook + + --- + + apiVersion: v1 + kind: Service + metadata: + name: livebook-loadbalancer + spec: + type: LoadBalancer + ports: + - port: 80 + targetPort: 8080 + selector: + app: livebook + + --- + + apiVersion: apps/v1 + kind: Deployment + metadata: + name: livebook + spec: + replicas: <%= replicas %> + selector: + matchLabels: + app: livebook + template: + metadata: + labels: + app: livebook + spec: + containers: + - name: livebook + image: <%= image %> + ports: + - containerPort: 8080 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LIVEBOOK_NODE + value: "livebook@$(POD_IP)"<%= for {k, v} <- envs do %> + - name: <%= k %> + value: <%= inspect(v) %><% end %><%= for {k, _} <- secrets do %> + - name: <%= k %> + valueFrom: + secretKeyRef: + name: livebook-secret + key: <%= k %><% end %> + + --- + + apiVersion: v1 + kind: Secret + metadata: + name: livebook-secret + namespace: livebook-namespace + type: Opaque + data: + # LIVEBOOK_PASSWORD: <%= for {k, v} <- secrets do %> + <%= k %>: <%= Base.encode64(v) %><% end %> + """, + [:image, :envs, :secrets, :replicas] + ) end From 0d78307078b84bce749206405e53ad49e3c41347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Tue, 2 Jul 2024 11:24:05 +0200 Subject: [PATCH 2/2] Update lib/livebook_web/components/core_components.ex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Kłosko --- lib/livebook_web/components/core_components.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/livebook_web/components/core_components.ex b/lib/livebook_web/components/core_components.ex index 661d1735b6c..681c0b19b3f 100644 --- a/lib/livebook_web/components/core_components.ex +++ b/lib/livebook_web/components/core_components.ex @@ -555,7 +555,7 @@ defmodule LivebookWeb.CoreComponents do -
+
<.code_preview source_id={@source_id} language={@language} source={@source} />