From 1a8fd7517e6e977046028ae9c821790600bc3186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 22 Jul 2021 13:58:51 +0200 Subject: [PATCH 1/5] Implement table widget for Ecto queries --- lib/kino.ex | 7 ++ lib/kino/data_table.ex | 93 ++---------------------- lib/kino/ecto.ex | 152 +++++++++++++++++++++++++++++++++++++++ lib/kino/ets.ex | 17 ++--- lib/kino/render.ex | 6 ++ lib/kino/utils.ex | 21 ++++++ lib/kino/utils/table.ex | 121 +++++++++++++++++++++++++++++++ mix.exs | 1 + mix.lock | 6 ++ test/kino/utils_test.exs | 5 ++ 10 files changed, 330 insertions(+), 99 deletions(-) create mode 100644 lib/kino/ecto.ex create mode 100644 lib/kino/utils.ex create mode 100644 lib/kino/utils/table.ex create mode 100644 test/kino/utils_test.exs diff --git a/lib/kino.ex b/lib/kino.ex index 3c32189a..27250cfe 100644 --- a/lib/kino.ex +++ b/lib/kino.ex @@ -83,6 +83,13 @@ defmodule Kino do | 2 | Erlang | https://www.erlang.org | \"\"\") + ### Kino.Ecto + + `Kino.Ecto` implements a data table output for arbitrary + `Ecto` queries: + + Kino.Ecto.new(Weather, Repo) + ### All others All other data structures are rendered as text using Elixir's diff --git a/lib/kino/data_table.ex b/lib/kino/data_table.ex index ccc716a2..090463c3 100644 --- a/lib/kino/data_table.ex +++ b/lib/kino/data_table.ex @@ -27,6 +27,8 @@ defmodule Kino.DataTable do use GenServer, restart: :temporary + alias Kino.Utils.Table + defstruct [:pid] @type t :: %__MODULE__{pid: pid()} @@ -146,15 +148,12 @@ defmodule Kino.DataTable do def handle_info({:connect, pid}, state) do columns = if state.keys do - Enum.map(state.keys, &key_to_column/1) + Table.keys_to_columns(state.keys) else [] end - features = - [pagination: true, sorting: state.sorting_enabled] - |> Enum.filter(&elem(&1, 1)) - |> Keyword.keys() + features = Kino.Utils.truthy_keys(pagination: true, sorting: state.sorting_enabled) send(pid, {:connect_reply, %{name: "Data", columns: columns, features: features}}) @@ -168,7 +167,7 @@ defmodule Kino.DataTable do if state.keys do {:initial, state.keys} else - columns = columns_structure(records) + columns = Table.columns_for_records(records) columns = if state.show_underscored, @@ -179,7 +178,7 @@ defmodule Kino.DataTable do {columns, keys} end - rows = Enum.map(records, &record_to_row(&1, keys)) + rows = Enum.map(records, &Table.record_to_row(&1, keys)) send(pid, {:rows, %{rows: rows, total_rows: state.total_rows, columns: columns}}) @@ -190,61 +189,10 @@ defmodule Kino.DataTable do {:stop, :shutdown, state} end - defp columns_structure(records) do - case Enum.at(records, 0) do - nil -> - [] - - first_record -> - first_record_columns = columns_structure_for_record(first_record) - - all_columns = - records - |> Enum.reduce(MapSet.new(), fn record, columns -> - record - |> columns_structure_for_record() - |> MapSet.new() - |> MapSet.union(columns) - end) - |> MapSet.to_list() - |> Enum.sort_by(& &1.key) - - # If all records have the same structure, keep the order, - # otherwise sort the accumulated columns - if length(first_record_columns) == length(all_columns) do - first_record_columns - else - all_columns - end - end - end - - defp columns_structure_for_record(record) when is_tuple(record) do - record - |> Tuple.to_list() - |> Enum.with_index() - |> Enum.map(fn {_, idx} -> key_to_column(idx) end) - end - - defp columns_structure_for_record(record) when is_map(record) do - record - |> Map.keys() - |> Enum.sort() - |> Enum.map(&key_to_column/1) - end - - defp columns_structure_for_record(record) when is_list(record) do - record - |> Keyword.keys() - |> Enum.map(&key_to_column/1) - end - - defp key_to_column(key), do: %{key: key, label: inspect(key)} - defp get_records(data, rows_spec) do sorted_data = if order_by = rows_spec[:order_by] do - Enum.sort_by(data, fn record -> get_field(record, order_by) end, rows_spec.order) + Enum.sort_by(data, &Table.get_field(&1, order_by), rows_spec.order) else data end @@ -252,33 +200,6 @@ defmodule Kino.DataTable do Enum.slice(sorted_data, rows_spec.offset, rows_spec.limit) end - defp get_field(record, key) when is_tuple(record) do - if key < tuple_size(record) do - elem(record, key) - else - nil - end - end - - defp get_field(record, key) when is_list(record) do - record[key] - end - - defp get_field(record, key) when is_map(record) do - Map.get(record, key) - end - - defp record_to_row(record, keys) do - fields = - Map.new(keys, fn key -> - value = get_field(record, key) - {key, inspect(value)} - end) - - # Note: id is opaque to the client, and we don't need it for now - %{id: nil, fields: fields} - end - defp underscored?(key) when is_atom(key) do key |> Atom.to_string() |> String.starts_with?("_") end diff --git a/lib/kino/ecto.ex b/lib/kino/ecto.ex new file mode 100644 index 00000000..5bdd6fca --- /dev/null +++ b/lib/kino/ecto.ex @@ -0,0 +1,152 @@ +defmodule Kino.Ecto do + @moduledoc """ + A widget for interactively viewing `Ecto` query results. + + The data must be an enumerable of records, where each + record is either map, struct, keyword list or tuple. + + ## Examples + + The widget primarly allows for viewing a database table + given a schema: + + Kino.Ecto.new(Weather, Repo) + + However, the first argument can be any queryable, so + you can pipe arbitrary queries directly to the widget: + + from(w in Weather, where: w.city == "New York") + |> Kino.Ecto.new(Repo) + """ + + use GenServer, restart: :temporary + + alias Kino.Utils.Table + + defstruct [:pid] + + import Ecto.Query, only: [from: 2] + + @type t :: %__MODULE__{pid: pid()} + + @typedoc false + @type state :: %{ + parent_monitor_ref: reference(), + repo: Ecto.Repo.t(), + queryable: Ecto.Queryable.t() + } + + @doc """ + Starts a widget process with the given queryable as + the data source. + """ + @spec new(Ecto.Queryable.t(), Ecto.Repo.t()) :: t() + def new(queryable, repo) when is_atom(repo) do + unless queryable?(queryable) do + raise ArgumentError, + "expected a term implementing the Ecto.Queryable protocol, got: #{inspect(queryable)}" + end + + parent = self() + opts = [repo: repo, queryable: queryable, parent: parent] + + {:ok, pid} = DynamicSupervisor.start_child(Kino.WidgetSupervisor, {__MODULE__, opts}) + + %__MODULE__{pid: pid} + end + + defp queryable?(term) do + Ecto.Queryable.impl_for(term) != nil + end + + @doc false + def start_link(opts) do + GenServer.start_link(__MODULE__, opts) + end + + @impl true + def init(opts) do + repo = Keyword.fetch!(opts, :repo) + queryable = Keyword.fetch!(opts, :queryable) + parent = Keyword.fetch!(opts, :parent) + + parent_monitor_ref = Process.monitor(parent) + + {:ok, %{parent_monitor_ref: parent_monitor_ref, repo: repo, queryable: queryable}} + end + + @impl true + def handle_info({:connect, pid}, state) do + {name, columns} = + if Table.ecto_schema?(state.queryable) do + name = state.queryable.__schema__(:source) |> to_string() + columns = state.queryable.__schema__(:fields) |> Table.keys_to_columns() + {name, columns} + else + {"Query", []} + end + + features = + Kino.Utils.truthy_keys( + refetch: true, + pagination: true, + # If the user specifies custom select, the record keys + # are not valid "order by" fields, so we disable sorting + sorting: default_select_query?(state.queryable) + ) + + send( + pid, + {:connect_reply, %{name: name, columns: columns, features: features}} + ) + + {:noreply, state} + end + + def handle_info({:get_rows, pid, rows_spec}, state) do + {total_rows, records} = get_records(state.repo, state.queryable, rows_spec) + + {columns, keys} = + if Table.ecto_schema?(state.queryable) do + keys = state.queryable.__schema__(:fields) + {:initial, keys} + else + columns = Table.columns_for_records(records) + keys = Enum.map(columns, & &1.key) + {columns, keys} + end + + rows = Enum.map(records, &Table.record_to_row(&1, keys)) + + send(pid, {:rows, %{rows: rows, total_rows: total_rows, columns: columns}}) + + {:noreply, state} + end + + def handle_info({:DOWN, ref, :process, _object, _reason}, %{parent_monitor_ref: ref} = state) do + {:stop, :shutdown, state} + end + + defp get_records(repo, queryable, rows_spec) do + count = repo.aggregate(queryable, :count) + + query = from(q in queryable, limit: ^rows_spec.limit, offset: ^rows_spec.offset) + + query = + if rows_spec[:order_by] do + order_by = [{rows_spec.order, rows_spec.order_by}] + from(q in query, order_by: ^order_by) + else + query + end + + records = repo.all(query) + + {count, records} + end + + defp default_select_query?(queryable) do + query = from(q in queryable, []) + query.select == nil + end +end diff --git a/lib/kino/ets.ex b/lib/kino/ets.ex index 7319b9fd..999b712b 100644 --- a/lib/kino/ets.ex +++ b/lib/kino/ets.ex @@ -12,6 +12,8 @@ defmodule Kino.ETS do use GenServer, restart: :temporary + alias Kino.Utils.Table + defstruct [:pid] @type t :: %__MODULE__{pid: pid()} @@ -77,7 +79,7 @@ defmodule Kino.ETS do columns = case :ets.match_object(state.tid, :_, 1) do - {[record], _} -> columns_structure_for_records([record]) + {[record], _} -> Table.columns_for_records([record]) :"$end_of_table" -> [] end @@ -97,7 +99,7 @@ defmodule Kino.ETS do columns = case records do [] -> :initial - records -> columns_structure_for_records(records) + records -> Table.columns_for_records(records) end send(pid, {:rows, %{rows: rows, total_rows: total_rows, columns: columns}}) @@ -109,17 +111,6 @@ defmodule Kino.ETS do {:stop, :shutdown, state} end - defp columns_structure_for_records(records) do - max_columns = - records - |> Enum.map(&tuple_size/1) - |> Enum.max() - - for idx <- 0..(max_columns - 1) do - %{key: idx, label: to_string(idx)} - end - end - defp get_records(tid, rows_spec) do query = :ets.table(tid) cursor = :qlc.cursor(query) diff --git a/lib/kino/render.ex b/lib/kino/render.ex index e85c840b..4f9d9892 100644 --- a/lib/kino/render.ex +++ b/lib/kino/render.ex @@ -50,6 +50,12 @@ defimpl Kino.Render, for: Kino.Markdown do end end +defimpl Kino.Render, for: Kino.Ecto do + def to_livebook(widget) do + Kino.Output.table_dynamic(widget.pid) + end +end + # Elixir built-ins defimpl Kino.Render, for: Reference do diff --git a/lib/kino/utils.ex b/lib/kino/utils.ex new file mode 100644 index 00000000..2c890eb8 --- /dev/null +++ b/lib/kino/utils.ex @@ -0,0 +1,21 @@ +defmodule Kino.Utils do + @moduledoc false + + @doc """ + Returns keyword list keys that hold a truthy value. + + ## Examples + + iex> Kino.Utils.truthy_keys(cat: true, dog: false) + [:cat] + + iex> Kino.Utils.truthy_keys(tea: :ok, coffee: nil) + [:tea] + """ + @spec truthy_keys(keyword()) :: list(atom()) + def truthy_keys(keywords) when is_list(keywords) do + keywords + |> Enum.filter(&elem(&1, 1)) + |> Keyword.keys() + end +end diff --git a/lib/kino/utils/table.ex b/lib/kino/utils/table.ex new file mode 100644 index 00000000..ca9ba1c3 --- /dev/null +++ b/lib/kino/utils/table.ex @@ -0,0 +1,121 @@ +defmodule Kino.Utils.Table do + @moduledoc false + + # Common functions for handling various Elixir + # terms as table records. + + @doc """ + Computes table columns that accomodate for all the given records. + """ + def columns_for_records(records) do + case Enum.at(records, 0) do + nil -> + [] + + first_record -> + first_record_columns = columns_for_record(first_record) + + all_columns = + records + |> Enum.reduce(MapSet.new(), fn record, columns -> + record + |> columns_for_record() + |> MapSet.new() + |> MapSet.union(columns) + end) + |> MapSet.to_list() + |> Enum.sort_by(& &1.key) + + # If all records have the same structure, keep the order, + # otherwise return the sorted accumulated columns + if length(first_record_columns) == length(all_columns) do + first_record_columns + else + all_columns + end + end + end + + defp columns_for_record(record) when is_tuple(record) do + record + |> Tuple.to_list() + |> Enum.with_index() + |> Enum.map(&elem(&1, 1)) + |> keys_to_columns() + end + + defp columns_for_record(record) when is_map(record) do + if is_struct(record) and ecto_schema?(record.__struct__) do + record.__struct__.__schema__(:fields) + else + record |> Map.keys() |> Enum.sort() + end + |> keys_to_columns() + end + + defp columns_for_record(record) when is_list(record) do + record + |> Keyword.keys() + |> keys_to_columns() + end + + defp columns_for_record(_record) do + # If the record is neither of the expected enumerables, + # we treat it as a single column value + keys_to_columns([:item]) + end + + @doc """ + Converts keys to column specifications. + """ + def keys_to_columns(keys) do + Enum.map(keys, fn key -> %{key: key, label: inspect(key)} end) + end + + @doc """ + Looks up record field value by key. + """ + def get_field(record, key) + + def get_field(record, key) when is_tuple(record) do + if key < tuple_size(record) do + elem(record, key) + else + nil + end + end + + def get_field(record, key) when is_list(record) do + record[key] + end + + def get_field(record, key) when is_map(record) do + Map.get(record, key) + end + + def get_field(record, :item) do + record + end + + @doc """ + Converts a record to row specification given a list + of desired keys. + """ + def record_to_row(record, keys) do + fields = + Map.new(keys, fn key -> + value = get_field(record, key) + {key, inspect(value)} + end) + + # Note: id is opaque to the client, and we don't need it for now + %{id: nil, fields: fields} + end + + @doc """ + Checks if the given term is an Ecto.Schema. + """ + def ecto_schema?(queryable) do + is_atom(queryable) and function_exported?(queryable, :__schema__, 1) + end +end diff --git a/mix.exs b/mix.exs index 2cfed787..78aa32cd 100644 --- a/mix.exs +++ b/mix.exs @@ -29,6 +29,7 @@ defmodule Kino.MixProject do defp deps do [ {:vega_lite, "~> 0.1.0", optional: true}, + {:ecto_sql, "~> 3.0", optional: true}, {:ex_doc, "~> 0.24", only: :dev, runtime: false} ] end diff --git a/mix.lock b/mix.lock index 1dadc3b8..b6662e9d 100644 --- a/mix.lock +++ b/mix.lock @@ -1,9 +1,15 @@ %{ + "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, + "db_connection": {:hex, :db_connection, "2.4.0", "d04b1b73795dae60cead94189f1b8a51cc9e1f911c234cc23074017c43c031e5", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad416c21ad9f61b3103d254a71b63696ecadb6a917b36f563921e0de00d7d7c8"}, + "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, + "ecto": {:hex, :ecto, "3.6.2", "efdf52acfc4ce29249bab5417415bd50abd62db7b0603b8bab0d7b996548c2bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "efad6dfb04e6f986b8a3047822b0f826d9affe8e4ebdd2aeedbfcb14fd48884e"}, + "ecto_sql": {:hex, :ecto_sql, "3.6.2", "9526b5f691701a5181427634c30655ac33d11e17e4069eff3ae1176c764e0ba3", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.6.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5ec9d7e6f742ea39b63aceaea9ac1d1773d574ea40df5a53ef8afbd9242fdb6b"}, "ex_doc": {:hex, :ex_doc, "0.24.2", "e4c26603830c1a2286dae45f4412a4d1980e1e89dc779fcd0181ed1d5a05c8d9", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "e134e1d9e821b8d9e4244687fb2ace58d479b67b282de5158333b0d57c6fb7da"}, "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, + "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "vega_lite": {:hex, :vega_lite, "0.1.0", "e55d0479f7088f817ad3ae087eb39195446a0df7f4eaa63078565b5a3a5fde24", [:mix], [], "hexpm", "4512f6841351b5b999c317dc1e47986422c873e64588404332efbbeda57f0573"}, } diff --git a/test/kino/utils_test.exs b/test/kino/utils_test.exs new file mode 100644 index 00000000..0403dce7 --- /dev/null +++ b/test/kino/utils_test.exs @@ -0,0 +1,5 @@ +defmodule Kino.UtilsTest do + use ExUnit.Case, async: true + + doctest Kino.Utils +end From 8b4e472b5e220ea18b6654538d578f52e3e9119e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 22 Jul 2021 16:57:57 +0200 Subject: [PATCH 2/5] Use to_query instead of empty from --- lib/kino/ecto.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kino/ecto.ex b/lib/kino/ecto.ex index 5bdd6fca..ab390dea 100644 --- a/lib/kino/ecto.ex +++ b/lib/kino/ecto.ex @@ -146,7 +146,7 @@ defmodule Kino.Ecto do end defp default_select_query?(queryable) do - query = from(q in queryable, []) + query = Ecto.Queryable.to_query(queryable) query.select == nil end end From ca7064448bf3f42f05374a15484f6c4cee52852c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Fri, 23 Jul 2021 10:42:26 +0200 Subject: [PATCH 3/5] Improve schema extraction --- lib/kino/ecto.ex | 10 +++++----- lib/kino/utils/table.ex | 28 +++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/kino/ecto.ex b/lib/kino/ecto.ex index ab390dea..1f26b24c 100644 --- a/lib/kino/ecto.ex +++ b/lib/kino/ecto.ex @@ -78,9 +78,9 @@ defmodule Kino.Ecto do @impl true def handle_info({:connect, pid}, state) do {name, columns} = - if Table.ecto_schema?(state.queryable) do - name = state.queryable.__schema__(:source) |> to_string() - columns = state.queryable.__schema__(:fields) |> Table.keys_to_columns() + if schema = Table.ecto_schema(state.queryable) do + name = schema.__schema__(:source) |> to_string() + columns = schema.__schema__(:fields) |> Table.keys_to_columns() {name, columns} else {"Query", []} @@ -107,8 +107,8 @@ defmodule Kino.Ecto do {total_rows, records} = get_records(state.repo, state.queryable, rows_spec) {columns, keys} = - if Table.ecto_schema?(state.queryable) do - keys = state.queryable.__schema__(:fields) + if schema = Table.ecto_schema(state.queryable) do + keys = schema.__schema__(:fields) {:initial, keys} else columns = Table.columns_for_records(records) diff --git a/lib/kino/utils/table.ex b/lib/kino/utils/table.ex index ca9ba1c3..c4aa3b64 100644 --- a/lib/kino/utils/table.ex +++ b/lib/kino/utils/table.ex @@ -45,8 +45,8 @@ defmodule Kino.Utils.Table do end defp columns_for_record(record) when is_map(record) do - if is_struct(record) and ecto_schema?(record.__struct__) do - record.__struct__.__schema__(:fields) + if schema = ecto_schema(record) do + schema.__schema__(:fields) else record |> Map.keys() |> Enum.sort() end @@ -113,9 +113,27 @@ defmodule Kino.Utils.Table do end @doc """ - Checks if the given term is an Ecto.Schema. + Extracts schema module from the given struct or queryable. + + If no schema found, `nil` is returned. """ - def ecto_schema?(queryable) do - is_atom(queryable) and function_exported?(queryable, :__schema__, 1) + def ecto_schema(queryable) + + def ecto_schema(struct) when is_struct(struct) do + ecto_schema(struct.__struct__) + end + + def ecto_schema(queryable) when is_atom(queryable) do + if function_exported?(queryable, :__schema__, 1) do + queryable + else + nil + end end + + def ecto_schema(%{from: %{source: {_source, schema}}}) when schema != nil do + schema + end + + def ecto_schema(_queryable), do: nil end From aafb63c79e18ccbd65e961edd0da44a1ae4e14a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Fri, 23 Jul 2021 10:49:59 +0200 Subject: [PATCH 4/5] Update lib/kino/utils/table.ex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Valim --- lib/kino/utils/table.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kino/utils/table.ex b/lib/kino/utils/table.ex index c4aa3b64..2bfde17c 100644 --- a/lib/kino/utils/table.ex +++ b/lib/kino/utils/table.ex @@ -124,7 +124,7 @@ defmodule Kino.Utils.Table do end def ecto_schema(queryable) when is_atom(queryable) do - if function_exported?(queryable, :__schema__, 1) do + if Code.ensure_loaded?(queryable) and function_exported?(queryable, :__schema__, 1) do queryable else nil From 72b62c447dfa52ab1b5f90dde5e551c85f8e84f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Fri, 23 Jul 2021 12:39:19 +0200 Subject: [PATCH 5/5] Add tests for the Ecto widget --- lib/kino/ecto.ex | 41 +++-- lib/kino/utils/table.ex | 8 +- test/kino/ecto_test.exs | 322 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 352 insertions(+), 19 deletions(-) create mode 100644 test/kino/ecto_test.exs diff --git a/lib/kino/ecto.ex b/lib/kino/ecto.ex index 1f26b24c..e59f7c29 100644 --- a/lib/kino/ecto.ex +++ b/lib/kino/ecto.ex @@ -77,14 +77,8 @@ defmodule Kino.Ecto do @impl true def handle_info({:connect, pid}, state) do - {name, columns} = - if schema = Table.ecto_schema(state.queryable) do - name = schema.__schema__(:source) |> to_string() - columns = schema.__schema__(:fields) |> Table.keys_to_columns() - {name, columns} - else - {"Query", []} - end + name = state.queryable |> query_source() |> to_string() + columns = state.queryable |> keys_from_queryable() |> Table.keys_to_columns() features = Kino.Utils.truthy_keys( @@ -107,13 +101,14 @@ defmodule Kino.Ecto do {total_rows, records} = get_records(state.repo, state.queryable, rows_spec) {columns, keys} = - if schema = Table.ecto_schema(state.queryable) do - keys = schema.__schema__(:fields) - {:initial, keys} - else - columns = Table.columns_for_records(records) - keys = Enum.map(columns, & &1.key) - {columns, keys} + case keys_from_queryable(state.queryable) do + [] -> + columns = Table.columns_for_records(records) + keys = Enum.map(columns, & &1.key) + {columns, keys} + + keys -> + {:initial, keys} end rows = Enum.map(records, &Table.record_to_row(&1, keys)) @@ -134,6 +129,7 @@ defmodule Kino.Ecto do query = if rows_spec[:order_by] do + query = Ecto.Query.exclude(query, :order_by) order_by = [{rows_spec.order, rows_spec.order_by}] from(q in query, order_by: ^order_by) else @@ -145,8 +141,23 @@ defmodule Kino.Ecto do {count, records} end + defp query_source(queryable) do + %{from: %{source: {source, _schema}}} = Ecto.Queryable.to_query(queryable) + source + end + defp default_select_query?(queryable) do query = Ecto.Queryable.to_query(queryable) query.select == nil end + + defp keys_from_queryable(queryable) do + schema = Table.ecto_schema(queryable) + + if schema != nil and default_select_query?(queryable) do + schema.__schema__(:fields) + else + [] + end + end end diff --git a/lib/kino/utils/table.ex b/lib/kino/utils/table.ex index 2bfde17c..458cb7ea 100644 --- a/lib/kino/utils/table.ex +++ b/lib/kino/utils/table.ex @@ -119,8 +119,8 @@ defmodule Kino.Utils.Table do """ def ecto_schema(queryable) - def ecto_schema(struct) when is_struct(struct) do - ecto_schema(struct.__struct__) + def ecto_schema(%{from: %{source: {_source, schema}}}) do + schema end def ecto_schema(queryable) when is_atom(queryable) do @@ -131,8 +131,8 @@ defmodule Kino.Utils.Table do end end - def ecto_schema(%{from: %{source: {_source, schema}}}) when schema != nil do - schema + def ecto_schema(struct) when is_struct(struct) do + ecto_schema(struct.__struct__) end def ecto_schema(_queryable), do: nil diff --git a/test/kino/ecto_test.exs b/test/kino/ecto_test.exs new file mode 100644 index 00000000..f67a3a97 --- /dev/null +++ b/test/kino/ecto_test.exs @@ -0,0 +1,322 @@ +defmodule Kino.EctoTest do + use ExUnit.Case, async: true + + import Ecto.Query, only: [from: 2] + + describe "new/1" do + test "raises an error when an invalid queryable is given" do + assert_raise ArgumentError, + "expected a term implementing the Ecto.Queryable protocol, got: 1", + fn -> + Kino.Ecto.new(1, Repo) + end + end + end + + defmodule User do + use Ecto.Schema + + schema "users" do + field(:name, :string) + + timestamps() + end + end + + describe "connecting" do + test "connect reply contains columns definition if a schema is given" do + widget = Kino.Ecto.new(User, MockRepo) + + send(widget.pid, {:connect, self()}) + + assert_receive {:connect_reply, + %{ + name: "users", + columns: [ + %{key: :id, label: ":id"}, + %{key: :name, label: ":name"}, + %{key: :inserted_at, label: ":inserted_at"}, + %{key: :updated_at, label: ":updated_at"} + ], + features: _features + }} + end + + test "connect reply contains columns definition if a query with schema source is given" do + query = from(u in User, where: like(u.name, "%Jake%")) + widget = Kino.Ecto.new(query, MockRepo) + + send(widget.pid, {:connect, self()}) + + assert_receive {:connect_reply, + %{ + name: "users", + columns: [ + %{key: :id, label: ":id"}, + %{key: :name, label: ":name"}, + %{key: :inserted_at, label: ":inserted_at"}, + %{key: :updated_at, label: ":updated_at"} + ], + features: _features + }} + end + + test "connect reply contains empty columns if a query without schema is given" do + query = from(u in "users", where: like(u.name, "%Jake%")) + widget = Kino.Ecto.new(query, MockRepo) + + send(widget.pid, {:connect, self()}) + + assert_receive {:connect_reply, + %{ + name: "users", + columns: [], + features: _features + }} + end + + test "connect reply contains empty columns if a query with custom select is given" do + query = from(u in User, select: {u.id, u.name}) + widget = Kino.Ecto.new(query, MockRepo) + + send(widget.pid, {:connect, self()}) + + assert_receive {:connect_reply, + %{ + name: "users", + columns: [], + features: _features + }} + end + + test "sorting is enabled when a regular query is given" do + query = from(u in User, where: like(u.name, "%Jake%")) + widget = Kino.Ecto.new(query, MockRepo) + + send(widget.pid, {:connect, self()}) + + assert_receive {:connect_reply, %{features: [:refetch, :pagination, :sorting]}} + end + + test "sorting is disabled when a query with custom select is given" do + query = from(u in User, where: like(u.name, "%Jake%"), select: {u.id, u.name}) + widget = Kino.Ecto.new(query, MockRepo) + + send(widget.pid, {:connect, self()}) + + assert_receive {:connect_reply, %{features: [:refetch, :pagination]}} + end + end + + defmodule MockRepo do + @moduledoc false + + # Allows tests to verify or refute that a query was run + # and also substitute the final result + + def all(query, opts \\ []) do + report_call([__MODULE__, :all, query: query, opts: opts]) + end + + def aggregate(query, :count, opts \\ []) do + report_call([__MODULE__, :aggregate, query: query, aggregate: :count, opts: opts]) + end + + # Test API + + @report_to :repo_report_to + + # Subscribes the caller to call info messages + def subscribe() do + Process.register(self(), @report_to) + end + + # Reports call info to the subscriber and waits for resolution message + defp report_call(info) do + ref = make_ref() + + send(@report_to, {{self(), ref}, info}) + + receive do + {:resolve, ^ref, value} -> value + after + 1_000 -> raise RuntimeError, "the following call hasn't been resolved: #{inspect(info)}" + end + end + + # Resolves the given call + def resolve_call({pid, ref}, value) do + send(pid, {:resolve, ref, value}) + end + end + + describe "querying rows" do + test "returns rows received from repo" do + widget = Kino.Ecto.new(User, MockRepo) + connect_self(widget) + + spec = %{offset: 0, limit: 10, order_by: nil, order: :asc} + + MockRepo.subscribe() + + send(widget.pid, {:get_rows, self(), spec}) + + assert_receive {from, [MockRepo, :aggregate, query: _, aggregate: :count, opts: []]} + MockRepo.resolve_call(from, 3) + + users = [ + %User{ + id: 1, + name: "Amy Santiago", + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }, + %User{ + id: 2, + name: "Jake Peralta", + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }, + %User{ + id: 3, + name: "Terry Jeffords", + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + } + ] + + assert_receive {from, [MockRepo, :all, query: _, opts: []]} + MockRepo.resolve_call(from, users) + + assert_receive {:rows, + %{ + rows: [ + %{ + id: _, + fields: %{ + id: "1", + name: ~s/"Amy Santiago"/, + inserted_at: _, + updated_at: _ + } + }, + %{ + id: _, + fields: %{ + id: "2", + name: ~s/"Jake Peralta"/, + inserted_at: _, + updated_at: _ + } + }, + %{ + id: _, + fields: %{ + id: "3", + name: ~s/"Terry Jeffords"/, + inserted_at: _, + updated_at: _ + } + } + ], + total_rows: 3, + columns: _columns + }} + end + + test "query limit and offset are overridden" do + query = from(u in User, offset: 2, limit: 2) + widget = Kino.Ecto.new(query, MockRepo) + connect_self(widget) + + spec = %{offset: 0, limit: 10, order_by: nil, order: :asc} + + MockRepo.subscribe() + + send(widget.pid, {:get_rows, self(), spec}) + + assert_receive {from, [MockRepo, :aggregate, query: _, aggregate: :count, opts: []]} + MockRepo.resolve_call(from, 3) + + assert_receive {_from, [MockRepo, :all, query: %{offset: offset, limit: limit}, opts: []]} + assert Macro.to_string(offset.expr) == "^0" + assert offset.params == [{0, :integer}] + assert Macro.to_string(limit.expr) == "^0" + assert limit.params == [{10, :integer}] + end + + test "query order by is kept if request doesn't specify any" do + query = from(u in User, order_by: u.name) + widget = Kino.Ecto.new(query, MockRepo) + connect_self(widget) + + spec = %{offset: 0, limit: 10, order_by: nil, order: :asc} + + MockRepo.subscribe() + + send(widget.pid, {:get_rows, self(), spec}) + + assert_receive {from, [MockRepo, :aggregate, query: _, aggregate: :count, opts: []]} + MockRepo.resolve_call(from, 3) + + assert_receive {_from, [MockRepo, :all, query: %{order_bys: [order_by]}, opts: []]} + assert Macro.to_string(order_by.expr) == "[asc: &0.name()]" + end + + test "query order by is overriden if specified in the request" do + query = from(u in User, order_by: u.name) + widget = Kino.Ecto.new(query, MockRepo) + connect_self(widget) + + spec = %{offset: 0, limit: 10, order_by: :id, order: :desc} + + MockRepo.subscribe() + + send(widget.pid, {:get_rows, self(), spec}) + + assert_receive {from, [MockRepo, :aggregate, query: _, aggregate: :count, opts: []]} + MockRepo.resolve_call(from, 3) + + assert_receive {_from, [MockRepo, :all, query: %{order_bys: [order_by]}, opts: []]} + assert Macro.to_string(order_by.expr) == "[desc: &0.id()]" + end + + test "handles custom select results" do + query = from(u in User, select: {u.id, u.name}) + widget = Kino.Ecto.new(query, MockRepo) + connect_self(widget) + + spec = %{offset: 0, limit: 10, order_by: :id, order: :desc} + + MockRepo.subscribe() + + send(widget.pid, {:get_rows, self(), spec}) + + assert_receive {from, [MockRepo, :aggregate, query: _, aggregate: :count, opts: []]} + MockRepo.resolve_call(from, 3) + + data = [{1, "Amy Santiago"}, {2, "Jake Peralta"}] + + assert_receive {from, [MockRepo, :all, query: _, opts: []]} + MockRepo.resolve_call(from, data) + + assert_receive {:rows, + %{ + rows: [ + %{id: _, fields: %{0 => "1", 1 => ~s/"Amy Santiago"/}}, + %{id: _, fields: %{0 => "2", 1 => ~s/"Jake Peralta"/}} + ], + total_rows: 3, + columns: [ + %{key: 0, label: "0"}, + %{key: 1, label: "1"} + ] + }} + end + end + + defp connect_self(widget) do + send(widget.pid, {:connect, self()}) + assert_receive {:connect_reply, %{}} + end +end