diff --git a/config/dev.exs b/config/dev.exs index 8297eee630..e04e4e4661 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -58,6 +58,9 @@ config :sanbase, Sanbase.KafkaExporter, producer: Sanbase.InMemoryKafka.Producer # with. When running the app locally these values are overridden by the values # in the .env.dev or dev.secret.exs files, which are ignored by git and not # published in the repository. Please do not report these as security issues. +# To create the user for your local env: +# In psql: CREATE ROLE postgres WITH LOGIN SUPERUSER PASSWORD 'postgres'; +# In the terminal: mix ecto.setup config :sanbase, Sanbase.Repo, username: "postgres", password: "postgres", diff --git a/lib/mix/tasks/clone_metric_registry.ex b/lib/mix/tasks/clone_metric_registry.ex new file mode 100644 index 0000000000..d0d20c6e1f --- /dev/null +++ b/lib/mix/tasks/clone_metric_registry.ex @@ -0,0 +1,60 @@ +defmodule Mix.Tasks.CloneMetricRegistry do + @shortdoc "Make sure the destructive operations are not executed on production databases" + + @moduledoc """ + #{@shortdoc} + + Check the MIX_ENV environment variable, the DATABASE_URL environment variable and + the database configuration to determine if the operation is executed in dev + or test environment against a production database. + """ + + use Mix.Task + + @impl Mix.Task + def run(args) do + opts = parse_args(args) + file = Path.join([__DIR__, "metric_registry_pg_dump.sql"]) + db_url = Keyword.get(opts, :database_url) |> String.replace_prefix("ecto://", "postgres://") + + try do + {"", 0} = + System.cmd( + "pg_dump", + [ + "--data-only", + "--table=metric_registry", + "--file=#{file}", + "--dbname=#{db_url}", + # So it can be used by pg_restore + # "--format=custom", + "--inserts" + ] + ) + + contents = File.read!(file) + contents = String.replace(contents, "sanbase2.metric_registry", "public.metric_registry") + File.write!(file, contents) + + System.cmd( + "psql", + [ + "--dbname=sanbase_dev", + "--file=#{file}", + "--echo-all" + ] + ) + after + File.rm!(file) + end + + :ok + end + + defp parse_args(args) do + {options, _, _} = + OptionParser.parse(args, strict: [database_url: :string, drop_existing: :boolean]) + + options + end +end diff --git a/lib/sanbase/metric/registry/change_suggestion.ex b/lib/sanbase/metric/registry/change_suggestion.ex index 0808999afd..9ac19898b4 100644 --- a/lib/sanbase/metric/registry/change_suggestion.ex +++ b/lib/sanbase/metric/registry/change_suggestion.ex @@ -138,10 +138,16 @@ defmodule Sanbase.Metric.Registry.ChangeSuggestion do # only after the DB changes are commited and not from insite the transaction. If the event # is emitted from inside the transaction, the event handler can be invoked before the DB # changes are commited and this handler will have no effect. - Sanbase.Metric.Registry.update(metric_registry, params, emit_event?: false) + Sanbase.Metric.Registry.update(metric_registry, params, emit_event: true) end def create_change_suggestion(%Registry{} = registry, params, notes, submitted_by) do + # After change suggestion is applied, put the metric in a unverified state and mark + # is as not synced. Someone needs to manually verify the metric after it is tested. + # When the data is synced between stage and prod, the sync status will be updated. + # Note: Keep the keys as strings, not atoms, so the map is not mixed + params = Map.merge(params, %{"is_verified" => false, "sync_status" => "not_synced"}) + case Registry.changeset(registry, params) do %{valid?: false} = changeset -> {:error, changeset} diff --git a/lib/sanbase/metric/registry/embedded_schemas.ex b/lib/sanbase/metric/registry/embedded_schemas.ex new file mode 100644 index 0000000000..6622e4aa3a --- /dev/null +++ b/lib/sanbase/metric/registry/embedded_schemas.ex @@ -0,0 +1,75 @@ +defmodule Sanbase.Metric.Registry.Selector do + use Ecto.Schema + import Ecto.Changeset + + @derive {Jason.Encoder, only: [:type]} + + @primary_key false + embedded_schema do + field(:type, :string) + end + + def changeset(%__MODULE__{} = struct, attrs) do + struct + |> cast(attrs, [:type]) + |> validate_required([:type]) + end +end + +defmodule Sanbase.Metric.Registry.Table do + use Ecto.Schema + import Ecto.Changeset + + @derive {Jason.Encoder, only: [:name]} + + @primary_key false + embedded_schema do + field(:name, :string) + end + + def changeset(%__MODULE__{} = struct, attrs) do + struct + |> cast(attrs, [:name]) + |> validate_required([:name]) + |> validate_format(:name, ~r/[a-z0-9_\-]/) + end +end + +defmodule Sanbase.Metric.Registry.Alias do + use Ecto.Schema + import Ecto.Changeset + + @derive {Jason.Encoder, only: [:name]} + + @primary_key false + embedded_schema do + field(:name, :string) + end + + def changeset(%__MODULE__{} = struct, attrs) do + struct + |> cast(attrs, [:name]) + |> validate_required(:name) + |> validate_format(:name, Sanbase.Metric.Registry.metric_regex()) + |> validate_length(:name, min: 3, max: 100) + end +end + +defmodule Sanbase.Metric.Registry.Doc do + use Ecto.Schema + import Ecto.Changeset + + @derive {Jason.Encoder, only: [:link]} + + @primary_key false + embedded_schema do + field(:link, :string) + end + + def changeset(%__MODULE__{} = struct, attrs) do + struct + |> cast(attrs, [:link]) + |> validate_required([:link]) + |> validate_format(:link, ~r|https://academy.santiment.net|) + end +end diff --git a/lib/sanbase/metric/registry/registry.ex b/lib/sanbase/metric/registry/registry.ex index 593304a9c7..348111fd56 100644 --- a/lib/sanbase/metric/registry/registry.ex +++ b/lib/sanbase/metric/registry/registry.ex @@ -5,9 +5,14 @@ defmodule Sanbase.Metric.Registry do import Ecto.Changeset import Sanbase.Metric.Registry.EventEmitter, only: [emit_event: 3] - alias Sanbase.Repo alias __MODULE__.Validation alias __MODULE__.ChangeSuggestion + alias __MODULE__.Doc + alias __MODULE__.Alias + alias __MODULE__.Selector + alias __MODULE__.Table + + alias Sanbase.Repo alias Sanbase.TemplateEngine # Matches letters, digits, _, -, :, ., {, }, (, ), \, / and space @@ -18,74 +23,6 @@ defmodule Sanbase.Metric.Registry do @metric_regex ~r/^[a-z0-9_{}:]+$/ def metric_regex(), do: @metric_regex - defmodule Selector do - use Ecto.Schema - import Ecto.Changeset - - @primary_key false - embedded_schema do - field(:type, :string) - end - - def changeset(%__MODULE__{} = struct, attrs) do - struct - |> cast(attrs, [:type]) - |> validate_required([:type]) - end - end - - defmodule Table do - use Ecto.Schema - import Ecto.Changeset - - @primary_key false - embedded_schema do - field(:name, :string) - end - - def changeset(%__MODULE__{} = struct, attrs) do - struct - |> cast(attrs, [:name]) - |> validate_required([:name]) - |> validate_format(:name, ~r/[a-z0-9_\-]/) - end - end - - defmodule Alias do - use Ecto.Schema - import Ecto.Changeset - - @primary_key false - embedded_schema do - field(:name, :string) - end - - def changeset(%__MODULE__{} = struct, attrs) do - struct - |> cast(attrs, [:name]) - |> validate_required(:name) - |> validate_format(:name, Sanbase.Metric.Registry.metric_regex()) - |> validate_length(:name, min: 3, max: 100) - end - end - - defmodule Doc do - use Ecto.Schema - import Ecto.Changeset - - @primary_key false - embedded_schema do - field(:link, :string) - end - - def changeset(%__MODULE__{} = struct, attrs) do - struct - |> cast(attrs, [:link]) - |> validate_required([:link]) - |> validate_format(:link, ~r|https://academy.santiment.net|) - end - end - @type t :: %__MODULE__{ id: integer(), metric: String.t(), @@ -112,9 +49,15 @@ defmodule Sanbase.Metric.Registry do deprecation_note: String.t(), data_type: String.t(), docs: [%Doc{}], + is_verified: boolean(), + sync_status: String.t(), inserted_at: DateTime.t(), updated_at: DateTime.t() } + + @derive {Jason.Encoder, + except: [:__struct__, :__meta__, :id, :inserted_at, :updated_at, :change_suggestions]} + @timestamps_opts [type: :utc_datetime] schema "metric_registry" do # How the metric is exposed to external users @@ -146,9 +89,12 @@ defmodule Sanbase.Metric.Registry do field(:is_deprecated, :boolean, default: false) field(:hard_deprecate_after, :utc_datetime, default: nil) field(:deprecation_note, :string, default: nil) - field(:data_type, :string, default: "timeseries") + # Sync-related fields + field(:is_verified, :boolean) + field(:sync_status, :string) + embeds_many(:tables, Table, on_replace: :delete) embeds_many(:selectors, Selector, on_replace: :delete) embeds_many(:required_selectors, Selector, on_replace: :delete) @@ -179,10 +125,12 @@ defmodule Sanbase.Metric.Registry do :is_hidden, :is_template, :is_timebound, + :is_verified, :metric, :min_interval, :sanbase_min_plan, :sanapi_min_plan, + :sync_status, :parameters ]) |> cast_embed(:selectors, @@ -235,6 +183,7 @@ defmodule Sanbase.Metric.Registry do |> validate_inclusion(:exposed_environments, ["all", "none", "stage", "prod"]) |> validate_inclusion(:access, ["free", "restricted"]) |> validate_change(:min_interval, &Validation.validate_min_interval/2) + |> validate_change(:sync_status, &Validation.validate_sync_status/2) |> validate_inclusion(:sanbase_min_plan, ["free", "pro", "max"]) |> validate_inclusion(:sanapi_min_plan, ["free", "pro", "max"]) |> Validation.validate_template_fields() @@ -297,7 +246,14 @@ defmodule Sanbase.Metric.Registry do need to be resolved, or aliases need to be applied. """ @spec all() :: [t()] - def all(), do: Sanbase.Repo.all(__MODULE__) + def all() do + query = + from(m in __MODULE__, + order_by: [asc: m.id] + ) + + Sanbase.Repo.all(query) + end @doc ~s""" Resolve all the metric registry records. @@ -324,7 +280,7 @@ defmodule Sanbase.Metric.Registry do :update_metric_registry, :delete_metric_registry ] do - if Keyword.get(opts, :emit_event?, true) do + if Keyword.get(opts, :emit_event, true) do emit_event(result, event_type, %{}) else result diff --git a/lib/sanbase/metric/registry/validation.ex b/lib/sanbase/metric/registry/validation.ex index d468de1508..69fdf22700 100644 --- a/lib/sanbase/metric/registry/validation.ex +++ b/lib/sanbase/metric/registry/validation.ex @@ -1,6 +1,17 @@ defmodule Sanbase.Metric.Registry.Validation do import Ecto.Changeset + @sync_statuses ["synced", "not_synced"] + def validate_sync_status(:sync_status, status) do + if status in @sync_statuses do + [] + else + [ + sync_status: "The provided sync_status #{status} is not supported." + ] + end + end + def validate_min_interval(:min_interval, min_interval) do if Sanbase.DateTimeUtils.valid_compound_duration?(min_interval) do if Sanbase.DateTimeUtils.str_to_days(min_interval) > 30 do @@ -19,31 +30,6 @@ defmodule Sanbase.Metric.Registry.Validation do end end - def validate_min_plan(:min_plan, min_plan) do - map_keys = Map.keys(min_plan) |> Enum.sort() - map_values = Map.values(min_plan) |> Enum.uniq() |> Enum.sort() - - cond do - map_size(min_plan) == 0 -> - [] - - map_keys != ["SANAPI", "SANBASE"] -> - [ - min_plan: - "The keys for min_plan must be 'SANBASE' and 'SANAPI'. Got #{Enum.join(map_keys, ", ")} instead" - ] - - Enum.any?(map_values, &(&1 not in ["free", "pro"])) -> - [ - min_plan: - "The values for the min_plan elements must be 'free' or 'restricted'. Got #{Enum.join(map_values, ", ")} instead" - ] - - true -> - [] - end - end - def validate_template_fields(%Ecto.Changeset{} = changeset) do is_template = get_field(changeset, :is_template) parameters = get_field(changeset, :parameters) diff --git a/lib/sanbase_web/components/core_components.ex b/lib/sanbase_web/components/core_components.ex index d67690ea32..aaaf4ec7dd 100644 --- a/lib/sanbase_web/components/core_components.ex +++ b/lib/sanbase_web/components/core_components.ex @@ -39,6 +39,7 @@ defmodule SanbaseWeb.CoreComponents do attr(:id, :string, required: true) attr(:show, :boolean, default: false) attr(:on_cancel, JS, default: %JS{}) + attr(:max_modal_width, :string, default: "max-w-3xl") slot(:inner_block, required: true) def modal(assigns) do @@ -60,7 +61,7 @@ defmodule SanbaseWeb.CoreComponents do tabindex="0" >
-
+
<.focus_wrap id={"#{@id}-container"} phx-window-keydown={JS.exec("data-cancel", to: "##{@id}")} diff --git a/lib/sanbase_web/controllers/metric_registry_controller.ex b/lib/sanbase_web/controllers/metric_registry_controller.ex new file mode 100644 index 0000000000..e49d300380 --- /dev/null +++ b/lib/sanbase_web/controllers/metric_registry_controller.ex @@ -0,0 +1,36 @@ +defmodule SanbaseWeb.MetricRegistryController do + use SanbaseWeb, :controller + + def export_json(conn, _params) do + conn + |> resp(200, get_metric_registry_json()) + |> send_resp() + end + + defp get_metric_registry_json() do + Sanbase.Metric.Registry.all() + |> Enum.take(1) + |> Enum.map(&transform/1) + + # |> Enum.map(&Jason.encode!/1) + # |> Enum.intersperse("\n") + end + + defp transform(struct) when is_struct(struct) do + struct + |> Map.from_struct() + |> Map.drop([:__meta__, :inserted_at, :updated_at, :change_suggestions]) + |> Map.new(fn + {k, v} when is_list(v) -> + {k, Enum.map(v, &transform/1)} + + {k, v} when is_map(v) -> + {k, transform(v)} + + {k, v} -> + {k, v} + end) + end + + defp transform(data), do: data +end diff --git a/lib/sanbase_web/live/available_metrics/available_metrics_description.ex b/lib/sanbase_web/live/available_metrics/available_metrics_description.ex index aca911dc36..953be53255 100644 --- a/lib/sanbase_web/live/available_metrics/available_metrics_description.ex +++ b/lib/sanbase_web/live/available_metrics/available_metrics_description.ex @@ -50,6 +50,17 @@ defmodule SanbaseWeb.AvailableMetricsDescription do """ end + def get_popover_text(%{key: "Human Readable Name"} = assigns) do + ~H""" + 
+    The name of the metric formatted in a way that is suitable for showing
+    in Web UI and in texts shown to end clients. The name is capitalized, underscores
+    are replaced with spaces and other formatting is applied.
+    Example: The human readable name of 'price_usd' is 'USD Price'
+
+ """ + end + def get_popover_text(%{key: "Clickhouse Table"} = assigns) do ~H""" 
@@ -378,4 +389,35 @@ defmodule SanbaseWeb.AvailableMetricsDescription do
""" end + + def get_popover_text(%{key: "Verified Status"} = assigns) do + ~H""" + 
+    After a change request is approved and its changes are applied to the databse
+    record, the metric is moved into a Unverified state. Someone will need to manually
+    verify the metric via the UI (after testing the changes).
+    Only verified metrics can be deployed from stage to prod.
+
+ """ + end + + def get_popover_text(%{key: "Exposed Environments"} = assigns) do + ~H""" + 
+    One of: none, all, stage, prod.
+    Controls on which deployment environment the metric is visible.
+
+ """ + end + + def get_popover_text(%{key: "Sync Status"} = assigns) do + ~H""" + 
+    Controls the deployment process of metrics. When a change request is approved
+    and applied, the metric is moved to 'not_synced' state indicating that the change
+    is not synced with the other environment.
+    Only metrics in not_synced state are deployed from stage to prod.
+
+ """ + end end diff --git a/lib/sanbase_web/live/metric_registry/metric_registry_change_suggestions_live.ex b/lib/sanbase_web/live/metric_registry/metric_registry_change_suggestions_live.ex index 5d7257ec91..fd42e685f2 100644 --- a/lib/sanbase_web/live/metric_registry/metric_registry_change_suggestions_live.ex +++ b/lib/sanbase_web/live/metric_registry/metric_registry_change_suggestions_live.ex @@ -3,6 +3,7 @@ defmodule SanbaseWeb.MetricRegistryChangeSuggestionsLive do alias SanbaseWeb.AdminFormsComponents alias Sanbase.Metric.Registry.ChangeSuggestion + alias SanbaseWeb.AvailableMetricsComponents @impl true def mount(_params, _session, socket) do @@ -16,12 +17,17 @@ defmodule SanbaseWeb.MetricRegistryChangeSuggestionsLive do def render(assigns) do ~H"""
+
<.table id="metric_registry_changes_suggestions" rows={@rows}> <:col :let={row} label="Status"> - <:col :let={row} label="Metric"> + <:col :let={row} label="Metric" col_class="max-w-[320px] break-words"> <.link class="underline text-blue-600" href={~p"/admin2/metric_registry/show/#{row.metric_registry_id}"} diff --git a/lib/sanbase_web/live/metric_registry/metric_registry_index_live.ex b/lib/sanbase_web/live/metric_registry/metric_registry_index_live.ex index 220ac6780b..d12c1c92ba 100644 --- a/lib/sanbase_web/live/metric_registry/metric_registry_index_live.ex +++ b/lib/sanbase_web/live/metric_registry/metric_registry_index_live.ex @@ -11,8 +11,11 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do {:ok, socket |> assign( - visible_metrics: metrics, + show_verified_changes_modal: false, + visible_metrics_ids: Enum.map(metrics, & &1.id), metrics: metrics, + changed_metrics_ids: [], + verified_metrics_updates_map: %{}, filter: %{} )} end @@ -20,19 +23,34 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do @impl true def render(assigns) do ~H""" + <.modal + :if={@show_verified_changes_modal} + show + id="verified_changes_modal" + max_modal_width="max-w-6xl" + on_cancel={JS.push("hide_show_verified_changes_modal")} + > + <.list_metrics_verified_status_changed + changed_metrics_ids={@changed_metrics_ids} + metrics={@metrics} + /> +
- Showing <%= length(@visible_metrics) %> metrics + Showing <%= length(@visible_metrics_ids) %> metrics
<.navigation /> - <.filters filter={@filter} /> - + <.filters filter={@filter} changed_metrics_ids={@changed_metrics_ids} /> + <:col :let={row} label="ID"> <%= row.id %> - <:col :let={row} label="Metric Names" col_class="max-w-[480px] break-all"> + <:col :let={row} label="Metric Names" col_class="max-w-[720px] break-all"> <.metric_names metric={row.metric} internal_metric={row.internal_metric} @@ -41,28 +59,12 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do <:col :let={row} - label="Min Interval" + label="Frequency" popover_target="popover-min-interval" popover_target_text={get_popover_text(%{key: "Frequency"})} > <%= row.min_interval %> - <:col - :let={row} - label="Table" - popover_target="popover-table" - popover_target_text={get_popover_text(%{key: "Clickhouse Table"})} - > - <.embeded_schema_show list={row.tables} key={:name} /> - - <:col - :let={row} - label="Default Aggregation" - popover_target="popover-default-aggregation" - popover_target_text={get_popover_text(%{key: "Default Aggregation"})} - > - <%= row.default_aggregation %> - <:col :let={row} label="Access" @@ -71,6 +73,9 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do > <%= if is_map(row.access), do: Jason.encode!(row.access), else: row.access %> + <:col :let={row} label="Status"> + <.verified_toggle row={row} /> + <:col :let={row} popover_target="popover-metric-details" @@ -84,31 +89,149 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do """ end - def embeded_schema_show(assigns) do - ~H""" -
-
- <%= Map.get(item, @key) %> -
-
- """ - end - @impl true def handle_event("apply_filters", params, socket) do - visible_metrics = + visible_metrics_ids = socket.assigns.metrics |> maybe_apply_filter(:match_metric, params) |> maybe_apply_filter(:match_table, params) + |> maybe_apply_filter(:only_unverified, params) + |> Enum.map(& &1.id) {:noreply, socket |> assign( - visible_metrics: visible_metrics, + visible_metrics_ids: visible_metrics_ids, filter: params )} end + def handle_event("show_verified_changes_modal", _params, socket) do + {:noreply, + socket + |> assign(show_verified_changes_modal: true)} + end + + def handle_event("hide_show_verified_changes_modal", _params, socket) do + {:noreply, + socket + |> assign(show_verified_changes_modal: false)} + end + + def handle_event( + "update_status_is_verified", + %{"metric_registry_id" => id, "is_verified" => bool}, + socket + ) do + verified_metrics_updates_map = + Map.update( + socket.assigns.verified_metrics_updates_map, + id, + # This will be invoked only the first time the metric registry is updated + %{old: Enum.find(socket.assigns.metrics, &(&1.id == id)).is_verified, new: bool}, + fn map -> Map.put(map, :new, bool) end + ) + + # Keep only the IDs and not the full metric list otherwise this list needs to be + # updated after each time update_metric/4 is called or the metrics list is mutated + # in any other way. + changed_metrics_ids = + verified_metrics_updates_map + |> Enum.reduce([], fn {id, map}, acc -> + if map.new != map.old, do: [id | acc], else: acc + end) + + {:noreply, + assign(socket, + changed_metrics_ids: changed_metrics_ids, + verified_metrics_updates_map: verified_metrics_updates_map, + metrics: update_metric(socket.assigns.metrics, id, :is_verified, bool) + )} + end + + def handle_event("confirm_verified_changes_update", _params, socket) do + for metric <- socket.assigns.metrics, metric.id in socket.assigns.changed_metrics_ids do + map = Map.get(socket.assigns.verified_metrics_updates_map, metric.id) + + # Explicitly put the old is_verified status in the first argument otherwise Ecto + # will decide that the value does not change and will not mutate the DB + {:ok, _} = + Sanbase.Metric.Registry.update( + %{metric | is_verified: map.old}, + %{is_verified: map.new}, + emit_event: false + ) + end + + {:noreply, + socket + |> assign( + changed_metrics_ids: [], + verified_metrics_updates_map: %{}, + show_verified_changes_modal: false + ) + |> put_flash( + :info, + "Suggessfully updated the is_verified status of #{length(socket.assigns.changed_metrics_ids)} metric metricss" + )} + end + + defp list_metrics_verified_status_changed(assigns) do + ~H""" +
+ No changes +
+
+ <.table id="uploaded_images" rows={Enum.filter(@metrics, &(&1.id in @changed_metrics_ids))}> + <:col :let={row} label="Metric"> + <.metric_names + metric={row.metric} + internal_metric={row.internal_metric} + human_readable_name={row.human_readable_name} + /> + + <:col :let={row} label="New Status"> + VERIFIED + UNVERIFIED + + +
+ <.phx_click_button + phx_click="confirm_verified_changes_update" + class="bg-green-500 hover:bg-green-900 text-white" + text="Confirm Changes" + /> + <.phx_click_button + phx_click="hide_show_verified_changes_modal" + class="bg-white hover:bg-gray-100 text-gray-800" + text="Close" + /> +
+
+ """ + end + + defp verified_toggle(assigns) do + ~H""" + + """ + end + defp navigation(assigns) do ~H"""
@@ -123,6 +246,12 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do text="See Change Suggestions" href={~p"/admin2/metric_registry/change_suggestions"} /> + + <.action_button + icon="hero-arrow-path-rounded-square" + text="Sync Metrics" + href={~p"/admin2/metric_registry/sync"} + />
""" @@ -132,24 +261,48 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do ~H"""
Filters -
- <.filter_input - id="metric-name-search" - value={@filter["match_metric"]} - name="match_metric" - placeholder="Filter by metric name" - /> + +
+ <.filter_input + id="metric-name-search" + value={@filter["match_metric"]} + name="match_metric" + placeholder="Filter by metric name" + /> - <.filter_input - id="table-search" - value={@filter["match_table"]} - name="match_table" - placeholder="Filter by table" - /> + <.filter_input + id="table-search" + value={@filter["match_table"]} + name="match_table" + placeholder="Filter by table" + /> +
+
+ <.filter_unverified /> +
+ <.phx_click_button + phx_click="show_verified_changes_modal" + class="text-gray-900 bg-white hover:bg-gray-100" + text="Apply Verified Status Changes" + count={length(@changed_metrics_ids)} + /> +
+ """ + end + + defp filter_unverified(assigns) do + ~H""" +
+ +
""" end @@ -184,6 +337,27 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do """ end + attr :phx_click, :string, required: true + attr :text, :string, required: true + attr :count, :integer, required: false, default: nil + attr :class, :string, required: true + + defp phx_click_button(assigns) do + ~H""" + + """ + end + defp metric_names(assigns) do ~H"""
@@ -194,6 +368,16 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do """ end + defp update_metric(metrics, id, key, value) do + Enum.map(metrics, fn metric -> + if metric.id == id do + %{metric | key => value} + else + metric + end + end) + end + defp maybe_apply_filter(metrics, :match_metric, %{"match_metric" => query}) when query != "" do query = String.downcase(query) @@ -218,5 +402,17 @@ defmodule SanbaseWeb.MetricRegistryIndexLive do end) end + defp maybe_apply_filter(metrics, :match_table, %{"unverified_only" => "on"}) do + metrics + |> Enum.filter(fn m -> + m.is_verified == false + end) + end + defp maybe_apply_filter(metrics, _, _), do: metrics + + defp take_ordered(metrics, ids) do + metrics_map = Map.new(metrics, &{&1.id, &1}) + Enum.map(ids, &Map.get(metrics_map, &1)) + end end diff --git a/lib/sanbase_web/live/metric_registry/metric_registry_show_live.ex b/lib/sanbase_web/live/metric_registry/metric_registry_show_live.ex index f7a8cfe5ef..1eb4990c8f 100644 --- a/lib/sanbase_web/live/metric_registry/metric_registry_show_live.ex +++ b/lib/sanbase_web/live/metric_registry/metric_registry_show_live.ex @@ -100,6 +100,24 @@ defmodule SanbaseWeb.MetricRegistryShowLive do popover_target: "popover-internal-name", popover_target_text: get_popover_text(%{key: "Internal Name"}) }, + %{ + key: "Human Readable Name", + value: metric_registry.human_readable_name, + popover_target: "popover-human-readable-name", + popover_target_text: get_popover_text(%{key: "Human Readable Name"}) + }, + %{ + key: "Verified Status", + value: to_verified_status(metric_registry.is_verified), + popover_target: "popover-verified-status", + popover_target_text: get_popover_text(%{key: "Verified Status"}) + }, + %{ + key: "Sync Status", + value: metric_registry.sync_status, + popover_target: "popover-sync-status", + popover_target_text: get_popover_text(%{key: "Sync Status"}) + }, %{ key: "Aliases", value: metric_registry.aliases |> Enum.map(& &1.name) |> Enum.join(", "), @@ -112,6 +130,12 @@ defmodule SanbaseWeb.MetricRegistryShowLive do popover_target: "popover-clickhouse-table", popover_target_text: get_popover_text(%{key: "Clickhouse Table"}) }, + %{ + key: "Exposed Environments", + value: metric_registry.exposed_environments, + popover_target: "popover-exposed-environments", + popover_target_text: get_popover_text(%{key: "Exposed Environments"}) + }, %{ key: "Min Interval", value: metric_registry.min_interval, @@ -228,4 +252,8 @@ defmodule SanbaseWeb.MetricRegistryShowLive do } ] end + + defp to_verified_status(is_verified) do + if is_verified, do: "VERIFIED", else: "UNVERIFIED" + end end diff --git a/lib/sanbase_web/live/metric_registry/metric_registry_sync_live.ex b/lib/sanbase_web/live/metric_registry/metric_registry_sync_live.ex new file mode 100644 index 0000000000..007fa5c966 --- /dev/null +++ b/lib/sanbase_web/live/metric_registry/metric_registry_sync_live.ex @@ -0,0 +1,147 @@ +defmodule SanbaseWeb.MetricRegistrySyncLive do + use SanbaseWeb, :live_view + + alias SanbaseWeb.AvailableMetricsComponents + @impl true + def mount(_params, _session, socket) do + metrics = Sanbase.Metric.Registry.all() |> Enum.filter(&(&1.sync_status == "not_synced")) + + {:ok, + socket + |> assign( + metrics: metrics, + sync_metric_ids: Enum.map(metrics, & &1.id) |> MapSet.new() + )} + end + + @impl true + def render(assigns) do + ~H""" +
+
+
+ Showing <%= length(@metrics) %> metrics that are not synced +
+
+
+ +
+
+ <.phx_click_button + text="Select All" + phx_click="select_all" + class="bg-white hover:bg-gray-100 text-zync-900" + /> + <.phx_click_button + text="Deselect All" + phx_click="deselect_all" + class="bg-white hover:bg-gray-100 text-zync-900" + /> +
+ <.table id="metrics_registry" rows={@metrics}> + <:col :let={row} label="Should Sync"> + <.checkbox row={row} sync_metric_ids={@sync_metric_ids} /> + + <:col :let={row} label="ID"> + <%= row.id %> + + <:col :let={row} label="Metric Names" col_class="max-w-[720px] break-all"> + <.metric_names + metric={row.metric} + internal_metric={row.internal_metric} + human_readable_name={row.human_readable_name} + /> + + + <.phx_click_button + text="Sync Metrics" + phx_click="sync" + class="bg-blue-700 hover:bg-blue-800 text-white" + count={MapSet.size(@sync_metric_ids)} + phx_disable_with="Syncing..." + /> +
+ """ + end + + attr :phx_click, :string, required: true + attr :text, :string, required: true + attr :count, :integer, required: false, default: nil + attr :class, :string, required: true + attr :phx_disable_with, :string, required: false, default: nil + + defp phx_click_button(assigns) do + ~H""" + + """ + end + + @impl true + def handle_event("sync", _params, socket) do + Process.sleep(5000) + {:noreply, socket} + end + + def handle_event("update_should_sync", %{"metric_registry_id" => id} = params, socket) do + checked = Map.get(params, "value") == "on" + + sync_metric_ids = + if checked do + MapSet.put(socket.assigns.sync_metric_ids, id) + else + MapSet.delete(socket.assigns.sync_metric_ids, id) + end + + {:noreply, assign(socket, sync_metric_ids: sync_metric_ids)} + end + + def handle_event("select_all", _params, socket) do + {:noreply, + assign(socket, sync_metric_ids: Enum.map(socket.assigns.metrics, & &1.id) |> MapSet.new())} + end + + def handle_event("deselect_all", _params, socket) do + {:noreply, assign(socket, sync_metric_ids: MapSet.new())} + end + + defp metric_names(assigns) do + ~H""" +
+
<%= @human_readable_name %>
+
<%= @metric %> (API)
+
<%= @internal_metric %> (DB)
+
+ """ + end + + defp checkbox(assigns) do + ~H""" +
+ +
+ """ + end +end diff --git a/lib/sanbase_web/router.ex b/lib/sanbase_web/router.ex index cbcf2a1038..df998e6746 100644 --- a/lib/sanbase_web/router.ex +++ b/lib/sanbase_web/router.ex @@ -87,6 +87,7 @@ defmodule SanbaseWeb.Router do live("/metric_registry/show/:id", MetricRegistryShowLive) live("/metric_registry/edit/:id", MetricRegistryFormLive, :edit) live("/metric_registry/new", MetricRegistryFormLive, :new) + live("/metric_registry/sync", MetricRegistrySyncLive, :new) end scope "/" do @@ -180,6 +181,7 @@ defmodule SanbaseWeb.Router do end scope "/", SanbaseWeb do + get("/metric_registry_export", MetricRegistryController, :export_json) get("/api_metric_name_mapping", MetricNameController, :api_metric_name_mapping) get("/projects_data", DataController, :projects_data) get("/projects_twitter_handles", DataController, :projects_twitter_handles) diff --git a/priv/repo/migrations/20241128113958_add_metric_registry_is_verified.exs b/priv/repo/migrations/20241128113958_add_metric_registry_is_verified.exs new file mode 100644 index 0000000000..ba2229a8d7 --- /dev/null +++ b/priv/repo/migrations/20241128113958_add_metric_registry_is_verified.exs @@ -0,0 +1,10 @@ +defmodule Sanbase.Repo.Migrations.AddMetricRegistryIsVerified do + use Ecto.Migration + + def change do + alter table(:metric_registry) do + add(:is_verified, :boolean, null: false, default: true) + add(:sync_status, :string, null: false, default: "synced") + end + end +end diff --git a/priv/repo/structure.sql b/priv/repo/structure.sql index d5ea62b7b8..83fc57aaaa 100644 --- a/priv/repo/structure.sql +++ b/priv/repo/structure.sql @@ -2308,7 +2308,9 @@ CREATE TABLE public.metric_registry ( hard_deprecate_after timestamp(0) without time zone DEFAULT NULL::timestamp without time zone, deprecation_note text, inserted_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL + updated_at timestamp with time zone NOT NULL, + is_verified boolean DEFAULT true NOT NULL, + sync_status character varying(255) DEFAULT 'synced'::character varying NOT NULL ); @@ -9729,6 +9731,7 @@ INSERT INTO public."schema_migrations" (version) VALUES (20241112094924); INSERT INTO public."schema_migrations" (version) VALUES (20241114140339); INSERT INTO public."schema_migrations" (version) VALUES (20241114141110); INSERT INTO public."schema_migrations" (version) VALUES (20241116104556); +INSERT INTO public."schema_migrations" (version) VALUES (20241128113958); INSERT INTO public."schema_migrations" (version) VALUES (20241128161315); INSERT INTO public."schema_migrations" (version) VALUES (20241202104812); INSERT INTO public."schema_migrations" (version) VALUES (20241212054904); diff --git a/test/sanbase/metric_registry/change_suggestion_test.exs b/test/sanbase/metric_registry/change_suggestion_test.exs index 85929a7f63..4fe3a5dea2 100644 --- a/test/sanbase/metric_registry/change_suggestion_test.exs +++ b/test/sanbase/metric_registry/change_suggestion_test.exs @@ -34,6 +34,9 @@ defmodule Sanbase.MetricRegisty.ChangeSuggestionTest do test "accepting a change suggestion updates the metric" do assert {:ok, metric} = Registry.by_name("price_usd_5m", "timeseries") + assert metric.is_verified == true + assert metric.sync_status == "synced" + assert {:ok, struct} = ChangeSuggestion.create_change_suggestion( metric, @@ -42,6 +45,10 @@ defmodule Sanbase.MetricRegisty.ChangeSuggestionTest do _submitted_by = "ivan@santiment.net" ) + # Creating a change suggestion does not change the metric is_verified and sync_status + assert {:ok, metric} = Registry.by_name("price_usd_5m", "timeseries") + assert metric.is_verified == true + assert metric.sync_status == "synced" assert {:ok, _} = ChangeSuggestion.update_status(struct.id, "approved") assert {:ok, metric} = Registry.by_name("price_usd_5m", "timeseries") @@ -53,6 +60,10 @@ defmodule Sanbase.MetricRegisty.ChangeSuggestionTest do assert metric.selectors |> Enum.map(&Map.get(&1, :type)) == ["slug", "slugs", "quote_asset"] assert metric.required_selectors |> hd() |> Map.get(:type) == "slug|slugs|quote_asset" assert metric.tables |> hd() |> Map.get(:name) == "new_intraday_table" + + # Approving a change puts the metric in a unverified and unsynced state + assert metric.is_verified == false + assert metric.sync_status == "not_synced" end test "declining a change suggestion does not update the metric" do