Sync Metric Registry stage -> prod

config/dev.exs

@@ -58,6 +58,9 @@ config :sanbase, Sanbase.KafkaExporter, producer: Sanbase.InMemoryKafka.Producer
 # with. When running the app locally these values are overridden by the values
 # in the .env.dev or dev.secret.exs files, which are ignored by git and not
 # published in the repository. Please do not report these as security issues.
+# To create the user for your local env:
+# In psql: CREATE ROLE postgres WITH LOGIN SUPERUSER PASSWORD 'postgres';
+# In the terminal: mix ecto.setup
 config :sanbase, Sanbase.Repo,
   username: "postgres",
   password: "postgres",

lib/mix/tasks/clone_metric_registry.ex

@@ -0,0 +1,60 @@
+defmodule Mix.Tasks.CloneMetricRegistry do
+  @shortdoc "Make sure the destructive operations are not executed on production databases"
+
+  @moduledoc """
+  #{@shortdoc}
+
+  Check the MIX_ENV environment variable, the DATABASE_URL environment variable and
+  the database configuration to determine if the operation is executed in dev
+  or test environment against a production database.
+  """
+
+  use Mix.Task
+
+  @impl Mix.Task
+  def run(args) do
+    opts = parse_args(args)
+    file = Path.join([__DIR__, "metric_registry_pg_dump.sql"])
+    db_url = Keyword.get(opts, :database_url) |> String.replace_prefix("ecto://", "postgres://")
+
+    try do
+      {"", 0} =
+        System.cmd(
+          "pg_dump",
+          [
+            "--data-only",
+            "--table=metric_registry",
+            "--file=#{file}",
+            "--dbname=#{db_url}",
+            # So it can be used by pg_restore
+            # "--format=custom",
+            "--inserts"
+          ]
+        )
+
+      contents = File.read!(file)
+      contents = String.replace(contents, "sanbase2.metric_registry", "public.metric_registry")
+      File.write!(file, contents)
+
+      System.cmd(
+        "psql",
+        [
+          "--dbname=sanbase_dev",
+          "--file=#{file}",
+          "--echo-all"
+        ]
+      )
+    after
+      File.rm!(file)
+    end
+
+    :ok
+  end
+
+  defp parse_args(args) do
+    {options, _, _} =
+      OptionParser.parse(args, strict: [database_url: :string, drop_existing: :boolean])
+
+    options
+  end
+end

lib/sanbase/metric/registry/change_suggestion.ex

@@ -138,10 +138,16 @@ defmodule Sanbase.Metric.Registry.ChangeSuggestion do
     # only after the DB changes are commited and not from insite the transaction. If the event
     # is emitted from inside the transaction, the event handler can be invoked before the DB
     # changes are commited and this handler will have no effect.
-    Sanbase.Metric.Registry.update(metric_registry, params, emit_event?: false)
+    Sanbase.Metric.Registry.update(metric_registry, params, emit_event: true)
   end
 
   def create_change_suggestion(%Registry{} = registry, params, notes, submitted_by) do
+    # After change suggestion is applied, put the metric in a unverified state and mark
+    # is as not synced. Someone needs to manually verify the metric after it is tested.
+    # When the data is synced between stage and prod, the sync status will be updated.
+    # Note: Keep the keys as strings, not atoms, so the map is not mixed
+    params = Map.merge(params, %{"is_verified" => false, "sync_status" => "not_synced"})
+
     case Registry.changeset(registry, params) do
       %{valid?: false} = changeset ->
         {:error, changeset}

lib/sanbase/metric/registry/embedded_schemas.ex

@@ -0,0 +1,75 @@
+defmodule Sanbase.Metric.Registry.Selector do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  @derive {Jason.Encoder, only: [:type]}
+
+  @primary_key false
+  embedded_schema do
+    field(:type, :string)
+  end
+
+  def changeset(%__MODULE__{} = struct, attrs) do
+    struct
+    |> cast(attrs, [:type])
+    |> validate_required([:type])
+  end
+end
+
+defmodule Sanbase.Metric.Registry.Table do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  @derive {Jason.Encoder, only: [:name]}
+
+  @primary_key false
+  embedded_schema do
+    field(:name, :string)
+  end
+
+  def changeset(%__MODULE__{} = struct, attrs) do
+    struct
+    |> cast(attrs, [:name])
+    |> validate_required([:name])
+    |> validate_format(:name, ~r/[a-z0-9_\-]/)
+  end
+end
+
+defmodule Sanbase.Metric.Registry.Alias do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  @derive {Jason.Encoder, only: [:name]}
+
+  @primary_key false
+  embedded_schema do
+    field(:name, :string)
+  end
+
+  def changeset(%__MODULE__{} = struct, attrs) do
+    struct
+    |> cast(attrs, [:name])
+    |> validate_required(:name)
+    |> validate_format(:name, Sanbase.Metric.Registry.metric_regex())
+    |> validate_length(:name, min: 3, max: 100)
+  end
+end
+
+defmodule Sanbase.Metric.Registry.Doc do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  @derive {Jason.Encoder, only: [:link]}
+
+  @primary_key false
+  embedded_schema do
+    field(:link, :string)
+  end
+
+  def changeset(%__MODULE__{} = struct, attrs) do
+    struct
+    |> cast(attrs, [:link])
+    |> validate_required([:link])
+    |> validate_format(:link, ~r|https://academy.santiment.net|)
+  end
+end

lib/sanbase/metric/registry/registry.ex

@@ -5,9 +5,14 @@ defmodule Sanbase.Metric.Registry do
   import Ecto.Changeset
   import Sanbase.Metric.Registry.EventEmitter, only: [emit_event: 3]
 
-  alias Sanbase.Repo
   alias __MODULE__.Validation
   alias __MODULE__.ChangeSuggestion
+  alias __MODULE__.Doc
+  alias __MODULE__.Alias
+  alias __MODULE__.Selector
+  alias __MODULE__.Table
+
+  alias Sanbase.Repo
   alias Sanbase.TemplateEngine
 
   # Matches letters, digits, _, -, :, ., {, }, (, ), \, /  and space
@@ -18,74 +23,6 @@ defmodule Sanbase.Metric.Registry do
   @metric_regex ~r/^[a-z0-9_{}:]+$/
   def metric_regex(), do: @metric_regex
 
-  defmodule Selector do
-    use Ecto.Schema
-    import Ecto.Changeset
-
-    @primary_key false
-    embedded_schema do
-      field(:type, :string)
-    end
-
-    def changeset(%__MODULE__{} = struct, attrs) do
-      struct
-      |> cast(attrs, [:type])
-      |> validate_required([:type])
-    end
-  end
-
-  defmodule Table do
-    use Ecto.Schema
-    import Ecto.Changeset
-
-    @primary_key false
-    embedded_schema do
-      field(:name, :string)
-    end
-
-    def changeset(%__MODULE__{} = struct, attrs) do
-      struct
-      |> cast(attrs, [:name])
-      |> validate_required([:name])
-      |> validate_format(:name, ~r/[a-z0-9_\-]/)
-    end
-  end
-
-  defmodule Alias do
-    use Ecto.Schema
-    import Ecto.Changeset
-
-    @primary_key false
-    embedded_schema do
-      field(:name, :string)
-    end
-
-    def changeset(%__MODULE__{} = struct, attrs) do
-      struct
-      |> cast(attrs, [:name])
-      |> validate_required(:name)
-      |> validate_format(:name, Sanbase.Metric.Registry.metric_regex())
-      |> validate_length(:name, min: 3, max: 100)
-    end
-  end
-
-  defmodule Doc do
-    use Ecto.Schema
-    import Ecto.Changeset
-
-    @primary_key false
-    embedded_schema do
-      field(:link, :string)
-    end
-
-    def changeset(%__MODULE__{} = struct, attrs) do
-      struct
-      |> cast(attrs, [:link])
-      |> validate_required([:link])
-      |> validate_format(:link, ~r|https://academy.santiment.net|)
-    end
-  end
-
   @type t :: %__MODULE__{
           id: integer(),
           metric: String.t(),
@@ -112,9 +49,15 @@ defmodule Sanbase.Metric.Registry do
           deprecation_note: String.t(),
           data_type: String.t(),
           docs: [%Doc{}],
+          is_verified: boolean(),
+          sync_status: String.t(),
           inserted_at: DateTime.t(),
           updated_at: DateTime.t()
         }
+
+  @derive {Jason.Encoder,
+           except: [:__struct__, :__meta__, :id, :inserted_at, :updated_at, :change_suggestions]}
+
   @timestamps_opts [type: :utc_datetime]
   schema "metric_registry" do
     # How the metric is exposed to external users
@@ -146,9 +89,12 @@ defmodule Sanbase.Metric.Registry do
     field(:is_deprecated, :boolean, default: false)
     field(:hard_deprecate_after, :utc_datetime, default: nil)
     field(:deprecation_note, :string, default: nil)
-
     field(:data_type, :string, default: "timeseries")
 
+    # Sync-related fields
+    field(:is_verified, :boolean)
+    field(:sync_status, :string)
+
     embeds_many(:tables, Table, on_replace: :delete)
     embeds_many(:selectors, Selector, on_replace: :delete)
     embeds_many(:required_selectors, Selector, on_replace: :delete)
@@ -179,10 +125,12 @@ defmodule Sanbase.Metric.Registry do
       :is_hidden,
       :is_template,
       :is_timebound,
+      :is_verified,
       :metric,
       :min_interval,
       :sanbase_min_plan,
       :sanapi_min_plan,
+      :sync_status,
       :parameters
     ])
     |> cast_embed(:selectors,
@@ -235,6 +183,7 @@ defmodule Sanbase.Metric.Registry do
     |> validate_inclusion(:exposed_environments, ["all", "none", "stage", "prod"])
     |> validate_inclusion(:access, ["free", "restricted"])
     |> validate_change(:min_interval, &Validation.validate_min_interval/2)
+    |> validate_change(:sync_status, &Validation.validate_sync_status/2)
     |> validate_inclusion(:sanbase_min_plan, ["free", "pro", "max"])
     |> validate_inclusion(:sanapi_min_plan, ["free", "pro", "max"])
     |> Validation.validate_template_fields()
@@ -297,7 +246,14 @@ defmodule Sanbase.Metric.Registry do
   need to be resolved, or aliases need to be applied.
   """
   @spec all() :: [t()]
-  def all(), do: Sanbase.Repo.all(__MODULE__)
+  def all() do
+    query =
+      from(m in __MODULE__,
+        order_by: [asc: m.id]
+      )
+
+    Sanbase.Repo.all(query)
+  end
 
   @doc ~s"""
   Resolve all the metric registry records.
@@ -324,7 +280,7 @@ defmodule Sanbase.Metric.Registry do
                 :update_metric_registry,
                 :delete_metric_registry
               ] do
-    if Keyword.get(opts, :emit_event?, true) do
+    if Keyword.get(opts, :emit_event, true) do
       emit_event(result, event_type, %{})
     else
       result

lib/sanbase/metric/registry/validation.ex

@@ -1,6 +1,17 @@
 defmodule Sanbase.Metric.Registry.Validation do
   import Ecto.Changeset
 
+  @sync_statuses ["synced", "not_synced"]
+  def validate_sync_status(:sync_status, status) do
+    if status in @sync_statuses do
+      []
+    else
+      [
+        sync_status: "The provided sync_status #{status} is not supported."
+      ]
+    end
+  end
+
   def validate_min_interval(:min_interval, min_interval) do
     if Sanbase.DateTimeUtils.valid_compound_duration?(min_interval) do
       if Sanbase.DateTimeUtils.str_to_days(min_interval) > 30 do
@@ -19,31 +30,6 @@ defmodule Sanbase.Metric.Registry.Validation do
     end
   end
 
-  def validate_min_plan(:min_plan, min_plan) do
-    map_keys = Map.keys(min_plan) |> Enum.sort()
-    map_values = Map.values(min_plan) |> Enum.uniq() |> Enum.sort()
-
-    cond do
-      map_size(min_plan) == 0 ->
-        []
-
-      map_keys != ["SANAPI", "SANBASE"] ->
-        [
-          min_plan:
-            "The keys for min_plan must be 'SANBASE' and 'SANAPI'. Got #{Enum.join(map_keys, ", ")} instead"
-        ]
-
-      Enum.any?(map_values, &(&1 not in ["free", "pro"])) ->
-        [
-          min_plan:
-            "The values for the min_plan elements must be 'free' or 'restricted'. Got #{Enum.join(map_values, ", ")} instead"
-        ]
-
-      true ->
-        []
-    end
-  end
-
   def validate_template_fields(%Ecto.Changeset{} = changeset) do
     is_template = get_field(changeset, :is_template)
     parameters = get_field(changeset, :parameters)

lib/sanbase_web/components/core_components.ex

@@ -39,6 +39,7 @@ defmodule SanbaseWeb.CoreComponents do
   attr(:id, :string, required: true)
   attr(:show, :boolean, default: false)
   attr(:on_cancel, JS, default: %JS{})
+  attr(:max_modal_width, :string, default: "max-w-3xl")
   slot(:inner_block, required: true)
 
   def modal(assigns) do
@@ -60,7 +61,7 @@ defmodule SanbaseWeb.CoreComponents do
         tabindex="0"
       >
         <div class="flex min-h-full items-center justify-center">
-          <div class="w-full max-w-3xl p-4 sm:p-6 lg:py-8">
+          <div class={["w-full p-4 sm:p-6 lg:py-8", @max_modal_width]}>
             <.focus_wrap
               id={"#{@id}-container"}
               phx-window-keydown={JS.exec("data-cancel", to: "##{@id}")}