Get catalog from runner

config/config.exs

@@ -90,6 +90,7 @@ config :trento, Trento.Scheduler,
   debug_logging: false
 
 config :trento, Trento.Integration.Telemetry, adapter: Trento.Integration.Telemetry.Suse
+config :trento, Trento.Integration.Checks, adapter: Trento.Integration.Checks.Runner
 
 config :trento,
   uuid_namespace: "fb92284e-aa5e-47f6-a883-bf9469e7a0dc",

config/dev.exs

@@ -93,6 +93,7 @@ config :trento, Trento.Scheduler,
 
 config :trento, Trento.Integration.Telemetry, adapter: Trento.Integration.Telemetry.ToLogger
 config :trento, Trento.Integration.Checks, adapter: Trento.Integration.Checks.MockRunner
+config :trento, Trento.Integration.Checks.Runner, runner_url: ""
 
 # Do not include metadata nor timestamps in development logs
 config :logger, :console, format: "[$level] $message\n"

config/runtime.exs

@@ -54,6 +54,15 @@ if config_env() == :prod do
     ],
     secret_key_base: secret_key_base
 
+  runner_url =
+    System.get_env("RUNNER_URL") ||
+      raise """
+      environment variable RUNNER_URL is missing.
+      For example: http://localhost:8080
+      """
+
+  config :trento, Trento.Integration.Checks.Runner, runner_url: runner_url
+
   # ## Using releases
   #
   # If you are doing OTP releases, you need to instruct Phoenix

lib/trento/application/integration/checks/adapter/gen.ex

@@ -1,6 +1,6 @@
 defmodule Trento.Integration.Checks.Gen do
   @moduledoc """
-  Behaviour of a telemetry adapter.
+  Behaviour of a runner adapter.
   """
 
   @callback request_execution(
@@ -10,4 +10,7 @@ defmodule Trento.Integration.Checks.Gen do
               selected_checks :: [String.t()]
             ) ::
               :ok | {:error, any}
+
+  @callback get_catalog() ::
+              {:ok, any} | {:error, any}
 end

lib/trento/application/integration/checks/adapter/mock_runner.ex

@@ -9,6 +9,10 @@ defmodule Trento.Integration.Checks.MockRunner do
 
   require Logger
 
+  @json_path Path.join(File.cwd!(), "priv/data/catalog.json")
+  @catalog @json_path |> File.read!() |> Jason.decode!()
+  @external_resource @json_path
+
   defstruct [:expected_results]
 
   @type t :: %__MODULE__{
@@ -60,6 +64,11 @@ defmodule Trento.Integration.Checks.MockRunner do
     )
   end
 
+  @impl true
+  def get_catalog do
+    {:ok, @catalog}
+  end
+
   @doc """
   Set the expected results for the next execution.
   """

lib/trento/application/integration/checks/adapter/runner.ex

@@ -0,0 +1,41 @@
+defmodule Trento.Integration.Checks.Runner do
+  @moduledoc """
+  Trento runner integration adapter
+  """
+
+  @behaviour Trento.Integration.Checks.Gen
+
+  @impl true
+  def request_execution(_execution_id, _cluster_id, _hosts, _selected_checks) do
+    :ok
+  end
+
+  @impl true
+  def get_catalog do
+    runner_url = runner_url()
+
+    case HTTPoison.get("#{runner_url}/api/catalog") do
+      {:ok, %HTTPoison.Response{status_code: 200, body: catalog}} ->
+        Jason.decode(catalog)
+
+      {:ok, %HTTPoison.Response{status_code: 204}} ->
+        {:error, :not_ready}
+
+      {:error, %HTTPoison.Error{reason: :econnrefused}} ->
+        {:error, "Connection to the runner component on #{runner_url} was refused."}
+
+      {:error, %HTTPoison.Error{reason: :nxdomain}} ->
+        {:error,
+         "Connection url to the runner component host on #{runner_url} could not be resolved."}
+
+      {:error, %HTTPoison.Error{reason: reason}} ->
+        {:error, reason}
+
+      _ ->
+        {:error, :unexpected_response}
+    end
+  end
+
+  defp runner_url,
+    do: Application.fetch_env!(:trento, __MODULE__)[:runner_url]
+end

lib/trento/application/integration/checks/checks.ex

@@ -1,4 +1,9 @@
 defmodule Trento.Integration.Checks do
+  alias Trento.Integration.Checks.Models.{
+    Catalog,
+    FlatCatalog
+  }
+
   @moduledoc """
   Checks runner service integration
   """
@@ -7,6 +12,52 @@ defmodule Trento.Integration.Checks do
   def request_execution(execution_id, cluster_id, hosts, selected_checks),
     do: adapter().request_execution(execution_id, cluster_id, hosts, selected_checks)
 
+  @spec get_catalog ::
+          {:ok, FlatCatalog.t()} | {:error, any}
+  def get_catalog do
+    case adapter().get_catalog() do
+      {:ok, catalog} ->
+        FlatCatalog.new(%{checks: catalog})
+
+      {:error, :not_ready} ->
+        {:error, "The catalog is still being built. Try again in some moments"}
+
+      {:error, reason} ->
+        {:error, reason}
+
+      _ ->
+        {:error, :unexpected_responses}
+    end
+  end
+
+  @spec get_catalog_by_provider ::
+          {:ok, Catalog.t()} | {:error, any}
+  def get_catalog_by_provider do
+    case get_catalog() do
+      {:ok, content} ->
+        group_by_provider_by_group(content.checks)
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  defp group_by_provider_by_group(flat_catalog) do
+    normalized_catalog =
+      flat_catalog
+      |> Enum.map(&Map.from_struct/1)
+      |> Enum.group_by(&Map.take(&1, [:provider]), &Map.drop(&1, [:provider]))
+      |> Enum.map(fn {key, value} -> Map.put(key, :groups, group_by_group(value)) end)
+
+    Catalog.new(%{providers: normalized_catalog})
+  end
+
+  defp group_by_group(groups) do
+    groups
+    |> Enum.group_by(&Map.take(&1, [:group]), &Map.drop(&1, [:group]))
+    |> Enum.map(fn {key, value} -> Map.put(key, :checks, value) end)
+  end
+
   defp adapter,
     do: Application.fetch_env!(:trento, __MODULE__)[:adapter]
 end

lib/trento/application/integration/checks/models/catalog.ex

@@ -0,0 +1,17 @@
+defmodule Trento.Integration.Checks.Models.Catalog do
+  @moduledoc """
+  Checks catalog
+  """
+
+  @required_fields [
+    :providers
+  ]
+
+  use Trento.Type
+
+  alias Trento.Integration.Checks.Models.Provider
+
+  deftype do
+    embeds_many :providers, Provider
+  end
+end

lib/trento/application/integration/checks/models/check.ex

@@ -0,0 +1,25 @@
+defmodule Trento.Integration.Checks.Models.Check do
+  @moduledoc """
+  Catalog check
+  """
+
+  @required_fields [
+    :id,
+    :name,
+    :description,
+    :remediation,
+    :implementation,
+    :labels
+  ]
+
+  use Trento.Type
+
+  deftype do
+    field :id, :string
+    field :name, :string
+    field :description, :string
+    field :remediation, :string
+    field :implementation, :string
+    field :labels, :string
+  end
+end

lib/trento/application/integration/checks/models/flat_catalog.ex

@@ -0,0 +1,17 @@
+defmodule Trento.Integration.Checks.Models.FlatCatalog do
+  @moduledoc """
+  Flat checks catalog
+  """
+
+  @required_fields [
+    :checks
+  ]
+
+  use Trento.Type
+
+  alias Trento.Integration.Checks.Models.FlatCheck
+
+  deftype do
+    embeds_many :checks, FlatCheck
+  end
+end

lib/trento/application/integration/checks/models/flat_check.ex

@@ -0,0 +1,29 @@
+defmodule Trento.Integration.Checks.Models.FlatCheck do
+  @moduledoc """
+  Flat catalog check
+  """
+
+  @required_fields [
+    :id,
+    :provider,
+    :group,
+    :name,
+    :description,
+    :remediation,
+    :implementation,
+    :labels
+  ]
+
+  use Trento.Type
+
+  deftype do
+    field :id, :string
+    field :provider, Ecto.Enum, values: [:azure, :aws, :gcp, :dev, :unknown]
+    field :group, :string
+    field :name, :string
+    field :description, :string
+    field :remediation, :string
+    field :implementation, :string
+    field :labels, :string
+  end
+end

lib/trento/application/integration/checks/models/group.ex

@@ -0,0 +1,20 @@
+defmodule Trento.Integration.Checks.Models.Group do
+  @moduledoc """
+  Catalog entry by group
+  """
+
+  @required_fields [
+    :group,
+    :checks
+  ]
+
+  use Trento.Type
+
+  alias Trento.Integration.Checks.Models.Check
+
+  deftype do
+    field :group, :string
+
+    embeds_many :checks, Check
+  end
+end

lib/trento/application/integration/checks/models/provider.ex

@@ -0,0 +1,20 @@
+defmodule Trento.Integration.Checks.Models.Provider do
+  @moduledoc """
+  Catalog entry by provider
+  """
+
+  @required_fields [
+    :provider,
+    :groups
+  ]
+
+  use Trento.Type
+
+  alias Trento.Integration.Checks.Models.Group
+
+  deftype do
+    field :provider, Ecto.Enum, values: [:azure, :aws, :gcp, :dev, :unknown]
+
+    embeds_many :groups, Group
+  end
+end

lib/trento_web/controllers/catalog_controller.ex

@@ -0,0 +1,30 @@
+defmodule TrentoWeb.CatalogController do
+  use TrentoWeb, :controller
+
+  alias Trento.Integration.Checks
+
+  @spec checks_catalog(Plug.Conn.t(), map) :: Plug.Conn.t()
+  def checks_catalog(conn, %{"flat" => ""}) do
+    case Checks.get_catalog() do
+      {:ok, catalog} ->
+        json(conn, catalog.checks)
+
+      {:error, reason} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{error: reason})
+    end
+  end
+
+  def checks_catalog(conn, _) do
+    case Checks.get_catalog_by_provider() do
+      {:ok, catalog} ->
+        json(conn, catalog.providers)
+
+      {:error, reason} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{error: reason})
+    end
+  end
+end

lib/trento_web/controllers/cluster_controller.ex

@@ -6,10 +6,6 @@ defmodule TrentoWeb.ClusterController do
     Tags
   }
 
-  @json_path Path.join(File.cwd!(), "priv/data/catalog.json")
-  @catalog @json_path |> File.read!() |> Jason.decode!()
-  @external_resource @json_path
-
   @spec list(Plug.Conn.t(), map) :: Plug.Conn.t()
   def list(conn, _) do
     clusters = Clusters.get_all_clusters()
@@ -98,11 +94,6 @@ defmodule TrentoWeb.ClusterController do
     end
   end
 
-  @spec checks_catalog(Plug.Conn.t(), map) :: Plug.Conn.t()
-  def checks_catalog(conn, _) do
-    json(conn, @catalog)
-  end
-
   @spec select_checks(Plug.Conn.t(), map) :: Plug.Conn.t()
   def select_checks(conn, %{"cluster_id" => cluster_id, "checks" => checks}) do
     case Clusters.select_checks(cluster_id, checks) do

lib/trento_web/router.ex

@@ -61,7 +61,7 @@ defmodule TrentoWeb.Router do
          ClusterController,
          :request_checks_execution
 
-    get "/checks/catalog", ClusterController, :checks_catalog
+    get "/checks/catalog", CatalogController, :checks_catalog
   end
 
   # Other scopes may use custom stacks.