From f46372122669e6d58f409f873dd09939164e7186 Mon Sep 17 00:00:00 2001
From: Diego Calero <dcalero@fiqus.coop>
Date: Tue, 25 Feb 2020 21:51:19 -0300
Subject: [PATCH] [#48] Moving all the backend calls to Coophub.Backends and
 changing the behaviour to just have the minimal needed specific
 implementation.

---
 lib/coophub/backends/backends.ex  | 143 +++++++++++++++++++++++--
 lib/coophub/backends/behaviour.ex |  39 ++++---
 lib/coophub/backends/github.ex    | 172 ++++++++++++------------------
 lib/coophub/cache_warmer.ex       |  10 +-
 4 files changed, 228 insertions(+), 136 deletions(-)

diff --git a/lib/coophub/backends/backends.ex b/lib/coophub/backends/backends.ex
index af2a49d..e75113c 100644
--- a/lib/coophub/backends/backends.ex
+++ b/lib/coophub/backends/backends.ex
@@ -1,20 +1,147 @@
 defmodule Coophub.Backends do
   alias Coophub.Backends
+  alias Coophub.Schemas.{Organization, Repository}
 
+  require Logger
+
+  @type url :: String.t()
   @type headers :: [{String.t(), String.t()}]
-  @type languages :: [%{String.t() => integer()}]
+  @type request :: {String.t(), url, headers}
+
+  @type org :: Organization.t()
+  @type repo :: Repository.t()
+  @type langs :: %{String.t() => integer()}
+  @type topics :: [String.t()]
+
+  ## Backends implementations
+  defp get_backend!("github"), do: Backends.Github
+  defp get_backend!(source), do: raise("Unknown backend source: #{source}")
+
+  ########
+  ## CALLS TO BACKENDS RESOURCES
+  ########
+
+  @spec get_org(String.t(), String.t(), map) :: org | :error
+  def get_org(source, key, yml_data) do
+    backend = get_backend!(source)
+    bname = backend.name()
+    {name, url, headers} = backend.request_org(key, yml_data)
+    Logger.info("Fetching '#{name}' organization from #{bname}..", ansi_color: :yellow)
+
+    case call_api_get(url, headers) do
+      {:ok, data, ms} ->
+        Logger.info("Fetched '#{name}' organization! (#{ms}ms)", ansi_color: :green)
+        backend.parse_org(data)
+
+      {:error, reason} ->
+        Logger.error("Error getting '#{name}' organization from #{bname}: #{inspect(reason)}")
+        :error
+    end
+  end
+
+  @spec get_members(String.t(), org) :: [map]
+  def get_members(source, org) do
+    backend = get_backend!(source)
+    bname = backend.name()
+    {name, url, headers} = backend.request_members(org)
+    Logger.info("Fetching '#{name}' members from #{bname}..", ansi_color: :yellow)
+
+    case call_api_get(url, headers) do
+      {:ok, members, ms} ->
+        Logger.info("Fetched #{length(members)} '#{name}' members! (#{ms}ms)", ansi_color: :green)
+        backend.parse_members(members)
+
+      {:error, reason} ->
+        Logger.error("Error getting '#{name}' members from #{bname}: #{inspect(reason)}")
+        []
+    end
+  end
+
+  @spec get_repos(String.t(), org) :: [repo]
+  def get_repos(source, org) do
+    backend = get_backend!(source)
+    bname = backend.name()
+    limit = Application.get_env(:coophub, :fetch_max_repos)
+    {name, url, headers} = backend.request_repos(org, limit)
+    Logger.info("Fetching '#{name}' repos from #{bname} (max=#{limit})..", ansi_color: :yellow)
+
+    case call_api_get(url, headers) do
+      {:ok, repos, ms} ->
+        Logger.info("Fetched #{length(repos)} '#{name}' repos! (#{ms}ms)", ansi_color: :green)
+
+        Enum.map(repos, fn repo_data ->
+          get_repo(backend, org, repo_data)
+        end)
 
-  ## Backends definitions
-  defp get_backend("github"), do: Backends.Github
-  defp get_backend(source), do: raise("Unknown backend source: #{source}")
+      {:error, reason} ->
+        Logger.error("Error getting '#{name}' repos from #{bname}: #{inspect(reason)}")
+        []
+    end
+  end
+
+  @spec get_repo(module, org, map) :: [repo]
+  def get_repo(backend, org, repo_data) do
+    bname = backend.name()
+    {name, url, headers} = backend.request_repo(org, repo_data)
+    Logger.info("Fetching '#{name}' repo data from #{bname}..", ansi_color: :cyan)
+
+    case call_api_get(url, headers) do
+      {:ok, data, ms} ->
+        Logger.info("Fetched '#{name}' repo data! (#{ms}ms)", ansi_color: :green)
+        backend.parse_repo(data)
+
+      {:error, reason} ->
+        Logger.error("Error getting '#{name}' repo data from #{bname}: #{inspect(reason)}")
+        ## Fallback to repo_data we've fetched before
+        backend.parse_repo(repo_data)
+    end
+  end
+
+  @spec get_topics(String.t(), org, repo) :: topics
+  def get_topics(source, org, repo) do
+    backend = get_backend!(source)
+    bname = backend.name()
+    {name, url, headers} = backend.request_topics(org, repo)
+    Logger.info("Fetching '#{name}' topics from #{bname}..", ansi_color: :cyan)
+
+    case call_api_get(url, headers) do
+      {:ok, data, ms} ->
+        topics = backend.parse_topics(data)
 
-  @spec call_backend!(String.t(), atom, list) :: Backends.Behaviour.results()
-  def call_backend!(source, func, params) do
-    get_backend(source) |> apply(func, params)
+        Logger.info("Fetched #{length(topics)} '#{name}' topics! (#{ms}ms)",
+          ansi_color: :green
+        )
+
+        topics
+
+      {:error, reason} ->
+        Logger.error("Error getting '#{name}' topics from #{bname}: #{inspect(reason)}")
+        []
+    end
+  end
+
+  @spec get_languages(String.t(), org, repo) :: langs
+  def get_languages(source, org, repo) do
+    backend = get_backend!(source)
+    bname = backend.name()
+    {name, url, headers} = backend.request_languages(org, repo)
+    Logger.info("Fetching '#{name}' languages from #{bname}..", ansi_color: :cyan)
+
+    case call_api_get(url, headers) do
+      {:ok, data, ms} ->
+        langs = backend.parse_languages(data)
+        count = langs |> Map.keys() |> length()
+        Logger.info("Fetched #{count} '#{name}' languages! (#{ms}ms)", ansi_color: :green)
+        langs
+
+      {:error, reason} ->
+        Logger.error("Error getting '#{name}' languages from #{bname}: #{inspect(reason)}")
+        []
+    end
   end
 
   @spec call_api_get(String.t(), headers) :: {:ok, map | [map], integer} | {:error, any}
-  def call_api_get(url, headers) do
+  defp call_api_get(url, headers) do
     start_ms = take_time()
 
     case HTTPoison.get(url, headers) do
diff --git a/lib/coophub/backends/behaviour.ex b/lib/coophub/backends/behaviour.ex
index 13c6b1c..a47c495 100644
--- a/lib/coophub/backends/behaviour.ex
+++ b/lib/coophub/backends/behaviour.ex
@@ -1,22 +1,27 @@
 defmodule Coophub.Backends.Behaviour do
   alias Coophub.Backends
-  alias Coophub.Schemas.{Organization, Repository}
 
-  @type org :: Organization.t()
-  @type repo :: Repository.t()
-  @type langs :: Backends.languages()
-  @type results ::
-          :error
-          | org
-          | repo
-          | [repo]
-          | [map]
-          | [String.t()]
-          | langs
+  @type request :: Backends.request()
+  @type org :: Backends.org()
+  @type repo :: Backends.repo()
+  @type langs :: Backends.langs()
+  @type topics :: Backends.topics()
 
-  @callback get_org(String.t(), map) :: org | :error
-  @callback get_members(org) :: [map]
-  @callback get_repos(org) :: [repo]
-  @callback get_topics(org, repo) :: [String.t()]
-  @callback get_languages(org, repo) :: langs
+  @callback name() :: String.t()
+
+  @callback request_org(String.t(), map) :: request
+  @callback parse_org(map) :: org
+
+  @callback request_members(org) :: request
+  @callback parse_members([map]) :: [map]
+
+  @callback request_repos(org, integer) :: request
+  @callback request_repo(org, map) :: request
+  @callback parse_repo(map) :: repo
+
+  @callback request_topics(org, repo) :: request
+  @callback parse_topics(any) :: topics
+
+  @callback request_languages(org, repo) :: request
+  @callback parse_languages(any) :: langs
 end
diff --git a/lib/coophub/backends/github.ex b/lib/coophub/backends/github.ex
index 3f11464..6061203 100644
--- a/lib/coophub/backends/github.ex
+++ b/lib/coophub/backends/github.ex
@@ -1,143 +1,106 @@
 defmodule Coophub.Backends.Github do
   alias Coophub.Repos
-  alias Coophub.Backends.Behaviour, as: BackendBehaviour
+  alias Coophub.Backends
   alias Coophub.Schemas.{Organization, Repository}
 
   require Logger
 
-  @type org :: BackendBehaviour.org()
-  @type repo :: BackendBehaviour.repo()
-  @type langs :: BackendBehaviour.langs()
+  @type request :: Backends.request()
+  @type org :: Backends.org()
+  @type repo :: Backends.repo()
+  @type langs :: Backends.langs()
+  @type topics :: Backends.topics()
 
-  @behaviour BackendBehaviour
+  @behaviour Backends.Behaviour
 
   ########
   ## BEHAVIOUR IMPLEMENTATION
   ########
 
-  @impl BackendBehaviour
-  @spec get_org(String.t(), map) :: org | :error
-  def get_org(key, _yml_data) do
-    Logger.info("Fetching '#{key}' organization from github..", ansi_color: :yellow)
+  @impl Backends.Behaviour
+  @spec name() :: String.t()
+  def name(), do: "github"
 
-    case call_api_get("orgs/#{key}") do
-      {:ok, org, ms} ->
-        Logger.info("Fetched '#{key}' organization! (#{ms}ms)", ansi_color: :green)
-        Repos.to_struct(Organization, org)
+  @impl Backends.Behaviour
+  @spec request_org(String.t(), map) :: request
+  def request_org(key, _yml_data) do
+    {key, full_url("orgs/#{key}"), headers()}
+  end
 
-      {:error, reason} ->
-        Logger.error("Error getting '#{key}' organization from github: #{inspect(reason)}")
-        :error
-    end
+  @impl Backends.Behaviour
+  @spec parse_org(map) :: org
+  def parse_org(data) do
+    Repos.to_struct(Organization, data)
   end
 
-  @impl BackendBehaviour
-  @spec get_members(org) :: [map]
+  @impl Backends.Behaviour
+  @spec request_members(org) :: request
   # @TODO Isn't fetching all the org members (ie: just 5 for fiqus)
-  def get_members(%Organization{key: key}) do
-    Logger.info("Fetching '#{key}' members from github..", ansi_color: :yellow)
-
-    case call_api_get("orgs/#{key}/members") do
-      {:ok, members, ms} ->
-        Logger.info("Fetched #{length(members)} '#{key}' members! (#{ms}ms)", ansi_color: :green)
-        members
+  def request_members(%Organization{key: key}) do
+    {key, full_url("orgs/#{key}/members"), headers()}
+  end
 
-      {:error, reason} ->
-        Logger.error("Error getting '#{key}' members from github: #{inspect(reason)}")
-        []
-    end
+  @impl Backends.Behaviour
+  @spec parse_members([map]) :: [map]
+  def parse_members(members) do
+    members
   end
 
-  @impl BackendBehaviour
-  @spec get_repos(org) :: [repo]
-  def get_repos(%Organization{key: key} = org) do
-    limit = Application.get_env(:coophub, :fetch_max_repos)
-    Logger.info("Fetching '#{key}' repos from github (max=#{limit})..", ansi_color: :yellow)
+  @impl Backends.Behaviour
+  @spec request_repos(org, integer) :: request
+  def request_repos(%Organization{key: key}, limit) do
     path = "orgs/#{key}/repos?per_page=#{limit}&type=public&sort=pushed&direction=desc"
-
-    case call_api_get(path) do
-      {:ok, repos, ms} ->
-        Logger.info("Fetched #{length(repos)} '#{key}' repos! (#{ms}ms)", ansi_color: :green)
-
-        Enum.map(repos, fn repo_data ->
-          get_repo(org, repo_data)
-        end)
-
-      {:error, reason} ->
-        Logger.error("Error getting '#{key}' repos from github: #{inspect(reason)}")
-        []
-    end
+    {key, full_url(path), headers()}
   end
 
-  @impl BackendBehaviour
-  @spec get_topics(org, repo) :: [String.t()]
-  def get_topics(%Organization{key: key}, %Repository{name: name}) do
-    Logger.info("Fetching '#{key}/#{name}' topics from github..", ansi_color: :cyan)
-
-    case call_api_get("repos/#{key}/#{name}/topics") do
-      {:ok, data, ms} ->
-        topics = Map.get(data, "names", [])
+  @impl Backends.Behaviour
+  @spec request_repo(org, map) :: request
+  def request_repo(%Organization{key: key}, %{"name" => name}) do
+    {"#{key}/#{name}", full_url("repos/#{key}/#{name}"), headers()}
+  end
 
-        Logger.info("Fetched #{length(topics)} '#{key}/#{name}' topics! (#{ms}ms)",
-          ansi_color: :green
-        )
+  @impl Backends.Behaviour
+  @spec parse_repo(map) :: repo
+  def parse_repo(data) do
+    repo = Repos.to_struct(Repository, data)
 
-        topics
+    case repo.parent do
+      %{"full_name" => name, "html_url" => url} ->
+        Map.put(repo, :parent, %{name: name, url: url})
 
-      {:error, reason} ->
-        Logger.error("Error getting '#{key}/#{name}' topics from github: #{inspect(reason)}")
-        []
+      _ ->
+        repo
     end
   end
 
-  @impl BackendBehaviour
-  @spec get_languages(org, repo) :: langs
-  def get_languages(%Organization{key: key}, %Repository{name: name}) do
-    Logger.info("Fetching '#{key}/#{name}' languages from github..", ansi_color: :cyan)
+  @impl Backends.Behaviour
+  @spec request_topics(org, repo) :: request
+  def request_topics(%Organization{key: key}, %Repository{name: name}) do
+    {"#{key}/#{name}", full_url("repos/#{key}/#{name}/topics"), headers()}
+  end
 
-    case call_api_get("repos/#{key}/#{name}/languages") do
-      {:ok, languages, ms} ->
-        Logger.info(
-          "Fetched #{length(Map.keys(languages))} '#{key}/#{name}' languages! (#{ms}ms)",
-          ansi_color: :green
-        )
+  @impl Backends.Behaviour
+  @spec parse_topics(map) :: topics
+  def parse_topics(data) do
+    Map.get(data, "names", [])
+  end
 
-        languages
+  @impl Backends.Behaviour
+  @spec request_languages(org, repo) :: request
+  def request_languages(%Organization{key: key}, %Repository{name: name}) do
+    {"#{key}/#{name}", full_url("repos/#{key}/#{name}/languages"), headers()}
+  end
 
-      {:error, reason} ->
-        Logger.error("Error getting '#{key}/#{name}' languages from github: #{inspect(reason)}")
-        []
-    end
+  @impl Backends.Behaviour
+  @spec parse_languages(langs) :: langs
+  def parse_languages(languages) do
+    languages
   end
 
   ########
   ## INTERNALS
   ########
 
-  defp get_repo(%Organization{key: key}, %{"name" => name} = repo_data) do
-    Logger.info("Fetching '#{key}/#{name}' repo data from github..", ansi_color: :cyan)
-
-    case call_api_get("repos/#{key}/#{name}") do
-      {:ok, data, ms} ->
-        Logger.info("Fetched '#{key}/#{name}' repo data! (#{ms}ms)", ansi_color: :green)
-
-        repo = Repos.to_struct(Repository, data)
-
-        case repo.parent do
-          %{"full_name" => name, "html_url" => url} ->
-            Map.put(repo, :parent, %{name: name, url: url})
-
-          _ ->
-            repo
-        end
-
-      {:error, reason} ->
-        Logger.error("Error getting '#{key}/#{name}' repo data from github: #{inspect(reason)}")
-        ## Fallback to repo_data we've fetched before
-        Repos.to_struct(Repository, repo_data)
-    end
-  end
-
   defp headers() do
     headers = [{"Accept", "application/vnd.github.mercy-preview+json"}]
     token = System.get_env("GITHUB_OAUTH_TOKEN")
@@ -147,8 +110,5 @@ defmodule Coophub.Backends.Github do
       else: headers
   end
 
-  defp call_api_get(path) do
-    url = "https://api.github.com/#{path}"
-    Coophub.Backends.call_api_get(url, headers())
-  end
+  defp full_url(path), do: "https://api.github.com/#{path}"
 end
diff --git a/lib/coophub/cache_warmer.ex b/lib/coophub/cache_warmer.ex
index 937711a..14e1d3b 100644
--- a/lib/coophub/cache_warmer.ex
+++ b/lib/coophub/cache_warmer.ex
@@ -117,7 +117,7 @@ defmodule Coophub.CacheWarmer do
   ##
 
   defp get_org(key, %{"source" => source} = yml_data) do
-    case Backends.call_backend!(source, :get_org, [key, yml_data]) do
+    case Backends.get_org(source, key, yml_data) do
       %Organization{} = org ->
         org
         |> Map.put(:key, key)
@@ -130,13 +130,13 @@ defmodule Coophub.CacheWarmer do
   end
 
   defp get_members(%Organization{yml_data: %{"source" => source}} = org) do
-    members = Backends.call_backend!(source, :get_members, [org])
+    members = Backends.get_members(source, org)
     Map.put(org, :members, members)
   end
 
   defp get_repos(%Organization{key: key, yml_data: %{"source" => source}} = org) do
     repos =
-      Backends.call_backend!(source, :get_repos, [org])
+      Backends.get_repos(source, org)
       |> put_key(key)
       |> put_popularities()
       |> put_topics(org)
@@ -164,14 +164,14 @@ defmodule Coophub.CacheWarmer do
 
   defp put_topics(repos, %Organization{yml_data: %{"source" => source}} = org) do
     Enum.map(repos, fn repo ->
-      topics = Backends.call_backend!(source, :get_topics, [org, repo])
+      topics = Backends.get_topics(source, org, repo)
       Map.put(repo, :topics, topics)
     end)
   end
 
   defp put_languages(repos, %Organization{yml_data: %{"source" => source}} = org) do
     Enum.map(repos, fn repo ->
-      languages = Backends.call_backend!(source, :get_languages, [org, repo])
+      languages = Backends.get_languages(source, org, repo)
       stats = Repos.get_percentages_by_language(languages)
       Map.put(repo, :languages, stats)
     end)