[#48] Renaming Repos.Warmer to CacheWarmer and adding basic Backends.…

…Behaviour implemented at Backends.Github for get_org/2, get_members/1, get_repos/1, get_topics/2 and headers/0 (WIP).

cooperatives.yml

@@ -1,70 +1,82 @@
 fiqus:
+  source: github
   name: Fiqus
   url: https://www.fiqus.coop
   description: ""
   location: CABA/Villa La Angostura
 
 gcoop-libre:
+  source: github
   name: gcoop
   url: https://www.gcoop.coop
   description: ""
   location: CABA
 
 cambalab:
+  source: github
   name: Cambá
   url: https://www.camba.coop
   description: ""
   location: CABA
 
 devecoop:
+  source: github
   name: Devecoop
   url: http://www.devecoop.com
   description: ""
   location: CABA
 
 Unixono:
+  source: github
   name: Uníxono
   url: http://www.unixono.com
   description: ""
   location: Bahía Blanca
 
 geneos:
+  source: github
   name: Geneos
   url: http://www.geneos.com.ar
   description: ""
   location: Tandil
 
 bitson:
+  source: github
   name: Bitson
   url: https://bitson.group/
   description: ""
   location: CABA
 
 nayracoop:
+  source: github
   name: Nayra
   url: https://nayra.coop/
   description: ""
   location: Provincia de Buenos Aires
 
 hyphacoop:
+  source: github
   name: Hypha
   url: https://hypha.coop
   description: "We cultivate collective growth and meaningful livelihoods through learning and building technologies together."
   location: Toronto, Canada
 
 agaric:
+  source: github
   name: Agaric
   url: https://agaric.coop
   description: "We build websites and online tools that respect your freedom, and we provide training and consultation to meet your goals."
   location: Boston, USA; Minneapolis, USA; Denver, USA; Managua, Nicaragua; Puebla, Mexico
 
 seattledeveloperscoop:
+  source: github
   name: Seattle Developer's Cooperative
   url: https://www.seattledevelopers.coop
   description: "A worker-owned web developer cooperative based in Seattle"
   location: Seattle, USA
 
 camplight:
+  source: github
   name: Camplight
   url: https://camplight.net
   description: "A digital cooperative that creates experiences for the web, mobile and beyond."

lib/coophub/application.ex

@@ -54,7 +54,7 @@ defmodule Coophub.Application do
   defp main_cache_opts(_) do
     [
       warmers: [
-        warmer(module: Coophub.Repos.Warmer)
+        warmer(module: Coophub.CacheWarmer)
       ],
       expiration: expiration(default: :timer.minutes(@cache_interval))
     ]

lib/coophub/backends/behaviour.ex

@@ -0,0 +1,29 @@
+defmodule Coophub.Backends.Behaviour do
+  alias Coophub.Schemas.{Organization, Repository}
+
+  require Logger
+
+  @type headers :: [{String.t(), String.t()}]
+
+  @callback get_org(String.t(), Map.t()) :: Organization.t() | :error
+  @callback get_members(Organization.t()) :: [map]
+
+  @callback get_repos(Organization.t()) :: [Repository.t()]
+  @callback get_topics(Organization.t(), Repository.t()) :: [String.t()]
+
+  @callback headers() :: headers
+
+  @spec call_api_get(String.t(), headers()) :: {:ok, map | [map]} | {:error, any}
+  def call_api_get(url, headers) do
+    case HTTPoison.get(url, headers) do
+      {:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
+        {:ok, Jason.decode!(body)}
+
+      {:ok, %HTTPoison.Response{status_code: 404}} ->
+        {:error, "Not found: #{url}"}
+
+      {:error, %HTTPoison.Error{reason: reason}} ->
+        {:error, reason}
+    end
+  end
+end

lib/coophub/backends/github.ex

@@ -0,0 +1,107 @@
+defmodule Coophub.Backends.Github do
+  alias Coophub.Repos
+  alias Coophub.Backends
+  alias Coophub.Schemas.{Organization, Repository}
+
+  require Logger
+
+  @behaviour Backends.Behaviour
+
+  @repos_max_fetch Application.get_env(:coophub, :fetch_max_repos)
+
+  ########
+  ## BEHAVIOUR IMPLEMENTATION
+  ########
+
+  @impl Backends.Behaviour
+  @spec get_org(String.t(), Map.t()) :: Organization.t() | :error
+  def get_org(key, _yml_data) do
+    Logger.info("Fetching '#{key}' organization from github..", ansi_color: :yellow)
+
+    case call_api_get("orgs/#{key}") do
+      {:ok, org} ->
+        Logger.info("Fetched '#{key}' organization!", ansi_color: :yellow)
+        Repos.to_struct(Organization, org)
+
+      {:error, reason} ->
+        Logger.error("Error getting '#{key}' organization from github: #{inspect(reason)}")
+        :error
+    end
+  end
+
+  @impl Backends.Behaviour
+  @spec get_members(Organization.t()) :: [map]
+  # @TODO Isn't fetching all the org members (ie: just 5 for fiqus)
+  def get_members(%Organization{key: key}) do
+    Logger.info("Fetching '#{key}' members from github..", ansi_color: :yellow)
+
+    case call_api_get("orgs/#{key}/members") do
+      {:ok, members} ->
+        Logger.info("Fetched #{length(members)} '#{key}' members!", ansi_color: :yellow)
+        members
+
+      {:error, reason} ->
+        Logger.error("Error getting '#{key}' members from github: #{inspect(reason)}")
+        []
+    end
+  end
+
+  @impl Backends.Behaviour
+  @spec get_repos(Organization.t()) :: [Repository.t()]
+  def get_repos(%Organization{key: key}) do
+    Logger.info("Fetching '#{key}' repos from github..", ansi_color: :yellow)
+    path = "orgs/#{key}/repos?per_page=#{@repos_max_fetch}&type=public&sort=pushed&direction=desc"
+
+    case call_api_get(path) do
+      {:ok, repos} ->
+        Logger.info("Fetched #{length(repos)} '#{key}' repos!", ansi_color: :yellow)
+        Repos.to_struct(Repository, repos)
+
+      {:error, reason} ->
+        Logger.error("Error getting '#{key}' repos from github: #{inspect(reason)}")
+        []
+    end
+  end
+
+  @impl Backends.Behaviour
+  @spec get_topics(Organization.t(), Repository.t()) :: [String.t()]
+  def get_topics(%Organization{key: key}, %Repository{name: name}) do
+    Logger.info("Fetching '#{key}/#{name}' topics from github..", ansi_color: :yellow)
+
+    case call_api_get("repos/#{key}/#{name}/topics") do
+      {:ok, data} ->
+        topics = Map.get(data, "names", [])
+        Logger.info("Fetched #{length(topics)} '#{key}/#{name}' topics!", ansi_color: :yellow)
+        topics
+
+      {:error, reason} ->
+        Logger.error("Error getting '#{key}/#{name}' topics from github: #{inspect(reason)}")
+        []
+    end
+  end
+
+  @impl Backends.Behaviour
+  @spec headers() :: Backends.Behaviour.headers()
+  def headers() do
+    headers = [
+      {"Accept", "application/vnd.github.mercy-preview+json"}
+    ]
+
+    token = System.get_env("GITHUB_OAUTH_TOKEN")
+
+    if is_binary(token) do
+      [{"Authorization", "token #{token}"} | headers]
+    else
+      headers
+    end
+  end
+
+  ########
+  ## INTERNALS
+  ########
+
+  defp call_api_get(path) do
+    url = "https://api.github.com/#{path}"
+    Backends.Behaviour.call_api_get(url, headers())
+  end
+end

lib/coophub/repos_warmer.ex → lib/coophub/cache_warmer.ex

@@ -1,12 +1,12 @@
-defmodule Coophub.Repos.Warmer do
+defmodule Coophub.CacheWarmer do
   use Cachex.Warmer
 
   alias Coophub.Repos
+  alias Coophub.Backends
   alias Coophub.Schemas.{Organization, Repository}
 
   require Logger
 
-  @repos_max_fetch Application.get_env(:coophub, :fetch_max_repos)
   @repos_cache_name Application.get_env(:coophub, :main_cache_name)
   @repos_cache_interval Application.get_env(:coophub, :cache_interval)
   @repos_cache_dump_file Application.get_env(:coophub, :main_cache_dump_file)
@@ -42,10 +42,10 @@ defmodule Coophub.Repos.Warmer do
 
     repos =
       read_yml()
-      |> Enum.reduce([], fn {name, yml_data}, acc ->
-        case get_org(name, yml_data) do
+      |> Enum.reduce([], fn {key, yml_data}, acc ->
+        case get_org(key, yml_data) do
           :error -> acc
-          org -> [get_repos(name, org) | acc]
+          org -> [get_repos(org) | acc]
         end
       end)
 
@@ -116,75 +116,54 @@ defmodule Coophub.Repos.Warmer do
   ## Github API calls and handling functions
   ##
 
-  defp get_repos(org_name, org) do
-    org_repos =
-      case call_api_get(
-             "orgs/#{org_name}/repos?per_page=#{@repos_max_fetch}&type=public&sort=pushed&direction=desc"
-           ) do
-        {:ok, body} ->
-          repos =
-            Repos.to_struct(Repository, body)
-            |> put_key(org_name)
-            |> put_popularities()
-            |> put_topics(org_name)
-            |> put_languages(org_name)
-            |> put_repo_data(org_name)
-
-          Logger.info("Fetched #{length(repos)} repos for #{org_name}", ansi_color: :yellow)
-          repos
-
-        {:error, reason} ->
-          Logger.error(
-            "Error getting the repos for '#{org_name}' from github: #{inspect(reason)}"
-          )
+  defp get_org(key, %{"source" => source} = yml_data) do
+    case call_backend(source, :get_org, [key, yml_data]) do
+      %Organization{} = org ->
+        org
+        |> Map.put(:key, key)
+        |> Map.put(:yml_data, yml_data)
+        |> get_members()
 
-          []
-      end
+      _ ->
+        :error
+    end
+  end
+
+  defp get_members(%Organization{yml_data: %{"source" => source}} = org) do
+    members = call_backend(source, :get_members, [org])
+    Map.put(org, :members, members)
+  end
+
+  defp get_repos(%Organization{key: key, yml_data: %{"source" => source}} = org) do
+    repos =
+      call_backend(source, :get_repos, [org])
+      |> put_key(key)
+      |> put_popularities()
+      |> put_topics(org)
+      |> put_languages(key)
+      |> put_repo_data(key)
 
     org =
       org
-      |> Map.put(:repos, org_repos)
-      |> Map.put(:repo_count, Enum.count(org_repos))
+      |> Map.put(:repos, repos)
+      |> Map.put(:repo_count, Enum.count(repos))
       |> put_org_languages_stats()
       |> put_org_popularity()
       |> put_org_last_activity()
 
-    {org_name, org}
-  end
-
-  defp get_members(%Organization{:key => key} = org) do
-    members =
-      case call_api_get("orgs/#{key}/members") do
-        {:ok, body} ->
-          body
-
-        {:error, reason} ->
-          Logger.error("Error getting members for '#{key}' from github: #{inspect(reason)}")
-          []
-      end
-
-    Map.put(org, :members, members)
+    {key, org}
   end
 
-  defp get_org(name, yml_data) do
-    case call_api_get("orgs/#{name}") do
-      {:ok, org} ->
-        msg =
-          "Fetched '#{name}' organization! Getting members and repos (max=#{@repos_max_fetch}).."
-
-        Logger.info(msg, ansi_color: :yellow)
-
-        Repos.to_struct(Organization, org)
-        |> Map.put(:key, name)
-        |> Map.put(:yml_data, yml_data)
-        |> get_members()
-
-      {:error, reason} ->
-        Logger.error("Error getting the organization '#{name}' from github: #{inspect(reason)}")
-        :error
+  defp call_backend(source, func, params) do
+    case get_backend(source) do
+      :unknown -> {:error, "Unknown backend source: #{source}"}
+      module -> apply(module, func, params)
     end
   end
 
+  defp get_backend("github"), do: Backends.Github
+  defp get_backend(_), do: :unknown
+
   defp put_key(repos, key) do
     Enum.map(repos, &Map.put(&1, :key, key))
   end
@@ -193,26 +172,10 @@ defmodule Coophub.Repos.Warmer do
     Enum.map(repos, &Map.put(&1, :popularity, Repos.get_repo_popularity(&1)))
   end
 
-  defp put_topics(repos, org_name) do
+  defp put_topics(repos, %Organization{yml_data: %{"source" => source}} = org) do
     Enum.map(repos, fn repo ->
-      repo_name = repo.name
-
-      topics =
-        case call_api_get("repos/#{org_name}/#{repo_name}/topics") do
-          {:ok, body} ->
-            body
-
-          {:error, reason} ->
-            Logger.error(
-              "Error getting the topics for '#{org_name}/#{repo_name}' from github: #{
-                inspect(reason)
-              }"
-            )
-
-            %{}
-        end
-
-      Map.put(repo, :topics, Map.get(topics, "names", []))
+      topics = call_backend(source, :get_topics, [org, repo])
+      Map.put(repo, :topics, topics)
     end)
   end