feat: use Stow for writing metadata and scrobbles + file read refac…

…toring

lib/lastfm_archive/archive/derived_archive.ex

@@ -10,6 +10,7 @@ defmodule LastfmArchive.Archive.DerivedArchive do
 
   import LastfmArchive.Utils.Archive, only: [derived_archive_dir: 1, metadata_filepath: 2, user_dir: 2]
   import LastfmArchive.Utils.DateTime, only: [year_range: 1]
+  import LastfmArchive.Utils.File, only: [source: 2]
 
   require Explorer.DataFrame
 
@@ -21,15 +22,14 @@ defmodule LastfmArchive.Archive.DerivedArchive do
   end
 
   @impl true
-  def describe(user, options) do
+  def describe(user, opts) do
     {:ok, metadata} = super(user, [])
 
-    # Fix this: use Utils.File read
-    metadata_filepath(user, options)
-    |> @file_io.read()
+    metadata_filepath(user, opts)
+    |> source(opts)
     |> case do
       {:ok, data} -> update_derived_archive_metadata(data |> Jason.decode!(keys: :atoms) |> Metadata.new(), metadata)
-      {:error, :enoent} -> metadata |> create_derived_archive_metadata(options)
+      {:error, :enoent} -> metadata |> create_derived_archive_metadata(opts)
     end
   end
 

lib/lastfm_archive/behaviour/archive.ex

@@ -52,7 +52,7 @@ defmodule LastfmArchive.Behaviour.Archive do
 
       import LastfmArchive.Behaviour.Archive
       import LastfmArchive.Utils.Archive, only: [metadata_filepath: 2, user_dir: 2]
-      import LastfmArchive.Utils.File, only: [write: 2]
+      import LastfmArchive.Utils.File, only: [write: 2, source: 2]
 
       alias LastfmArchive.Archive.Metadata
 
@@ -61,12 +61,16 @@ defmodule LastfmArchive.Behaviour.Archive do
       @impl true
       def update_metadata(%Metadata{creator: user} = metadata, options)
           when user != nil and is_binary(user) do
-        write(metadata, options)
+        case Keyword.get(options, :reset, false) do
+          false -> metadata
+          true -> %{metadata | created: DateTime.utc_now(), date: nil, modified: nil}
+        end
+        |> write(options)
       end
 
       @impl true
       def describe(user, options \\ []) do
-        case @file_io.read(metadata_filepath(user, options)) do
+        case source(metadata_filepath(user, options), options) do
           {:ok, metadata} -> {:ok, Jason.decode!(metadata, keys: :atoms) |> Metadata.new()}
           {:error, :enoent} -> {:ok, Metadata.new(user, options)}
         end

lib/lastfm_archive/configs.ex

@@ -8,4 +8,7 @@ defmodule LastfmArchive.Configs do
   def lastfm_api_key, do: System.get_env("LB_LFM_API_KEY") || lastfm_api()[:key] || ""
 
   def lastfm_user, do: System.get_env("LB_LFM_USER") || Application.get_env(:lastfm_archive, :user)
+
+  def data_dir, do: Application.get_env(:lastfm_archive, :data_dir, "./lastfm_data/")
+  def file_io, do: Application.get_env(:lastfm_archive, :file_io, Elixir.File)
 end

lib/lastfm_archive/lastfm_client/impl.ex

@@ -36,7 +36,7 @@ defmodule LastfmArchive.LastfmClient.Impl do
 
   # TODO: need Stow to have API to better return plug private (error) status
   defp read_body(%Conn{resp_body: body}) when not is_nil(body), do: body |> Jason.decode!()
-  defp read_body(%{private: %{stow: %{source: %{status: status}}}}), do: status
+  defp read_body(%Conn{private: %{stow: %{source: %{status: status}}}}), do: status
 
   @doc """
   Returns the scrobbles of a user for a given time range.

lib/lastfm_archive/utils/archive.ex

@@ -2,22 +2,25 @@ defmodule LastfmArchive.Utils.Archive do
   @moduledoc false
 
   require Logger
+  import LastfmArchive.Configs, only: [data_dir: 0]
 
   @metadata_dir ".metadata"
-  @data_dir Application.compile_env(:lastfm_archive, :data_dir, "./lastfm_data/")
   @file_io Application.compile_env(:lastfm_archive, :file_io, Elixir.File)
 
-  def metadata_filepath(user, opts \\ []) do
-    Path.join([
-      Keyword.get(opts, :data_dir) || data_dir(),
-      user,
-      "#{@metadata_dir}/#{Keyword.get(opts, :facet, "scrobbles")}/#{Keyword.get(opts, :format, "json")}_archive"
-    ])
+  def metadata_filepath(user, opts) when is_binary(user) do
+    [user | metadata_filepath(opts)] |> Path.join()
   end
 
-  def check_existing_archive(user, options) do
-    case metadata_filepath(user, options) |> @file_io.exists?() do
-      true -> {:ok, metadata_filepath(user, options)}
+  def metadata_filepath(opts) when is_list(opts) do
+    [@metadata_dir, "#{Keyword.get(opts, :facet, :scrobbles)}", "#{Keyword.get(opts, :format, "json")}_archive"]
+  end
+
+  def check_existing_archive(user, opts) do
+    [data_dir(opts), metadata_filepath(user, opts)]
+    |> Path.join()
+    |> @file_io.exists?()
+    |> case do
+      true -> {:ok, metadata_filepath(user, opts)}
       false -> {:error, :archive_not_found}
     end
   end
@@ -41,6 +44,5 @@ defmodule LastfmArchive.Utils.Archive do
   def user_dir(user), do: Path.join([data_dir(), user])
   def user_dir(user, opts), do: Path.join([data_dir(opts), user])
 
-  defp data_dir(), do: @data_dir
-  defp data_dir(opts), do: Keyword.get(opts, :data_dir, data_dir())
+  def data_dir(opts), do: Keyword.get(opts, :data_dir, data_dir())
 end

lib/lastfm_archive/utils/file.ex

@@ -3,13 +3,17 @@ defmodule LastfmArchive.Utils.File do
 
   alias Explorer.DataFrame
   alias LastfmArchive.Archive.Metadata
+  alias Stow.Plug.Sink
+
+  import LastfmArchive.Configs, only: [file_io: 0]
+  import LastfmArchive.Utils.Archive, only: [metadata_filepath: 2, user_dir: 1, data_dir: 1]
+
+  import Stow, only: [status: 2]
 
-  import LastfmArchive.Utils.Archive, only: [metadata_filepath: 2, user_dir: 1, user_dir: 2]
   require Logger
 
   @file_io Application.compile_env(:lastfm_archive, :file_io, Elixir.File)
   @path_io Application.compile_env(:lastfm_archive, :path_io, Elixir.Path)
-  @reset Application.compile_env(:lastfm_archive, :reset, false)
 
   def check_filepath(:csv, path), do: check_filepath(path <> ".gz")
   def check_filepath(_format, path), do: check_filepath(path)
@@ -48,6 +52,7 @@ defmodule LastfmArchive.Utils.File do
 
   @doc """
   Read and unzip a file from the archive of a Lastfm user.
+  TODO: stop gap pending Stow file source
   """
   def read(filepath) do
     case @file_io.read(filepath) do
@@ -60,36 +65,46 @@ defmodule LastfmArchive.Utils.File do
     end
   end
 
+  # TODO: stop gap pending Stow file source
+  def source(filepath, opts), do: [data_dir(opts), filepath] |> Path.join() |> @file_io.read()
+
   @doc """
   Writes data frame or metadata to a file given a write function or options.
   """
   def write(%DataFrame{} = dataframe, write_fun), do: :ok = dataframe |> write_fun.()
 
-  def write(%Metadata{creator: creator} = metadata, options) when is_list(options) do
-    metadata =
-      case Keyword.get(options, :reset, @reset) do
-        false -> metadata
-        true -> %{metadata | created: DateTime.utc_now(), date: nil, modified: nil}
-      end
-
-    filepath = metadata_filepath(creator, options)
-    filepath |> Path.dirname() |> @file_io.mkdir_p()
-
-    case @file_io.write(filepath, Jason.encode!(metadata)) do
+  def write(%Metadata{creator: user} = metadata, opts) when is_list(opts) do
+    [
+      uri: Path.join(["file:/", metadata_filepath(user, opts)]),
+      data: Jason.encode!(metadata),
+      options: %{
+        "file" => [base_dir: data_dir(opts), file_io: file_io()]
+      }
+    ]
+    |> init_plug(Sink)
+    |> call_plug(Sink)
+    |> status(Sink)
+    |> case do
       :ok -> {:ok, metadata}
       error -> error
     end
   end
 
-  def write(%Metadata{creator: user}, scrobbles, options) when is_map(scrobbles) do
-    full_path =
-      Keyword.fetch!(options, :filepath)
-      |> then(fn path -> user_dir(user, options) |> Path.join("#{path}.gz") end)
-
-    full_path_dir = Path.dirname(full_path)
-    unless @file_io.exists?(full_path_dir), do: @file_io.mkdir_p(full_path_dir)
-    @file_io.write(full_path, scrobbles |> Jason.encode!(), [:compressed])
+  def write(%Metadata{creator: user}, scrobbles, opts) when is_map(scrobbles) do
+    [
+      uri: Path.join(["file:/", user, "#{Keyword.fetch!(opts, :filepath)}.gz"]),
+      data: Jason.encode!(scrobbles),
+      options: %{
+        "file" => [base_dir: data_dir(opts), file_io: file_io(), modes: [:compressed]]
+      }
+    ]
+    |> init_plug(Sink)
+    |> call_plug(Sink)
+    |> status(Sink)
   end
 
   def write(_metadata, {:error, api_message}, _options), do: {:error, api_message}
+
+  defp init_plug(opts, plug_type), do: apply(plug_type, :init, [opts])
+  defp call_plug(opts, plug_type), do: apply(plug_type, :call, [%Plug.Conn{}, opts])
 end

test/lastfm_archive/archive/derived_archive_test.exs

@@ -5,7 +5,7 @@ defmodule LastfmArchive.Archive.DerivedArchiveTest do
   import Hammox
 
   import LastfmArchive.Factory, only: [build: 2, dataframe: 0, dataframe: 1]
-  import LastfmArchive.Utils.Archive, only: [derived_archive_dir: 1, user_dir: 1, metadata_filepath: 2]
+  import LastfmArchive.Utils.Archive, only: [data_dir: 1, derived_archive_dir: 1, user_dir: 1, metadata_filepath: 2]
 
   alias Explorer.DataFrame
   alias LastfmArchive.Archive.DerivedArchive
@@ -44,7 +44,8 @@ defmodule LastfmArchive.Archive.DerivedArchiveTest do
       facet_dataframes: Transformer.facets() |> Enum.into(%{}, &{&1, dataframe(&1)}),
       dataframe: dataframe(),
       file_archive_metadata: metadata,
-      user: user
+      user: user,
+      data_dir: data_dir([])
     }
   end
 
@@ -81,14 +82,18 @@ defmodule LastfmArchive.Archive.DerivedArchiveTest do
 
   describe "describe/2" do
     for format <- Transformer.formats(), facet <- Transformer.facets() do
-      test "existing #{format} derived #{facet} archive", %{user: user, file_archive_metadata: metadata} do
+      test "existing #{format} derived #{facet} archive", %{
+        user: user,
+        data_dir: data_dir,
+        file_archive_metadata: metadata
+      } do
         facet = unquote(facet)
         format = unquote(format)
         opts = [format: format, facet: facet]
 
         derived_archive_metadata = build(:derived_archive_metadata, file_archive_metadata: metadata, options: opts)
-        derived_archive_metadata_filepath = metadata_filepath(user, format: format)
-        file_archive_metadata_filepath = metadata_filepath(user, [])
+        derived_archive_metadata_filepath = [data_dir, metadata_filepath(user, format: format)] |> Path.join()
+        file_archive_metadata_filepath = [data_dir, metadata_filepath(user, [])] |> Path.join()
         mimetype = Transformer.mimetype(format)
 
         FileIOMock
@@ -118,13 +123,17 @@ defmodule LastfmArchive.Archive.DerivedArchiveTest do
 
       test "returns metadata when archive exists for #{facet} archive in #{format}", %{
         user: user,
+        data_dir: data_dir,
         file_archive_metadata: metadata
       } do
         facet = unquote(facet)
         format = unquote(format)
 
-        file_archive_metadata_filepath = metadata_filepath(user, [])
-        derived_archive_metadata_filepath = metadata_filepath(user, format: format, facet: facet)
+        file_archive_metadata_filepath = [data_dir, metadata_filepath(user, [])] |> Path.join()
+
+        derived_archive_metadata_filepath =
+          [data_dir, metadata_filepath(user, format: format, facet: facet)] |> Path.join()
+
         mimetype = Transformer.mimetype(format)
 
         FileIOMock

test/lastfm_archive/archive/file_archive_test.exs

@@ -76,8 +76,8 @@ defmodule LastfmArchive.Archive.FileArchiveTest do
       |> expect(:exists?, 3, fn _page_dir -> false end)
       |> expect(:mkdir_p, 3, fn _page_dir -> :ok end)
       |> expect(:write, 3, fn path, ^scrobbles, [:compressed] ->
-        assert path =~ "./lastfm_data/test/#{user}/2021/04"
-        assert path =~ "/200_001.gz"
+        assert path |> Path.join() =~ "./lastfm_data/test/#{user}/2021/04"
+        assert path |> Path.join() =~ "/200_001.gz"
         :ok
       end)
 
@@ -89,7 +89,7 @@ defmodule LastfmArchive.Archive.FileArchiveTest do
 
       LastfmArchive.FileIOMock
       |> expect(:write, 3, fn path, ^scrobbles, [:compressed] ->
-        assert path =~ "/2021/"
+        assert path |> Path.join() =~ "/2021/"
         :ok
       end)
 
@@ -101,7 +101,7 @@ defmodule LastfmArchive.Archive.FileArchiveTest do
 
       LastfmArchive.FileIOMock
       |> expect(:write, fn path, ^scrobbles, [:compressed] ->
-        assert path =~ "/2021/04/01"
+        assert path |> Path.join() =~ "/2021/04/01"
         :ok
       end)
 

test/lastfm_archive/behaviour/archive_test.exs

@@ -11,7 +11,7 @@ defmodule LastfmArchive.Behaviour.ArchiveTest do
 
   import Hammox
   import LastfmArchive.Factory, only: [build: 2]
-  import LastfmArchive.Utils.Archive, only: [metadata_filepath: 2]
+  import LastfmArchive.Utils.Archive, only: [metadata_filepath: 2, data_dir: 1]
 
   alias LastfmArchive.Archive.Metadata
 
@@ -21,19 +21,22 @@ defmodule LastfmArchive.Behaviour.ArchiveTest do
     %{
       archive: LastfmArchive.TestArchive,
       metadata: build(:file_archive_metadata, creator: "a_user"),
-      metadata_path: "test_data_dir/a_user/.metadata/scrobbles/json_archive",
+      metadata_path: "/a_user/.metadata/scrobbles/json_archive",
       type: :scrobbles
     }
   end
 
   describe "update_metadata/2" do
     test "writes metadata to file", %{archive: archive, metadata: metadata, metadata_path: path, type: type} do
+      data_dir = "test_data_dir"
+      path = [data_dir, path]
       metadata_encoded = Jason.encode!(metadata)
-      dir = path |> Path.dirname()
+      dir = path |> Path.join() |> Path.dirname()
 
       LastfmArchive.FileIOMock
+      |> expect(:exists?, fn ^dir -> false end)
       |> expect(:mkdir_p, fn ^dir -> :ok end)
-      |> expect(:write, fn ^path, ^metadata_encoded -> :ok end)
+      |> expect(:write, fn ^path, ^metadata_encoded, [] -> :ok end)
 
       assert {
                :ok,
@@ -47,16 +50,17 @@ defmodule LastfmArchive.Behaviour.ArchiveTest do
                  title: "Lastfm archive of a_user",
                  type: ^type
                }
-             } = archive.update_metadata(metadata, data_dir: "test_data_dir")
+             } = archive.update_metadata(metadata, data_dir: data_dir)
     end
 
     test "reset an existing archive metadata via 'reset' option", %{archive: archive} do
       earlier_created_datetime = DateTime.add(DateTime.utc_now(), -3600, :second)
       metadata = build(:file_archive_metadata, creator: "a_user", created: earlier_created_datetime)
 
       LastfmArchive.FileIOMock
+      |> expect(:exists?, fn _dir -> false end)
       |> expect(:mkdir_p, fn _dir -> :ok end)
-      |> expect(:write, fn _path, _ -> :ok end)
+      |> expect(:write, fn _path, _, [] -> :ok end)
 
       assert {
                :ok,
@@ -84,6 +88,8 @@ defmodule LastfmArchive.Behaviour.ArchiveTest do
 
     test "an existing file archive", %{archive: archive, metadata: metadata, metadata_path: path, type: type} do
       user = metadata.creator
+      path = [data_dir([]), path] |> Path.join()
+
       LastfmArchive.FileIOMock |> expect(:read, fn ^path -> {:ok, metadata |> Jason.encode!()} end)
 
       assert {

test/lastfm_archive/utils/archive_test.exs

@@ -22,14 +22,14 @@ defmodule LastfmArchive.Utils.ArchiveTest do
   test "metadata_filepath/2" do
     opts = []
     filepath = ".metadata/scrobbles/json_archive"
-    assert ArchiveUtils.metadata_filepath("user", opts) == "#{ArchiveUtils.user_dir("user", opts)}/#{filepath}"
+    assert ArchiveUtils.metadata_filepath("user", opts) == "user/#{filepath}"
 
     opts = [format: :ipc_stream]
     filepath = ".metadata/scrobbles/ipc_stream_archive"
-    assert ArchiveUtils.metadata_filepath("user", opts) == "#{ArchiveUtils.user_dir("user", opts)}/#{filepath}"
+    assert ArchiveUtils.metadata_filepath("user", opts) == "user/#{filepath}"
 
     opts = [format: :ipc_stream, facet: :artists]
     filepath = ".metadata/artists/ipc_stream_archive"
-    assert ArchiveUtils.metadata_filepath("user", opts) == "#{ArchiveUtils.user_dir("user", opts)}/#{filepath}"
+    assert ArchiveUtils.metadata_filepath("user", opts) == "user/#{filepath}"
   end
 end