Merge pull request #38 from boonious/update-latest-function

feat: convenient function to sync and transform latest scrobbles

lib/lastfm_archive.ex

@@ -16,7 +16,10 @@ defmodule LastfmArchive do
   alias LastfmArchive.LastfmClient.LastfmApi
 
   import LastfmArchive.Archive.Transformers.Transformer, only: [transformer: 1]
+  import LastfmArchive.Archive.Transformers.TransformerConfigs, only: [default_opts: 0]
+  import LastfmArchive.Utils.Archive, only: [check_existing_archive: 2]
 
+  @facets LastfmArchive.Archive.Transformers.TransformerConfigs.facets()
   @path_io Application.compile_env(:lastfm_archive, :path_io, Elixir.Path)
 
   @type metadata :: Metadata.t()
@@ -97,13 +100,49 @@ defmodule LastfmArchive do
   ```
 
   """
-  @spec sync(binary, keyword) :: {:ok, metadata()} | {:error, :file.posix()}
+  @spec sync(binary(), keyword()) :: {:ok, metadata()} | {:error, :file.posix()}
   def sync(user \\ default_user(), options \\ []) do
     user
     |> impl().describe(options)
     |> then(fn {:ok, metadata} -> impl().archive(metadata, options, LastfmApi.new()) end)
   end
 
+  @doc """
+  Convenient update function to sync the latest scrobbles and transforms them into existing faceted archives.
+
+  Options:
+  - `:year` - limit sync to this particular year, default: the current year
+  - `:format` - transform archive format: `:csv`, `:parquet`, `:ipc`, `:ipc_stream` (default)
+  """
+  @spec update_latest(binary(), keyword()) :: list({:ok, metadata()} | {:error, :archive_not_found})
+  def update_latest(user \\ default_user(), options \\ []) do
+    year = Keyword.get(options, :year, this_year())
+    overwrite = Keyword.get(options, :overwrite, true)
+
+    options =
+      Keyword.validate!(options, default_opts())
+      |> Keyword.put(:year, year)
+      |> Keyword.put(:overwrite, overwrite)
+
+    sync_resp =
+      with {:ok, _} <- check_existing_archive(user, Keyword.delete(options, :format)) do
+        sync(user, year: year)
+      end
+
+    transform_resp =
+      for facet <- @facets do
+        options = Keyword.put(options, :facet, facet)
+
+        with {:ok, _} <- check_existing_archive(user, options) do
+          transform(user, options)
+        end
+      end
+
+    [sync_resp] ++ transform_resp
+  end
+
+  defp this_year(), do: Date.utc_today().year
+
   @doc """
   Read from an archive of a Lastfm user.
 
@@ -138,13 +177,13 @@ defmodule LastfmArchive do
   ```
 
   Options:
-  - `:format` (required) - derived archive format: `:csv`, `:parquet`, `:ipc`, `:ipc_stream`
+  - `:format` - derived archive format: `:csv`, `:parquet`, `:ipc`, `:ipc_stream` (default)
   - `:facet` - type of archive: `:scrobbles` (default), `:albums`, `:artists` or `:tracks`
-  - `:year` - only read scrobbles for this particular year
+  - `:year` - only read scrobbles for this particular year (default - all years)
   - `:columns` - an atom list for retrieving only a columns subset, available columns:
   #{%LastfmArchive.Archive.Scrobble{} |> Map.keys() |> List.delete(:__struct__) |> Enum.map_join(", ", &(("`:" <> Atom.to_string(&1)) <> "`"))}
   """
-  @spec read(binary, keyword()) :: {:ok, Explorer.DataFrame} | {:error, term()}
+  @spec read(binary(), keyword()) :: {:ok, Explorer.DataFrame} | {:error, term()}
   def read(user \\ default_user(), options) do
     user
     |> impl(options).describe(options)
@@ -174,19 +213,20 @@ defmodule LastfmArchive do
   - `:format` - format into which file archive is transformed: `:csv`, `:parquet`, `:ipc`, `:ipc_stream` (default)
   - `:facet` - type of archive: `:scrobbles` (default), `:albums`, `:artists` or `:tracks`
   - `:overwrite` existing data, default: false
-  - `:year` - optionally transform data from this particular year
+  - `:year` - transform data from this particular year
   """
-  @spec transform(binary, options) :: any
-  def transform(user \\ default_user(), options \\ [format: :ipc_stream])
+  @spec transform(binary(), options) :: any
+  def transform(user \\ default_user(), options \\ [])
 
   def transform(user, options) when is_binary(user) do
-    facet = Keyword.get(options, :facet, :scrobbles)
-    options = options |> Keyword.merge(facet: facet)
+    options = Keyword.validate!(options, default_opts()) |> Enum.sort()
+    facet = Keyword.get(options, :facet)
 
-    user
-    |> impl(options).describe(options)
-    |> then(fn {:ok, metadata} -> impl(options).post_archive(metadata, transformer(facet), options) end)
-    |> then(fn {:ok, metadata} -> impl(options).update_metadata(metadata, options) end)
+    with {:ok, facet} <- validate_facet(facet),
+         {:ok, metadata} <- impl(options).describe(user, options),
+         {:ok, metadata} <- impl(options).post_archive(metadata, transformer(facet), options) do
+      impl(options).update_metadata(metadata, options)
+    end
   end
 
   defp impl(options \\ []) do
@@ -196,6 +236,9 @@ defmodule LastfmArchive do
     end
   end
 
+  defp validate_facet(facet) when facet in @facets, do: {:ok, facet}
+  defp validate_facet(_), do: {:error, :invalid_facet}
+
   # return all archive file paths in a list
   defp ls_archive_files(user, options \\ []) do
     LastfmArchive.Utils.Archive.user_dir(user, options)

lib/lastfm_archive/archive/transformers/transformer.ex

@@ -57,10 +57,14 @@ defmodule LastfmArchive.Archive.Transformers.Transformer do
     Path.join(user_dir(metadata.creator, opts), derived_archive_dir(opts |> validate_opts()))
     |> maybe_create_dir()
 
-    run_pipeline(transformer, metadata, opts, Keyword.get(opts, :year, year_range(metadata.temporal) |> Enum.to_list()))
+    run_pipeline(transformer, metadata, opts, Keyword.get(opts, :year))
     {:ok, %{metadata | modified: DateTime.utc_now()}}
   end
 
+  defp run_pipeline(transformer, metadata, opts, year) when is_nil(year) do
+    run_pipeline(transformer, metadata, opts, year_range(metadata.temporal) |> Enum.to_list())
+  end
+
   defp run_pipeline(transformer, metadata, opts, year) when is_integer(year) do
     run_pipeline(transformer, metadata, opts)
   end

lib/lastfm_archive/archive/transformers/transformer_configs.ex

@@ -27,7 +27,7 @@ defmodule LastfmArchive.Archive.Transformers.TransformerConfigs do
     end
   end
 
-  def default_opts, do: [format: :ipc_stream, facet: :scrobbles, overwrite: false]
+  def default_opts, do: [facet: :scrobbles, format: :ipc_stream, overwrite: false, year: nil]
   def facets, do: facet_transformers_configs() |> Map.keys()
   def formats, do: format_configs() |> Map.keys()
 

lib/lastfm_archive/utils/archive.ex

@@ -5,6 +5,7 @@ defmodule LastfmArchive.Utils.Archive do
 
   @metadata_dir ".metadata"
   @data_dir Application.compile_env(:lastfm_archive, :data_dir, "./lastfm_data/")
+  @file_io Application.compile_env(:lastfm_archive, :file_io, Elixir.File)
 
   def metadata_filepath(user, opts \\ []) do
     Path.join([
@@ -14,6 +15,13 @@ defmodule LastfmArchive.Utils.Archive do
     ])
   end
 
+  def check_existing_archive(user, options) do
+    case metadata_filepath(user, options) |> @file_io.exists?() do
+      true -> {:ok, metadata_filepath(user, options)}
+      false -> {:error, :archive_not_found}
+    end
+  end
+
   def num_pages(playcount, per_page), do: (playcount / per_page) |> :math.ceil() |> round
 
   # returns 2021/12/31/200_001 type paths

test/lastfm_archive_test.exs

@@ -7,6 +7,8 @@ defmodule LastfmArchiveTest do
   alias LastfmArchive.Archive.DerivedArchiveMock
   alias LastfmArchive.Archive.FileArchiveMock
   alias LastfmArchive.Archive.Transformers.Transformer
+  alias LastfmArchive.Archive.Transformers.TransformerConfigs
+  alias LastfmArchive.FileIOMock
 
   setup :verify_on_exit!
 
@@ -53,6 +55,64 @@ defmodule LastfmArchiveTest do
     end
   end
 
+  describe "update_latest/2" do
+    setup context do
+      metadata =
+        build(:derived_archive_metadata,
+          file_archive_metadat: context.file_archive_metadata,
+          options: TransformerConfigs.default_opts()
+        )
+
+      DerivedArchiveMock
+      |> stub(:describe, fn _user, _opts -> {:ok, metadata} end)
+      |> stub(:post_archive, fn _metadata, _transformer, _opts -> {:ok, metadata} end)
+      |> stub(:update_metadata, fn _metadata, _opts -> {:ok, metadata} end)
+
+      :ok
+    end
+
+    test "sync and transform scrobbles in default format", %{user: user, file_archive_metadata: metadata} do
+      FileIOMock |> stub(:exists?, fn _path -> true end)
+
+      FileArchiveMock
+      |> expect(:describe, fn _user, _opts -> {:ok, metadata} end)
+      |> expect(:archive, fn _metadata, _opts, _api_client -> {:ok, metadata} end)
+
+      LastfmArchive.update_latest(user, year: 2023, format: :ipc_stream)
+    end
+
+    test "when file archive not available", %{user: user, file_archive_metadata: metadata} do
+      FileIOMock
+      |> expect(:exists?, fn _path -> false end)
+      |> stub(:exists?, fn _path -> true end)
+
+      FileArchiveMock
+      |> expect(:describe, 0, fn _user, _opts -> {:ok, metadata} end)
+      |> expect(:archive, 0, fn _metadata, _opts, _api_client -> {:ok, metadata} end)
+
+      [sync_resp | _transform_resp] = LastfmArchive.update_latest(user)
+
+      assert sync_resp == {:error, :archive_not_found}
+    end
+
+    test "when facet archives not available", %{user: user, file_archive_metadata: metadata} do
+      FileIOMock |> stub(:exists?, fn _path -> false end)
+
+      FileArchiveMock
+      |> expect(:describe, 0, fn _user, _opts -> {:ok, metadata} end)
+      |> expect(:archive, 0, fn _metadata, _opts, _api_client -> {:ok, metadata} end)
+
+      DerivedArchiveMock
+      |> expect(:describe, 0, fn _user, _opts -> {:ok, metadata} end)
+      |> expect(:post_archive, 0, fn _metadata, _transformer, _opts -> {:ok, metadata} end)
+      |> expect(:update_metadata, 0, fn _metadata, _opts -> {:ok, metadata} end)
+
+      resp = LastfmArchive.update_latest(user)
+
+      assert Enum.all?(resp, &(&1 == {:error, :archive_not_found}))
+    end
+  end
+
   describe "read/2" do
     test "scrobbles of a user from a file archive", %{user: user, file_archive_metadata: metadata} do
       date = ~D[2023-06-01]
@@ -106,7 +166,7 @@ defmodule LastfmArchiveTest do
       test "#{facet} into #{format} files", %{user: user, file_archive_metadata: file_archive_metadata} do
         facet = unquote(facet)
         format = unquote(format)
-        opts = [format: format, facet: facet]
+        opts = [facet: facet, format: format] |> Keyword.validate!(TransformerConfigs.default_opts()) |> Enum.sort()
 
         metadata = build(:derived_archive_metadata, file_archive_metadat: file_archive_metadata, options: opts)
         transformer = Transformer.facet_transformer_config(facet)[:transformer]
@@ -123,7 +183,7 @@ defmodule LastfmArchiveTest do
     test "scrobbles of default user with default (Arrow IPC stream) format", %{file_archive_metadata: metadata} do
       user = Application.get_env(:lastfm_archive, :user)
       transformer = Transformer.facet_transformer_config(:scrobbles)[:transformer]
-      opts = [format: :ipc_stream, facet: :scrobbles]
+      opts = [] |> Keyword.validate!(TransformerConfigs.default_opts()) |> Enum.sort()
 
       DerivedArchiveMock
       |> expect(:describe, fn ^user, _options -> {:ok, metadata} end)