From babb874d4883f06f772df914570b044d58f8c380 Mon Sep 17 00:00:00 2001 From: Boon Low Date: Fri, 17 Nov 2023 13:55:30 +0000 Subject: [PATCH] feat: update_latest/2 --- lib/lastfm_archive.ex | 69 +++++++++++++++---- .../archive/transformers/transformer.ex | 6 +- .../transformers/transformer_configs.ex | 2 +- lib/lastfm_archive/utils/archive.ex | 8 +++ test/lastfm_archive_test.exs | 64 ++++++++++++++++- 5 files changed, 132 insertions(+), 17 deletions(-) diff --git a/lib/lastfm_archive.ex b/lib/lastfm_archive.ex index 859decb..1f6d987 100644 --- a/lib/lastfm_archive.ex +++ b/lib/lastfm_archive.ex @@ -16,7 +16,10 @@ defmodule LastfmArchive do alias LastfmArchive.LastfmClient.LastfmApi import LastfmArchive.Archive.Transformers.Transformer, only: [transformer: 1] + import LastfmArchive.Archive.Transformers.TransformerConfigs, only: [default_opts: 0] + import LastfmArchive.Utils.Archive, only: [check_existing_archive: 2] + @facets LastfmArchive.Archive.Transformers.TransformerConfigs.facets() @path_io Application.compile_env(:lastfm_archive, :path_io, Elixir.Path) @type metadata :: Metadata.t() @@ -97,13 +100,49 @@ defmodule LastfmArchive do ``` """ - @spec sync(binary, keyword) :: {:ok, metadata()} | {:error, :file.posix()} + @spec sync(binary(), keyword()) :: {:ok, metadata()} | {:error, :file.posix()} def sync(user \\ default_user(), options \\ []) do user |> impl().describe(options) |> then(fn {:ok, metadata} -> impl().archive(metadata, options, LastfmApi.new()) end) end + @doc """ + Convenient update function to sync the latest scrobbles and transforms them into existing faceted archives. + + Options: + - `:year` - limit sync to this particular year, default: the current year + - `:format` - transform archive format: `:csv`, `:parquet`, `:ipc`, `:ipc_stream` (default) + """ + @spec update_latest(binary(), keyword()) :: list({:ok, metadata()} | {:error, :archive_not_found}) + def update_latest(user \\ default_user(), options \\ []) do + year = Keyword.get(options, :year, this_year()) + overwrite = Keyword.get(options, :overwrite, true) + + options = + Keyword.validate!(options, default_opts()) + |> Keyword.put(:year, year) + |> Keyword.put(:overwrite, overwrite) + + sync_resp = + with {:ok, _} <- check_existing_archive(user, Keyword.delete(options, :format)) do + sync(user, year: year) + end + + transform_resp = + for facet <- @facets do + options = Keyword.put(options, :facet, facet) + + with {:ok, _} <- check_existing_archive(user, options) do + transform(user, options) + end + end + + [sync_resp] ++ transform_resp + end + + defp this_year(), do: Date.utc_today().year + @doc """ Read from an archive of a Lastfm user. @@ -138,13 +177,13 @@ defmodule LastfmArchive do ``` Options: - - `:format` (required) - derived archive format: `:csv`, `:parquet`, `:ipc`, `:ipc_stream` + - `:format` - derived archive format: `:csv`, `:parquet`, `:ipc`, `:ipc_stream` (default) - `:facet` - type of archive: `:scrobbles` (default), `:albums`, `:artists` or `:tracks` - - `:year` - only read scrobbles for this particular year + - `:year` - only read scrobbles for this particular year (default - all years) - `:columns` - an atom list for retrieving only a columns subset, available columns: #{%LastfmArchive.Archive.Scrobble{} |> Map.keys() |> List.delete(:__struct__) |> Enum.map_join(", ", &(("`:" <> Atom.to_string(&1)) <> "`"))} """ - @spec read(binary, keyword()) :: {:ok, Explorer.DataFrame} | {:error, term()} + @spec read(binary(), keyword()) :: {:ok, Explorer.DataFrame} | {:error, term()} def read(user \\ default_user(), options) do user |> impl(options).describe(options) @@ -174,19 +213,20 @@ defmodule LastfmArchive do - `:format` - format into which file archive is transformed: `:csv`, `:parquet`, `:ipc`, `:ipc_stream` (default) - `:facet` - type of archive: `:scrobbles` (default), `:albums`, `:artists` or `:tracks` - `:overwrite` existing data, default: false - - `:year` - optionally transform data from this particular year + - `:year` - transform data from this particular year """ - @spec transform(binary, options) :: any - def transform(user \\ default_user(), options \\ [format: :ipc_stream]) + @spec transform(binary(), options) :: any + def transform(user \\ default_user(), options \\ []) def transform(user, options) when is_binary(user) do - facet = Keyword.get(options, :facet, :scrobbles) - options = options |> Keyword.merge(facet: facet) + options = Keyword.validate!(options, default_opts()) |> Enum.sort() + facet = Keyword.get(options, :facet) - user - |> impl(options).describe(options) - |> then(fn {:ok, metadata} -> impl(options).post_archive(metadata, transformer(facet), options) end) - |> then(fn {:ok, metadata} -> impl(options).update_metadata(metadata, options) end) + with {:ok, facet} <- validate_facet(facet), + {:ok, metadata} <- impl(options).describe(user, options), + {:ok, metadata} <- impl(options).post_archive(metadata, transformer(facet), options) do + impl(options).update_metadata(metadata, options) + end end defp impl(options \\ []) do @@ -196,6 +236,9 @@ defmodule LastfmArchive do end end + defp validate_facet(facet) when facet in @facets, do: {:ok, facet} + defp validate_facet(_), do: {:error, :invalid_facet} + # return all archive file paths in a list defp ls_archive_files(user, options \\ []) do LastfmArchive.Utils.Archive.user_dir(user, options) diff --git a/lib/lastfm_archive/archive/transformers/transformer.ex b/lib/lastfm_archive/archive/transformers/transformer.ex index cbc659e..7a5bbab 100644 --- a/lib/lastfm_archive/archive/transformers/transformer.ex +++ b/lib/lastfm_archive/archive/transformers/transformer.ex @@ -57,10 +57,14 @@ defmodule LastfmArchive.Archive.Transformers.Transformer do Path.join(user_dir(metadata.creator, opts), derived_archive_dir(opts |> validate_opts())) |> maybe_create_dir() - run_pipeline(transformer, metadata, opts, Keyword.get(opts, :year, year_range(metadata.temporal) |> Enum.to_list())) + run_pipeline(transformer, metadata, opts, Keyword.get(opts, :year)) {:ok, %{metadata | modified: DateTime.utc_now()}} end + defp run_pipeline(transformer, metadata, opts, year) when is_nil(year) do + run_pipeline(transformer, metadata, opts, year_range(metadata.temporal) |> Enum.to_list()) + end + defp run_pipeline(transformer, metadata, opts, year) when is_integer(year) do run_pipeline(transformer, metadata, opts) end diff --git a/lib/lastfm_archive/archive/transformers/transformer_configs.ex b/lib/lastfm_archive/archive/transformers/transformer_configs.ex index 7158cbd..61f0b82 100644 --- a/lib/lastfm_archive/archive/transformers/transformer_configs.ex +++ b/lib/lastfm_archive/archive/transformers/transformer_configs.ex @@ -27,7 +27,7 @@ defmodule LastfmArchive.Archive.Transformers.TransformerConfigs do end end - def default_opts, do: [format: :ipc_stream, facet: :scrobbles, overwrite: false] + def default_opts, do: [facet: :scrobbles, format: :ipc_stream, overwrite: false, year: nil] def facets, do: facet_transformers_configs() |> Map.keys() def formats, do: format_configs() |> Map.keys() diff --git a/lib/lastfm_archive/utils/archive.ex b/lib/lastfm_archive/utils/archive.ex index ecff308..3012800 100644 --- a/lib/lastfm_archive/utils/archive.ex +++ b/lib/lastfm_archive/utils/archive.ex @@ -5,6 +5,7 @@ defmodule LastfmArchive.Utils.Archive do @metadata_dir ".metadata" @data_dir Application.compile_env(:lastfm_archive, :data_dir, "./lastfm_data/") + @file_io Application.compile_env(:lastfm_archive, :file_io, Elixir.File) def metadata_filepath(user, opts \\ []) do Path.join([ @@ -14,6 +15,13 @@ defmodule LastfmArchive.Utils.Archive do ]) end + def check_existing_archive(user, options) do + case metadata_filepath(user, options) |> @file_io.exists?() do + true -> {:ok, metadata_filepath(user, options)} + false -> {:error, :archive_not_found} + end + end + def num_pages(playcount, per_page), do: (playcount / per_page) |> :math.ceil() |> round # returns 2021/12/31/200_001 type paths diff --git a/test/lastfm_archive_test.exs b/test/lastfm_archive_test.exs index 702603d..94a33ef 100644 --- a/test/lastfm_archive_test.exs +++ b/test/lastfm_archive_test.exs @@ -7,6 +7,8 @@ defmodule LastfmArchiveTest do alias LastfmArchive.Archive.DerivedArchiveMock alias LastfmArchive.Archive.FileArchiveMock alias LastfmArchive.Archive.Transformers.Transformer + alias LastfmArchive.Archive.Transformers.TransformerConfigs + alias LastfmArchive.FileIOMock setup :verify_on_exit! @@ -53,6 +55,64 @@ defmodule LastfmArchiveTest do end end + describe "update_latest/2" do + setup context do + metadata = + build(:derived_archive_metadata, + file_archive_metadat: context.file_archive_metadata, + options: TransformerConfigs.default_opts() + ) + + DerivedArchiveMock + |> stub(:describe, fn _user, _opts -> {:ok, metadata} end) + |> stub(:post_archive, fn _metadata, _transformer, _opts -> {:ok, metadata} end) + |> stub(:update_metadata, fn _metadata, _opts -> {:ok, metadata} end) + + :ok + end + + test "sync and transform scrobbles in default format", %{user: user, file_archive_metadata: metadata} do + FileIOMock |> stub(:exists?, fn _path -> true end) + + FileArchiveMock + |> expect(:describe, fn _user, _opts -> {:ok, metadata} end) + |> expect(:archive, fn _metadata, _opts, _api_client -> {:ok, metadata} end) + + LastfmArchive.update_latest(user, year: 2023, format: :ipc_stream) + end + + test "when file archive not available", %{user: user, file_archive_metadata: metadata} do + FileIOMock + |> expect(:exists?, fn _path -> false end) + |> stub(:exists?, fn _path -> true end) + + FileArchiveMock + |> expect(:describe, 0, fn _user, _opts -> {:ok, metadata} end) + |> expect(:archive, 0, fn _metadata, _opts, _api_client -> {:ok, metadata} end) + + [sync_resp | _transform_resp] = LastfmArchive.update_latest(user) + + assert sync_resp == {:error, :archive_not_found} + end + + test "when facet archives not available", %{user: user, file_archive_metadata: metadata} do + FileIOMock |> stub(:exists?, fn _path -> false end) + + FileArchiveMock + |> expect(:describe, 0, fn _user, _opts -> {:ok, metadata} end) + |> expect(:archive, 0, fn _metadata, _opts, _api_client -> {:ok, metadata} end) + + DerivedArchiveMock + |> expect(:describe, 0, fn _user, _opts -> {:ok, metadata} end) + |> expect(:post_archive, 0, fn _metadata, _transformer, _opts -> {:ok, metadata} end) + |> expect(:update_metadata, 0, fn _metadata, _opts -> {:ok, metadata} end) + + resp = LastfmArchive.update_latest(user) + + assert Enum.all?(resp, &(&1 == {:error, :archive_not_found})) + end + end + describe "read/2" do test "scrobbles of a user from a file archive", %{user: user, file_archive_metadata: metadata} do date = ~D[2023-06-01] @@ -106,7 +166,7 @@ defmodule LastfmArchiveTest do test "#{facet} into #{format} files", %{user: user, file_archive_metadata: file_archive_metadata} do facet = unquote(facet) format = unquote(format) - opts = [format: format, facet: facet] + opts = [facet: facet, format: format] |> Keyword.validate!(TransformerConfigs.default_opts()) |> Enum.sort() metadata = build(:derived_archive_metadata, file_archive_metadat: file_archive_metadata, options: opts) transformer = Transformer.facet_transformer_config(facet)[:transformer] @@ -123,7 +183,7 @@ defmodule LastfmArchiveTest do test "scrobbles of default user with default (Arrow IPC stream) format", %{file_archive_metadata: metadata} do user = Application.get_env(:lastfm_archive, :user) transformer = Transformer.facet_transformer_config(:scrobbles)[:transformer] - opts = [format: :ipc_stream, facet: :scrobbles] + opts = [] |> Keyword.validate!(TransformerConfigs.default_opts()) |> Enum.sort() DerivedArchiveMock |> expect(:describe, fn ^user, _options -> {:ok, metadata} end)