Add stats reporting when merging (#201)

lib/gettext/merger.ex

@@ -49,35 +49,59 @@ defmodule Gettext.Merger do
         by the references in the POT file
 
   """
-  @spec merge(PO.t(), PO.t(), String.t(), Keyword.t()) :: PO.t()
+  @spec merge(PO.t(), PO.t(), String.t(), Keyword.t()) :: {PO.t(), map()}
   def merge(%PO{} = old, %PO{} = new, locale, opts) when is_binary(locale) and is_list(opts) do
     opts = put_plural_forms_opt(opts, old.headers, locale)
+    stats = %{new: 0, exact_matches: 0, fuzzy_matches: 0, removed: 0}
 
-    %PO{
+    {translations, stats} = merge_translations(old.translations, new.translations, opts, stats)
+
+    po = %PO{
       top_of_the_file_comments: old.top_of_the_file_comments,
       headers: old.headers,
       file: old.file,
-      translations: merge_translations(old.translations, new.translations, opts)
+      translations: translations
     }
+
+    {po, stats}
   end
 
-  defp merge_translations(old, new, opts) do
+  defp merge_translations(old, new, opts, stats) do
     fuzzy? = Keyword.fetch!(opts, :fuzzy)
     fuzzy_threshold = Keyword.fetch!(opts, :fuzzy_threshold)
     plural_forms = Keyword.fetch!(opts, :plural_forms)
 
     old = Map.new(old, &{PO.Translations.key(&1), &1})
 
-    Enum.map(new, fn t ->
-      key = PO.Translations.key(t)
-      t = adjust_number_of_plural_forms(t, plural_forms)
-
-      case Map.fetch(old, key) do
-        {:ok, exact_match} -> merge_two_translations(exact_match, t)
-        :error when fuzzy? -> maybe_merge_fuzzy(t, old, key, fuzzy_threshold)
-        :error -> t
-      end
-    end)
+    {translations, {stats, unused}} =
+      Enum.map_reduce(new, {stats, _unused = old}, fn t, {stats_acc, unused} ->
+        key = PO.Translations.key(t)
+        t = adjust_number_of_plural_forms(t, plural_forms)
+
+        case Map.fetch(old, key) do
+          {:ok, exact_match} ->
+            stats = update_in(stats_acc.exact_matches, &(&1 + 1))
+            {merge_two_translations(exact_match, t), {stats, Map.delete(unused, key)}}
+
+          :error when fuzzy? ->
+            case maybe_merge_fuzzy(t, old, key, fuzzy_threshold) do
+              {:matched, match, fuzzy_merged} ->
+                stats_acc = update_in(stats_acc.fuzzy_matches, &(&1 + 1))
+                unused = Map.delete(unused, PO.Translations.key(match))
+                {fuzzy_merged, {stats_acc, unused}}
+
+              :nomatch ->
+                stats_acc = update_in(stats_acc.new, &(&1 + 1))
+                {t, {stats_acc, unused}}
+            end
+
+          :error ->
+            stats_acc = update_in(stats_acc.new, &(&1 + 1))
+            {t, {stats_acc, unused}}
+        end
+      end)
+
+    {translations, put_in(stats.removed, map_size(unused))}
   end
 
   defp adjust_number_of_plural_forms(%PluralTranslation{} = t, plural_forms)
@@ -92,9 +116,9 @@ defmodule Gettext.Merger do
 
   defp maybe_merge_fuzzy(t, old, key, fuzzy_threshold) do
     if matched = find_fuzzy_match(old, key, fuzzy_threshold) do
-      Fuzzy.merge(t, matched)
+      {:matched, matched, Fuzzy.merge(t, matched)}
     else
-      t
+      :nomatch
     end
   end
 
@@ -162,12 +186,16 @@ defmodule Gettext.Merger do
     opts = put_plural_forms_opt(opts, pot.headers, locale)
     plural_forms = Keyword.fetch!(opts, :plural_forms)
 
-    %PO{
+    po = %PO{
       top_of_the_file_comments: String.split(@new_po_informative_comment, "\n", trim: true),
       headers: headers_for_new_po_file(locale, plural_forms),
       file: po_file,
       translations: Enum.map(pot.translations, &prepare_new_translation(&1, plural_forms))
     }
+
+    stats = %{new: length(po.translations), exact_matches: 0, fuzzy_matches: 0, removed: 0}
+
+    {po, stats}
   end
 
   defp headers_for_new_po_file(locale, plural_forms) do

lib/mix/tasks/gettext.merge.ex

@@ -139,8 +139,11 @@ defmodule Mix.Tasks.Gettext.Merge do
       ensure_file_exists!(po_file)
       ensure_file_exists!(reference_file)
       locale = locale_from_path(po_file)
-      contents = merge_files(po_file, reference_file, locale, merging_opts, gettext_config)
-      write_file(po_file, contents)
+
+      {contents, stats} =
+        merge_files(po_file, reference_file, locale, merging_opts, gettext_config)
+
+      write_file(po_file, contents, stats)
     else
       Mix.raise("Arguments must be a PO file and a PO/POT file")
     end
@@ -176,8 +179,8 @@ defmodule Mix.Tasks.Gettext.Merge do
   defp merge_dirs(po_dir, pot_dir, locale, opts, gettext_config) do
     merger = fn pot_file ->
       po_file = find_matching_po(pot_file, po_dir)
-      contents = merge_or_create(pot_file, po_file, locale, opts, gettext_config)
-      write_file(po_file, contents)
+      {contents, stats} = merge_or_create(pot_file, po_file, locale, opts, gettext_config)
+      write_file(po_file, contents, stats)
     end
 
     pot_dir
@@ -198,19 +201,21 @@ defmodule Mix.Tasks.Gettext.Merge do
     if File.regular?(po_file) do
       merge_files(po_file, pot_file, locale, opts, gettext_config)
     else
-      new_po = Merger.new_po_file(po_file, pot_file, locale, opts)
-      PO.dump(new_po, gettext_config)
+      {new_po, stats} = Merger.new_po_file(po_file, pot_file, locale, opts)
+      {PO.dump(new_po, gettext_config), stats}
     end
   end
 
   defp merge_files(po_file, pot_file, locale, opts, gettext_config) do
-    merged = Merger.merge(PO.parse_file!(po_file), PO.parse_file!(pot_file), locale, opts)
-    PO.dump(merged, gettext_config)
+    {merged, stats} =
+      Merger.merge(PO.parse_file!(po_file), PO.parse_file!(pot_file), locale, opts)
+
+    {PO.dump(merged, gettext_config), stats}
   end
 
-  defp write_file(path, contents) do
+  defp write_file(path, contents, stats) do
     File.write!(path, contents)
-    Mix.shell().info("Wrote #{path}")
+    Mix.shell().info("Wrote #{path} (#{format_stats(stats)})")
   end
 
   # Warns for every PO file that has no matching POT file.
@@ -268,4 +273,11 @@ defmodule Mix.Tasks.Gettext.Merge do
     index = Enum.find_index(parts, &(&1 == "LC_MESSAGES"))
     Enum.at(parts, index - 1)
   end
+
+  defp format_stats(stats) do
+    pluralized = if stats.new == 1, do: "translation", else: "translations"
+
+    "#{stats.new} new #{pluralized}, #{stats.removed} removed, " <>
+      "#{stats.exact_matches} unchanged, #{stats.fuzzy_matches} reworded (fuzzy)"
+  end
 end

test/gettext/merger_test.exs

@@ -15,7 +15,8 @@ defmodule Gettext.MergerTest do
       old_po = %PO{headers: [~S(Language: it\n), ~S(My-Header: my-value\n)]}
       new_pot = %PO{headers: ["foo"]}
 
-      assert Merger.merge(old_po, new_pot, "en", @opts).headers == old_po.headers
+      assert {new_po, _stats} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert new_po.headers == old_po.headers
     end
 
     test "obsolete translations are discarded (even the manually entered ones)" do
@@ -29,16 +30,19 @@ defmodule Gettext.MergerTest do
 
       new_pot = %PO{translations: [%Translation{msgid: "tomerge", msgstr: ""}]}
 
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, stats} = Merger.merge(old_po, new_pot, "en", @opts)
+
       assert t.msgid == "tomerge"
       assert t.msgstr == "foo"
+
+      assert stats == %{exact_matches: 1, fuzzy_matches: 0, new: 0, removed: 2}
     end
 
     test "when translations match, the msgstr of the old one is preserved" do
       old_po = %PO{translations: [%Translation{msgid: "foo", msgstr: "bar"}]}
       new_pot = %PO{translations: [%Translation{msgid: "foo", msgstr: ""}]}
 
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, _stats} = Merger.merge(old_po, new_pot, "en", @opts)
       assert t.msgid == "foo"
       assert t.msgstr == "bar"
     end
@@ -49,7 +53,7 @@ defmodule Gettext.MergerTest do
       old_po = %PO{translations: [%Translation{msgid: "foo", comments: ["# existing comment"]}]}
       new_pot = %PO{translations: [%Translation{msgid: "foo", comments: ["# new comment"]}]}
 
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, _stats} = Merger.merge(old_po, new_pot, "en", @opts)
       assert t.msgid == "foo"
       assert t.comments == ["# existing comment"]
     end
@@ -68,15 +72,15 @@ defmodule Gettext.MergerTest do
         translations: [%Translation{msgid: "foo", extracted_comments: ["#. new comment"]}]
       }
 
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, _stats} = Merger.merge(old_po, new_pot, "en", @opts)
       assert t.extracted_comments == ["#. new comment"]
     end
 
     test "when translations match, existing references are replaced by new ones" do
       old_po = %PO{translations: [%Translation{msgid: "foo", references: [{"foo.ex", 1}]}]}
       new_pot = %PO{translations: [%Translation{msgid: "foo", references: [{"bar.ex", 1}]}]}
 
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, _stats} = Merger.merge(old_po, new_pot, "en", @opts)
       assert t.references == [{"bar.ex", 1}]
     end
 
@@ -87,7 +91,7 @@ defmodule Gettext.MergerTest do
         translations: [%Translation{msgid: "foo", flags: @autogenerated_flags}]
       }
 
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, _stats} = Merger.merge(old_po, new_pot, "en", @opts)
       assert t.flags == @autogenerated_flags
     end
 
@@ -115,7 +119,9 @@ defmodule Gettext.MergerTest do
         ]
       }
 
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, stats} = Merger.merge(old_po, new_pot, "en", @opts)
+
+      assert stats == %{exact_matches: 0, fuzzy_matches: 1, new: 0, removed: 0}
 
       assert t.msgid == "hello worlds!"
       assert t.msgstr == ["foo"]
@@ -137,10 +143,13 @@ defmodule Gettext.MergerTest do
 
       # Let's check that the "hello worlds!" translation is discarded even if it's
       # a fuzzy match for "hello world!".
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, stats} = Merger.merge(old_po, new_pot, "en", @opts)
+
       refute "fuzzy" in t.flags
       assert t.msgid == ["hello world!"]
       assert t.msgstr == ["foo"]
+
+      assert stats == %{exact_matches: 1, fuzzy_matches: 0, new: 0, removed: 1}
     end
 
     test "exact matches do not prevent fuzzy matches for other translations" do
@@ -155,7 +164,7 @@ defmodule Gettext.MergerTest do
         ]
       }
 
-      assert %PO{translations: [t1, t2]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t1, t2]}, stats} = Merger.merge(old_po, new_pot, "en", @opts)
 
       assert t1.msgid == ["hello world"]
       assert t1.msgstr == ["foo"]
@@ -164,6 +173,11 @@ defmodule Gettext.MergerTest do
       assert t2.msgid == ["hello world!"]
       assert t2.msgstr == ["foo"]
       assert "fuzzy" in t2.flags
+
+      assert stats.new == 0
+      assert stats.removed == 0
+      assert stats.fuzzy_matches == 1
+      assert stats.exact_matches == 1
     end
 
     test "multiple translations can fuzzy match against a single translation" do
@@ -176,7 +190,7 @@ defmodule Gettext.MergerTest do
         ]
       }
 
-      assert %PO{translations: [t1, t2]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t1, t2]}, stats} = Merger.merge(old_po, new_pot, "en", @opts)
 
       assert t1.msgid == ["hello world 1"]
       assert t1.msgstr == ["foo"]
@@ -185,6 +199,8 @@ defmodule Gettext.MergerTest do
       assert t2.msgid == ["hello world 2"]
       assert t2.msgstr == ["foo"]
       assert "fuzzy" in t2.flags
+
+      assert stats == %{exact_matches: 0, new: 0, fuzzy_matches: 2, removed: 0}
     end
 
     test "filling in a fuzzy translation preserves references" do
@@ -203,7 +219,7 @@ defmodule Gettext.MergerTest do
         translations: [%Translation{msgid: ["hello worlds!"], references: [{"new_file.txt", 2}]}]
       }
 
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, _stats} = Merger.merge(old_po, new_pot, "en", @opts)
       assert MapSet.member?(t.flags, "fuzzy")
       assert t.msgid == ["hello worlds!"]
       assert t.msgstr == ["foo"]
@@ -233,13 +249,16 @@ defmodule Gettext.MergerTest do
         ]
       }
 
-      assert %PO{translations: [t]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t]}, stats} = Merger.merge(old_po, new_pot, "en", @opts)
+
       assert MapSet.member?(t.flags, "fuzzy")
       assert t.msgid == ["Here is a cocoa ball."]
       assert t.msgid_plural == ["Here are {count} cocoa balls."]
       assert t.msgstr[0] == ["Hier sind {count} Kakaokugeln."]
       assert t.comments == ["# Guyanese Cocoballs"]
       assert t.references == [{"new_file.txt", 2}]
+
+      assert stats == %{exact_matches: 0, fuzzy_matches: 1, new: 0, removed: 0}
     end
 
     test "if there's a Plural-Forms header, it's used to determine number of plural forms" do
@@ -255,7 +274,7 @@ defmodule Gettext.MergerTest do
         ]
       }
 
-      assert %PO{translations: [t, pt]} = Merger.merge(old_po, new_pot, "en", @opts)
+      assert {%PO{translations: [t, pt]}, _stats} = Merger.merge(old_po, new_pot, "en", @opts)
 
       assert t.msgid == "a"
 
@@ -275,7 +294,7 @@ defmodule Gettext.MergerTest do
       }
 
       opts = [plural_forms: 1] ++ @opts
-      assert %PO{translations: [t, pt]} = Merger.merge(old_po, new_pot, "en", opts)
+      assert {%PO{translations: [t, pt]}, _stats} = Merger.merge(old_po, new_pot, "en", opts)
 
       assert t.msgid == "a"
 
@@ -301,7 +320,7 @@ defmodule Gettext.MergerTest do
     msgstr[1] ""
     """)
 
-    new_po = Merger.new_po_file(new_po_path, pot_path, "it", [plural_forms: 1] ++ @opts)
+    {new_po, _stats} = Merger.new_po_file(new_po_path, pot_path, "it", [plural_forms: 1] ++ @opts)
 
     assert new_po.file == new_po_path
     assert new_po.headers == ["Language: it\n", "Plural-Forms: nplurals=1\n"]

test/mix/tasks/gettext.merge_test.exs

@@ -58,6 +58,7 @@ defmodule Mix.Tasks.Gettext.MergeTest do
       end)
 
     assert output =~ "Wrote tmp/gettext.merge/it/LC_MESSAGES/foo.po"
+    assert output =~ "(1 new translation, 0 removed, 0 unchanged, 0 reworded (fuzzy))"
 
     # The POT file is left unchanged
     assert read_file("foo.pot") == pot_contents