From c2dc31100fb764ad4b0c4df6bf94bdef0bbc8984 Mon Sep 17 00:00:00 2001 From: Kevin Webster Date: Wed, 2 Oct 2019 10:12:11 -0700 Subject: [PATCH] Breadcrumbs (#235) * RingBuffer * Basic working version * Added Collector * Adding error breadcrumb in notice * Fixing some specs, ignore DateTime in to_encodable * Added Utils.sanitize * Moved notice breadcrumb creation out of notice.new * Now sanitizing breadcrumb metadata * Filter breadcrumbs * Convert structs to Map in sanitizer * Dropped elixir 1.7 and added 1.9 to the matrix * Storing error breadcrumb * Basic setup for telemetry events * Added specs for telemetry * Updated docs --- .travis.yml | 2 +- README.md | 68 ++++++++++ dummy/mixapp/mix.lock | 1 + lib/honeybadger.ex | 116 +++++++++++++++++- lib/honeybadger/breadcrumbs/breadcrumb.ex | 43 +++++++ lib/honeybadger/breadcrumbs/collector.ex | 59 +++++++++ lib/honeybadger/breadcrumbs/ring_buffer.ex | 24 ++++ lib/honeybadger/breadcrumbs/telemetry.ex | 91 ++++++++++++++ lib/honeybadger/client.ex | 9 +- lib/honeybadger/filter.ex | 15 +++ lib/honeybadger/filter/mixin.ex | 19 +-- lib/honeybadger/json.ex | 3 + lib/honeybadger/logger.ex | 19 ++- lib/honeybadger/notice.ex | 16 ++- lib/honeybadger/notice_filter/default.ex | 5 + lib/honeybadger/plug.ex | 2 + lib/honeybadger/utils.ex | 76 ++++++++++++ mix.exs | 8 +- mix.lock | 2 + .../breadcrumbs/collector_test.exs | 67 ++++++++++ .../breadcrumbs/ring_buffer_test.exs | 21 ++++ .../breadcrumbs/telemetry_test.exs | 78 ++++++++++++ test/honeybadger/json_test.exs | 1 + test/honeybadger/logger_test.exs | 31 +++-- test/honeybadger/notice_test.exs | 25 +++- test/honeybadger/plug_test.exs | 6 +- test/honeybadger/utils_test.exs | 45 +++++++ 27 files changed, 821 insertions(+), 31 deletions(-) create mode 100644 lib/honeybadger/breadcrumbs/breadcrumb.ex create mode 100644 lib/honeybadger/breadcrumbs/collector.ex create mode 100644 lib/honeybadger/breadcrumbs/ring_buffer.ex create mode 100644 lib/honeybadger/breadcrumbs/telemetry.ex create mode 100644 test/honeybadger/breadcrumbs/collector_test.exs create mode 100644 test/honeybadger/breadcrumbs/ring_buffer_test.exs create mode 100644 test/honeybadger/breadcrumbs/telemetry_test.exs diff --git a/.travis.yml b/.travis.yml index a1a1e020..02199e11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: elixir sudo: false elixir: - - 1.7 - 1.8 + - 1.9 otp_release: - 21.2 - 22.0 diff --git a/README.md b/README.md index 99a1f6f4..9dcd78a1 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,53 @@ rescue end ``` +## Breadcrumbs + +Breadcrumbs allow you to record events along a processes execution path. If +an error is thrown, the set of breadcrumb events will be sent along with the +notice. These breadcrumbs can contain useful hints while debugging. + +Breadcrumbs are stored in the logger context, referenced by the calling +process. If you are sending messages between processes, breadcrumbs will not +transfer automatically. Since a typical system might have many processes, it +is advised that you be conservative when storing breadcrumbs as each +breadcrumb consumes memory. + +### Enabling Breadcrumbs + +As of version `0.13.0`, Breadcrumbs are _available_ yet _disabled_. You must +explicitly enable them if you want breadcrumbs to be reported. We plan on +enabling this by default in a future release. + +Toggle `breadcrumbs_enabled` in the config to start sending Breadcrumbs with +notices: + +```elixir +config :honeybadger, + breadcrumbs_enabled: true +``` + +### Automatic Breadcrumbs + +We leverage the `telemetry` library to automatically create breadcrumbs from +specific events. + +__Phoenix__ + +If you are using `phoenix` (>= v1.4.7) we add a breadcrumb from the router +start event. + +__Ecto__ + +We can create breadcrumbs from Ecto SQL calls if you are using `ecto_sql` (>= +v3.1.0). You also must specify in the config which ecto adapters you want to +be instrumented: + +```elixir +config :honeybadger, + ecto_repos: [MyApp.Repo] +``` + ## Sample Application If you'd like to see the module in action before you integrate it with your apps, check out our [sample Phoenix application](https://github.com/honeybadger-io/crywolf-elixir). @@ -187,6 +234,8 @@ Here are all of the options you can pass in the keyword list: | `filter_disable_params` | If true, will remove the request params | `false` | | `notice_filter` | Module implementing `Honeybadger.NoticeFilter`. If `nil`, no filtering is done. | `Honeybadger.NoticeFilter.Default` | | `use_logger` | Enable the Honeybadger Logger for handling errors outside of web requests | `true` | +| `breadcrumbs_enabled` | Enable breadcrumb event tracking | `false` | +| `ecto_repos` | Modules with implemented Ecto.Repo behaviour for tracking SQL breadcrumb events | `[]` | ## Public Interface @@ -248,6 +297,25 @@ end) --- +### `Honeybadger.add_breadcrumb/2`: Store breadcrumb within process + +Appends a breadcrumb to the notice. Use this when you want to add some custom +data to your breadcrumb trace in effort to help debugging. If a notice is +reported to Honeybadger, all breadcrumbs within the execution path will be +appended to the notice. You will be able to view the breadcrumb trace in the +Honeybadger interface to see what events led up to the notice. + +#### Examples: + +```elixir +Honeybadger.add_breadcrumb("Email sent", metadata: %{ + user: user.id, + message: message +}) +``` + +--- + ## Proxy configuration If your server needs a proxy to access honeybadger, add the following to your config diff --git a/dummy/mixapp/mix.lock b/dummy/mixapp/mix.lock index 59b11999..03b4e818 100644 --- a/dummy/mixapp/mix.lock +++ b/dummy/mixapp/mix.lock @@ -9,5 +9,6 @@ "plug": {:hex, :plug, "1.4.3", "236d77ce7bf3e3a2668dc0d32a9b6f1f9b1f05361019946aae49874904be4aed", [], [{:cowboy, "~> 1.0.1 or ~> 1.1", [hex: :cowboy, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.3.1", "a1f612a7b512638634a603c8f401892afbf99b8ce93a45041f8aaca99cadb85e", [:rebar3], [], "hexpm"}, } diff --git a/lib/honeybadger.ex b/lib/honeybadger.ex index 32701d62..bd4bade7 100644 --- a/lib/honeybadger.ex +++ b/lib/honeybadger.ex @@ -16,6 +16,8 @@ defmodule Honeybadger do environment_name: :prod, app: :my_app_name, exclude_envs: [:dev, :test], + breadcrumbs_enabled: false, + ecto_repos: [MyAppName.Ecto.Repo], hostname: "myserver.domain.com", origin: "https://api.honeybadger.io", proxy: "http://proxy.net:PORT", @@ -121,11 +123,51 @@ defmodule Honeybadger do end See `Honeybadger.Filter` for details on implementing your own filter. + + ### Breadcrumbs + + Breadcrumbs allow you to record events along a processes execution path. If + an error is thrown, the set of breadcrumb events will be sent along with the + notice. These breadcrumbs can contain useful hints while debugging. + + Breadcrumbs are stored in the logger context, referenced by the calling + process. If you are sending messages between processes, breadcrumbs will not + transfer automatically. Since a typical system might have many processes, it + is advised that you be conservative when storing breadcrumbs as each + breadcrumb consumes memory. + + Ensure that you enable breadcrumbs in the config (as it is disabled by + default): + + config :honeybadger, + breadcrumbs_enabled: true + + See `Honeybadger.add_breadcrumb` for info on how to add custom breadcrumbs. + + ### Automatic Breadcrumbs + + We leverage the `telemetry` library to automatically create breadcrumbs from + specific events. + + #### Phoenix + + If you are using `phoenix` (>= v1.4.7) we add a breadcrumb from the router + start event. + + #### Ecto + + We can create breadcrumbs from Ecto SQL calls if you are using `ecto_sql` (>= + v3.1.0). You also must specify in the config which ecto adapters you want to + be instrumented: + + config :honeybadger, + ecto_repos: [MyApp.Repo] """ use Application alias Honeybadger.{Client, Notice} + alias Honeybadger.Breadcrumbs.{Collector, Breadcrumb} defmodule MissingEnvironmentNameError do defexception message: """ @@ -152,6 +194,10 @@ defmodule Honeybadger do _ = Logger.add_backend(Honeybadger.Logger) end + if config[:breadcrumbs_enabled] do + Honeybadger.Breadcrumbs.Telemetry.attach() + end + children = [ worker(Client, [config]) ] @@ -203,11 +249,53 @@ defmodule Honeybadger do """ @spec notify(Notice.noticeable(), map(), list()) :: :ok def notify(exception, metadata \\ %{}, stacktrace \\ []) do + # Grab process local breadcrumbs if not passed with call and add notice breadcrumb + breadcrumbs = + metadata + |> Map.get(:breadcrumbs, Collector.breadcrumbs()) + |> Collector.put(notice_breadcrumb(exception)) + |> Collector.output() + + metadata_with_breadcrumbs = + metadata + |> Map.delete(:breadcrumbs) + |> contextual_metadata() + |> Map.put(:breadcrumbs, breadcrumbs) + exception - |> Notice.new(contextual_metadata(metadata), stacktrace) + |> Notice.new(metadata_with_breadcrumbs, stacktrace) |> Client.send_notice() end + @doc """ + Stores a breadcrumb item. + + Appends a breadcrumb to the notice. Use this when you want to add some custom + data to your breadcrumb trace in effort to help debugging. If a notice is + reported to Honeybadger, all breadcrumbs within the execution path will be + appended to the notice. You will be able to view the breadcrumb trace in the + Honeybadger interface to see what events led up to the notice. + + ## Breadcrumb with metadata + + Honeybadger.add_breadcrumb("email sent", metadata: %{ + user: user.id, message: message + }) + => :ok + + ## Breadcrumb with specified category. This will display a query icon in the interface + + Honeybadger.add_breadcrumb("ETS Lookup", category: "query", metadata: %{ + key: key, + value: value + }) + => :ok + """ + @spec add_breadcrumb(String.t(), Breadcrumb.opts()) :: :ok + def add_breadcrumb(message, opts \\ []) when is_binary(message) and is_list(opts) do + Collector.add(Breadcrumb.new(message, opts)) + end + @doc """ Retrieves the context that will be sent to the Honeybadger API when an exception occurs in the current process. @@ -216,7 +304,7 @@ defmodule Honeybadger do """ @spec context() :: map() def context do - Logger.metadata() |> Map.new() + Logger.metadata() |> Map.new() |> Map.delete(Collector.metadata_key()) end @doc """ @@ -291,6 +379,30 @@ defmodule Honeybadger do # Helpers + # Allows for Notice breadcrumb to have custom text as message if an error is + # not passed to the notice function. We can assume if it was passed an error + # then there will be an error breadcrumb right before this one. + defp notice_breadcrumb(exception) do + reason = + case exception do + title when is_binary(title) -> + title + + error when is_atom(error) and not is_nil(error) -> + :error + |> Exception.normalize(error) + |> Map.get(:message, to_string(error)) + + _ -> + nil + end + + ["Honeybadger Notice", reason] + |> Enum.reject(&is_nil/1) + |> Enum.join(": ") + |> Breadcrumb.new(category: "notice") + end + defp put_dynamic_env(config) do hostname = fn -> :inet.gethostname() diff --git a/lib/honeybadger/breadcrumbs/breadcrumb.ex b/lib/honeybadger/breadcrumbs/breadcrumb.ex new file mode 100644 index 00000000..744ffeb8 --- /dev/null +++ b/lib/honeybadger/breadcrumbs/breadcrumb.ex @@ -0,0 +1,43 @@ +defmodule Honeybadger.Breadcrumbs.Breadcrumb do + @moduledoc false + + @derive Jason.Encoder + + @type t :: %__MODULE__{ + message: String.t(), + category: String.t(), + timestamp: DateTime.t(), + metadata: map() + } + + @type opts :: [{:metadata, map()} | {:category, String.t()}] + @enforce_keys [:message, :category, :timestamp, :metadata] + + @default_category "custom" + @default_metadata %{} + + defstruct [:message, :category, :timestamp, :metadata] + + @spec new(String.t(), opts()) :: t() + def new(message, opts) do + %__MODULE__{ + message: message, + category: opts[:category] || @default_category, + timestamp: DateTime.utc_now(), + metadata: opts[:metadata] || @default_metadata + } + end + + @spec from_error(any()) :: t() + def from_error(error) do + error = Exception.normalize(:error, error, []) + + %{__struct__: error_mod} = error + + new( + Honeybadger.Utils.module_to_string(error_mod), + metadata: %{message: error_mod.message(error)}, + category: "error" + ) + end +end diff --git a/lib/honeybadger/breadcrumbs/collector.ex b/lib/honeybadger/breadcrumbs/collector.ex new file mode 100644 index 00000000..fe06da7b --- /dev/null +++ b/lib/honeybadger/breadcrumbs/collector.ex @@ -0,0 +1,59 @@ +defmodule Honeybadger.Breadcrumbs.Collector do + @moduledoc false + + @doc """ + The Collector provides an interface for accessing and affecting the current + set of breadcrumbs. Most operations are delegated to the supplied Buffer + implementation. This is mainly for internal use. + """ + + alias Honeybadger.Breadcrumbs.{RingBuffer, Breadcrumb} + alias Honeybadger.Utils + + @buffer_impl RingBuffer + @buffer_size 40 + @metadata_key :hb_breadcrumbs + + @type t :: %{enabled: boolean(), trail: [Breadcrumb.t()]} + + @spec output() :: t() + def output(), do: output(breadcrumbs()) + + @spec output(@buffer_impl.t()) :: t() + def output(breadcrumbs) do + %{ + enabled: Honeybadger.get_env(:breadcrumbs_enabled), + trail: @buffer_impl.to_list(breadcrumbs) + } + end + + @spec put(@buffer_impl.t(), Breadcrumb.t()) :: @buffer_impl.t() + def put(breadcrumbs, breadcrumb) do + @buffer_impl.add( + breadcrumbs, + Map.update(breadcrumb, :metadata, %{}, &Utils.sanitize(&1, max_depth: 1)) + ) + end + + @spec add(Breadcrumb.t()) :: :ok + def add(breadcrumb) do + if Honeybadger.get_env(:breadcrumbs_enabled) do + Logger.metadata([{@metadata_key, put(breadcrumbs(), breadcrumb)}]) + end + + :ok + end + + @spec clear() :: :ok + def clear() do + Logger.metadata([{@metadata_key, @buffer_impl.new(@buffer_size)}]) + end + + def metadata_key(), do: @metadata_key + + @spec breadcrumbs() :: @buffer_impl.t() + def breadcrumbs() do + Logger.metadata() + |> Keyword.get(@metadata_key, @buffer_impl.new(@buffer_size)) + end +end diff --git a/lib/honeybadger/breadcrumbs/ring_buffer.ex b/lib/honeybadger/breadcrumbs/ring_buffer.ex new file mode 100644 index 00000000..c1362aa1 --- /dev/null +++ b/lib/honeybadger/breadcrumbs/ring_buffer.ex @@ -0,0 +1,24 @@ +defmodule Honeybadger.Breadcrumbs.RingBuffer do + @moduledoc false + + @type t :: %__MODULE__{buffer: [any()], size: pos_integer(), ct: non_neg_integer()} + + defstruct [:size, buffer: [], ct: 0] + + @spec new(pos_integer()) :: t() + def new(size) do + %__MODULE__{size: size} + end + + @spec add(t(), any()) :: t() + def add(ring = %{ct: ct, size: ct, buffer: [_head | rest]}, item) do + %__MODULE__{ring | buffer: rest ++ [item]} + end + + def add(ring = %{ct: ct, buffer: buffer}, item) do + %__MODULE__{ring | buffer: buffer ++ [item], ct: ct + 1} + end + + @spec to_list(t()) :: [any()] + def to_list(%{buffer: buffer}), do: buffer +end diff --git a/lib/honeybadger/breadcrumbs/telemetry.ex b/lib/honeybadger/breadcrumbs/telemetry.ex new file mode 100644 index 00000000..d15d8bf0 --- /dev/null +++ b/lib/honeybadger/breadcrumbs/telemetry.ex @@ -0,0 +1,91 @@ +defmodule Honeybadger.Breadcrumbs.Telemetry do + @moduledoc false + + @spec telemetry_events() :: [[atom()]] + def telemetry_events do + [] + |> append_phoenix_events() + |> append_ecto_events() + end + + @spec attach() :: :ok + def attach do + :telemetry.attach_many( + "hb-telemetry", + telemetry_events(), + &handle_telemetry/4, + nil + ) + + :ok + end + + @spec append_phoenix_events([[atom()]]) :: [[atom()]] + defp append_phoenix_events(events) do + Enum.concat( + events, + [[:phoenix, :router_dispatch, :start]] + ) + end + + @spec append_ecto_events([[atom()]]) :: [[atom()]] + defp append_ecto_events(events) do + Honeybadger.get_env(:ecto_repos) + |> Enum.map(&get_telemetry_prefix/1) + |> Enum.concat(events) + end + + @spec get_telemetry_prefix(Ecto.Repo.t()) :: [atom()] + defp get_telemetry_prefix(repo) do + case Keyword.get(repo.config(), :telemetry_prefix) do + nil -> + [] + + telemetry_prefix -> + telemetry_prefix ++ [:query] + end + end + + def handle_telemetry(_path, %{decode_time: _} = time, %{query: _} = meta, _) do + Map.merge(time, meta) + |> handle_sql() + end + + def handle_telemetry(_path, _time, %{query: _} = meta, _) do + handle_sql(meta) + end + + def handle_telemetry([:phoenix, :router_dispatch, :start], _timing, meta, _) do + metadata = + meta + |> Map.take([:plug, :plug_opts, :route, :pipe_through]) + |> Map.update(:pipe_through, "", &inspect/1) + + Honeybadger.add_breadcrumb("Phoenix Router Dispatch", + metadata: metadata, + category: "request" + ) + end + + defp handle_sql(meta) do + metadata = + meta + |> Map.take([:query, :decode_time, :query_time, :queue_time, :source]) + |> Map.update(:decode_time, nil, &time_format/1) + |> Map.update(:query_time, nil, &time_format/1) + |> Map.update(:queue_time, nil, &time_format/1) + + Honeybadger.add_breadcrumb("Ecto SQL Query (#{meta[:source]})", + metadata: metadata, + category: "query" + ) + end + + defp time_format(nil), do: nil + + defp time_format(time) do + us = System.convert_time_unit(time, :native, :microsecond) + ms = div(us, 100) / 10 + "#{:io_lib_format.fwrite_g(ms)}ms" + end +end diff --git a/lib/honeybadger/client.ex b/lib/honeybadger/client.ex index ae61e318..d9b7a0b6 100644 --- a/lib/honeybadger/client.ex +++ b/lib/honeybadger/client.ex @@ -24,7 +24,14 @@ defmodule Honeybadger.Client do url: binary() } - defstruct [:api_key, :enabled, :headers, :proxy, :proxy_auth, :url] + defstruct [ + :api_key, + :enabled, + :headers, + :proxy, + :proxy_auth, + :url + ] # API diff --git a/lib/honeybadger/filter.ex b/lib/honeybadger/filter.ex index e6582ddb..0dac99a2 100644 --- a/lib/honeybadger/filter.ex +++ b/lib/honeybadger/filter.ex @@ -1,4 +1,6 @@ defmodule Honeybadger.Filter do + alias Honeybadger.Breadcrumbs.Breadcrumb + @moduledoc """ Specification of user overrideable filter functions. @@ -38,4 +40,17 @@ defmodule Honeybadger.Filter do recently thrown error. """ @callback filter_error_message(String.t()) :: String.t() + + @doc """ + Filter breadcrumbs. This filter function recieves a list of Breadcrumb + structs. You could use any Enum function to constrain the set. Let's say you + want to remove any breadcrumb that have metadata that contain SSN: + + def filter_breadcrumbs(breadcrumbs) do + Enum.reject(breadcrumbs, fn breadcrumb -> do + Map.has_key?(breadcrumb.metadata, :ssn) + end) + end + """ + @callback filter_breadcrumbs([Breadcrumb.t()]) :: [Breadcrumb.t()] end diff --git a/lib/honeybadger/filter/mixin.ex b/lib/honeybadger/filter/mixin.ex index d267c712..ac3dc39b 100644 --- a/lib/honeybadger/filter/mixin.ex +++ b/lib/honeybadger/filter/mixin.ex @@ -36,6 +36,7 @@ defmodule Honeybadger.Filter.Mixin do def filter_cgi_data(cgi_data), do: filter_map(cgi_data) def filter_session(session), do: filter_map(session) def filter_error_message(message), do: message + def filter_breadcrumbs(breadcrumbs), do: breadcrumbs @doc false def filter_map(map) do @@ -43,8 +44,13 @@ defmodule Honeybadger.Filter.Mixin do end def filter_map(map, keys) when is_list(keys) do - filter_keys = Enum.map(keys, &canonicalize(&1)) - drop_keys = Enum.filter(Map.keys(map), &Enum.member?(filter_keys, canonicalize(&1))) + filter_keys = Enum.map(keys, &Honeybadger.Utils.canonicalize/1) + + drop_keys = + Enum.filter( + Map.keys(map), + &Enum.member?(filter_keys, Honeybadger.Utils.canonicalize(&1)) + ) Map.drop(map, drop_keys) end @@ -53,17 +59,12 @@ defmodule Honeybadger.Filter.Mixin do map end - defp canonicalize(key) do - key - |> to_string() - |> String.downcase() - end - defoverridable filter_context: 1, filter_params: 1, filter_cgi_data: 1, filter_session: 1, - filter_error_message: 1 + filter_error_message: 1, + filter_breadcrumbs: 1 end end end diff --git a/lib/honeybadger/json.ex b/lib/honeybadger/json.ex index 327f51d5..861ae934 100644 --- a/lib/honeybadger/json.ex +++ b/lib/honeybadger/json.ex @@ -15,6 +15,9 @@ defmodule Honeybadger.JSON do end end + # Keep from converting DateTime to a map + defp to_encodeable(%DateTime{} = datetime), do: datetime + # struct defp to_encodeable(%_{} = struct) do struct diff --git a/lib/honeybadger/logger.ex b/lib/honeybadger/logger.ex index ab6256de..090d1a6c 100644 --- a/lib/honeybadger/logger.ex +++ b/lib/honeybadger/logger.ex @@ -1,4 +1,6 @@ defmodule Honeybadger.Logger do + alias Honeybadger.Breadcrumbs.{Collector, Breadcrumb} + @moduledoc false @behaviour :gen_event @@ -32,10 +34,10 @@ defmodule Honeybadger.Logger do case Keyword.get(metadata, :crash_reason) do {reason, stacktrace} -> - Honeybadger.notify(reason, full_context, stacktrace) + notify(reason, full_context, stacktrace) reason when is_atom(reason) and not is_nil(reason) -> - Honeybadger.notify(reason, full_context, []) + notify(reason, full_context, []) _ -> :ok @@ -65,6 +67,19 @@ defmodule Honeybadger.Logger do ## Helpers + defp notify(reason, metadata, stacktrace) do + breadcrumbs = + Map.get(metadata, Collector.metadata_key(), Collector.breadcrumbs()) + |> Collector.put(Breadcrumb.from_error(reason)) + + metadata_with_breadcrumbs = + metadata + |> Map.delete(Collector.metadata_key()) + |> Map.put(:breadcrumbs, breadcrumbs) + + Honeybadger.notify(reason, metadata_with_breadcrumbs, stacktrace) + end + @standard_metadata ~w(ancestors callers crash_reason file function line module pid)a defp extract_context(metadata) do diff --git a/lib/honeybadger/notice.ex b/lib/honeybadger/notice.ex index db3164ae..fe214dbb 100644 --- a/lib/honeybadger/notice.ex +++ b/lib/honeybadger/notice.ex @@ -2,6 +2,7 @@ defmodule Honeybadger.Notice do @doc false alias Honeybadger.{Backtrace, Utils} + alias Honeybadger.Breadcrumbs.{Collector} @type error :: %{class: atom | iodata, message: iodata, tags: list, backtrace: list} @type notifier :: %{name: String.t(), url: String.t(), version: String.t()} @@ -19,16 +20,17 @@ defmodule Honeybadger.Notice do notifier: notifier(), server: server(), error: error(), + breadcrumbs: Collector.t(), request: map() } @url get_in(Honeybadger.Mixfile.project(), [:package, :links, "GitHub"]) @version Honeybadger.Mixfile.project()[:version] - @notifier %{name: "Honeybadger Elixir Notifier", url: @url, version: @version} + @notifier %{name: "honeybadger-elixir", language: "elixir", url: @url, version: @version} @derive Jason.Encoder - @enforce_keys [:notifier, :server, :error, :request] - defstruct [:notifier, :server, :error, :request] + @enforce_keys [:breadcrumbs, :notifier, :server, :error, :request] + defstruct [:breadcrumbs, :notifier, :server, :error, :request] @spec new(noticeable(), map(), list()) :: t() def new(error, metadata, stacktrace) @@ -55,7 +57,13 @@ defmodule Honeybadger.Notice do |> Map.get(:plug_env, %{}) |> Map.put(:context, Map.get(metadata, :context, %{})) - filter(%__MODULE__{error: error, request: request, notifier: @notifier, server: server()}) + filter(%__MODULE__{ + breadcrumbs: Map.get(metadata, :breadcrumbs, %{}), + error: error, + request: request, + notifier: @notifier, + server: server() + }) end defp filter(notice) do diff --git a/lib/honeybadger/notice_filter/default.ex b/lib/honeybadger/notice_filter/default.ex index 1bd61102..78d8f1bc 100644 --- a/lib/honeybadger/notice_filter/default.ex +++ b/lib/honeybadger/notice_filter/default.ex @@ -8,6 +8,7 @@ defmodule Honeybadger.NoticeFilter.Default do notice |> Map.put(:request, filter_request(notice.request, filter)) |> Map.put(:error, filter_error(notice.error, filter)) + |> Map.put(:breadcrumbs, filter_breadcrumbs(notice.breadcrumbs, filter)) else notice end @@ -28,6 +29,10 @@ defmodule Honeybadger.NoticeFilter.Default do Map.put(error, :message, filter.filter_error_message(message)) end + defp filter_breadcrumbs(breadcrumbs, filter) do + Map.update(breadcrumbs, :trail, %{}, &filter.filter_breadcrumbs/1) + end + defp apply_filter(request, key, filter_fn) do case Map.get(request, key) do nil -> request diff --git a/lib/honeybadger/plug.ex b/lib/honeybadger/plug.ex index 2cf7f29b..51659d7d 100644 --- a/lib/honeybadger/plug.ex +++ b/lib/honeybadger/plug.ex @@ -37,6 +37,7 @@ if Code.ensure_loaded?(Plug) do """ alias Honeybadger.PlugData + alias Honeybadger.Breadcrumbs.{Breadcrumb, Collector} @doc false defmacro __using__(opts) do @@ -60,6 +61,7 @@ if Code.ensure_loaded?(Plug) do # 404 errors are not reported :ok else + Collector.add(Breadcrumb.from_error(reason)) metadata = @plug_data.metadata(conn, __MODULE__) Honeybadger.notify(reason, metadata, stack) end diff --git a/lib/honeybadger/utils.ex b/lib/honeybadger/utils.ex index 10a75146..89b13a89 100644 --- a/lib/honeybadger/utils.ex +++ b/lib/honeybadger/utils.ex @@ -17,4 +17,80 @@ defmodule Honeybadger.Utils do |> Module.split() |> Enum.join(".") end + + @doc """ + Transform value into a consistently cased string representation + + # Example + + iex> Honeybadger.Utils.canonicalize(:User_SSN) + "user_ssn" + + """ + def canonicalize(val) do + val + |> to_string() + |> String.downcase() + end + + @doc """ + Configurable data sanitization. This currently: + + - recursively truncates deep structures (to a depth of 20) + - constrains large string values (to 64k) + - filters out any map keys that might contain sensitive information. + """ + @depth_token "[DEPTH]" + @truncated_token "[TRUNCATED]" + @filtered_token "[FILTERED]" + + # 64k with enough space to concat truncated_token + @default_max_string_size 64 * 1024 - 11 + @default_max_depth 20 + + def sanitize(value, opts \\ []) do + base = %{ + max_depth: @default_max_depth, + max_string_size: @default_max_string_size, + filter_keys: Honeybadger.get_env(:filter_keys) + } + + opts = + Enum.into(opts, base) + |> Map.update!(:filter_keys, fn v -> MapSet.new(v, &canonicalize/1) end) + + sanitize_val(value, Map.put(opts, :depth, 0)) + end + + defp sanitize_val(v, %{depth: depth, max_depth: depth}) when is_map(v) or is_list(v) do + @depth_token + end + + defp sanitize_val(%{__struct__: _} = struct, opts) do + sanitize_val(Map.from_struct(struct), opts) + end + + defp sanitize_val(v, %{depth: depth, filter_keys: filter_keys} = opts) when is_map(v) do + for {key, val} <- v, into: %{} do + if MapSet.member?(filter_keys, canonicalize(key)) do + {key, @filtered_token} + else + {key, sanitize_val(val, Map.put(opts, :depth, depth + 1))} + end + end + end + + defp sanitize_val(v, %{depth: depth} = opts) when is_list(v) do + Enum.map(v, &sanitize_val(&1, Map.put(opts, :depth, depth + 1))) + end + + defp sanitize_val(v, %{max_string_size: max_string_size}) when is_binary(v) do + if String.valid?(v) and String.length(v) > max_string_size do + String.slice(v, 0, max_string_size) <> @truncated_token + else + v + end + end + + defp sanitize_val(v, _), do: v end diff --git a/mix.exs b/mix.exs index 2e93d7d2..e7785b0d 100644 --- a/mix.exs +++ b/mix.exs @@ -22,7 +22,7 @@ defmodule Honeybadger.Mixfile do # Dialyzer dialyzer: [ - plt_add_apps: [:plug, :mix], + plt_add_apps: [:plug, :mix, :ecto], flags: [:error_handling, :race_conditions, :underspecs] ], @@ -38,10 +38,12 @@ defmodule Honeybadger.Mixfile do def application do [ - applications: [:hackney, :logger, :jason], + applications: [:hackney, :logger, :jason, :telemetry], env: [ api_key: {:system, "HONEYBADGER_API_KEY"}, app: nil, + breadcrumbs_enabled: false, + ecto_repos: [], environment_name: Mix.env(), exclude_envs: [:dev, :test], origin: "https://api.honeybadger.io", @@ -65,7 +67,9 @@ defmodule Honeybadger.Mixfile do {:hackney, "~> 1.1"}, {:jason, "~> 1.0"}, {:plug, ">= 1.0.0 and < 2.0.0", optional: true}, + {:ecto, ">= 2.0.0", optional: true}, {:phoenix, ">= 1.0.0 and < 2.0.0", optional: true}, + {:telemetry, "~> 0.4"}, # Dev dependencies {:ex_doc, "~> 0.7", only: :dev, runtime: false}, diff --git a/mix.lock b/mix.lock index c5f3ebf7..822a58d6 100644 --- a/mix.lock +++ b/mix.lock @@ -2,8 +2,10 @@ "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"}, + "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.5", "0db71c8290b5bc81cb0101a2a507a76dca659513984d683119ee722828b424f6", [:mix], [], "hexpm"}, + "ecto": {:hex, :ecto, "3.2.1", "a0f9af0fb50b19d3bb6237e512ac0ba56ea222c2bbea92e7c6c94897932c76ba", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/honeybadger/breadcrumbs/collector_test.exs b/test/honeybadger/breadcrumbs/collector_test.exs new file mode 100644 index 00000000..b2668aa4 --- /dev/null +++ b/test/honeybadger/breadcrumbs/collector_test.exs @@ -0,0 +1,67 @@ +defmodule Honeybadger.Breadcrumbs.CollectorTest do + use Honeybadger.Case, async: true + + alias Honeybadger.Breadcrumbs.{Collector, Breadcrumb} + + test "stores and outputs data" do + with_config([breadcrumbs_enabled: true], fn -> + bc1 = Breadcrumb.new("test1", []) + bc2 = Breadcrumb.new("test2", []) + Collector.add(bc1) + Collector.add(bc2) + + assert Collector.output() == %{ + enabled: true, + trail: [bc1, bc2] + } + end) + end + + test "runs metadata through sanitizer" do + with_config([breadcrumbs_enabled: true], fn -> + bc1 = + Breadcrumb.new("test1", + metadata: %{ + key1: %{key2: 12} + } + ) + + Collector.add(bc1) + + assert List.first(Collector.output()[:trail]).metadata == %{ + key1: "[DEPTH]" + } + end) + end + + test "ignores when breadcrumbs are disabled" do + Collector.add("test1") + Collector.add("test2") + + assert Collector.output() == %{ + enabled: false, + trail: [] + } + end + + test "clearing data" do + with_config([breadcrumbs_enabled: true], fn -> + Collector.add(Breadcrumb.new("test1", [])) + Collector.clear() + + assert Collector.output()[:trail] == [] + end) + end + + test "allows put operation on supplied breadcrumb buffer" do + with_config([breadcrumbs_enabled: true], fn -> + bc = Breadcrumb.new("test1", []) + + breadcrumbs = + Collector.breadcrumbs() + |> Collector.put(bc) + + assert Collector.output(breadcrumbs)[:trail] == [bc] + end) + end +end diff --git a/test/honeybadger/breadcrumbs/ring_buffer_test.exs b/test/honeybadger/breadcrumbs/ring_buffer_test.exs new file mode 100644 index 00000000..2bebfdd1 --- /dev/null +++ b/test/honeybadger/breadcrumbs/ring_buffer_test.exs @@ -0,0 +1,21 @@ +defmodule Honeybadger.Breadcrumbs.RingBufferTest do + use ExUnit.Case, async: true + + alias Honeybadger.Breadcrumbs.RingBuffer + + test "adds items" do + buffer = RingBuffer.new(2) |> RingBuffer.add(:item) |> RingBuffer.to_list() + assert buffer == [:item] + end + + test "shifts when limit is hit" do + buffer = + RingBuffer.new(2) + |> RingBuffer.add(:a) + |> RingBuffer.add(:b) + |> RingBuffer.add(:c) + |> RingBuffer.to_list() + + assert buffer == [:b, :c] + end +end diff --git a/test/honeybadger/breadcrumbs/telemetry_test.exs b/test/honeybadger/breadcrumbs/telemetry_test.exs new file mode 100644 index 00000000..5e95a845 --- /dev/null +++ b/test/honeybadger/breadcrumbs/telemetry_test.exs @@ -0,0 +1,78 @@ +defmodule Honeybadger.Breadcrumbs.TelemetryTest do + use Honeybadger.Case, async: true + + alias Honeybadger.Breadcrumbs.{Telemetry, Collector} + + defmodule Test.MockEctoConfig do + def config() do + [telemetry_prefix: [:a, :b]] + end + end + + test "inserts ecto telemetry events" do + with_config([ecto_repos: [Test.MockEctoConfig]], fn -> + assert Telemetry.telemetry_events() == [ + [:a, :b, :query], + [:phoenix, :router_dispatch, :start] + ] + end) + end + + test "works without ecto" do + assert Telemetry.telemetry_events() == [[:phoenix, :router_dispatch, :start]] + end + + test "produces merged sql breadcrumb" do + with_config([breadcrumbs_enabled: true], fn -> + query = "SELECT * from table" + + Telemetry.handle_telemetry( + [], + %{decode_time: 66_000_000}, + %{query: query, source: "here"}, + nil + ) + + bc = latest_breadcrumb() + assert bc.message == "Ecto SQL Query (here)" + assert bc.metadata[:decode_time] == "66.0ms" + end) + end + + test "produces sql breadcrumb without telemetry measurements" do + with_config([breadcrumbs_enabled: true], fn -> + query = "SELECT * from table" + + Telemetry.handle_telemetry( + [], + 4000, + %{query: query, source: "table", query_time: 4_000_000}, + nil + ) + + bc = latest_breadcrumb() + assert bc.message == "Ecto SQL Query (table)" + assert bc.metadata[:query_time] == "4.0ms" + end) + end + + test "produces phoenix router breadcrumb" do + with_config([breadcrumbs_enabled: true], fn -> + Telemetry.handle_telemetry( + [:phoenix, :router_dispatch, :start], + 4000, + %{plug: "Test.Controller", pipe_through: [:a, :b]}, + nil + ) + + bc = latest_breadcrumb() + assert bc.message == "Phoenix Router Dispatch" + assert bc.metadata[:plug] == "Test.Controller" + assert bc.metadata[:pipe_through] == "[:a, :b]" + end) + end + + defp latest_breadcrumb() do + hd(Collector.breadcrumbs().buffer) + end +end diff --git a/test/honeybadger/json_test.exs b/test/honeybadger/json_test.exs index bc444459..bff14b7f 100644 --- a/test/honeybadger/json_test.exs +++ b/test/honeybadger/json_test.exs @@ -17,6 +17,7 @@ defmodule Honeybadger.JSONTest do assert encoded =~ ~s|"server"| assert encoded =~ ~s|"error"| assert encoded =~ ~s|"request"| + assert encoded =~ ~s|"breadcrumbs"| end test "encodes notice when context has structs" do diff --git a/test/honeybadger/logger_test.exs b/test/honeybadger/logger_test.exs index 7ccf56c2..70a1122a 100644 --- a/test/honeybadger/logger_test.exs +++ b/test/honeybadger/logger_test.exs @@ -6,7 +6,7 @@ defmodule Honeybadger.LoggerTest do setup do {:ok, _} = Honeybadger.API.start(self()) - restart_with_config(exclude_envs: []) + restart_with_config(exclude_envs: [], breadcrumbs_enabled: true) on_exit(&Honeybadger.API.stop/0) end @@ -32,7 +32,10 @@ defmodule Honeybadger.LoggerTest do GenServer.cast(pid, :raise_error) - assert_receive {:api_request, %{"error" => error, "request" => request}} + assert_receive {:api_request, + %{"breadcrumbs" => breadcrumbs, "error" => error, "request" => request}} + + assert List.first(breadcrumbs["trail"])["message"] == "KeyError" assert error["class"] == "KeyError" @@ -63,7 +66,10 @@ defmodule Honeybadger.LoggerTest do :gen_event.notify(manager, :raise_error) - assert_receive {:api_request, %{"error" => error, "request" => request}} + assert_receive {:api_request, + %{"breadcrumbs" => breadcrumbs, "error" => error, "request" => request}} + + assert List.first(breadcrumbs["trail"])["message"] == "RuntimeError" assert error["class"] == "RuntimeError" @@ -75,7 +81,10 @@ defmodule Honeybadger.LoggerTest do test "process raising an error" do pid = spawn(fn -> raise "Oops" end) - assert_receive {:api_request, %{"error" => error, "request" => request}} + assert_receive {:api_request, + %{"breadcrumbs" => breadcrumbs, "error" => error, "request" => request}} + + assert List.first(breadcrumbs["trail"])["message"] == "RuntimeError" assert error["class"] == "RuntimeError" @@ -85,7 +94,10 @@ defmodule Honeybadger.LoggerTest do test "task with anonymous function raising an error" do Task.start(fn -> raise "Oops" end) - assert_receive {:api_request, %{"error" => error, "request" => request}} + assert_receive {:api_request, + %{"breadcrumbs" => breadcrumbs, "error" => error, "request" => request}} + + assert List.first(breadcrumbs["trail"])["message"] == "RuntimeError" assert error["class"] == "RuntimeError" assert error["message"] == "Oops" @@ -101,7 +113,10 @@ defmodule Honeybadger.LoggerTest do Task.start(MyModule, :raise_error, ["my message"]) - assert_receive {:api_request, %{"error" => error, "request" => request}} + assert_receive {:api_request, + %{"breadcrumbs" => breadcrumbs, "error" => error, "request" => request}} + + assert List.first(breadcrumbs["trail"])["metadata"]["message"] == "my message" assert request["context"]["function"] =~ "&Honeybadger.LoggerTest.MyModule.raise_error/1" assert request["context"]["args"] == ~s(["my message"]) @@ -114,7 +129,9 @@ defmodule Honeybadger.LoggerTest do raise "Oops" end) - assert_receive {:api_request, %{"request" => request}} + assert_receive {:api_request, %{"breadcrumbs" => breadcrumbs, "request" => request}} + + assert List.first(breadcrumbs["trail"])["metadata"]["message"] == "Oops" assert request["context"]["age"] == 2 assert request["context"]["name"] == "Danny" diff --git a/test/honeybadger/notice_test.exs b/test/honeybadger/notice_test.exs index 2750a114..c0efd825 100644 --- a/test/honeybadger/notice_test.exs +++ b/test/honeybadger/notice_test.exs @@ -4,6 +4,7 @@ defmodule Honeybadger.NoticeTest do doctest Honeybadger.Notice alias Honeybadger.Notice + alias Honeybadger.Breadcrumbs.Breadcrumb setup do exception = %RuntimeError{message: "Oops"} @@ -25,7 +26,8 @@ defmodule Honeybadger.NoticeTest do test "notifier information", %{notice: %Notice{notifier: notifier}} do assert "https://github.com/honeybadger-io/honeybadger-elixir" == notifier[:url] - assert "Honeybadger Elixir Notifier" == notifier[:name] + assert "honeybadger-elixir" == notifier[:name] + assert "elixir" == notifier[:language] assert Honeybadger.Mixfile.project()[:version] == notifier[:version] end @@ -45,6 +47,18 @@ defmodule Honeybadger.NoticeTest do end) end + test "with breadcrumbs", _ do + breadcrumbs = %{ + enabled: true, + trail: [] + } + + %Notice{breadcrumbs: to_breadcrumbs} = + Notice.new(%RuntimeError{message: "Oops"}, %{breadcrumbs: breadcrumbs}, []) + + assert breadcrumbs == to_breadcrumbs + end + test "error information", %{notice: %Notice{error: error}} do assert "RuntimeError" == error[:class] assert "Oops" == error[:message] @@ -95,6 +109,8 @@ defmodule Honeybadger.NoticeTest do def filter_error_message(message), do: Regex.replace(~r/(Secret data: )(\w+)/, message, "\\1 xxx") + + def filter_breadcrumbs(_breadcrumbs), do: [999] end with_config([filter: TestFilter], fn -> @@ -107,6 +123,7 @@ defmodule Honeybadger.NoticeTest do assert get_in(notice.request, [:params, "credit_card"]) refute notice.error.message =~ "XYZZY" refute get_in(notice.request, [:params, "token"]) + assert notice.breadcrumbs.trail == [999] end) end @@ -219,6 +236,12 @@ defmodule Honeybadger.NoticeTest do metadata = %{ context: %{password: "123", foo: "foo"}, + breadcrumbs: %{ + active: true, + trail: [ + Breadcrumb.new("my message", %{}) + ] + }, plug_env: %{ url: "/some/secret/place", component: SomeApp.PageController, diff --git a/test/honeybadger/plug_test.exs b/test/honeybadger/plug_test.exs index 9a04afdb..8061389e 100644 --- a/test/honeybadger/plug_test.exs +++ b/test/honeybadger/plug_test.exs @@ -38,7 +38,7 @@ defmodule Honeybadger.PlugTest do on_exit(&Honeybadger.API.stop/0) - restart_with_config(exclude_envs: []) + restart_with_config(exclude_envs: [], breadcrumbs_enabled: true) end test "errors are reported" do @@ -47,7 +47,9 @@ defmodule Honeybadger.PlugTest do assert %WrapperError{reason: reason} = catch_error(PlugApp.call(conn, [])) assert %RuntimeError{message: "Oops"} = reason - assert_receive {:api_request, _} + assert_receive {:api_request, %{"breadcrumbs" => breadcrumbs}} + + assert List.first(breadcrumbs["trail"])["metadata"]["message"] == "Oops" end test "not found errors for plug are ignored" do diff --git a/test/honeybadger/utils_test.exs b/test/honeybadger/utils_test.exs index 33291481..51615976 100644 --- a/test/honeybadger/utils_test.exs +++ b/test/honeybadger/utils_test.exs @@ -2,4 +2,49 @@ defmodule Honeybadger.UtilsTest do use ExUnit.Case, async: true doctest Honeybadger.Utils + alias Honeybadger.Utils + + test "sanitize drops nested hash based on depth" do + item = %{ + a: %{ + b: 12, + m: %{ + j: 3 + } + }, + c: "string" + } + + assert Utils.sanitize(item, max_depth: 2) == %{ + a: %{ + b: 12, + m: "[DEPTH]" + }, + c: "string" + } + end + + test "sanitize drops nested lists based on depth" do + item = [[[a: 12]], 1, 2, 3] + + assert Utils.sanitize(item, max_depth: 2) == [["[DEPTH]"], 1, 2, 3] + end + + test "sanitize truncates strings" do + item = "123456789" + + assert Utils.sanitize(item, max_string_size: 3) == "123[TRUNCATED]" + end + + test "sanitize removes filtered_keys" do + item = %{ + filter_me: "secret stuff", + okay: "not a secret at all" + } + + assert Utils.sanitize(item, filter_keys: [:filter_me]) == %{ + filter_me: "[FILTERED]", + okay: "not a secret at all" + } + end end