From d6facc7809bef1e31996855d845bff9bed599198 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 22 Jun 2024 21:30:49 -0400 Subject: [PATCH 1/4] Autocomplete logic extraction --- lib/philomena/autocomplete.ex | 212 +++-------------------- lib/philomena/autocomplete/generator.ex | 147 ++++++++++++++++ lib/philomena/tags/local_autocomplete.ex | 101 +++++++++++ 3 files changed, 270 insertions(+), 190 deletions(-) create mode 100644 lib/philomena/autocomplete/generator.ex create mode 100644 lib/philomena/tags/local_autocomplete.ex diff --git a/lib/philomena/autocomplete.ex b/lib/philomena/autocomplete.ex index ec5e04ec0..f9cc9f445 100644 --- a/lib/philomena/autocomplete.ex +++ b/lib/philomena/autocomplete.ex @@ -1,19 +1,32 @@ defmodule Philomena.Autocomplete do @moduledoc """ Pregenerated autocomplete files. + + These are used to eliminate the latency of looking up search results on the server. + A script can parse the binary and generate results directly as the user types, without + incurring any roundtrip penalty. """ import Ecto.Query, warn: false alias Philomena.Repo - alias Philomena.Tags.Tag - alias Philomena.Images.Tagging alias Philomena.Autocomplete.Autocomplete + alias Philomena.Autocomplete.Generator + + @doc """ + Gets the current local autocompletion binary. + + Returns nil if the binary is not currently generated. + + ## Examples + + iex> get_artist_link() + nil - @type tags_list() :: [{String.t(), number(), number(), String.t() | nil}] - @type assoc_map() :: %{String.t() => [number()]} + iex> get_autocomplete() + %Autocomplete{} - @spec get_autocomplete() :: Autocomplete.t() | nil + """ def get_autocomplete do Autocomplete |> order_by(desc: :created_at) @@ -21,103 +34,11 @@ defmodule Philomena.Autocomplete do |> Repo.one() end + @doc """ + Creates a new local autocompletion binary, replacing any which currently exist. + """ def generate_autocomplete! do - tags = get_tags() - associations = get_associations(tags) - - # Tags are already sorted, so just add them to the file directly - # - # struct tag { - # uint8_t key_length; - # uint8_t key[]; - # uint8_t association_length; - # uint32_t associations[]; - # }; - # - - {ac_file, name_locations} = - Enum.reduce(tags, {<<>>, %{}}, fn {name, _, _, _}, {file, name_locations} -> - pos = byte_size(file) - assn = Map.get(associations, name, []) - assn_bin = for id <- assn, into: <<>>, do: <> - - { - <>, - Map.put(name_locations, name, pos) - } - end) - - # Link reference list; self-referential, so must be preprocessed to deal with aliases - # - # struct tag_reference { - # uint32_t tag_location; - # uint8_t is_aliased : 1; - # union { - # uint32_t num_uses : 31; - # uint32_t alias_index : 31; - # }; - # }; - # - - ac_file = int32_align(ac_file) - reference_start = byte_size(ac_file) - - reference_indexes = - tags - |> Enum.with_index() - |> Enum.map(fn {{name, _, _, _}, index} -> {name, index} end) - |> Map.new() - - references = - Enum.reduce(tags, <<>>, fn {name, images_count, _, alias_target}, references -> - pos = Map.fetch!(name_locations, name) - - if not is_nil(alias_target) do - target = Map.fetch!(reference_indexes, alias_target) - - <> - else - <> - end - end) - - # Reorder tags by name in their namespace to provide a secondary ordering - # - # struct secondary_reference { - # uint32_t primary_location; - # }; - # - - secondary_references = - tags - |> Enum.map(&{name_in_namespace(elem(&1, 0)), elem(&1, 0)}) - |> Enum.sort() - |> Enum.reduce(<<>>, fn {_k, v}, secondary_references -> - target = Map.fetch!(reference_indexes, v) - - <> - end) - - # Finally add the reference start and number of tags in the footer - # - # struct autocomplete_file { - # struct tag tags[]; - # struct tag_reference primary_references[]; - # struct secondary_reference secondary_references[]; - # uint32_t format_version; - # uint32_t reference_start; - # uint32_t num_tags; - # }; - # - - ac_file = << - ac_file::binary, - references::binary, - secondary_references::binary, - 2::32-little, - reference_start::32-little, - length(tags)::32-little - >> + ac_file = Generator.generate() # Insert the autocomplete binary new_ac = @@ -130,93 +51,4 @@ defmodule Philomena.Autocomplete do |> where([ac], ac.created_at < ^new_ac.created_at) |> Repo.delete_all() end - - # - # Get the names of tags and their number of uses as a map. - # Sort is done in the application to avoid collation. - # - @spec get_tags() :: tags_list() - defp get_tags do - top_tags = - Tag - |> select([t], {t.name, t.images_count, t.id, nil}) - |> where([t], t.images_count > 0) - |> order_by(desc: :images_count) - |> limit(50_000) - |> Repo.all() - - aliases_of_top_tags = - Tag - |> where([t], t.aliased_tag_id in ^Enum.map(top_tags, fn {_, _, id, _} -> id end)) - |> join(:inner, [t], _ in assoc(t, :aliased_tag)) - |> select([t, a], {t.name, 0, 0, a.name}) - |> Repo.all() - - (aliases_of_top_tags ++ top_tags) - |> Enum.filter(fn {name, _, _, _} -> byte_size(name) < 255 end) - |> Enum.sort() - end - - # - # Get up to eight associated tag ids for each returned tag. - # - @spec get_associations(tags_list()) :: assoc_map() - defp get_associations(tags) do - tags - |> Enum.filter(fn {_, _, _, aliased} -> is_nil(aliased) end) - |> Enum.map(fn {name, images_count, id, _} -> - # Randomly sample 100 images with this tag - image_sample = - Tagging - |> where(tag_id: ^id) - |> select([it], it.image_id) - |> order_by(asc: fragment("random()")) - |> limit(100) - - # Select the tags from those images which have more uses than - # the current one being considered, and overlap more than 50% - assoc_ids = - Tagging - |> join(:inner, [it], _ in assoc(it, :tag)) - |> where([_, t], t.images_count > ^images_count) - |> where([it, _], it.image_id in subquery(image_sample)) - |> group_by([_, t], t.id) - |> order_by(desc: fragment("count(*)")) - |> having([_, t], fragment("(100 * count(*)::float / LEAST(?, 100)) > 50", ^images_count)) - |> select([_, t], t.id) - |> limit(8) - |> Repo.all(timeout: 120_000) - - {name, assoc_ids} - end) - |> Map.new() - end - - # - # Right-pad a binary to be a multiple of 4 bytes. - # - @spec int32_align(binary()) :: binary() - defp int32_align(bin) do - pad_bits = 8 * (4 - rem(byte_size(bin), 4)) - - <> - end - - # - # Remove the artist:, oc: etc. prefix from a tag name, - # if one is present. - # - @spec name_in_namespace(String.t()) :: String.t() - defp name_in_namespace(s) do - case String.split(s, ":", parts: 2, trim: true) do - [_namespace, name] -> - name - - [name] -> - name - - _unknown -> - s - end - end end diff --git a/lib/philomena/autocomplete/generator.ex b/lib/philomena/autocomplete/generator.ex new file mode 100644 index 000000000..6493027d0 --- /dev/null +++ b/lib/philomena/autocomplete/generator.ex @@ -0,0 +1,147 @@ +defmodule Philomena.Autocomplete.Generator do + @moduledoc """ + Compiled autocomplete binary for frontend usage. + + See assets/js/utils/local-autocompleter.ts for how this should be used. + The file follows the following binary format: + + struct tag { + uint8_t key_length; + uint8_t key[]; + uint8_t association_length; + uint32_t associations[]; + }; + + struct tag_reference { + uint32_t tag_location; + union { + int32_t raw; + uint32_t num_uses; ///< when positive + uint32_t alias_index; ///< when negative, -alias_index - 1 + }; + }; + + struct secondary_reference { + uint32_t primary_location; + }; + + struct autocomplete_file { + struct tag tags[]; + struct tag_reference primary_references[]; + struct secondary_reference secondary_references[]; + uint32_t format_version; + uint32_t reference_start; + uint32_t num_tags; + }; + + """ + + alias Philomena.Tags.LocalAutocomplete + + @format_version 2 + @top_tags 50_000 + @max_associations 8 + + @doc """ + Create the compiled autocomplete binary. + + See module documentation for the format. This is not expected to be larger + than a few megabytes on average. + """ + @spec generate() :: binary() + def generate do + {tags, associations} = tags_and_associations() + + # Tags are already sorted, so just add them to the file directly + {tag_block, name_locations} = + Enum.reduce(tags, {<<>>, %{}}, fn %{name: name}, {data, name_locations} -> + pos = byte_size(data) + assn = Map.get(associations, name, []) + assn_bin = for id <- assn, into: <<>>, do: <> + + { + <>, + Map.put(name_locations, name, pos) + } + end) + + # Link reference list; self-referential, so must be preprocessed to deal with aliases + tag_block = int32_align(tag_block) + reference_start = byte_size(tag_block) + + reference_indexes = + tags + |> Enum.with_index() + |> Enum.map(fn {entry, index} -> {entry.name, index} end) + |> Map.new() + + references = + Enum.reduce(tags, <<>>, fn entry, references -> + pos = Map.fetch!(name_locations, entry.name) + + if not is_nil(entry.alias_name) do + target = Map.fetch!(reference_indexes, entry.alias_name) + + <> + else + <> + end + end) + + # Reorder tags by name in their namespace to provide a secondary ordering + secondary_references = + tags + |> Enum.map(&{name_in_namespace(&1.name), &1.name}) + |> Enum.sort() + |> Enum.reduce(<<>>, fn {_k, v}, secondary_references -> + target = Map.fetch!(reference_indexes, v) + + <> + end) + + # Finally add the reference start and number of tags in the footer + << + tag_block::binary, + references::binary, + secondary_references::binary, + @format_version::32-little, + reference_start::32-little, + length(tags)::32-little + >> + end + + defp tags_and_associations do + # Names longer than 255 bytes do not fit and will break parsing. + # Sort is done in the application to avoid collation. + tags = + LocalAutocomplete.get_tags(@top_tags) + |> Enum.filter(&(byte_size(&1.name) < 255)) + |> Enum.sort_by(& &1.name) + + associations = + LocalAutocomplete.get_associations(tags, @max_associations) + + {tags, associations} + end + + defp int32_align(bin) do + # Right-pad a binary to be a multiple of 4 bytes. + pad_bits = 8 * (4 - rem(byte_size(bin), 4)) + + <> + end + + defp name_in_namespace(s) do + # Remove the artist:, oc: etc. prefix from a tag name, if one is present. + case String.split(s, ":", parts: 2, trim: true) do + [_namespace, name] -> + name + + [name] -> + name + + _unknown -> + s + end + end +end diff --git a/lib/philomena/tags/local_autocomplete.ex b/lib/philomena/tags/local_autocomplete.ex new file mode 100644 index 000000000..6f1785edf --- /dev/null +++ b/lib/philomena/tags/local_autocomplete.ex @@ -0,0 +1,101 @@ +defmodule Philomena.Tags.LocalAutocomplete do + alias Philomena.Images.Tagging + alias Philomena.Tags.Tag + alias Philomena.Repo + import Ecto.Query + + defmodule Entry do + @moduledoc """ + An individual entry record for autocomplete generation. + """ + + @type t :: %__MODULE__{ + name: String.t(), + images_count: integer(), + id: integer(), + alias_name: String.t() | nil + } + + defstruct name: "", + images_count: 0, + id: 0, + alias_name: nil + end + + @type entry_list() :: [Entry.t()] + + @type tag_id :: integer() + @type assoc_map() :: %{optional(String.t()) => [tag_id()]} + + @doc """ + Get a flat list of entry records for all of the top `amount` tags, and all of their + aliases. + """ + @spec get_tags(integer()) :: entry_list() + def get_tags(amount) do + tags = top_tags(amount) + aliases = aliases_of_tags(tags) + aliases ++ tags + end + + @doc """ + Get a map of tag names to their most associated tag ids. + + For every tag entry, its associated tags satisfy the following properties: + - is not the same as the entry's tag id + - of a sample of 100 images, appear simultaneously more than 50% of the time + """ + @spec get_associations(entry_list(), integer()) :: assoc_map() + def get_associations(tags, amount) do + tags + |> Enum.filter(&is_nil(&1.alias_name)) + |> Map.new(&{&1.name, associated_tag_ids(&1, amount)}) + end + + defp top_tags(amount) do + query = + from t in Tag, + where: t.images_count > 0, + select: %Entry{name: t.name, images_count: t.images_count, id: t.id}, + order_by: [desc: :images_count], + limit: ^amount + + Repo.all(query) + end + + defp aliases_of_tags(tags) do + ids = Enum.map(tags, & &1.id) + + query = + from t in Tag, + where: t.aliased_tag_id in ^ids, + inner_join: a in assoc(t, :aliased_tag), + select: %Entry{name: t.name, images_count: 0, id: 0, alias_name: a.name} + + Repo.all(query) + end + + defp associated_tag_ids(entry, amount) do + image_sample_query = + from it in Tagging, + where: it.tag_id == ^entry.id, + select: it.image_id, + order_by: [asc: fragment("random()")], + limit: 100 + + # Select the tags from those images which have more uses than + # the current one being considered, and overlap more than 50% + assoc_query = + from it in Tagging, + inner_join: t in assoc(it, :tag), + where: t.images_count > ^entry.images_count, + where: it.image_id in subquery(image_sample_query), + group_by: t.id, + order_by: [desc: fragment("count(*)")], + having: fragment("(100 * count(*)::float / LEAST(?, 100)) > 50", ^entry.images_count), + select: t.id, + limit: ^amount + + Repo.all(assoc_query, timeout: 120_000) + end +end From dbbc067679593f013aa63d8443a92a880862a3ea Mon Sep 17 00:00:00 2001 From: Liam Date: Mon, 24 Jun 2024 19:47:03 -0400 Subject: [PATCH 2/4] Fix documentation errors --- lib/philomena/adverts.ex | 2 +- lib/philomena/autocomplete.ex | 2 +- lib/philomena/badges.ex | 14 +++++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/philomena/adverts.ex b/lib/philomena/adverts.ex index 06a1104e8..f1794d8d8 100644 --- a/lib/philomena/adverts.ex +++ b/lib/philomena/adverts.ex @@ -162,7 +162,7 @@ defmodule Philomena.Adverts do iex> update_advert_image(advert, %{image: new_value}) {:ok, %Advert{}} - iex> update_advert(advert, %{image: bad_value}) + iex> update_advert_image(advert, %{image: bad_value}) {:error, %Ecto.Changeset{}} """ diff --git a/lib/philomena/autocomplete.ex b/lib/philomena/autocomplete.ex index f9cc9f445..e496ffc49 100644 --- a/lib/philomena/autocomplete.ex +++ b/lib/philomena/autocomplete.ex @@ -20,7 +20,7 @@ defmodule Philomena.Autocomplete do ## Examples - iex> get_artist_link() + iex> get_autocomplete() nil iex> get_autocomplete() diff --git a/lib/philomena/badges.ex b/lib/philomena/badges.ex index 494dd7525..3cd0cd550 100644 --- a/lib/philomena/badges.ex +++ b/lib/philomena/badges.ex @@ -84,7 +84,7 @@ defmodule Philomena.Badges do end @doc """ - Updates a badge. + Updates a badge without updating its image. ## Examples @@ -101,6 +101,18 @@ defmodule Philomena.Badges do |> Repo.update() end + @doc """ + Updates the image for a badge. + + ## Examples + + iex> update_badge_image(badge, %{image: new_value}) + {:ok, %Badge{}} + + iex> update_badge_image(badge, %{image: bad_value}) + {:error, %Ecto.Changeset{}} + + """ def update_badge_image(%Badge{} = badge, attrs) do badge |> Badge.changeset(attrs) From 318ef681de260efe0cbdddcf01a597f8d6c3c5ca Mon Sep 17 00:00:00 2001 From: Liam Date: Mon, 24 Jun 2024 20:45:56 -0400 Subject: [PATCH 3/4] Bans logic cleanup --- lib/philomena/bans.ex | 167 +++++------------- lib/philomena/bans/finder.ex | 86 +++++++++ lib/philomena/bans/subnet_creator.ex | 27 +++ .../controllers/admin/user_ban_controller.ex | 3 - .../plugs/api_require_authorization_plug.ex | 2 +- lib/philomena_web/plugs/current_ban_plug.ex | 2 +- 6 files changed, 157 insertions(+), 130 deletions(-) create mode 100644 lib/philomena/bans/finder.ex create mode 100644 lib/philomena/bans/subnet_creator.ex diff --git a/lib/philomena/bans.ex b/lib/philomena/bans.ex index 8b6daa6e8..4b4bdcc8e 100644 --- a/lib/philomena/bans.ex +++ b/lib/philomena/bans.ex @@ -4,13 +4,17 @@ defmodule Philomena.Bans do """ import Ecto.Query, warn: false + alias Ecto.Multi alias Philomena.Repo - alias Philomena.UserIps + alias Philomena.Bans.Finder alias Philomena.Bans.Fingerprint + alias Philomena.Bans.SubnetCreator + alias Philomena.Bans.Subnet + alias Philomena.Bans.User @doc """ - Returns the list of fingerprint_bans. + Returns the list of fingerprint bans. ## Examples @@ -23,9 +27,9 @@ defmodule Philomena.Bans do end @doc """ - Gets a single fingerprint. + Gets a single fingerprint ban. - Raises `Ecto.NoResultsError` if the Fingerprint does not exist. + Raises `Ecto.NoResultsError` if the fingerprint ban does not exist. ## Examples @@ -39,7 +43,7 @@ defmodule Philomena.Bans do def get_fingerprint!(id), do: Repo.get!(Fingerprint, id) @doc """ - Creates a fingerprint. + Creates a fingerprint ban. ## Examples @@ -57,7 +61,7 @@ defmodule Philomena.Bans do end @doc """ - Updates a fingerprint. + Updates a fingerprint ban. ## Examples @@ -75,7 +79,7 @@ defmodule Philomena.Bans do end @doc """ - Deletes a Fingerprint. + Deletes a fingerprint ban. ## Examples @@ -91,7 +95,7 @@ defmodule Philomena.Bans do end @doc """ - Returns an `%Ecto.Changeset{}` for tracking fingerprint changes. + Returns an `%Ecto.Changeset{}` for tracking fingerprint ban changes. ## Examples @@ -103,10 +107,8 @@ defmodule Philomena.Bans do Fingerprint.changeset(fingerprint, %{}) end - alias Philomena.Bans.Subnet - @doc """ - Returns the list of subnet_bans. + Returns the list of subnet bans. ## Examples @@ -119,9 +121,9 @@ defmodule Philomena.Bans do end @doc """ - Gets a single subnet. + Gets a single subnet ban. - Raises `Ecto.NoResultsError` if the Subnet does not exist. + Raises `Ecto.NoResultsError` if the subnet ban does not exist. ## Examples @@ -135,7 +137,7 @@ defmodule Philomena.Bans do def get_subnet!(id), do: Repo.get!(Subnet, id) @doc """ - Creates a subnet. + Creates a subnet ban. ## Examples @@ -153,7 +155,7 @@ defmodule Philomena.Bans do end @doc """ - Updates a subnet. + Updates a subnet ban. ## Examples @@ -171,7 +173,7 @@ defmodule Philomena.Bans do end @doc """ - Deletes a Subnet. + Deletes a subnet ban. ## Examples @@ -187,7 +189,7 @@ defmodule Philomena.Bans do end @doc """ - Returns an `%Ecto.Changeset{}` for tracking subnet changes. + Returns an `%Ecto.Changeset{}` for tracking subnet ban changes. ## Examples @@ -199,10 +201,8 @@ defmodule Philomena.Bans do Subnet.changeset(subnet, %{}) end - alias Philomena.Bans.User - @doc """ - Returns the list of user_bans. + Returns the list of user bans. ## Examples @@ -215,9 +215,9 @@ defmodule Philomena.Bans do end @doc """ - Gets a single user. + Gets a single user ban. - Raises `Ecto.NoResultsError` if the User does not exist. + Raises `Ecto.NoResultsError` if the user ban does not exist. ## Examples @@ -231,7 +231,7 @@ defmodule Philomena.Bans do def get_user!(id), do: Repo.get!(User, id) @doc """ - Creates a user. + Creates a user ban. ## Examples @@ -243,31 +243,27 @@ defmodule Philomena.Bans do """ def create_user(creator, attrs \\ %{}) do - %User{banning_user_id: creator.id} - |> User.save_changeset(attrs) - |> Repo.insert() + changeset = + %User{banning_user_id: creator.id} + |> User.save_changeset(attrs) + + Multi.new() + |> Multi.insert(:user_ban, changeset) + |> Multi.run(:subnet_ban, fn _repo, %{user_ban: %{user_id: user_id}} -> + SubnetCreator.create_for_user(creator, user_id, attrs) + end) + |> Repo.transaction() |> case do - {:ok, user_ban} -> - ip = UserIps.get_ip_for_user(user_ban.user_id) - - if ip do - # Automatically create associated IP ban. - ip = UserIps.masked_ip(ip) - - %Subnet{banning_user_id: creator.id, specification: ip} - |> Subnet.save_changeset(attrs) - |> Repo.insert() - end - + {:ok, %{user_ban: user_ban}} -> {:ok, user_ban} - error -> - error + {:error, :user_ban, changeset, _changes} -> + {:error, changeset} end end @doc """ - Updates a user. + Updates a user ban. ## Examples @@ -285,7 +281,7 @@ defmodule Philomena.Bans do end @doc """ - Deletes a User. + Deletes a user ban. ## Examples @@ -301,7 +297,7 @@ defmodule Philomena.Bans do end @doc """ - Returns an `%Ecto.Changeset{}` for tracking user changes. + Returns an `%Ecto.Changeset{}` for tracking user ban changes. ## Examples @@ -314,88 +310,9 @@ defmodule Philomena.Bans do end @doc """ - Returns the first ban, if any, that matches the specified request - attributes. + Returns the first ban, if any, that matches the specified request attributes. """ - def exists_for?(user, ip, fingerprint) do - now = DateTime.utc_now() - - queries = - subnet_query(ip, now) ++ - fingerprint_query(fingerprint, now) ++ - user_query(user, now) - - bans = - queries - |> union_all_queries() - |> Repo.all() - - # Don't return a ban if the user is currently signed in. - case is_nil(user) do - true -> Enum.at(bans, 0) - false -> user_ban(bans) - end - end - - defp fingerprint_query(nil, _now), do: [] - - defp fingerprint_query(fingerprint, now) do - [ - Fingerprint - |> select([f], %{ - reason: f.reason, - valid_until: f.valid_until, - generated_ban_id: f.generated_ban_id, - type: ^"FingerprintBan" - }) - |> where([f], f.enabled and f.valid_until > ^now) - |> where([f], f.fingerprint == ^fingerprint) - ] - end - - defp subnet_query(nil, _now), do: [] - - defp subnet_query(ip, now) do - {:ok, inet} = EctoNetwork.INET.cast(ip) - - [ - Subnet - |> select([s], %{ - reason: s.reason, - valid_until: s.valid_until, - generated_ban_id: s.generated_ban_id, - type: ^"SubnetBan" - }) - |> where([s], s.enabled and s.valid_until > ^now) - |> where(fragment("specification >>= ?", ^inet)) - ] - end - - defp user_query(nil, _now), do: [] - - defp user_query(user, now) do - [ - User - |> select([u], %{ - reason: u.reason, - valid_until: u.valid_until, - generated_ban_id: u.generated_ban_id, - type: ^"UserBan" - }) - |> where([u], u.enabled and u.valid_until > ^now) - |> where([u], u.user_id == ^user.id) - ] - end - - defp union_all_queries([query]), - do: query - - defp union_all_queries([query | rest]), - do: query |> union_all(^union_all_queries(rest)) - - defp user_ban(bans) do - bans - |> Enum.filter(&(&1.type == "UserBan")) - |> Enum.at(0) + def find(user, ip, fingerprint) do + Finder.find(user, ip, fingerprint) end end diff --git a/lib/philomena/bans/finder.ex b/lib/philomena/bans/finder.ex new file mode 100644 index 000000000..f44e1754c --- /dev/null +++ b/lib/philomena/bans/finder.ex @@ -0,0 +1,86 @@ +defmodule Philomena.Bans.Finder do + @moduledoc """ + Helper to find a bans associated with a set of request attributes. + """ + + import Ecto.Query, warn: false + alias Philomena.Repo + + alias Philomena.Bans.Fingerprint + alias Philomena.Bans.Subnet + alias Philomena.Bans.User + + @fingerprint "Fingerprint" + @subnet "Subnet" + @user "User" + + @doc """ + Returns the first ban, if any, that matches the specified request attributes. + """ + def find(user, ip, fingerprint) do + bans = + generate_valid_queries([ + {ip, &subnet_query/2}, + {fingerprint, &fingerprint_query/2}, + {user, &user_query/2} + ]) + |> union_all_queries() + |> Repo.all() + + # Don't return a fingerprint or subnet ban if the user is currently signed in. + case is_nil(user) do + true -> Enum.at(bans, 0) + false -> user_ban(bans) + end + end + + defp query_base(schema, name, now) do + from b in schema, + where: b.enabled and b.valid_until > ^now, + select: %{ + reason: b.reason, + valid_until: b.valid_until, + generated_ban_id: b.generated_ban_id, + type: type(^name, :string) + } + end + + defp fingerprint_query(fingerprint, now) do + Fingerprint + |> query_base(@fingerprint, now) + |> where([f], f.fingerprint == ^fingerprint) + end + + defp subnet_query(ip, now) do + {:ok, inet} = EctoNetwork.INET.cast(ip) + + Subnet + |> query_base(@subnet, now) + |> where(fragment("specification >>= ?", ^inet)) + end + + defp user_query(user, now) do + User + |> query_base(@user, now) + |> where([u], u.user_id == ^user.id) + end + + defp generate_valid_queries(sources) do + now = DateTime.utc_now() + + Enum.flat_map(sources, fn + {nil, _cb} -> [] + {source, cb} -> [cb.(source, now)] + end) + end + + defp union_all_queries([query | rest]) do + Enum.reduce(rest, query, fn q, acc -> union_all(acc, ^q) end) + end + + defp user_ban(bans) do + bans + |> Enum.filter(&(&1.type == @user)) + |> Enum.at(0) + end +end diff --git a/lib/philomena/bans/subnet_creator.ex b/lib/philomena/bans/subnet_creator.ex new file mode 100644 index 000000000..3f54a3c54 --- /dev/null +++ b/lib/philomena/bans/subnet_creator.ex @@ -0,0 +1,27 @@ +defmodule Philomena.Bans.SubnetCreator do + @moduledoc """ + Handles automatic creation of subnet bans for an input user ban. + + This prevents trivial ban evasion with the creation of a new account from the same address. + The user must work around or wait out the subnet ban first. + """ + + alias Philomena.UserIps + alias Philomena.Bans + + @doc """ + Creates a subnet ban for the given user's last known IP address. + + Returns `{:ok, ban}`, `{:ok, nil}`, or `{:error, changeset}`. The return value is + suitable for use as the return value to an `Ecto.Multi.run/3` callback. + """ + def create_for_user(creator, user_id, attrs) do + ip = UserIps.get_ip_for_user(user_id) + + if ip do + Bans.create_subnet(creator, Map.put(attrs, "specification", UserIps.masked_ip(ip))) + else + {:ok, nil} + end + end +end diff --git a/lib/philomena_web/controllers/admin/user_ban_controller.ex b/lib/philomena_web/controllers/admin/user_ban_controller.ex index 24847076f..ff6833c0d 100644 --- a/lib/philomena_web/controllers/admin/user_ban_controller.ex +++ b/lib/philomena_web/controllers/admin/user_ban_controller.ex @@ -53,9 +53,6 @@ defmodule PhilomenaWeb.Admin.UserBanController do |> moderation_log(details: &log_details/2, data: user_ban) |> redirect(to: ~p"/admin/user_bans") - {:error, :user_ban, changeset, _changes} -> - render(conn, "new.html", changeset: changeset) - {:error, changeset} -> render(conn, "new.html", changeset: changeset) end diff --git a/lib/philomena_web/plugs/api_require_authorization_plug.ex b/lib/philomena_web/plugs/api_require_authorization_plug.ex index 118a59c2f..719ccf0b7 100755 --- a/lib/philomena_web/plugs/api_require_authorization_plug.ex +++ b/lib/philomena_web/plugs/api_require_authorization_plug.ex @@ -22,7 +22,7 @@ defmodule PhilomenaWeb.ApiRequireAuthorizationPlug do conn |> maybe_unauthorized(user) - |> maybe_forbidden(Bans.exists_for?(user, conn.remote_ip, "NOTAPI")) + |> maybe_forbidden(Bans.find(user, conn.remote_ip, "NOTAPI")) end defp maybe_unauthorized(conn, nil) do diff --git a/lib/philomena_web/plugs/current_ban_plug.ex b/lib/philomena_web/plugs/current_ban_plug.ex index 273a7889b..d112fa16d 100644 --- a/lib/philomena_web/plugs/current_ban_plug.ex +++ b/lib/philomena_web/plugs/current_ban_plug.ex @@ -20,7 +20,7 @@ defmodule PhilomenaWeb.CurrentBanPlug do user = conn.assigns.current_user ip = conn.remote_ip - ban = Bans.exists_for?(user, ip, fingerprint) + ban = Bans.find(user, ip, fingerprint) Conn.assign(conn, :current_ban, ban) end From 1b020a4fe94a70bedc923e47e37e51fb2dd2b400 Mon Sep 17 00:00:00 2001 From: Liam Date: Mon, 24 Jun 2024 23:23:43 -0400 Subject: [PATCH 4/4] Fix some credo issues --- .credo.exs | 11 +++++++++++ lib/philomena/galleries.ex | 2 +- lib/philomena/users/user.ex | 1 - lib/philomena_query/parse/parser.ex | 4 +--- lib/philomena_query/parse/string.ex | 3 +-- .../plugs/content_security_policy_plug.ex | 5 +---- lib/philomena_web/views/app_view.ex | 4 +--- 7 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 .credo.exs diff --git a/.credo.exs b/.credo.exs new file mode 100644 index 000000000..8e6be3030 --- /dev/null +++ b/.credo.exs @@ -0,0 +1,11 @@ +%{ + configs: %{ + name: "default", + checks: %{ + disabled: [ + {Credo.Check.Refactor.CondStatements, false}, + {Credo.Check.Refactor.NegatedConditionsWithElse, false} + ] + } + } +} diff --git a/lib/philomena/galleries.ex b/lib/philomena/galleries.ex index b0a61bd9c..1943fc25e 100644 --- a/lib/philomena/galleries.ex +++ b/lib/philomena/galleries.ex @@ -333,7 +333,7 @@ defmodule Philomena.Galleries do end) changes - |> Enum.map(fn change -> + |> Enum.each(fn change -> id = Keyword.fetch!(change, :id) change = Keyword.delete(change, :id) diff --git a/lib/philomena/users/user.ex b/lib/philomena/users/user.ex index 1972241d7..182412d73 100644 --- a/lib/philomena/users/user.ex +++ b/lib/philomena/users/user.ex @@ -535,7 +535,6 @@ defmodule Philomena.Users.User do "data:image/png;base64," <> png end - @spec totp_secret(%Philomena.Users.User{}) :: binary() def totp_secret(user) do Philomena.Users.Encryptor.decrypt_model( user.encrypted_otp_secret, diff --git a/lib/philomena_query/parse/parser.ex b/lib/philomena_query/parse/parser.ex index a9d402223..a89434d25 100644 --- a/lib/philomena_query/parse/parser.ex +++ b/lib/philomena_query/parse/parser.ex @@ -212,9 +212,7 @@ defmodule PhilomenaQuery.Parse.Parser do end defp debug_tokens(tokens) do - tokens - |> Enum.map(fn {_k, v} -> v end) - |> Enum.join("") + Enum.map_join(tokens, fn {_k, v} -> v end) end # diff --git a/lib/philomena_query/parse/string.ex b/lib/philomena_query/parse/string.ex index f6dc2fa03..497274d1d 100644 --- a/lib/philomena_query/parse/string.ex +++ b/lib/philomena_query/parse/string.ex @@ -26,7 +26,6 @@ defmodule PhilomenaQuery.Parse.String do str |> String.replace("\r", "") |> String.split("\n", trim: true) - |> Enum.map(fn s -> "(#{s})" end) - |> Enum.join(" || ") + |> Enum.map_join(" || ", &"(#{&1})") end end diff --git a/lib/philomena_web/plugs/content_security_policy_plug.ex b/lib/philomena_web/plugs/content_security_policy_plug.ex index 34cca2d6b..854cb6139 100644 --- a/lib/philomena_web/plugs/content_security_policy_plug.ex +++ b/lib/philomena_web/plugs/content_security_policy_plug.ex @@ -37,10 +37,7 @@ defmodule PhilomenaWeb.ContentSecurityPolicyPlug do {:media_src, ["'self'", "blob:", "data:", cdn_uri, camo_uri]} ] - csp_value = - csp_config - |> Enum.map(&cspify_element/1) - |> Enum.join("; ") + csp_value = Enum.map_join(csp_config, "; ", &cspify_element/1) csp_relaxed? do if conn.status == 500 do diff --git a/lib/philomena_web/views/app_view.ex b/lib/philomena_web/views/app_view.ex index 1b0febd5d..9feaf46dc 100644 --- a/lib/philomena_web/views/app_view.ex +++ b/lib/philomena_web/views/app_view.ex @@ -117,9 +117,7 @@ defmodule PhilomenaWeb.AppView do def escape_nl2br(text) do text |> String.split("\n") - |> Enum.map(&html_escape/1) - |> Enum.map(&safe_to_string/1) - |> Enum.join("
") + |> Enum.map_intersperse("
", &safe_to_string(html_escape(&1))) |> raw() end