Release

apps/cf/lib/actions/action_creator.ex

@@ -10,6 +10,7 @@ defmodule CF.Actions.ActionCreator do
   alias DB.Schema.Speaker
   alias DB.Schema.Statement
   alias DB.Schema.Comment
+  alias DB.Schema.VideoCaption
   alias CF.Actions.ReputationChange
 
   # Create
@@ -175,6 +176,19 @@ defmodule CF.Actions.ActionCreator do
     )
   end
 
+  def action_upload_video_captions(user_id, video_id, caption = %VideoCaption{}) do
+    action(
+      user_id,
+      :video_caption,
+      :upload,
+      video_id: video_id,
+      changes: %{
+        "format" => caption.format,
+        "parsed" => caption.parsed
+      }
+    )
+  end
+
   def action_revert_vote(user_id, video_id, vote_type, comment = %Comment{})
       when vote_type in [:revert_vote_up, :revert_vote_down, :revert_self_vote] do
     action(

apps/cf/lib/videos/captions_fetcher_youtube.ex

@@ -9,6 +9,8 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
   require Logger
   import SweetXml
 
+  @user_agent "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/135.0"
+
   @impl true
   def fetch(%{youtube_id: youtube_id, language: language}) do
     with {:ok, data} <- fetch_youtube_data(youtube_id),
@@ -27,7 +29,7 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
   defp fetch_youtube_data(video_id) do
     url = "https://www.youtube.com/watch?v=#{video_id}"
 
-    case HTTPoison.get(url, []) do
+    case HTTPoison.get(url, [{"User-Agent", @user_agent}]) do
       {:ok, %HTTPoison.Response{body: body}} ->
         {:ok, body}
 
@@ -59,7 +61,7 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
   end
 
   defp fetch_transcript(base_url) do
-    case HTTPoison.get(base_url, []) do
+    case HTTPoison.get(base_url, [{"User-Agent", @user_agent}]) do
       {:ok, %HTTPoison.Response{body: body}} ->
         {:ok, body}
 
@@ -69,30 +71,7 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
   end
 
   defp process_transcript(transcript) do
-    transcript
-    |> SweetXml.xpath(
-      ~x"//transcript/text"l,
-      text: ~x"./text()"s |> transform_by(&clean_text/1),
-      start: ~x"./@start"s |> transform_by(&parse_float/1),
-      duration: ~x"./@dur"os |> transform_by(&parse_float/1)
-    )
-    |> Enum.filter(fn %{text: text, start: start} ->
-      start != nil and text != nil and text != ""
-    end)
-  end
-
-  defp clean_text(text) do
-    text
-    |> String.replace("&amp;", "&")
-    |> HtmlEntities.decode()
-    |> String.trim()
-  end
-
-  defp parse_float(val) do
-    case Float.parse(val) do
-      {num, _} -> num
-      _ -> nil
-    end
+    CF.Videos.CaptionsSrv1Parser.parse_file(transcript)
   end
 
   # Below is an implementation using the official YouTube API, but it requires OAuth2 authentication.

apps/cf/lib/videos/captions_srv1_parser.ex

@@ -0,0 +1,37 @@
+defmodule CF.Videos.CaptionsSrv1Parser do
+  @moduledoc """
+  A captions parser for the srv1 format.
+  """
+
+  require Logger
+  import SweetXml
+
+  def parse_file(content) do
+    content
+    |> SweetXml.xpath(
+      ~x"//transcript/text"l,
+      text: ~x"./text()"s |> transform_by(&clean_text/1),
+      start: ~x"./@start"s |> transform_by(&parse_float/1),
+      duration: ~x"./@dur"os |> transform_by(&parse_float/1)
+    )
+    |> Enum.filter(fn %{text: text, start: start} ->
+      # Filter out text in brackets, like "[Music]"
+      start != nil and text != nil and text != "" and
+        String.match?(text, ~r/^\[.*\]$/) == false
+    end)
+  end
+
+  defp clean_text(text) do
+    text
+    |> String.replace("&", "&")
+    |> HtmlEntities.decode()
+    |> String.trim()
+  end
+
+  defp parse_float(val) do
+    case Float.parse(val) do
+      {num, _} -> num
+      _ -> nil
+    end
+  end
+end

apps/cf/test/comments/comments_test.exs

@@ -141,7 +141,7 @@ defmodule CF.Comments.CommentsTest do
         Comments.delete_comment(random_user, comment.statement.video_id, comment)
       end
 
-      refute_deleted(comment)
+      assert_not_deleted(comment)
 
       Comments.delete_comment(comment.user, comment.statement.video_id, comment)
       assert_deleted(comment)
@@ -168,7 +168,7 @@ defmodule CF.Comments.CommentsTest do
         Comments.delete_comment(comment.user, comment.statement.video_id, comment)
       end
 
-      refute_deleted(comment)
+      assert_not_deleted(comment)
     end
 
     test "but an admin can" do

apps/cf/test/support/test_utils.ex

@@ -36,7 +36,7 @@ defmodule CF.TestUtils do
     end
   end
 
-  def refute_deleted(%Comment{id: id}) do
+  def assert_not_deleted(%Comment{id: id}) do
     {comment, _} = get_comment_and_actions(id)
     assert comment != nil
 

apps/cf_graphql/lib/endpoint.ex

@@ -13,9 +13,9 @@ defmodule CF.GraphQLWeb.Endpoint do
 
   plug(
     Plug.Parsers,
-    parsers: [:urlencoded, :multipart, :json],
+    parsers: [:urlencoded, :multipart, :json, Absinthe.Plug.Parser],
     pass: ["*/*"],
-    json_decoder: Poison
+    json_decoder: Jason
   )
 
   plug(Plug.MethodOverride)

apps/cf_graphql/lib/resolvers/videos.ex

@@ -8,6 +8,7 @@ defmodule CF.Graphql.Resolvers.Videos do
   import Ecto.Query
   import Absinthe.Resolution.Helpers, only: [batch: 3]
 
+  alias Ecto.Multi
   alias DB.Repo
   alias DB.Schema.Video
   alias DB.Schema.VideoCaption
@@ -137,4 +138,48 @@ defmodule CF.Graphql.Resolvers.Videos do
         {:error, "Failed to update video"}
     end
   end
+
+  def set_captions(
+        _root,
+        %{video_id: video_id, captions: %{path: path, content_type: content_type}},
+        %{
+          context: %{user: user}
+        }
+      ) do
+    cond do
+      content_type != "text/xml" ->
+        {:error, "Unsupported content type: #{content_type}"}
+
+      File.stat!(path).size > 10_000_000 ->
+        {:error, "File must be < 10MB"}
+
+      true ->
+        video = DB.Repo.get!(DB.Schema.Video, video_id)
+        captions_file_content = File.read!(path)
+        parsed = CF.Videos.CaptionsSrv1Parser.parse_file(captions_file_content)
+
+        Multi.new()
+        |> Multi.insert(
+          :caption,
+          VideoCaption.changeset(%VideoCaption{
+            video_id: video.id,
+            raw: captions_file_content,
+            parsed: parsed,
+            format: "xml"
+          })
+        )
+        |> Multi.run(
+          :action,
+          fn _repo, %{caption: caption} ->
+            CF.Actions.ActionCreator.action_upload_video_captions(user.id, video.id, caption)
+            |> DB.Repo.insert!()
+
+            {:ok, caption}
+          end
+        )
+        |> DB.Repo.transaction()
+
+        {:ok, video}
+    end
+  end
 end

apps/cf_graphql/lib/schema/input_objects/statement_filter.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.InputObjects.StatementFilter do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   @desc "Props to filter statements on"
   input_object :statement_filter do

apps/cf_graphql/lib/schema/input_objects/video_filter.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.InputObjects.VideoFilter do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   @desc "Props to filter videos on"
   input_object :video_filter do

apps/cf_graphql/lib/schema/schema.ex

@@ -2,9 +2,42 @@ defmodule CF.Graphql.Schema do
   use Absinthe.Schema
   alias CF.Graphql.Resolvers
   alias CF.Graphql.Schema.Middleware
+  import Absinthe.Resolution.Helpers, only: [dataloader: 1]
+
+  import_types(Absinthe.Plug.Types)
+
+  import_types(CF.Graphql.Schema.Types.{
+    AppInfo,
+    Comment,
+    Notification,
+    Paginated,
+    Source,
+    Speaker,
+    Statement,
+    Statistics,
+    Subscription,
+    UserAction,
+    User,
+    Video,
+    VideoCaption
+  })
+
+  import_types(CF.Graphql.Schema.InputObjects.{
+    VideoFilter,
+    StatementFilter
+  })
+
+  def context(ctx) do
+    loader =
+      Dataloader.new()
+      |> Dataloader.add_source(DB.Repo, Dataloader.Ecto.new(DB.Repo))
+
+    Map.put(ctx, :loader, loader)
+  end
 
-  import_types(CF.Graphql.Schema.Types)
-  import_types(CF.Graphql.Schema.InputObjects)
+  def plugins do
+    [Absinthe.Middleware.Dataloader | Absinthe.Plugin.defaults()]
+  end
 
   # Query API
 
@@ -110,5 +143,15 @@ defmodule CF.Graphql.Schema do
 
       resolve(&Resolvers.Videos.edit/3)
     end
+
+    field :set_video_captions, :video do
+      middleware(Middleware.RequireAuthentication)
+      middleware(Middleware.RequireReputation, 450)
+
+      arg(:video_id, non_null(:id))
+      arg(:captions, non_null(:upload))
+
+      resolve(&Resolvers.Videos.set_captions/3)
+    end
   end
 end

apps/cf_graphql/lib/schema/types/app_info.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.Types.AppInfo do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   @desc "Information about the application"
   object :app_info do

apps/cf_graphql/lib/schema/types/comment.ex

@@ -4,7 +4,8 @@ defmodule CF.Graphql.Schema.Types.Comment do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
+
+  import Absinthe.Resolution.Helpers, only: [dataloader: 1]
   import CF.Graphql.Schema.Utils
   alias CF.Graphql.Resolvers
 
@@ -13,7 +14,7 @@ defmodule CF.Graphql.Schema.Types.Comment do
     field(:id, non_null(:id))
     @desc "User who made the comment"
     field :user, :user do
-      resolve(assoc(:user))
+      resolve(dataloader(DB.Repo))
       complexity(join_complexity())
     end
 
@@ -31,7 +32,7 @@ defmodule CF.Graphql.Schema.Types.Comment do
 
     @desc "Source of the scomment. If null, a text must be set"
     field :source, :source do
-      resolve(assoc(:source))
+      resolve(dataloader(DB.Repo))
       complexity(join_complexity())
     end
 

apps/cf_graphql/lib/schema/types/notification.ex

@@ -4,10 +4,9 @@ defmodule CF.Graphql.Schema.Types.Notification do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
-  import CF.Graphql.Schema.Utils
 
-  import_types(CF.Graphql.Schema.Types.Paginated)
+  import Absinthe.Resolution.Helpers, only: [dataloader: 1]
+  import CF.Graphql.Schema.Utils
 
   @desc "A user notification"
   object :notification do
@@ -20,7 +19,7 @@ defmodule CF.Graphql.Schema.Types.Notification do
     field(:seen_at, :string)
     @desc "Action the notification is referencing"
     field :action, :user_action do
-      resolve(assoc(:action))
+      resolve(dataloader(DB.Repo))
       complexity(join_complexity())
     end
   end

apps/cf_graphql/lib/schema/types/paginated.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.Types.Paginated do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   object :paginated do
     field(:page_number, :integer)