Plumb SubtitleParser.Factory into FragmentedMp4Extractor

We introduce SubtitleParser.Factory that supports no formats to be used FragmentedMp4Extractors that will not do any subtitle parsing on the extraction side. We also slowly move away from using SubtitleTranscodingExtractor to SubtitleTranscodingExtractorOutput (hence making it public). This is required by individual Extractor impls so that they can start using SubtitleTranscodingExtractorOutput rather than be wrapped by SubtitleTranscodingExtractor. The latter is to be deprecated after all the subtitle related Extractors have achieved this migration. PiperOrigin-RevId: 596942147
androidx · Jan 9, 2024 · da724c8 · da724c8
1 parent e54abaa
commit da724c8
Show file tree

Hide file tree

Showing 13 changed files with 340 additions and 88 deletions.
diff --git a/...exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java b/...exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java
@@ -105,19 +105,27 @@ public ChunkExtractor createProgressiveMediaExtractor(
       } else if (Objects.equals(containerMimeType, MimeTypes.IMAGE_PNG)) {
         extractor = new PngExtractor();
       } else {
-        int flags = 0;
+        @FragmentedMp4Extractor.Flags int flags = 0;
         if (enableEventMessageTrack) {
           flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
         }
+        if (subtitleParserFactory == null) {
+          flags |= FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA;
+        }
         extractor =
             new FragmentedMp4Extractor(
+                subtitleParserFactory != null
+                    ? subtitleParserFactory
+                    : SubtitleParser.Factory.UNSUPPORTED,
                 flags,
                 /* timestampAdjuster= */ null,
                 /* sideloadedTrack= */ null,
                 closedCaptionFormats,
                 playerEmsgTrackOutput);
       }
-      if (subtitleParserFactory != null && !MimeTypes.isText(containerMimeType)) {
+      if (subtitleParserFactory != null
+          && !MimeTypes.isText(containerMimeType)
+          && !(extractor.getUnderlyingImplementation() instanceof FragmentedMp4Extractor)) {
         extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
       }
       return new BundledChunkExtractor(extractor, primaryTrackType, representationFormat);

diff --git a/...exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java b/...exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java
@@ -39,6 +39,7 @@
 import androidx.media3.extractor.ts.AdtsExtractor;
 import androidx.media3.extractor.ts.DefaultTsPayloadReaderFactory;
 import androidx.media3.extractor.ts.TsExtractor;
+import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Ints;
 import java.io.EOFException;
 import java.io.IOException;
@@ -201,11 +202,8 @@ private Extractor createExtractorByFileType(
       case FileTypes.MP3:
         return new Mp3Extractor(/* flags= */ 0, /* forcedFirstSampleTimestampUs= */ 0);
       case FileTypes.MP4:
-        Extractor mp4Extractor =
-            createFragmentedMp4Extractor(timestampAdjuster, format, muxedCaptionFormats);
-        return subtitleParserFactory != null
-            ? new SubtitleTranscodingExtractor(mp4Extractor, subtitleParserFactory)
-            : mp4Extractor;
+        return createFragmentedMp4Extractor(
+            subtitleParserFactory, timestampAdjuster, format, muxedCaptionFormats);
       case FileTypes.TS:
         Extractor tsExtractor =
             createTsExtractor(
@@ -264,16 +262,25 @@ private static TsExtractor createTsExtractor(
   }
 
   private static FragmentedMp4Extractor createFragmentedMp4Extractor(
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       TimestampAdjuster timestampAdjuster,
       Format format,
       @Nullable List<Format> muxedCaptionFormats) {
     // Only enable the EMSG TrackOutput if this is the 'variant' track (i.e. the main one) to avoid
     // creating a separate EMSG track for every audio track in a video stream.
+    @FragmentedMp4Extractor.Flags
+    int flags = isFmp4Variant(format) ? FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK : 0;
+    if (subtitleParserFactory == null) {
+      subtitleParserFactory = SubtitleParser.Factory.UNSUPPORTED;
+      flags |= FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA;
+    }
     return new FragmentedMp4Extractor(
-        /* flags= */ isFmp4Variant(format) ? FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK : 0,
+        subtitleParserFactory,
+        flags,
         timestampAdjuster,
         /* sideloadedTrack= */ null,
-        muxedCaptionFormats != null ? muxedCaptionFormats : Collections.emptyList());
+        muxedCaptionFormats != null ? muxedCaptionFormats : ImmutableList.of(),
+        /* additionalEmsgTrackOutput= */ null);
   }
 
   /** Returns true if this {@code format} represents a 'variant' track (i.e. the main one). */

diff --git a/...reaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java b/...reaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/DefaultSsChunkSource.java
@@ -53,7 +53,7 @@
 import androidx.media3.extractor.mp4.Track;
 import androidx.media3.extractor.mp4.TrackEncryptionBox;
 import androidx.media3.extractor.text.SubtitleParser;
-import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
+import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.util.List;
 
@@ -173,15 +173,22 @@ public DefaultSsChunkSource(
               nalUnitLengthFieldLength,
               null,
               null);
+      @FragmentedMp4Extractor.Flags
+      int flags =
+          FragmentedMp4Extractor.FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME
+              | FragmentedMp4Extractor.FLAG_WORKAROUND_IGNORE_TFDT_BOX;
       Extractor extractor =
           new FragmentedMp4Extractor(
-              FragmentedMp4Extractor.FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME
-                  | FragmentedMp4Extractor.FLAG_WORKAROUND_IGNORE_TFDT_BOX,
+              subtitleParserFactory == null
+                  ? SubtitleParser.Factory.UNSUPPORTED
+                  : subtitleParserFactory,
+              subtitleParserFactory == null
+                  ? flags | FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA
+                  : flags,
               /* timestampAdjuster= */ null,
-              track);
-      if (subtitleParserFactory != null) {
-        extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
-      }
+              track,
+              /* closedCaptionFormats= */ ImmutableList.of(),
+              /* additionalEmsgTrackOutput= */ null);
       chunkExtractors[i] = new BundledChunkExtractor(extractor, streamElement.type, format);
     }
   }

diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java b/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java
@@ -277,7 +277,7 @@ public synchronized DefaultExtractorsFactory setMp4ExtractorFlags(@Mp4Extractor.
   /**
    * Sets flags for {@link FragmentedMp4Extractor} instances created by the factory.
    *
-   * @see FragmentedMp4Extractor#FragmentedMp4Extractor(int)
+   * @see FragmentedMp4Extractor#FragmentedMp4Extractor(SubtitleParser.Factory, int)
    * @param flags The flags to use.
    * @return The factory, for convenience.
    */
@@ -436,10 +436,12 @@ public synchronized Extractor[] createExtractors(
     }
     Extractor[] result = new Extractor[extractors.size()];
     for (int i = 0; i < extractors.size(); i++) {
+      Extractor extractor = extractors.get(i);
       result[i] =
           textTrackTranscodingEnabled
-              ? new SubtitleTranscodingExtractor(extractors.get(i), subtitleParserFactory)
-              : extractors.get(i);
+                  && !(extractor.getUnderlyingImplementation() instanceof FragmentedMp4Extractor)
+              ? new SubtitleTranscodingExtractor(extractor, subtitleParserFactory)
+              : extractor;
     }
     return result;
   }
@@ -500,7 +502,13 @@ private void addExtractorsForFileType(@FileTypes.Type int fileType, List<Extract
                         : 0)));
         break;
       case FileTypes.MP4:
-        extractors.add(new FragmentedMp4Extractor(fragmentedMp4Flags));
+        extractors.add(
+            new FragmentedMp4Extractor(
+                subtitleParserFactory,
+                fragmentedMp4Flags
+                    | (textTrackTranscodingEnabled
+                        ? 0
+                        : FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA)));
         extractors.add(new Mp4Extractor(mp4Flags));
         break;
       case FileTypes.OGG:

diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/Extractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/Extractor.java
@@ -25,6 +25,7 @@
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+import org.checkerframework.dataflow.qual.SideEffectFree;
 
 /** Extracts media data from a container format. */
 @UnstableApi
@@ -130,6 +131,7 @@ public interface Extractor {
    * <p>{@code Extractor} implementations that operate by delegating to another {@code Extractor}
    * should override this method to return that delegate.
    */
+  @SideEffectFree
   default Extractor getUnderlyingImplementation() {
     return this;
   }