diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
index 0337a58881..65fbdde27d 100644
--- a/lib/media/media_source_engine.js
+++ b/lib/media/media_source_engine.js
@@ -941,14 +941,18 @@ shaka.media.MediaSourceEngine = class {
             .box('moof', Mp4Parser.children)
             .box('traf', Mp4Parser.children)
             .fullBox('tfdt', (box) => {
-              goog.asserts.assert(
-                  box.version == 0 || box.version == 1,
-                  'TFDT version can only be 0 or 1');
-              const parsed = shaka.util.Mp4BoxParsers.parseTFDTInaccurate(
-                  box.reader, box.version);
-              startTime = parsed.baseMediaDecodeTime / timescale;
-              parsedMedia = true;
-              box.parser.stop();
+              if (!parsedMedia) {
+                goog.asserts.assert(
+                    box.version == 0 || box.version == 1,
+                    'TFDT version can only be 0 or 1');
+                const parsed = shaka.util.Mp4BoxParsers.parseTFDTInaccurate(
+                    box.reader, box.version);
+                startTime = parsed.baseMediaDecodeTime / timescale;
+                parsedMedia = true;
+                if (!hasEmsg) {
+                  box.parser.stop();
+                }
+              }
             }).parse(data, /* partialOkay= */ true);
         if (parsedMedia && reference.timestampOffset == 0) {
           timestamp = startTime;