diff --git a/lib/transmuxer/ts_transmuxer.js b/lib/transmuxer/ts_transmuxer.js
index 984cd467d8..9cb19f39bf 100644
--- a/lib/transmuxer/ts_transmuxer.js
+++ b/lib/transmuxer/ts_transmuxer.js
@@ -170,17 +170,7 @@ shaka.transmuxer.TsTransmuxer = class {
 
     const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
 
-    let timestamp = reference.endTime * 1000;
-
     const tsParser = this.tsParser_.parse(uint8ArrayData);
-    const startTime = tsParser.getStartTime();
-
-    if (startTime.audio != null) {
-      timestamp = startTime.audio;
-    }
-    if (startTime.video != null) {
-      timestamp = startTime.video;
-    }
     const streamInfos = [];
     const codecs = tsParser.getCodecs();
     try {
@@ -189,8 +179,7 @@ shaka.transmuxer.TsTransmuxer = class {
         switch (codecs.video) {
           case 'avc':
             streamInfo =
-              this.getAvcStreamInfo_(
-                  tsParser, timestamp, stream, duration);
+              this.getAvcStreamInfo_(tsParser, stream, duration);
             break;
         }
         if (streamInfo) {
@@ -202,23 +191,19 @@ shaka.transmuxer.TsTransmuxer = class {
         switch (codecs.audio) {
           case 'aac':
             streamInfo =
-              this.getAacStreamInfo_(
-                  tsParser, timestamp, stream, duration);
+              this.getAacStreamInfo_(tsParser, stream, duration);
             break;
           case 'ac3':
             streamInfo =
-              this.getAc3StreamInfo_(
-                  tsParser, timestamp, stream, duration);
+              this.getAc3StreamInfo_(tsParser, stream, duration);
             break;
           case 'ec3':
             streamInfo =
-              this.getEc3StreamInfo_(
-                  tsParser, timestamp, stream, duration);
+              this.getEc3StreamInfo_(tsParser, stream, duration);
             break;
           case 'mp3':
             streamInfo =
-              this.getMp3StreamInfo_(
-                  tsParser, timestamp, stream, duration);
+              this.getMp3StreamInfo_(tsParser, stream, duration);
             break;
         }
         if (streamInfo) {
@@ -255,20 +240,22 @@ shaka.transmuxer.TsTransmuxer = class {
 
   /**
    * @param {shaka.util.TsParser} tsParser
-   * @param {number} timestamp
    * @param {shaka.extern.Stream} stream
    * @param {number} duration
    * @return {shaka.util.Mp4Generator.StreamInfo}
    * @private
    */
-  getAacStreamInfo_(tsParser, timestamp, stream, duration) {
+  getAacStreamInfo_(tsParser, stream, duration) {
     const ADTS = shaka.transmuxer.ADTS;
+    const timescale = shaka.util.TsParser.Timescale;
 
     /** @type {!Array.<shaka.util.Mp4Generator.Mp4Sample>} */
     const samples = [];
 
     let info;
 
+    let firstPts = null;
+
     for (const audioData of tsParser.getAudioData()) {
       const data = audioData.data;
       if (!data) {
@@ -284,7 +271,9 @@ shaka.transmuxer.TsTransmuxer = class {
       }
       stream.audioSamplingRate = info.sampleRate;
       stream.channelsCount = info.channelCount;
-
+      if (firstPts == null && audioData.pts !== null) {
+        firstPts = audioData.pts;
+      }
       while (offset < data.length) {
         const header = ADTS.parseHeader(data, offset);
         if (!header) {
@@ -316,7 +305,7 @@ shaka.transmuxer.TsTransmuxer = class {
       }
     }
 
-    if (!info) {
+    if (!info || firstPts == null) {
       throw new shaka.util.Error(
           shaka.util.Error.Severity.CRITICAL,
           shaka.util.Error.Category.MEDIA,
@@ -326,7 +315,7 @@ shaka.transmuxer.TsTransmuxer = class {
     /** @type {number} */
     const sampleRate = info.sampleRate;
     /** @type {number} */
-    const baseMediaDecodeTime = Math.floor(timestamp * sampleRate / 1000);
+    const baseMediaDecodeTime = firstPts / timescale * sampleRate;
 
     return {
       id: stream.id,
@@ -350,14 +339,14 @@ shaka.transmuxer.TsTransmuxer = class {
 
   /**
    * @param {shaka.util.TsParser} tsParser
-   * @param {number} timestamp
    * @param {shaka.extern.Stream} stream
    * @param {number} duration
    * @return {shaka.util.Mp4Generator.StreamInfo}
    * @private
    */
-  getAc3StreamInfo_(tsParser, timestamp, stream, duration) {
+  getAc3StreamInfo_(tsParser, stream, duration) {
     const Ac3 = shaka.transmuxer.Ac3;
+    const timescale = shaka.util.TsParser.Timescale;
 
     /** @type {!Array.<shaka.util.Mp4Generator.Mp4Sample>} */
     const samples = [];
@@ -368,8 +357,13 @@ shaka.transmuxer.TsTransmuxer = class {
     /** @type {!Uint8Array} */
     let audioConfig = new Uint8Array([]);
 
+    let firstPts = null;
+
     for (const audioData of tsParser.getAudioData()) {
       const data = audioData.data;
+      if (firstPts == null && audioData.pts !== null) {
+        firstPts = audioData.pts;
+      }
       let offset = 0;
       while (offset < data.length) {
         const frame = Ac3.parseFrame(data, offset);
@@ -403,7 +397,7 @@ shaka.transmuxer.TsTransmuxer = class {
       }
     }
 
-    if (sampleRate == 0 || audioConfig.byteLength == 0) {
+    if (sampleRate == 0 || audioConfig.byteLength == 0 || firstPts == null) {
       throw new shaka.util.Error(
           shaka.util.Error.Severity.CRITICAL,
           shaka.util.Error.Category.MEDIA,
@@ -412,7 +406,7 @@ shaka.transmuxer.TsTransmuxer = class {
 
 
     /** @type {number} */
-    const baseMediaDecodeTime = Math.floor(timestamp * sampleRate / 1000);
+    const baseMediaDecodeTime = firstPts / timescale * sampleRate;
 
     return {
       id: stream.id,
@@ -436,14 +430,14 @@ shaka.transmuxer.TsTransmuxer = class {
 
   /**
    * @param {shaka.util.TsParser} tsParser
-   * @param {number} timestamp
    * @param {shaka.extern.Stream} stream
    * @param {number} duration
    * @return {shaka.util.Mp4Generator.StreamInfo}
    * @private
    */
-  getEc3StreamInfo_(tsParser, timestamp, stream, duration) {
+  getEc3StreamInfo_(tsParser, stream, duration) {
     const Ec3 = shaka.transmuxer.Ec3;
+    const timescale = shaka.util.TsParser.Timescale;
 
     /** @type {!Array.<shaka.util.Mp4Generator.Mp4Sample>} */
     const samples = [];
@@ -454,8 +448,13 @@ shaka.transmuxer.TsTransmuxer = class {
     /** @type {!Uint8Array} */
     let audioConfig = new Uint8Array([]);
 
+    let firstPts = null;
+
     for (const audioData of tsParser.getAudioData()) {
       const data = audioData.data;
+      if (firstPts == null && audioData.pts !== null) {
+        firstPts = audioData.pts;
+      }
       let offset = 0;
       while (offset < data.length) {
         const frame = Ec3.parseFrame(data, offset);
@@ -489,7 +488,7 @@ shaka.transmuxer.TsTransmuxer = class {
       }
     }
 
-    if (sampleRate == 0 || audioConfig.byteLength == 0) {
+    if (sampleRate == 0 || audioConfig.byteLength == 0 || firstPts == null) {
       throw new shaka.util.Error(
           shaka.util.Error.Severity.CRITICAL,
           shaka.util.Error.Category.MEDIA,
@@ -498,7 +497,7 @@ shaka.transmuxer.TsTransmuxer = class {
 
 
     /** @type {number} */
-    const baseMediaDecodeTime = Math.floor(timestamp * sampleRate / 1000);
+    const baseMediaDecodeTime = firstPts / timescale * sampleRate;
 
     return {
       id: stream.id,
@@ -522,25 +521,30 @@ shaka.transmuxer.TsTransmuxer = class {
 
   /**
    * @param {shaka.util.TsParser} tsParser
-   * @param {number} timestamp
    * @param {shaka.extern.Stream} stream
    * @param {number} duration
    * @return {shaka.util.Mp4Generator.StreamInfo}
    * @private
    */
-  getMp3StreamInfo_(tsParser, timestamp, stream, duration) {
+  getMp3StreamInfo_(tsParser, stream, duration) {
     const MpegAudio = shaka.transmuxer.MpegAudio;
+    const timescale = shaka.util.TsParser.Timescale;
 
     /** @type {!Array.<shaka.util.Mp4Generator.Mp4Sample>} */
     const samples = [];
 
     let firstHeader;
 
+    let firstPts = null;
+
     for (const audioData of tsParser.getAudioData()) {
       const data = audioData.data;
       if (!data) {
         continue;
       }
+      if (firstPts == null && audioData.pts !== null) {
+        firstPts = audioData.pts;
+      }
       let offset = 0;
       while (offset < data.length) {
         const header = MpegAudio.parseHeader(data, offset);
@@ -570,7 +574,7 @@ shaka.transmuxer.TsTransmuxer = class {
         offset += header.frameLength;
       }
     }
-    if (!firstHeader) {
+    if (!firstHeader || firstPts == null) {
       throw new shaka.util.Error(
           shaka.util.Error.Severity.CRITICAL,
           shaka.util.Error.Category.MEDIA,
@@ -579,7 +583,7 @@ shaka.transmuxer.TsTransmuxer = class {
     /** @type {number} */
     const sampleRate = firstHeader.sampleRate;
     /** @type {number} */
-    const baseMediaDecodeTime = Math.floor(timestamp * sampleRate / 1000);
+    const baseMediaDecodeTime = firstPts / timescale * sampleRate;
 
     return {
       id: stream.id,
@@ -603,13 +607,12 @@ shaka.transmuxer.TsTransmuxer = class {
 
   /**
    * @param {shaka.util.TsParser} tsParser
-   * @param {number} timestamp
    * @param {shaka.extern.Stream} stream
    * @param {number} duration
    * @return {shaka.util.Mp4Generator.StreamInfo}
    * @private
    */
-  getAvcStreamInfo_(tsParser, timestamp, stream, duration) {
+  getAvcStreamInfo_(tsParser, stream, duration) {
     const H264 = shaka.transmuxer.H264;
     const timescale = shaka.util.TsParser.Timescale;