diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js
index f71310e16c..07f8d65f6e 100644
--- a/lib/hls/hls_parser.js
+++ b/lib/hls/hls_parser.js
@@ -493,7 +493,11 @@ shaka.hls.HlsParser = class {
         // Now adjust timestamps back to begin at 0.
         const segmentN = segmentIndex.earliestReference();
         if (segmentN) {
-          this.offsetStream_(streamInfo, -segmentN.startTime);
+          const streamOffset = -segmentN.startTime;
+          // Modify all SegmentReferences equally.
+          streamInfo.stream.segmentIndex.offset(streamOffset);
+          // Update other parts of streamInfo the same way.
+          this.offsetStreamInfo_(streamInfo, streamOffset);
         }
       }
     }
@@ -535,22 +539,29 @@ shaka.hls.HlsParser = class {
     for (const streamInfo of this.uriToStreamInfosMap_.values()) {
       const segmentIndex = streamInfo.stream.segmentIndex;
       if (segmentIndex != null) {
+        // A segment's startTime should be based on its syncTime vs the lowest
+        // syncTime across all streams.  The earliest segment sync time from
+        // any stream will become presentation time 0.  If two streams start
+        // e.g. 6 seconds apart in syncTime, then their first segments will
+        // also start 6 seconds apart in presentation time.
+
         const segment0 = segmentIndex.earliestReference();
         if (segment0.syncTime == null) {
           shaka.log.alwaysError('Missing EXT-X-PROGRAM-DATE-TIME for stream',
               streamInfo.verbatimMediaPlaylistUri,
               'Expect AV sync issues!');
         } else {
-          // The first segment's target startTime should be based entirely on
-          // its syncTime.  The rest of the stream will be based on that
-          // starting point.  The earliest segment sync time from any stream
-          // will become presentation time 0.  If two streams start e.g. 6
-          // seconds apart in syncTime, then their first segments will also
-          // start 6 seconds apart in presentation time.
+          // Stream metadata are offset by a fixed amount based on the
+          // first segment.
           const segment0TargetTime = segment0.syncTime - lowestSyncTime;
           const streamOffset = segment0TargetTime - segment0.startTime;
+          this.offsetStreamInfo_(streamInfo, streamOffset);
 
-          this.offsetStream_(streamInfo, streamOffset);
+          // This is computed across all segments separately to manage
+          // accumulated drift in durations.
+          for (const segment of segmentIndex) {
+            segment.syncAgainst(lowestSyncTime);
+          }
         }
       }
     }
@@ -561,13 +572,13 @@ shaka.hls.HlsParser = class {
    * @param {number} offset
    * @private
    */
-  offsetStream_(streamInfo, offset) {
-    streamInfo.stream.segmentIndex.offset(offset);
-
+  offsetStreamInfo_(streamInfo, offset) {
+    // Adjust our accounting of the maximum timestamp.
     streamInfo.maxTimestamp += offset;
     goog.asserts.assert(streamInfo.maxTimestamp >= 0,
         'Negative maxTimestamp after adjustment!');
 
+    // Update our map from sequence number to start time.
     const mediaSequenceToStartTime =
         this.getMediaSequenceToStartTimeFor_(streamInfo);
     for (const [key, value] of mediaSequenceToStartTime) {
@@ -2754,6 +2765,16 @@ shaka.hls.HlsParser = class {
       }
     }
 
+    // lowestSyncTime is a value from a previous playlist update.  Use it to
+    // set reference start times.  If this is the first playlist parse, we will
+    // skip this step, and wait until we have sync time across stream types.
+    const lowestSyncTime = this.lowestSyncTime_;
+    if (someSyncTime && lowestSyncTime != Infinity) {
+      for (const reference of references) {
+        reference.syncAgainst(lowestSyncTime);
+      }
+    }
+
     return references;
   }
 
diff --git a/lib/media/segment_index.js b/lib/media/segment_index.js
index 913ecdb55f..bad96a0343 100644
--- a/lib/media/segment_index.js
+++ b/lib/media/segment_index.js
@@ -183,15 +183,7 @@ shaka.media.SegmentIndex = class {
   offset(offset) {
     if (!this.immutable_) {
       for (const ref of this.references) {
-        ref.startTime += offset;
-        ref.endTime += offset;
-        ref.trueEndTime += offset;
-
-        for (const partial of ref.partialReferences) {
-          partial.startTime += offset;
-          partial.endTime += offset;
-          partial.trueEndTime += offset;
-        }
+        ref.offset(offset);
       }
     }
   }
diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js
index 38d1851504..cfde258c8f 100644
--- a/lib/media/segment_reference.js
+++ b/lib/media/segment_reference.js
@@ -8,6 +8,7 @@ goog.provide('shaka.media.InitSegmentReference');
 goog.provide('shaka.media.SegmentReference');
 
 goog.require('goog.asserts');
+goog.require('shaka.log');
 goog.require('shaka.util.ArrayUtils');
 
 
@@ -383,6 +384,44 @@ shaka.media.SegmentReference = class {
   getThumbnailSprite() {
     return this.thumbnailSprite;
   }
+
+  /**
+   * Offset the segment reference by a fixed amount.
+   *
+   * @param {number} offset The amount to add to the segment's start and end
+   *   times.
+   * @export
+   */
+  offset(offset) {
+    this.startTime += offset;
+    this.endTime += offset;
+    this.trueEndTime += offset;
+
+    for (const partial of this.partialReferences) {
+      partial.startTime += offset;
+      partial.endTime += offset;
+      partial.trueEndTime += offset;
+    }
+  }
+
+  /**
+   * Sync this segment against a particular sync time that will serve as "0" in
+   * the presentation timeline.
+   *
+   * @param {number} lowestSyncTime
+   * @export
+   */
+  syncAgainst(lowestSyncTime) {
+    if (this.syncTime == null) {
+      shaka.log.alwaysError('Sync attempted without sync time!');
+      return;
+    }
+    const desiredStart = this.syncTime - lowestSyncTime;
+    const offset = desiredStart - this.startTime;
+    if (Math.abs(offset) >= 0.001) {
+      this.offset(offset);
+    }
+  }
 };