diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js
index f0a74c5742a..a27c33580f6 100644
--- a/lib/hls/hls_parser.js
+++ b/lib/hls/hls_parser.js
@@ -551,10 +551,13 @@ shaka.hls.HlsParser = class {
           // first segment.
           const segment0TargetTime = segment0.syncTime - lowestSyncTime;
           const streamOffset = segment0TargetTime - segment0.startTime;
-
-          // Modify all SegmentReferences equally.
-          streamInfo.stream.segmentIndex.offset(streamOffset);
           this.offsetStreamInfo_(streamInfo, streamOffset);
+
+          // This is computed across all segments separately to manage
+          // accumulated drift in durations.
+          for (const segment of segmentIndex) {
+            segment.syncAgainst(lowestSyncTime);
+          }
         }
       }
     }
@@ -2773,6 +2776,16 @@ shaka.hls.HlsParser = class {
       }
     }
 
+    // lowestSyncTime is a value from a previous playlist update.  Use it to
+    // set reference start times.  If this is the first playlist parse, we will
+    // skip this step, and wait until we have sync time across stream types.
+    const lowestSyncTime = this.lowestSyncTime_;
+    if (someSyncTime && lowestSyncTime != Infinity) {
+      for (const reference of references) {
+        reference.syncAgainst(lowestSyncTime);
+      }
+    }
+
     return references;
   }
 
diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js
index 66f5b3d7f55..615200cdfe6 100644
--- a/lib/media/segment_reference.js
+++ b/lib/media/segment_reference.js
@@ -8,6 +8,7 @@ goog.provide('shaka.media.InitSegmentReference');
 goog.provide('shaka.media.SegmentReference');
 
 goog.require('goog.asserts');
+goog.require('shaka.log');
 goog.require('shaka.util.ArrayUtils');
 
 
@@ -402,6 +403,23 @@ shaka.media.SegmentReference = class {
       partial.trueEndTime += offset;
     }
   }
+
+  /**
+   * Sync this segment against a particular sync time that will serve as "0" in
+   * the presentation timeline.
+   *
+   * @param {number} lowestSyncTime
+   * @export
+   */
+  syncAgainst(lowestSyncTime) {
+    if (this.syncTime == null) {
+      shaka.log.alwaysError('Sync attempted without sync time!');
+      return;
+    }
+    const desiredStart = this.syncTime - lowestSyncTime;
+    const offset = desiredStart - this.startTime;
+    this.offset(offset);
+  }
 };