diff --git a/src/remux/passthrough-remuxer.ts b/src/remux/passthrough-remuxer.ts
index c822b942f82..aa68758ac66 100644
--- a/src/remux/passthrough-remuxer.ts
+++ b/src/remux/passthrough-remuxer.ts
@@ -168,11 +168,12 @@ class PassThroughRemuxer implements Remuxer {
     }
 
     const startDTS = getStartDTS(initData, data);
+    const decodeTime = startDTS === null ? timeOffset : startDTS;
     if (
-      isInvalidInitPts(initPTS, startDTS, timeOffset) ||
+      isInvalidInitPts(initPTS, decodeTime, timeOffset) ||
       (initSegment.timescale !== initPTS.timescale && accurateTimeOffset)
     ) {
-      initSegment.initPTS = startDTS - timeOffset;
+      initSegment.initPTS = decodeTime - timeOffset;
       this.initPTS = initPTS = {
         baseTime: initSegment.initPTS,
         timescale: 1,
@@ -181,7 +182,7 @@ class PassThroughRemuxer implements Remuxer {
 
     const duration = getDuration(data, initData);
     const startTime = audioTrack
-      ? startDTS - initPTS.baseTime / initPTS.timescale
+      ? decodeTime - initPTS.baseTime / initPTS.timescale
       : (lastEndTime as number);
     const endTime = startTime + duration;
     offsetStartDTS(initData, data, initPTS.baseTime / initPTS.timescale);
diff --git a/src/utils/mp4-tools.ts b/src/utils/mp4-tools.ts
index 5fd52c3162b..8de7f4e3708 100644
--- a/src/utils/mp4-tools.ts
+++ b/src/utils/mp4-tools.ts
@@ -350,10 +350,13 @@ export function parseSinf(sinf: Uint8Array): Uint8Array | null {
  * @returns the earliest base media decode start time for the
  * fragment, in seconds
  */
-export function getStartDTS(initData: InitData, fmp4: Uint8Array): number {
+export function getStartDTS(
+  initData: InitData,
+  fmp4: Uint8Array
+): number | null {
   // we need info from two children of each track fragment box
-  return (
-    findBox(fmp4, ['moof', 'traf']).reduce((result: number | null, traf) => {
+  return findBox(fmp4, ['moof', 'traf']).reduce(
+    (result: number | null, traf) => {
       const tfdt = findBox(traf, ['tfdt'])[0];
       const version = tfdt[0];
       const start = findBox(traf, ['tfhd']).reduce(
@@ -364,7 +367,16 @@ export function getStartDTS(initData: InitData, fmp4: Uint8Array): number {
           if (track) {
             let baseTime = readUint32(tfdt, 4);
             if (version === 1) {
-              baseTime *= Math.pow(2, 32);
+              // If value is too large, assume signed 64-bit. Negative track fragment decode times are invalid, but they exist in the wild.
+              // This prevents large values from being used for initPTS, which can cause playlist sync issues.
+              // https://github.com/video-dev/hls.js/issues/5303
+              if (baseTime === UINT32_MAX) {
+                logger.warn(
+                  `[mp4-demuxer]: Ignoring assumed invalid signed 64-bit track fragment decode time`
+                );
+                return result;
+              }
+              baseTime *= UINT32_MAX + 1;
               baseTime += readUint32(tfdt, 8);
             }
             // assume a 90kHz clock if no timescale was specified
@@ -390,7 +402,8 @@ export function getStartDTS(initData: InitData, fmp4: Uint8Array): number {
         return start;
       }
       return result;
-    }, null) || 0
+    },
+    null
   );
 }