diff --git a/src/core/document.js b/src/core/document.js
index cb9e474d3bd21c..19bd28977bc660 100644
--- a/src/core/document.js
+++ b/src/core/document.js
@@ -76,6 +76,7 @@ class Page {
     fontCache,
     builtInCMapCache,
     globalImageCache,
+    nonBlendModesSet,
   }) {
     this.pdfManager = pdfManager;
     this.pageIndex = pageIndex;
@@ -85,6 +86,7 @@ class Page {
     this.fontCache = fontCache;
     this.builtInCMapCache = builtInCMapCache;
     this.globalImageCache = globalImageCache;
+    this.nonBlendModesSet = nonBlendModesSet;
     this.evaluatorOptions = pdfManager.evaluatorOptions;
     this.resourcesPromise = null;
 
@@ -312,7 +314,10 @@ class Page {
       const opList = new OperatorList(intent, sink);
 
       handler.send("StartRenderPage", {
-        transparency: partialEvaluator.hasBlendModes(this.resources),
+        transparency: partialEvaluator.hasBlendModes(
+          this.resources,
+          this.nonBlendModesSet
+        ),
         pageIndex: this.pageIndex,
         intent,
       });
@@ -917,6 +922,7 @@ class PDFDocument {
         fontCache: catalog.fontCache,
         builtInCMapCache: catalog.builtInCMapCache,
         globalImageCache: catalog.globalImageCache,
+        nonBlendModesSet: catalog.nonBlendModesSet,
       });
     }));
   }
diff --git a/src/core/evaluator.js b/src/core/evaluator.js
index 16bff8a9d8cf47..dc61e8f31265a1 100644
--- a/src/core/evaluator.js
+++ b/src/core/evaluator.js
@@ -239,12 +239,15 @@ class PartialEvaluator {
     return newEvaluator;
   }
 
-  hasBlendModes(resources) {
+  hasBlendModes(resources, nonBlendModesSet) {
     if (!(resources instanceof Dict)) {
       return false;
     }
+    if (resources.objId && nonBlendModesSet.has(resources.objId)) {
+      return false;
+    }
 
-    const processed = new RefSet();
+    const processed = new RefSet(nonBlendModesSet);
     if (resources.objId) {
       processed.put(resources.objId);
     }
@@ -344,6 +347,15 @@ class PartialEvaluator {
         }
       }
     }
+
+    if (processed.size > 0) {
+      // When no blend modes exist, there's no need check any of the parsed
+      // `Ref`s again for subsequent pages. This helps reduce redundant
+      // `XRef.fetch` calls for some documents (e.g. issue6961.pdf).
+      processed.forEach(ref => {
+        nonBlendModesSet.put(ref);
+      });
+    }
     return false;
   }
 
diff --git a/src/core/obj.js b/src/core/obj.js
index 7cd4193a69517d..b7565b7b0ed93f 100644
--- a/src/core/obj.js
+++ b/src/core/obj.js
@@ -76,6 +76,7 @@ class Catalog {
     this.builtInCMapCache = new Map();
     this.globalImageCache = new GlobalImageCache();
     this.pageKidsCountCache = new RefSetCache();
+    this.nonBlendModesSet = new RefSet();
   }
 
   get version() {
@@ -937,6 +938,7 @@ class Catalog {
     clearPrimitiveCaches();
     this.globalImageCache.clear(/* onlyData = */ manuallyTriggered);
     this.pageKidsCountCache.clear();
+    this.nonBlendModesSet.clear();
 
     const promises = [];
     this.fontCache.forEach(function (promise) {
diff --git a/src/core/primitives.js b/src/core/primitives.js
index a6083db99a47aa..1587308267ef7a 100644
--- a/src/core/primitives.js
+++ b/src/core/primitives.js
@@ -277,8 +277,20 @@ var Ref = (function RefClosure() {
 // The reference is identified by number and generation.
 // This structure stores only one instance of the reference.
 class RefSet {
-  constructor() {
-    this._set = new Set();
+  constructor(parent = null) {
+    if (
+      (typeof PDFJSDev === "undefined" ||
+        PDFJSDev.test("!PRODUCTION || TESTING")) &&
+      parent &&
+      !(parent instanceof RefSet)
+    ) {
+      unreachable('RefSet: Invalid "parent" value.');
+    }
+    this._set = new Set(parent && parent._set);
+  }
+
+  get size() {
+    return this._set.size;
   }
 
   has(ref) {
@@ -292,6 +304,16 @@ class RefSet {
   remove(ref) {
     this._set.delete(ref.toString());
   }
+
+  forEach(callback) {
+    for (const ref of this._set.values()) {
+      callback(ref);
+    }
+  }
+
+  clear() {
+    this._set.clear();
+  }
 }
 
 class RefSetCache {