diff --git a/src/main/java/org/opensearch/knn/index/KNNWeight.java b/src/main/java/org/opensearch/knn/index/KNNWeight.java
index 909e7222f..a6e0cd350 100644
--- a/src/main/java/org/opensearch/knn/index/KNNWeight.java
+++ b/src/main/java/org/opensearch/knn/index/KNNWeight.java
@@ -186,7 +186,10 @@ public Scorer scorer(LeafReaderContext context) throws IOException {
             .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), spaceType)));
         int maxDoc = Collections.max(scores.keySet()) + 1;
         DocIdSetBuilder docIdSetBuilder = new DocIdSetBuilder(maxDoc);
-        DocIdSetBuilder.BulkAdder setAdder = docIdSetBuilder.grow(maxDoc);
+
+        // The docIdSetIterator will contain the docids of the returned results. So, before adding results to
+        // the builder, we can grow to results.length
+        DocIdSetBuilder.BulkAdder setAdder = docIdSetBuilder.grow(results.length);
         Arrays.stream(results).forEach(result -> setAdder.add(result.getId()));
         DocIdSetIterator docIdSetIter = docIdSetBuilder.build().iterator();
         return new KNNScorer(this, docIdSetIter, scores, boost);