add offsetInBlock to avoid reading duplicate data from datanode.

apache · Apr 17, 2024 · 5d597a1 · 5d597a1
1 parent 7e4992b
commit 5d597a1
Showing 1 changed file with 6 additions and 5 deletions.
diff --git a/...op-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java b/...op-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
@@ -235,7 +235,8 @@ private ByteBufferStrategy[] getReadStrategies(StripingChunk chunk) {
 
   private int readToBuffer(BlockReader blockReader,
       DatanodeInfo currentNode, ByteBufferStrategy strategy,
-      LocatedBlock currentBlock, int chunkIndex) throws IOException {
+      LocatedBlock currentBlock, int chunkIndex, long offsetInBlock)
+      throws IOException {
     final int targetLength = strategy.getTargetLength();
     int curAttempts = 0;
     while (curAttempts < readDNMaxAttempts) {
@@ -267,7 +268,7 @@ private int readToBuffer(BlockReader blockReader,
             readerInfos[chunkIndex].reader.close();
           }
           if (dfsStripedInputStream.createBlockReader(currentBlock,
-              alignedStripe.getOffsetInBlock(), targetBlocks,
+              offsetInBlock, targetBlocks,
               readerInfos, chunkIndex, readTo)) {
             blockReader = readerInfos[chunkIndex].reader;
             String msg = "Reconnect to " + currentNode.getInfoAddr()
@@ -307,9 +308,9 @@ private Callable<BlockReadStats> readCells(final BlockReader reader,
 
       int ret = 0;
       for (ByteBufferStrategy strategy : strategies) {
-        int bytesReead = readToBuffer(reader, datanode, strategy, currentBlock,
-            chunkIndex);
-        ret += bytesReead;
+        int bytesRead = readToBuffer(reader, datanode, strategy, currentBlock,
+            chunkIndex, alignedStripe.getOffsetInBlock() + ret);
+        ret += bytesRead;
       }
       return new BlockReadStats(ret, reader.isShortCircuit(),
           reader.getNetworkDistance());