HDFS-16186 Datanode kicks out hard disk logic optimization

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeExCountPair.java

@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+/**
+ * Record volume's IOException total counts and update timestamp.
+ */
+public class VolumeExCountPair {
+
+  private long prevTs;
+  private long ioExceptionCnt;
+
+  public VolumeExCountPair() {
+  }
+
+  public VolumeExCountPair(long prevTimeStamp, long ioExceptionCnt) {
+    this.prevTs = prevTimeStamp;
+    this.ioExceptionCnt = ioExceptionCnt;
+  }
+
+  public void setNewPair(long now, long curExCnt) {
+    setPrevTs(now);
+    setIoExceptionCnt(curExCnt);
+  }
+
+  public VolumeExCountPair getPair() {
+    return new VolumeExCountPair(prevTs, ioExceptionCnt);
+  }
+
+  public void setPrevTs(long prevTs) {
+    this.prevTs = prevTs;
+  }
+
+  public void setIoExceptionCnt(long ioExceptionCnt) {
+    this.ioExceptionCnt = ioExceptionCnt;
+  }
+
+  public long getPrevTs() {
+    return prevTs;
+  }
+
+  public long getIoExceptionCnt() {
+    return ioExceptionCnt;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.datanode.checker;
 
+import org.apache.hadoop.hdfs.server.datanode.VolumeExCountPair;
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback;
@@ -37,6 +38,7 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.Timer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -392,7 +394,24 @@ public void onFailure(@Nonnull Throwable t) {
     }
 
     private void markHealthy() {
+
       synchronized (DatasetVolumeChecker.this) {
+        long totalFileIoErrors = reference.getVolume().getMetrics().getTotalFileIoErrors();
+        VolumeExCountPair volumeExCountPair = reference.getVolume().getExCountPair();
+        LOG.warn("Volume {} Exception Count is {}", reference.getVolume().toString(),
+                reference.getVolume().getExCountPair().getIoExceptionCnt());
+        if (volumeExCountPair.getIoExceptionCnt() == 0) {
+          volumeExCountPair.setNewPair(Time.monotonicNow(), totalFileIoErrors);
+        }else{
+          long lastTime = reference.getVolume().getExCountPair().getPrevTs();
+          if((Time.monotonicNow() - lastTime)/1000/60/60 > 2){
+            long cnt = reference.getVolume().getExCountPair().getPair().getIoExceptionCnt();
+            if(cnt > 30){
+              failedVolumes.add(reference.getVolume());
+              return;
+            }
+          }
+        }
         healthyVolumes.add(reference.getVolume());
       }
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java

@@ -36,6 +36,7 @@
 import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler;
 import org.apache.hadoop.hdfs.server.datanode.FileIoProvider;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.hdfs.server.datanode.VolumeExCountPair;
 import org.apache.hadoop.hdfs.server.datanode.checker.Checkable;
 import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
 
@@ -448,4 +449,6 @@ class VolumeCheckContext {
   FileIoProvider getFileIoProvider();
 
   DataNodeVolumeMetrics getMetrics();
+
+  VolumeExCountPair getExCountPair();
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java

@@ -57,7 +57,6 @@
 import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
 import org.apache.hadoop.hdfs.server.datanode.DataStorage;
 import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
-import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler;
 import org.apache.hadoop.hdfs.server.datanode.FileIoProvider;
 import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
 import org.apache.hadoop.hdfs.server.datanode.LocalReplica;
@@ -67,6 +66,8 @@
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.hdfs.server.datanode.VolumeExCountPair;
+import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler;
 import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -139,6 +140,8 @@ public class FsVolumeImpl implements FsVolumeSpi {
   private final String mount;
   private double reservedForArchive;
 
+  private VolumeExCountPair volumeExCountPair = new VolumeExCountPair();
+
   /**
    * Per-volume worker pool that processes new blocks to cache.
    * The maximum number of workers per volume is bounded (configurable via
@@ -1405,6 +1408,11 @@ public DataNodeVolumeMetrics getMetrics() {
     return metrics;
   }
 
+  @Override
+  public VolumeExCountPair getExCountPair() {
+    return this.volumeExCountPair;
+  }
+
   /**
    * Filter for block file names stored on the file system volumes.
    */

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java

@@ -563,6 +563,7 @@ static class SimulatedVolume implements FsVolumeSpi {
     private final SimulatedStorage storage;
     private final FileIoProvider fileIoProvider;
     private final DataNodeVolumeMetrics metrics;
+    private VolumeExCountPair volumeExCountPair = new VolumeExCountPair();
 
     SimulatedVolume(final SimulatedStorage storage,
                     final FileIoProvider fileIoProvider,
@@ -629,6 +630,11 @@ public void releaseLockedMemory(long bytesToRelease) {
     public void releaseReservedSpace(long bytesToRelease) {
     }
 
+    @Override
+    public VolumeExCountPair getExCountPair() {
+      return this.volumeExCountPair;
+    }
+
     @Override
     public BlockIterator newBlockIterator(String bpid, String name) {
       throw new UnsupportedOperationException();

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java

@@ -927,6 +927,12 @@ private void verifyStorageType(long blockId, boolean expectTransient) {
   }
 
   private static class TestFsVolumeSpi implements FsVolumeSpi {
+    private VolumeExCountPair volumeExCountPair;
+    @Override
+    public VolumeExCountPair getExCountPair() {
+      return this.volumeExCountPair;
+    }
+
     @Override
     public String[] getBlockPoolList() {
       return new String[0];

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java

@@ -30,13 +30,21 @@
 import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler;
 import org.apache.hadoop.hdfs.server.datanode.FileIoProvider;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.hdfs.server.datanode.VolumeExCountPair;
 import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 
 public class ExternalVolumeImpl implements FsVolumeSpi {
+  private VolumeExCountPair volumeExCountPair;
+
+  @Override
+  public VolumeExCountPair getExCountPair() {
+    return this.volumeExCountPair;
+  }
+
   @Override
   public FsVolumeReference obtainReference() throws ClosedChannelException {
     return null;