From 4da8cd91cbffffc6937544dcf4c06bf9151a9f22 Mon Sep 17 00:00:00 2001 From: Gang Liao Date: Sat, 28 Sep 2019 15:45:46 -0400 Subject: [PATCH 1/2] [FIX] awaitility schedule conflicts --- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 +- .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../server/namenode/INodeKeyedObjects.java | 101 ++++++++++-------- 3 files changed, 58 insertions(+), 51 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index fc5505fa193..a4447f42ae8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -36,12 +36,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - + org.apache.hadoop commons-pool2 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 015e1bffc45..260f20f9cc0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2552,7 +2552,7 @@ private HdfsFileStatus startFileInt(String src, String syncStr = System.getenv("SYNC_COMMAND_LOGGING"); if (syncStr != null && Boolean.parseBoolean(syncStr) == true) { - await().atMost(10, SECONDS).until(() -> INodeKeyedObjects.getBackupSet().size() < 1024); + INodeKeyedObjects.syncUpdateDB(); } return stat; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeKeyedObjects.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeKeyedObjects.java index adfcb09b3af..7e5c5e8ece8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeKeyedObjects.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeKeyedObjects.java @@ -33,59 +33,62 @@ public static Set getBackupSet() { return concurrentHashSet; } - public static void BackupSetToDB() { + public static void syncUpdateDB() { // In HDFS, the default log buffer size is 512 * 1024 bytes, or 512 KB. // We assume that each object size is 512 bytes, then the size of // concurrentHashSet should be 1024 which only records INode Id. // Note: Using INode Id, it's easy to find INode object in cache. - + int i = 0; + final int num = 1024; + if (concurrentHashSet.size() >= num) { + Iterator iterator = concurrentHashSet.iterator(); + if (LOG.isInfoEnabled()) { + LOG.info("Sync files/directories from cache to database."); + } + + List longAttr = new ArrayList<>(); + List strAttr = new ArrayList<>(); + + List fileIds = new ArrayList<>(); + List fileAttr = new ArrayList<>(); + while (iterator.hasNext()) { + INode inode = + INodeKeyedObjects.getCache().getIfPresent(Long.class, iterator.next()); + + strAttr.add(inode.getLocalName()); + longAttr.add(inode.getParentId()); + longAttr.add(inode.getId()); + longAttr.add(inode.getModificationTime()); + longAttr.add(inode.getAccessTime()); + longAttr.add(inode.getPermissionLong()); + if (inode.isDirectory()) { + longAttr.add(0L); + } else { + longAttr.add(inode.asFile().getHeaderLong()); + FileUnderConstructionFeature uc = + inode.asFile().getFileUnderConstructionFeature(); + if (uc != null) { + fileIds.add(inode.getId()); + fileAttr.add(uc.getClientName(inode.getId())); + fileAttr.add(uc.getClientMachine(inode.getId())); + } + } + iterator.remove(); + if (++i >= num) break; + } + try { + DatabaseINode.batchUpdateINodes(longAttr, strAttr, fileIds, fileAttr); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + public static void BackupSetToDB() { final Runnable updateToDB = new Runnable() { public void run() { - int i = 0; - final int num = 1024; - if (concurrentHashSet.size() >= num) { - Iterator iterator = concurrentHashSet.iterator(); - if (LOG.isInfoEnabled()) { - LOG.info("Sync files/directories from cache to database."); - } - - List longAttr = new ArrayList<>(); - List strAttr = new ArrayList<>(); - - List fileIds = new ArrayList<>(); - List fileAttr = new ArrayList<>(); - while (iterator.hasNext()) { - INode inode = - INodeKeyedObjects.getCache().getIfPresent(Long.class, iterator.next()); - - strAttr.add(inode.getLocalName()); - longAttr.add(inode.getParentId()); - longAttr.add(inode.getId()); - longAttr.add(inode.getModificationTime()); - longAttr.add(inode.getAccessTime()); - longAttr.add(inode.getPermissionLong()); - if (inode.isDirectory()) { - longAttr.add(0L); - } else { - longAttr.add(inode.asFile().getHeaderLong()); - FileUnderConstructionFeature uc = - inode.asFile().getFileUnderConstructionFeature(); - if (uc != null) { - fileIds.add(inode.getId()); - fileAttr.add(uc.getClientName(inode.getId())); - fileAttr.add(uc.getClientMachine(inode.getId())); - } - } - iterator.remove(); - if (++i >= num) break; - } - try { - DatabaseINode.batchUpdateINodes(longAttr, strAttr, fileIds, fileAttr); - } catch (Exception e) { - e.printStackTrace(); - } - } + syncUpdateDB(); } }; @@ -117,7 +120,11 @@ public void run() { public static IndexedCache getCache() { if (cache == null) { concurrentHashSet = ConcurrentHashMap.newKeySet(); - BackupSetToDB(); + + String syncStr = System.getenv("SYNC_COMMAND_LOGGING"); + if (syncStr == null || Boolean.parseBoolean(syncStr) == false) { + BackupSetToDB(); + } // Assuming each INode has 600 bytes, then // 10000000 * 600 / 2^30 = 5.58 GB. From 92f5eb7022b978ffe6244286913c03a301a7cf21 Mon Sep 17 00:00:00 2001 From: Gang Liao Date: Sat, 28 Sep 2019 15:47:43 -0400 Subject: [PATCH 2/2] update --- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index a4447f42ae8..3e0b72db51d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -36,12 +36,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - org.apache.hadoop commons-pool2