From e4dd3f93ee4c83e4cb394078560998d9795fef08 Mon Sep 17 00:00:00 2001 From: Gang Liao Date: Wed, 10 Jun 2020 15:53:08 +0800 Subject: [PATCH] fix: remote renmae logging failure (#267) (#268) --- .../hdfs/server/namenode/FSEditLog.java | 22 +++++++------------ .../namenode/FSEditLogProtocolImpl.java | 8 ++++++- .../server/namenode/FSImageFormatPBINode.java | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index f59f2370042..795f5debf31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -120,8 +120,8 @@ import java.io.ByteArrayOutputStream; import java.net.InetSocketAddress; -// import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Saver.buildAclEntries; -// import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Saver.buildXAttrs; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Saver.buildAclEntries; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.Saver.buildXAttrs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; @@ -820,8 +820,6 @@ public void logAppendFile(String path, INodeFile file, boolean newBlock, } public void remoteLogOpenFile(INodeFile newNode, String nameNodeAddress) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - INodeSection.INodeFile.Builder b = INodeSection.INodeFile.newBuilder() .setAccessTime(newNode.getAccessTime()) .setModificationTime(newNode.getModificationTime()) @@ -838,12 +836,12 @@ public void remoteLogOpenFile(INodeFile newNode, String nameNodeAddress) { AclFeature acl = newNode.getAclFeature(); if (acl != null) { - // b.setAcl(buildAclEntries(acl)); + b.setAcl(buildAclEntries(acl)); } XAttrFeature xAttrFeature = newNode.getXAttrFeature(); if (xAttrFeature != null) { - // b.setXAttrs(buildXAttrs(xAttrFeature)); + b.setXAttrs(buildXAttrs(xAttrFeature)); } BlockInfo[] blocks = newNode.getBlocks(); @@ -868,9 +866,8 @@ public void remoteLogOpenFile(INodeFile newNode, String nameNodeAddress) { .setId(newNode.getId()) .setName(ByteString.copyFrom(newNode.getLocalNameBytes())) .setType(INodeSection.INode.Type.FILE).setFile(b).build(); - r.writeDelimitedTo(out); - byte[] data = out.toByteArray(); + byte[] data = r.toByteArray(); FSEditLogProtocol proxy = (FSEditLogProtocol) RPC.getProxy( FSEditLogProtocol.class, FSEditLogProtocol.versionID, new InetSocketAddress(nameNodeAddress, 10086), new Configuration()); @@ -956,8 +953,6 @@ public void logUpdateBlocks(String path, INodeFile file, boolean toLogRpcIds) { } public void remoteLogMkDir(INodeDirectory newNode, String nameNodeAddress) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - INodeSection.INodeDirectory.Builder b = INodeSection.INodeDirectory .newBuilder() .setModificationTime(newNode.getModificationTime()) @@ -965,12 +960,12 @@ public void remoteLogMkDir(INodeDirectory newNode, String nameNodeAddress) { AclFeature f = newNode.getAclFeature(); if (f != null) { - // b.setAcl(buildAclEntries(f)); + b.setAcl(buildAclEntries(f)); } XAttrFeature xAttrFeature = newNode.getXAttrFeature(); if (xAttrFeature != null) { - // b.setXAttrs(buildXAttrs(xAttrFeature)); + b.setXAttrs(buildXAttrs(xAttrFeature)); } try { @@ -978,9 +973,8 @@ public void remoteLogMkDir(INodeDirectory newNode, String nameNodeAddress) { .setId(newNode.getId()) .setName(ByteString.copyFrom(newNode.getLocalNameBytes())) .setType(INodeSection.INode.Type.DIRECTORY).setDirectory(b).build(); - r.writeDelimitedTo(out); - byte[] data = out.toByteArray(); + byte[] data = r.toByteArray(); FSEditLogProtocol proxy = (FSEditLogProtocol) RPC.getProxy( FSEditLogProtocol.class, FSEditLogProtocol.versionID, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogProtocolImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogProtocolImpl.java index 0395a6739ca..dabab7c21b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogProtocolImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogProtocolImpl.java @@ -63,6 +63,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import com.google.common.base.Preconditions; +import com.google.protobuf.InvalidProtocolBufferException; public class FSEditLogProtocolImpl implements FSEditLogProtocol { @@ -157,7 +158,12 @@ public INodeDirectory loadINodeDirectory(INodeSection.INode n) { @Override public void logEdit(byte[] in) throws IOException { - INodeSection.INode p = INodeSection.INode.parseFrom(in); + INodeSection.INode p = NULL; + try { + p = INodeSection.INode.parseFrom(in); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + } INode n; switch (p.getType()) { case FILE: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index ec631c5ecd0..344ea415065 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -404,7 +404,7 @@ private static long buildPermissionStatus(INodeAttributes n) { return n.getPermissionLong(); } - private static AclFeatureProto.Builder buildAclEntries(AclFeature f) { + public static AclFeatureProto.Builder buildAclEntries(AclFeature f) { AclFeatureProto.Builder b = AclFeatureProto.newBuilder(); for (int pos = 0, e; pos < f.getEntriesSize(); pos++) { e = f.getEntryAt(pos); @@ -413,7 +413,7 @@ private static AclFeatureProto.Builder buildAclEntries(AclFeature f) { return b; } - private static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f) { + public static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f) { XAttrFeatureProto.Builder b = XAttrFeatureProto.newBuilder(); for (XAttr a : f.getXAttrs()) { XAttrCompactProto.Builder xAttrCompactBuilder = XAttrCompactProto.