diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 1ab7edd6adc00..0055a2eaf7c20 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -674,6 +674,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_ENABLED_DEFAULT = false; + public static final String + DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_REQUIRED = + "dfs.namenode.edits.qjournals.resolution-required"; + public static final boolean + DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_REQUIRED_DEFAULT = true; + public static final String DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_RESOLVER_IMPL = "dfs.namenode.edits.qjournals.resolver.impl"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index 4b7e59c51f13e..4577ae8b1db3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -599,7 +599,12 @@ public ListenableFuture getJournalCTime() { @Override public String toString() { - return InetAddresses.toAddrString(addr.getAddress()) + ':' + addr.getPort(); + if (addr.isUnresolved()) { + return addr.getHostName() + ":" + addr.getPort(); + } else { + return InetAddresses.toAddrString(addr.getAddress()) + ':' + + addr.getPort(); + } } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java index 6eef8ffd38620..0d57d06eb9660 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java @@ -104,7 +104,12 @@ static IPCLoggerChannelMetrics create(IPCLoggerChannel ch) { private static String getName(IPCLoggerChannel ch) { InetSocketAddress addr = ch.getRemoteAddress(); - String addrStr = addr.getAddress().getHostAddress(); + String addrStr; + if (addr.isUnresolved()) { + addrStr = addr.getHostName(); + } else { + addrStr = addr.getAddress().getHostAddress(); + } // IPv6 addresses have colons, which aren't allowed as part of // MBean names. Replace with '.' diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java index 5039db6ceb488..f81e547a3880f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java @@ -378,6 +378,11 @@ public static List getAddressesList(URI uri, Configuration co boolean resolveNeeded = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_ENABLED, DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_ENABLED_DEFAULT); + // If not required, the resulting address may be unresolved, but that can be corrected within + // the client once the server becomes available. + boolean resolveRequired = conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_REQUIRED, + DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_REQUIRED_DEFAULT); DomainNameResolver dnr = DomainNameResolverFactory.newInstance( conf, DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_RESOLVER_IMPL); @@ -394,7 +399,7 @@ public static List getAddressesList(URI uri, Configuration co // QJM should just use FQDN String[] hostnames = dnr .getAllResolvedHostnameByDomainName(isa.getHostName(), true); - if (hostnames.length == 0) { + if (hostnames.length == 0 && resolveRequired) { throw new UnknownHostException(addr); } for (String h : hostnames) { @@ -406,7 +411,7 @@ public static List getAddressesList(URI uri, Configuration co } else { InetSocketAddress isa = NetUtils.createSocketAddr( addr, DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT); - if (isa.isUnresolved()) { + if (isa.isUnresolved() && resolveRequired) { throw new UnknownHostException(addr); } addrs.add(isa); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 5643a9b5c5ee1..d04aaca1a3aa2 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -508,7 +508,20 @@ Determines if the given qjournals address is a domain name which needs to be resolved. - This is used by namenode to resolve qjournals. + This is used by namenode to resolve qjournals, and journalnode for syncing. + + + + + dfs.namenode.edits.qjournals.resolution-required + true + + Determines if the given qjournals address is a domain name which is + required to be resolved. + if resolution is required, then unresolvable addresses are ignored. + if resolution is not required, then the address may be temporarily + unresolved, allowing the client to resolve it later. + This is used by namenode to resolve qjournals, and journalnode for syncing. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java index e9f46e335c6e6..23a7bf9ae595a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java @@ -38,6 +38,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; @@ -61,8 +63,6 @@ import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.slf4j.event.Level; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java index 064dd9e5dd86e..92fa698cb8390 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java @@ -124,6 +124,11 @@ public void setup() throws Exception { "testJournalNodeSyncerNotStartWhenSyncEnabled")) { conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, "qjournal://jn0:9900;jn1:9901/" + journalId); + } else if (testName.getMethodName().equals( + "testJournalNodeSyncerStartWhenSyncEnabledAndUnresolved")) { + conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, + "qjournal://jn0:9900;jn1:9901/" + journalId); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_REQUIRED, false); } else if (testName.getMethodName().equals( "testJournalNodeSyncwithFederationTypeConfigWithNameServiceId")) { conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1", @@ -570,6 +575,12 @@ public void testJournalNodeSyncerNotStartWhenSyncEnabledIncorrectURI() } + /** + * Require resolution. Syncer must not start when there aren't any other known journal + * nodes due to the required resolution. + * + * @throws IOException if unable to get or create the journal + */ @Test public void testJournalNodeSyncerNotStartWhenSyncEnabled() throws IOException { @@ -593,6 +604,20 @@ public void testJournalNodeSyncerNotStartWhenSyncEnabled() } + /** + * Attempt resolution, but allow unresolved. Syncer must start because there are other potential + * journal nodes and syncing is enabled. The syncing will include the unresolved addresses in + * syncing so that once the target server is available it can be included in syncing. + * + * @throws IOException if unable to get or create the journal + */ + @Test + public void testJournalNodeSyncerStartWhenSyncEnabledAndUnresolved() + throws IOException { + jn.getOrCreateJournal(journalId); + Assert.assertEquals(true, + jn.getJournalSyncerStatus(journalId)); + } @Test public void testJournalNodeSyncwithFederationTypeConfigWithNameServiceId() diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java index 82ea34f2e0f20..d717429f495d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestGetConf.java @@ -484,8 +484,9 @@ public void testGetJournalNodes() throws Exception { conf.clear(); } - /* - ** Test for unknown journal node host exception. + /** + * Test handling of unresolvable journal node hosts. They are still configured assuming that + * they will be resolvable in the future. */ @Test(expected = UnknownHostException.class, timeout = 10000) public void testUnknownJournalNodeHost()