From 04c34a2171088d17ac4114df50c63a99aad9a1e9 Mon Sep 17 00:00:00 2001 From: WangYuanben <48795318+YuanbenWang@users.noreply.github.com> Date: Thu, 3 Aug 2023 22:22:06 +0800 Subject: [PATCH 01/48] HDFS-17113. Reconfig transfer and write bandwidth for datanode. (#5869) --- .../hadoop/hdfs/server/datanode/DataNode.java | 48 ++++++++++++++++--- .../server/datanode/DataXceiverServer.java | 15 +++++- .../datanode/TestDataNodeReconfiguration.java | 45 ++++++++++++++++- .../hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- 4 files changed, 99 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 0ed1304cb8f02..bb61e8037e9f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -36,6 +36,10 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ALLOW_SAME_DISK_TIERING; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ALLOW_SAME_DISK_TIERING_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_KEY; @@ -362,7 +366,9 @@ public class DataNode extends ReconfigurableBase FS_GETSPACEUSED_JITTER_KEY, FS_GETSPACEUSED_CLASSNAME, DFS_DISK_BALANCER_ENABLED, - DFS_DISK_BALANCER_PLAN_VALID_INTERVAL)); + DFS_DISK_BALANCER_PLAN_VALID_INTERVAL, + DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY, + DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY)); public static final String METRICS_LOG_NAME = "DataNodeMetricsLog"; @@ -694,6 +700,8 @@ public String reconfigurePropertyImpl(String property, String newVal) case DFS_BLOCKREPORT_INITIAL_DELAY_KEY: return reconfBlockReportParameters(property, newVal); case DFS_DATANODE_MAX_RECEIVER_THREADS_KEY: + case DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY: + case DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY: return reconfDataXceiverParameters(property, newVal); case DFS_CACHEREPORT_INTERVAL_MSEC_KEY: return reconfCacheReportParameters(property, newVal); @@ -724,14 +732,40 @@ public String reconfigurePropertyImpl(String property, String newVal) private String reconfDataXceiverParameters(String property, String newVal) throws ReconfigurationException { - String result; + String result = null; try { LOG.info("Reconfiguring {} to {}", property, newVal); - Preconditions.checkNotNull(getXferServer(), "DataXceiverServer has not been initialized."); - int threads = (newVal == null ? DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT : - Integer.parseInt(newVal)); - result = Integer.toString(threads); - getXferServer().setMaxXceiverCount(threads); + if (property.equals(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY)) { + Preconditions.checkNotNull(getXferServer(), "DataXceiverServer has not been initialized."); + int threads = (newVal == null ? DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT : + Integer.parseInt(newVal)); + result = Integer.toString(threads); + getXferServer().setMaxXceiverCount(threads); + } else if (property.equals(DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY)) { + Preconditions.checkNotNull(getXferServer(), "DataXceiverServer has not been initialized."); + long bandwidthPerSec = (newVal == null ? + DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_DEFAULT : Long.parseLong(newVal)); + DataTransferThrottler transferThrottler = null; + if (bandwidthPerSec > 0) { + transferThrottler = new DataTransferThrottler(bandwidthPerSec); + } else { + bandwidthPerSec = 0; + } + result = Long.toString(bandwidthPerSec); + getXferServer().setTransferThrottler(transferThrottler); + } else if (property.equals(DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY)) { + Preconditions.checkNotNull(getXferServer(), "DataXceiverServer has not been initialized."); + long bandwidthPerSec = (newVal == null ? DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_DEFAULT : + Long.parseLong(newVal)); + DataTransferThrottler writeThrottler = null; + if (bandwidthPerSec > 0) { + writeThrottler = new DataTransferThrottler(bandwidthPerSec); + } else { + bandwidthPerSec = 0; + } + result = Long.toString(bandwidthPerSec); + getXferServer().setWriteThrottler(writeThrottler); + } LOG.info("RECONFIGURE* changed {} to {}", property, newVal); return result; } catch (IllegalArgumentException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java index 3a67b76e43467..9b31dd3b21855 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java @@ -168,9 +168,9 @@ void release() { final BlockBalanceThrottler balanceThrottler; - private final DataTransferThrottler transferThrottler; + private volatile DataTransferThrottler transferThrottler; - private final DataTransferThrottler writeThrottler; + private volatile DataTransferThrottler writeThrottler; /** * Stores an estimate for block size to check if the disk partition has enough @@ -200,7 +200,10 @@ void release() { DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_DEFAULT), conf.getInt(DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY, DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT)); + initBandwidthPerSec(conf); + } + private void initBandwidthPerSec(Configuration conf) { long bandwidthPerSec = conf.getLongBytes( DFSConfigKeys.DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY, DFSConfigKeys.DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_DEFAULT); @@ -524,4 +527,12 @@ public void setMaxXceiverCount(int xceiverCount) { public int getMaxXceiverCount() { return maxXceiverCount; } + + public void setTransferThrottler(DataTransferThrottler transferThrottler) { + this.transferThrottler = transferThrottler; + } + + public void setWriteThrottler(DataTransferThrottler writeThrottler) { + this.writeThrottler = writeThrottler; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java index a14ee2554f03a..0ca5bedff8bbe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java @@ -31,6 +31,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT; @@ -443,20 +445,61 @@ public void testDataXceiverReconfiguration() assertTrue("expecting IllegalArgumentException", expected.getCause() instanceof IllegalArgumentException); } + try { + dn.reconfigureProperty(DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY, "text"); + fail("ReconfigurationException expected"); + } catch (ReconfigurationException expected) { + assertTrue("expecting NumberFormatException", + expected.getCause() instanceof NumberFormatException); + } + try { + dn.reconfigureProperty(DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY, "text"); + fail("ReconfigurationException expected"); + } catch (ReconfigurationException expected) { + assertTrue("expecting NumberFormatException", + expected.getCause() instanceof NumberFormatException); + } // Change properties and verify change. dn.reconfigureProperty(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, String.valueOf(123)); assertEquals(String.format("%s has wrong value", DFS_DATANODE_MAX_RECEIVER_THREADS_KEY), 123, dn.getXferServer().getMaxXceiverCount()); + dn.reconfigureProperty(DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY, + String.valueOf(1000)); + assertEquals(String.format("%s has wrong value", + DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY), + 1000, dn.getXferServer().getTransferThrottler().getBandwidth()); + + dn.reconfigureProperty(DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY, + String.valueOf(1000)); + assertEquals(String.format("%s has wrong value", + DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY), + 1000, dn.getXferServer().getWriteThrottler().getBandwidth()); + // Revert to default. dn.reconfigureProperty(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, null); assertEquals(String.format("%s has wrong value", DFS_DATANODE_MAX_RECEIVER_THREADS_KEY), DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT, dn.getXferServer().getMaxXceiverCount()); - assertNull(String.format("expect %s is not configured", DFS_DATANODE_MAX_RECEIVER_THREADS_KEY), dn.getConf().get(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY)); + + dn.reconfigureProperty(DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY, null); + assertEquals(String.format("%s has wrong value", + DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY), + null, dn.getXferServer().getTransferThrottler()); + assertNull(String.format("expect %s is not configured", + DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY), + dn.getConf().get(DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY)); + + dn.reconfigureProperty(DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY, null); + assertEquals(String.format("%s has wrong value", + DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY), + null, dn.getXferServer().getWriteThrottler()); + assertNull(String.format("expect %s is not configured", + DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY), + dn.getConf().get(DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 78664e27ca286..d7fee5f1f809e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -346,7 +346,7 @@ public void testDataNodeGetReconfigurableProperties() throws IOException, Interr final List outs = Lists.newArrayList(); final List errs = Lists.newArrayList(); getReconfigurableProperties("datanode", address, outs, errs); - assertEquals(22, outs.size()); + assertEquals(24, outs.size()); assertEquals(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, outs.get(1)); } From 6e6349c79301c84e1929fb4b41149dbb439fa707 Mon Sep 17 00:00:00 2001 From: Peter Szucs <116345192+p-szucs@users.noreply.github.com> Date: Thu, 3 Aug 2023 16:29:41 +0200 Subject: [PATCH 02/48] YARN-11545. Fixed FS2CS ACL conversion when all users are allowed. (#5910) --- .../scheduler/fair/converter/FSConfigToCSConfigConverter.java | 4 ++-- .../fair/converter/TestFSConfigToCSConfigConverter.java | 4 ++-- .../src/test/resources/fair-scheduler-conversion.xml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index 25832c7633bf1..646fb03798fb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -432,13 +432,13 @@ private void generateQueueAcl(String queue, AccessControlList adminAcls = access.get(AccessType.ADMINISTER_QUEUE); if (!submitAcls.getGroups().isEmpty() || - !submitAcls.getUsers().isEmpty()) { + !submitAcls.getUsers().isEmpty() || submitAcls.isAllAllowed()) { capacitySchedulerConfig.set(PREFIX + queue + ".acl_submit_applications", submitAcls.getAclString()); } if (!adminAcls.getGroups().isEmpty() || - !adminAcls.getUsers().isEmpty()) { + !adminAcls.getUsers().isEmpty() || adminAcls.isAllAllowed()) { capacitySchedulerConfig.set(PREFIX + queue + ".acl_administer_queue", adminAcls.getAclString()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index beb8f2f1de93f..86123f211f3f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -263,9 +263,9 @@ public void testConvertACLs() throws Exception { conf.get(PREFIX + "root.admins.alice.acl_administer_queue")); // root.users.john - assertEquals("root.users.john submit ACL", "john ", + assertEquals("root.users.john submit ACL", "*", conf.get(PREFIX + "root.users.john.acl_submit_applications")); - assertEquals("root.users.john admin ACL", "john ", + assertEquals("root.users.john admin ACL", "*", conf.get(PREFIX + "root.users.john.acl_administer_queue")); // root.users.joe diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml index 2c4f28930c417..0874c7c98d80b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml @@ -33,8 +33,8 @@ 1.0 memory-mb=4096, vcores=1 drf - john - john + * + * vcores=2,memory-mb=8192 From 35af8b9d021e240e5f0617fa332c00366d300be7 Mon Sep 17 00:00:00 2001 From: Susheel Gupta <38013283+susheelgupta7@users.noreply.github.com> Date: Thu, 3 Aug 2023 20:05:27 +0530 Subject: [PATCH 03/48] YARN-11535: Jackson-dataformat-yaml should be upgraded to 2.15.2 as it may cause transitive dependency issue with 2.12.7 (#5884) --- hadoop-project/pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 5504c51522526..02feba3d1e685 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -71,6 +71,7 @@ 2.12.7 2.12.7.1 + 2.15.2 4.5.13 @@ -1303,7 +1304,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - ${jackson2.version} + ${jackson2.dataformat-yaml.version} org.mockito From 600dd45e47cfda8c19e14baa76757b4fea250709 Mon Sep 17 00:00:00 2001 From: zhtttylz Date: Fri, 4 Aug 2023 13:07:19 +0800 Subject: [PATCH 04/48] HDFS-17122. Rectify the table length discrepancy in the DataNode UI (#5915) --- .../hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js index 42fb059f09ed1..8074755bfee24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js @@ -378,6 +378,7 @@ }); } }); + $("#table-datanodes").width('100%'); renderHistogram(data); $('#ui-tabs a[href="#tab-datanode"]').tab('show'); }); From 8eb58630ec12826fce765305787e325a547069f1 Mon Sep 17 00:00:00 2001 From: susheel-gupta <113329035+susheel-gupta@users.noreply.github.com> Date: Fri, 4 Aug 2023 14:02:55 +0530 Subject: [PATCH 05/48] YARN-11416. FS2CS should use CapacitySchedulerConfiguration in FSQueueConverterBuilder (#5320) Co-authored-by: Susheel Gupta <38013283+susheelgupta7@users.noreply.github.com> --- .../CapacitySchedulerConfiguration.java | 75 ++++- .../FSConfigToCSConfigConverter.java | 39 +-- .../fair/converter/FSQueueConverter.java | 66 ++-- .../converter/FSQueueConverterBuilder.java | 8 +- .../weightconversion/CapacityConverter.java | 4 +- .../WeightToPercentConverter.java | 12 +- .../WeightToWeightConverter.java | 26 +- .../TestFSConfigToCSConfigConverter.java | 133 ++++---- .../fair/converter/TestFSQueueConverter.java | 315 ++++++++++-------- .../TestWeightToPercentageConverter.java | 100 +++--- .../TestWeightToWeightConverter.java | 70 ++-- 11 files changed, 445 insertions(+), 403 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index b3934c695a968..417638516251a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -501,6 +501,10 @@ public int getMaximumSystemApplications() { return maxApplications; } + public void setMaximumApplicationMasterResourcePercent(float percent) { + setFloat(PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX, percent); + } + public float getMaximumApplicationMasterResourcePercent() { return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT); @@ -1222,6 +1226,16 @@ public void reinitializeConfigurationProperties() { configurationProperties = new ConfigurationProperties(props); } + public void setQueueMaximumAllocationMb(String queue, int value) { + String queuePrefix = getQueuePrefix(queue); + setInt(queuePrefix + MAXIMUM_ALLOCATION_MB, value); + } + + public void setQueueMaximumAllocationVcores(String queue, int value) { + String queuePrefix = getQueuePrefix(queue); + setInt(queuePrefix + MAXIMUM_ALLOCATION_VCORES, value); + } + public long getQueueMaximumAllocationMb(String queue) { String queuePrefix = getQueuePrefix(queue); return getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, (int)UNDEFINED); @@ -1496,6 +1510,14 @@ public List parseJSONMappingRules() throws IOException { return new ArrayList<>(); } + public void setMappingRuleFormat(String format) { + set(MAPPING_RULE_FORMAT, format); + } + + public void setMappingRuleJson(String json) { + set(MAPPING_RULE_JSON, json); + } + public List getMappingRules() throws IOException { String mappingFormat = get(MAPPING_RULE_FORMAT, MAPPING_RULE_FORMAT_DEFAULT); @@ -1712,6 +1734,14 @@ public boolean getIntraQueuePreemptionDisabled(String queue, + QUEUE_PREEMPTION_DISABLED, defaultVal); } + public void setPreemptionObserveOnly(boolean value) { + setBoolean(PREEMPTION_OBSERVE_ONLY, value); + } + + public boolean getPreemptionObserveOnly() { + return getBoolean(PREEMPTION_OBSERVE_ONLY, DEFAULT_PREEMPTION_OBSERVE_ONLY); + } + /** * Get configured node labels in a given queuePath. * @@ -1816,29 +1846,48 @@ public static boolean shouldAppFailFast(Configuration conf) { return conf.getBoolean(APP_FAIL_FAST, DEFAULT_APP_FAIL_FAST); } - public Integer getMaxParallelAppsForQueue(String queue) { - int defaultMaxParallelAppsForQueue = - getInt(PREFIX + MAX_PARALLEL_APPLICATIONS, + public void setDefaultMaxParallelApps(int value) { + setInt(PREFIX + MAX_PARALLEL_APPLICATIONS, value); + } + + public Integer getDefaultMaxParallelApps() { + return getInt(PREFIX + MAX_PARALLEL_APPLICATIONS, DEFAULT_MAX_PARALLEL_APPLICATIONS); + } - String maxParallelAppsForQueue = get(getQueuePrefix(queue) - + MAX_PARALLEL_APPLICATIONS); + public void setDefaultMaxParallelAppsPerUser(int value) { + setInt(PREFIX + "user." + MAX_PARALLEL_APPLICATIONS, value); + } - return (maxParallelAppsForQueue != null) ? - Integer.parseInt(maxParallelAppsForQueue) - : defaultMaxParallelAppsForQueue; + public Integer getDefaultMaxParallelAppsPerUser() { + return getInt(PREFIX + "user." + MAX_PARALLEL_APPLICATIONS, + DEFAULT_MAX_PARALLEL_APPLICATIONS); + } + + public void setMaxParallelAppsForUser(String user, int value) { + setInt(getUserPrefix(user) + MAX_PARALLEL_APPLICATIONS, value); } public Integer getMaxParallelAppsForUser(String user) { - int defaultMaxParallelAppsForUser = - getInt(PREFIX + "user." + MAX_PARALLEL_APPLICATIONS, - DEFAULT_MAX_PARALLEL_APPLICATIONS); String maxParallelAppsForUser = get(getUserPrefix(user) + MAX_PARALLEL_APPLICATIONS); return (maxParallelAppsForUser != null) ? - Integer.parseInt(maxParallelAppsForUser) - : defaultMaxParallelAppsForUser; + Integer.valueOf(maxParallelAppsForUser) + : getDefaultMaxParallelAppsPerUser(); + } + + public void setMaxParallelAppsForQueue(String queue, String value) { + set(getQueuePrefix(queue) + MAX_PARALLEL_APPLICATIONS, value); + } + + public Integer getMaxParallelAppsForQueue(String queue) { + String maxParallelAppsForQueue = get(getQueuePrefix(queue) + + MAX_PARALLEL_APPLICATIONS); + + return (maxParallelAppsForQueue != null) ? + Integer.valueOf(maxParallelAppsForQueue) + : getDefaultMaxParallelApps(); } public boolean getAllowZeroCapacitySum(String queue) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index 646fb03798fb1..6c8fed8182b80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -16,9 +16,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_JSON; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT_JSON; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSQueueConverter.QUEUE_MAX_AM_SHARE_DISABLED; @@ -35,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AccessType; @@ -342,14 +340,13 @@ private void performRuleConversion(FairScheduler fs) } writer.writeValue(mappingRulesOutputStream, desc); - capacitySchedulerConfig.set(MAPPING_RULE_FORMAT, - MAPPING_RULE_FORMAT_JSON); + capacitySchedulerConfig.setMappingRuleFormat(MAPPING_RULE_FORMAT_JSON); capacitySchedulerConfig.setOverrideWithQueueMappings(true); if (!rulesToFile) { String json = ((ByteArrayOutputStream)mappingRulesOutputStream) .toString(StandardCharsets.UTF_8.displayName()); - capacitySchedulerConfig.set(MAPPING_RULE_JSON, json); + capacitySchedulerConfig.setMappingRuleJson(json); } } else { LOG.info("No rules to convert"); @@ -377,38 +374,31 @@ private OutputStream getOutputStreamForJson() throws FileNotFoundException { private void emitDefaultQueueMaxParallelApplications() { if (queueMaxAppsDefault != Integer.MAX_VALUE) { - capacitySchedulerConfig.set( - PREFIX + "max-parallel-apps", - String.valueOf(queueMaxAppsDefault)); + capacitySchedulerConfig.setDefaultMaxParallelApps( + queueMaxAppsDefault); } } private void emitDefaultUserMaxParallelApplications() { if (userMaxAppsDefault != Integer.MAX_VALUE) { - capacitySchedulerConfig.set( - PREFIX + "user.max-parallel-apps", - String.valueOf(userMaxAppsDefault)); + capacitySchedulerConfig.setDefaultMaxParallelAppsPerUser( + userMaxAppsDefault); } } private void emitUserMaxParallelApplications() { userMaxApps .forEach((user, apps) -> { - capacitySchedulerConfig.setInt( - PREFIX + "user." + user + ".max-parallel-apps", apps); + capacitySchedulerConfig.setMaxParallelAppsForUser(user, apps); }); } private void emitDefaultMaxAMShare() { if (queueMaxAMShareDefault == QUEUE_MAX_AM_SHARE_DISABLED) { - capacitySchedulerConfig.setFloat( - CapacitySchedulerConfiguration. - MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + capacitySchedulerConfig.setMaximumApplicationMasterResourcePercent( 1.0f); } else { - capacitySchedulerConfig.setFloat( - CapacitySchedulerConfiguration. - MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + capacitySchedulerConfig.setMaximumApplicationMasterResourcePercent( queueMaxAMShareDefault); } } @@ -416,8 +406,7 @@ private void emitDisablePreemptionForObserveOnlyMode() { if (preemptionMode == FSConfigToCSConfigConverterParams .PreemptionMode.OBSERVE_ONLY) { capacitySchedulerConfig. - setBoolean(CapacitySchedulerConfiguration. - PREEMPTION_OBSERVE_ONLY, true); + setPreemptionObserveOnly(true); } } @@ -433,13 +422,13 @@ private void generateQueueAcl(String queue, if (!submitAcls.getGroups().isEmpty() || !submitAcls.getUsers().isEmpty() || submitAcls.isAllAllowed()) { - capacitySchedulerConfig.set(PREFIX + queue + ".acl_submit_applications", + capacitySchedulerConfig.setAcl(queue, QueueACL.SUBMIT_APPLICATIONS, submitAcls.getAclString()); } if (!adminAcls.getGroups().isEmpty() || !adminAcls.getUsers().isEmpty() || adminAcls.isAllAllowed()) { - capacitySchedulerConfig.set(PREFIX + queue + ".acl_administer_queue", + capacitySchedulerConfig.setAcl(queue, QueueACL.ADMINISTER_QUEUE, adminAcls.getAclString()); } } @@ -501,7 +490,7 @@ Configuration getYarnSiteConfig() { } @VisibleForTesting - Configuration getCapacitySchedulerConfig() { + CapacitySchedulerConfiguration getCapacitySchedulerConfig() { return capacitySchedulerConfig; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index 34400d229d247..1bbf056c148d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -16,16 +16,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_ENABLED; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DEFAULT_AUTO_QUEUE_CREATION_ENABLED; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR; import java.util.List; import java.util.stream.Collectors; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; @@ -50,7 +45,7 @@ public class FSQueueConverter { private static final String FIFO_POLICY = "fifo"; private final FSConfigToCSConfigRuleHandler ruleHandler; - private Configuration capacitySchedulerConfig; + private CapacitySchedulerConfiguration capacitySchedulerConfig; private final boolean preemptionEnabled; private final boolean sizeBasedWeight; @SuppressWarnings("unused") @@ -106,11 +101,10 @@ private void emitChildQueues(String queueName, List children) { ruleHandler.handleChildQueueCount(queueName, children.size()); if (children.size() > 0) { - String childQueues = children.stream() + List childQueues = children.stream() .map(child -> getQueueShortName(child.getName())) - .collect(Collectors.joining(",")); - - capacitySchedulerConfig.set(PREFIX + queueName + ".queues", childQueues); + .collect(Collectors.toList()); + capacitySchedulerConfig.setQueues(queueName, childQueues.toArray(new String[0])); } } @@ -127,14 +121,14 @@ private void emitMaxAMShare(String queueName, FSQueue queue) { if (queueMaxAmShare != 0.0f && queueMaxAmShare != queueMaxAMShareDefault && queueMaxAmShare != QUEUE_MAX_AM_SHARE_DISABLED) { - capacitySchedulerConfig.setFloat(PREFIX + queueName + - ".maximum-am-resource-percent", queueMaxAmShare); + capacitySchedulerConfig.setMaximumApplicationMasterResourcePerQueuePercent( + queueName, queueMaxAmShare); } if (queueMaxAmShare == QUEUE_MAX_AM_SHARE_DISABLED && queueMaxAmShare != queueMaxAMShareDefault) { - capacitySchedulerConfig.setFloat(PREFIX + queueName + - ".maximum-am-resource-percent", 1.0f); + capacitySchedulerConfig.setMaximumApplicationMasterResourcePerQueuePercent( + queueName, 1.0f); } } @@ -147,7 +141,7 @@ private void emitMaxAMShare(String queueName, FSQueue queue) { private void emitMaxParallelApps(String queueName, FSQueue queue) { if (queue.getMaxRunningApps() != MAX_RUNNING_APPS_UNSET && queue.getMaxRunningApps() != queueMaxAppsDefault) { - capacitySchedulerConfig.set(PREFIX + queueName + ".max-parallel-apps", + capacitySchedulerConfig.setMaxParallelAppsForQueue(queueName, String.valueOf(queue.getMaxRunningApps())); } } @@ -167,8 +161,8 @@ private void emitMaximumCapacity(String queueName, FSQueue queue) { ruleHandler.handleMaxResources(); } - capacitySchedulerConfig.set(PREFIX + queueName + ".maximum-capacity", - "100"); + capacitySchedulerConfig.setMaximumCapacity(queueName, + 100.0f); } /** @@ -182,7 +176,7 @@ private void emitMaxAllocations(String queueName, FSQueue queue) { Resource maxAllocation = queue.getMaximumContainerAllocation(); if (isNotUnboundedResource(maxAllocation)) { - long parentMaxVcores = Integer.MIN_VALUE; + int parentMaxVcores = Integer.MIN_VALUE; long parentMaxMemory = Integer.MIN_VALUE; if (queue.getParent() != null) { @@ -194,16 +188,16 @@ private void emitMaxAllocations(String queueName, FSQueue queue) { } } - long maxVcores = maxAllocation.getVirtualCores(); + int maxVcores = maxAllocation.getVirtualCores(); long maxMemory = maxAllocation.getMemorySize(); // only emit max allocation if it differs from the parent's setting if (maxVcores != parentMaxVcores || maxMemory != parentMaxMemory) { - capacitySchedulerConfig.set(PREFIX + queueName + - ".maximum-allocation-mb", String.valueOf(maxMemory)); + capacitySchedulerConfig.setQueueMaximumAllocationMb( + queueName, (int) maxMemory); - capacitySchedulerConfig.set(PREFIX + queueName + - ".maximum-allocation-vcores", String.valueOf(maxVcores)); + capacitySchedulerConfig.setQueueMaximumAllocationVcores( + queueName, maxVcores); } } } @@ -216,17 +210,14 @@ private void emitMaxAllocations(String queueName, FSQueue queue) { */ private void emitPreemptionDisabled(String queueName, FSQueue queue) { if (preemptionEnabled && !queue.isPreemptable()) { - capacitySchedulerConfig.set(PREFIX + queueName + ".disable_preemption", - "true"); + capacitySchedulerConfig.setPreemptionDisabled(queueName, true); } } public void emitDefaultUserLimitFactor(String queueName, List children) { - if (children.isEmpty() && checkAutoQueueCreationV2Disabled(queueName)) { - capacitySchedulerConfig.setFloat( - CapacitySchedulerConfiguration. - PREFIX + queueName + DOT + USER_LIMIT_FACTOR, - -1.0f); + if (children.isEmpty() && + !capacitySchedulerConfig.isAutoQueueCreationV2Enabled(queueName)) { + capacitySchedulerConfig.setUserLimitFactor(queueName, -1.0f); } } @@ -255,19 +246,16 @@ private void emitOrderingPolicy(String queueName, FSQueue queue) { switch (policy) { case DominantResourceFairnessPolicy.NAME: - capacitySchedulerConfig.set(PREFIX + queueName - + ".ordering-policy", FAIR_POLICY); + capacitySchedulerConfig.setOrderingPolicy(queueName, FAIR_POLICY); break; case FairSharePolicy.NAME: - capacitySchedulerConfig.set(PREFIX + queueName - + ".ordering-policy", FAIR_POLICY); + capacitySchedulerConfig.setOrderingPolicy(queueName, FAIR_POLICY); if (drfUsed) { ruleHandler.handleFairAsDrf(queueName); } break; case FifoPolicy.NAME: - capacitySchedulerConfig.set(PREFIX + queueName - + ".ordering-policy", FIFO_POLICY); + capacitySchedulerConfig.setOrderingPolicy(queueName, FIFO_POLICY); break; default: String msg = String.format("Unexpected ordering policy " + @@ -311,12 +299,6 @@ private void checkMaxChildCapacitySetting(FSQueue queue) { } } - private boolean checkAutoQueueCreationV2Disabled(String queueName) { - return !capacitySchedulerConfig.getBoolean( - PREFIX + queueName + DOT + AUTO_QUEUE_CREATION_V2_ENABLED, - DEFAULT_AUTO_QUEUE_CREATION_ENABLED); - } - private String getQueueShortName(String queueName) { int lastDot = queueName.lastIndexOf("."); return queueName.substring(lastDot + 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java index c01461747c7a1..2c0c113bd70e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java @@ -18,13 +18,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @SuppressWarnings({"checkstyle:visibilitymodifier", "checkstyle:hiddenfield"}) public final class FSQueueConverterBuilder { FSConfigToCSConfigRuleHandler ruleHandler; - Configuration capacitySchedulerConfig; + CapacitySchedulerConfiguration capacitySchedulerConfig; boolean preemptionEnabled; boolean sizeBasedWeight; Resource clusterResource; @@ -48,8 +48,8 @@ public FSQueueConverterBuilder withRuleHandler( } public FSQueueConverterBuilder withCapacitySchedulerConfig( - Configuration config) { - this.capacitySchedulerConfig = config; + CapacitySchedulerConfiguration capacitySchedulerConfig) { + this.capacitySchedulerConfig = capacitySchedulerConfig; return this; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/CapacityConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/CapacityConverter.java index 847473cd8e4bc..b28d48118614e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/CapacityConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/CapacityConverter.java @@ -18,9 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.weightconversion; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; public interface CapacityConverter { - void convertWeightsForChildQueues(FSQueue queue, Configuration csConfig); + void convertWeightsForChildQueues(FSQueue queue, CapacitySchedulerConfiguration csConfig); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java index c430a7c05263f..600b92626db9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.weightconversion; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; - import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Comparator; @@ -29,8 +27,8 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; public class WeightToPercentConverter @@ -41,7 +39,7 @@ public class WeightToPercentConverter @Override public void convertWeightsForChildQueues(FSQueue queue, - Configuration csConfig) { + CapacitySchedulerConfiguration csConfig) { List children = queue.getChildQueues(); int totalWeight = getTotalWeight(children); @@ -52,13 +50,11 @@ public void convertWeightsForChildQueues(FSQueue queue, boolean shouldAllowZeroSumCapacity = result.getRight(); capacities - .forEach((key, value) -> csConfig.set(PREFIX + key + - ".capacity", value.toString())); + .forEach((key, value) -> csConfig.setCapacity(key, value.toString())); if (shouldAllowZeroSumCapacity) { String queueName = queue.getName(); - csConfig.setBoolean( - PREFIX + queueName + ".allow-zero-capacity-sum", true); + csConfig.setAllowZeroCapacitySum(queueName, true); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java index cbc98e41a53d4..75554fc4d2e6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java @@ -18,11 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.weightconversion; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; - import java.util.List; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -32,29 +30,17 @@ public class WeightToWeightConverter @Override public void convertWeightsForChildQueues(FSQueue queue, - Configuration csConfig) { + CapacitySchedulerConfiguration csConfig) { List children = queue.getChildQueues(); if (queue instanceof FSParentQueue || !children.isEmpty()) { if (queue.getName().equals(ROOT_QUEUE)) { - csConfig.set(getProperty(queue), getWeightString(queue)); + csConfig.setNonLabeledQueueWeight(queue.getName(), queue.getWeight()); } - children.forEach(fsQueue -> csConfig.set( - getProperty(fsQueue), getWeightString(fsQueue))); - csConfig.setBoolean(getAutoCreateV2EnabledProperty(queue), true); + children.forEach(fsQueue -> csConfig.setNonLabeledQueueWeight( + fsQueue.getName(), fsQueue.getWeight())); + csConfig.setAutoQueueCreationV2Enabled(queue.getName(), true); } } - - private String getProperty(FSQueue queue) { - return PREFIX + queue.getName() + ".capacity"; - } - - private String getAutoCreateV2EnabledProperty(FSQueue queue) { - return PREFIX + queue.getName() + ".auto-queue-creation-v2.enabled"; - } - - private String getWeightString(FSQueue queue) { - return Float.toString(queue.getWeight()) + "w"; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 86123f211f3f7..55c43666cdb5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -16,8 +16,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_LIMIT_FACTOR; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.DYNAMIC_MAX_ASSIGN; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CAPACITY_PERCENTAGE; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_CAPACITY; @@ -33,7 +31,6 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.ABORT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.WARNING; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; @@ -49,6 +46,7 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.ServiceStateException; +import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; @@ -172,41 +170,44 @@ private void createConverter() { public void testDefaultMaxAMShare() throws Exception { converter.convert(config); - Configuration conf = converter.getCapacitySchedulerConfig(); - String maxAmShare = - conf.get(CapacitySchedulerConfiguration. - MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT); + CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); + Float maxAmShare = + conf.getMaximumApplicationMasterResourcePercent(); - assertEquals("Default max AM share", "0.16", maxAmShare); + assertEquals("Default max AM share", 0.16f, maxAmShare, 0.0f); - assertEquals("root.admins.alice max-am-resource-percent", "0.15", - conf.get(PREFIX + "root.admins.alice.maximum-am-resource-percent")); + assertEquals("root.admins.alice max-am-resource-percent", 0.15f, + conf.getMaximumApplicationMasterResourcePerQueuePercent("root.admins.alice"), + 0.0f); - assertNull("root.users.joe maximum-am-resource-percent should be null", - conf.get(PREFIX + "root.users.joe.maximum-am-resource-percent")); + //root.users.joe don’t have maximum-am-resource-percent set + // so falling back to the global value + assertEquals("root.users.joe maximum-am-resource-percent", 0.16f, + conf.getMaximumApplicationMasterResourcePerQueuePercent("root.users.joe"), + 0.0f); } @Test public void testDefaultUserLimitFactor() throws Exception { converter.convert(config); - Configuration conf = converter.getCapacitySchedulerConfig(); + CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); - assertNull("root.users user-limit-factor should be null", - conf.get(PREFIX + "root.users." + USER_LIMIT_FACTOR)); - assertEquals("root.users auto-queue-creation-v2.enabled", "true", - conf.get(PREFIX + "root.users.auto-queue-creation-v2.enabled")); + assertEquals("root.users user-limit-factor", 1.0f, + conf.getUserLimitFactor("root.users"), 0.0f); + assertEquals("root.users auto-queue-creation-v2.enabled", true, + conf.isAutoQueueCreationV2Enabled("root.users")); - assertEquals("root.default user-limit-factor", "-1.0", - conf.get(PREFIX + "root.default.user-limit-factor")); + assertEquals("root.default user-limit-factor", -1.0f, + conf.getUserLimitFactor("root.default"), 0.0f); - assertEquals("root.users.joe user-limit-factor", "-1.0", - conf.get(PREFIX + "root.users.joe.user-limit-factor")); + assertEquals("root.users.joe user-limit-factor", -1.0f, + conf.getUserLimitFactor("root.users.joe"), 0.0f); - assertEquals("root.admins.bob user-limit-factor", "-1.0", - conf.get(PREFIX + "root.admins.bob.user-limit-factor")); - assertNull("root.admin.bob auto-queue-creation-v2.enabled should be null", - conf.get(PREFIX + "root.admin.bob.auto-queue-creation-v2.enabled")); + assertEquals("root.admins.bob user-limit-factor", -1.0f, + conf.getUserLimitFactor("root.admins.bob"), 0.0f); + assertEquals("root.admin.bob auto-queue-creation-v2.enabled", false, + conf.isAutoQueueCreationV2Enabled("root.admin.bob")); } @Test @@ -218,110 +219,109 @@ public void testDefaultMaxAMShareDisabled() throws Exception { converter.convert(params); - Configuration conf = converter.getCapacitySchedulerConfig(); + CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); // -1.0 means disabled ==> 1.0 in CS - assertEquals("Default max-am-resource-percent", "1.0", - conf.get(CapacitySchedulerConfiguration. - MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT)); + assertEquals("Default max-am-resource-percent", 1.0f, + conf.getMaximumApplicationMasterResourcePercent(), 0.0f); - // root.admins.bob -1.0 equals to the default -1.0 - assertNull("root.admins.bob maximum-am-resource-percent should be null", - conf.get(PREFIX + "root.admins.bob.maximum-am-resource-percent")); + // root.admins.bob is unset,so falling back to the global value + assertEquals("root.admins.bob maximum-am-resource-percent", 1.0f, + conf.getMaximumApplicationMasterResourcePerQueuePercent("root.admins.bob"), 0.0f); // root.admins.alice 0.15 != -1.0 - assertEquals("root.admins.alice max-am-resource-percent", "0.15", - conf.get(PREFIX + "root.admins.alice.maximum-am-resource-percent")); + assertEquals("root.admins.alice max-am-resource-percent", 0.15f, + conf.getMaximumApplicationMasterResourcePerQueuePercent("root.admins.alice"), 0.0f); - // root.users.joe is unset, inherits -1.0 - assertNull("root.users.joe maximum-am-resource-percent should be null", - conf.get(PREFIX + "root.users.joe.maximum-am-resource-percent")); + // root.users.joe is unset,so falling back to the global value + assertEquals("root.users.joe maximum-am-resource-percent", 1.0f, + conf.getMaximumApplicationMasterResourcePerQueuePercent("root.users.joe"), 0.0f); } @Test public void testConvertACLs() throws Exception { converter.convert(config); - Configuration conf = converter.getCapacitySchedulerConfig(); + CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); // root assertEquals("root submit ACL", "alice,bob,joe,john hadoop_users", - conf.get(PREFIX + "root.acl_submit_applications")); + conf.getAcl("root", QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root admin ACL", "alice,bob,joe,john hadoop_users", - conf.get(PREFIX + "root.acl_administer_queue")); + conf.getAcl("root", QueueACL.ADMINISTER_QUEUE).getAclString()); // root.admins.bob assertEquals("root.admins.bob submit ACL", "bob ", - conf.get(PREFIX + "root.admins.bob.acl_submit_applications")); + conf.getAcl("root.admins.bob", QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root.admins.bob admin ACL", "bob ", - conf.get(PREFIX + "root.admins.bob.acl_administer_queue")); + conf.getAcl("root.admins.bob", QueueACL.ADMINISTER_QUEUE).getAclString()); // root.admins.alice assertEquals("root.admins.alice submit ACL", "alice ", - conf.get(PREFIX + "root.admins.alice.acl_submit_applications")); + conf.getAcl("root.admins.alice", QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root.admins.alice admin ACL", "alice ", - conf.get(PREFIX + "root.admins.alice.acl_administer_queue")); + conf.getAcl("root.admins.alice", QueueACL.ADMINISTER_QUEUE).getAclString()); // root.users.john assertEquals("root.users.john submit ACL", "*", - conf.get(PREFIX + "root.users.john.acl_submit_applications")); + conf.getAcl("root.users.john", QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root.users.john admin ACL", "*", - conf.get(PREFIX + "root.users.john.acl_administer_queue")); + conf.getAcl("root.users.john", QueueACL.ADMINISTER_QUEUE).getAclString()); // root.users.joe assertEquals("root.users.joe submit ACL", "joe ", - conf.get(PREFIX + "root.users.joe.acl_submit_applications")); + conf.getAcl("root.users.joe", QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root.users.joe admin ACL", "joe ", - conf.get(PREFIX + "root.users.joe.acl_administer_queue")); + conf.getAcl("root.users.joe", QueueACL.ADMINISTER_QUEUE).getAclString()); } @Test public void testDefaultQueueMaxParallelApps() throws Exception { converter.convert(config); - Configuration conf = converter.getCapacitySchedulerConfig(); + CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); assertEquals("Default max parallel apps", 15, - conf.getInt(PREFIX + "max-parallel-apps", -1)); + conf.getDefaultMaxParallelApps(), 0); } @Test public void testSpecificQueueMaxParallelApps() throws Exception { converter.convert(config); - Configuration conf = converter.getCapacitySchedulerConfig(); + CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); assertEquals("root.admins.alice max parallel apps", 2, - conf.getInt(PREFIX + "root.admins.alice.max-parallel-apps", -1)); + conf.getMaxParallelAppsForQueue("root.admins.alice"), 0); } @Test public void testDefaultUserMaxParallelApps() throws Exception { converter.convert(config); - Configuration conf = converter.getCapacitySchedulerConfig(); - int userMaxParallelApps = - conf.getInt( - PREFIX + "user.max-parallel-apps", -1); + CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); assertEquals("Default user max parallel apps", 10, - userMaxParallelApps); + conf.getDefaultMaxParallelAppsPerUser(), 0); } @Test public void testSpecificUserMaxParallelApps() throws Exception { converter.convert(config); - Configuration conf = converter.getCapacitySchedulerConfig(); + CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); assertEquals("Max parallel apps for alice", 30, - conf.getInt(PREFIX + "user.alice.max-parallel-apps", -1)); - assertNull("Max parallel apps should be undefined for user bob", - conf.get(PREFIX + "user.bob.max-parallel-apps")); - assertNull("Max parallel apps should be undefined for user joe", - conf.get(PREFIX + "user.joe.max-parallel-apps")); - assertNull("Max parallel apps should be undefined for user john", - conf.get(PREFIX + "user.john.max-parallel-apps")); + conf.getMaxParallelAppsForUser("alice"), 0); + + //users.bob, user.joe, user.john don’t have max-parallel-app set + // so falling back to the global value for .user to 10 + assertEquals("Max parallel apps for user bob", 10, + conf.getMaxParallelAppsForUser("bob"), 0); + assertEquals("Max parallel apps for user joe", 10, + conf.getMaxParallelAppsForUser("joe"), 0); + assertEquals("Max parallel apps for user john", 10, + conf.getMaxParallelAppsForUser("john"), 0); } @Test @@ -722,8 +722,7 @@ public void testSiteDisabledPreemptionWithObserveOnlyConversion() converter.convert(params); assertTrue("The observe only should be true", converter.getCapacitySchedulerConfig(). - getBoolean(CapacitySchedulerConfiguration. - PREEMPTION_OBSERVE_ONLY, false)); + getPreemptionObserveOnly()); } private boolean testConversionWithAsyncSchedulingOption(boolean enabled) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java index b1cc4a6ccdf59..edab194a91817 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -16,7 +16,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DEFAULT_MAX_PARALLEL_APPLICATIONS; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -35,6 +37,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; @@ -81,12 +85,13 @@ private static String prepareFileName(String f) { private FSQueueConverter converter; private Configuration yarnConfig; - private Configuration csConfig; + private CapacitySchedulerConfiguration csConfig; private FairScheduler fs; private FSQueue rootQueue; private ConversionOptions conversionOptions; private DryRunResultHolder dryRunResultHolder; private FSQueueConverterBuilder builder; + private String key; @Mock private FSConfigToCSConfigRuleHandler ruleHandler; @@ -100,7 +105,8 @@ public void setup() { yarnConfig.set(FairSchedulerConfiguration.ALLOCATION_FILE, FAIR_SCHEDULER_XML); yarnConfig.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true); - csConfig = new Configuration(false); + csConfig = new CapacitySchedulerConfiguration( + new Configuration(false)); dryRunResultHolder = new DryRunResultHolder(); conversionOptions = new ConversionOptions(dryRunResultHolder, false); @@ -149,20 +155,20 @@ public void testConvertQueueHierarchy() { converter.convertQueueHierarchy(rootQueue); // root children - assertEquals("root children", "admins,users,misc,default", - csConfig.get(PREFIX + "root.queues")); + assertArrayEquals("root children", new String[]{"admins", "users", "misc", "default"}, + csConfig.getQueues("root")); // root.admins children - assertEquals("root.admins children", "bob,alice", - csConfig.get(PREFIX + "root.admins.queues")); + assertArrayEquals("root.admins children", new String[]{"bob", "alice"}, + csConfig.getQueues("root.admins")); // root.default children - none - assertNull("root.default children", csConfig.get(PREFIX + "root.default" + - ".queues")); + assertNull("root.default children", + csConfig.getQueues("root.default")); // root.users children - assertEquals("root.users children", "john,joe", - csConfig.get(PREFIX + "root.users.queues")); + assertArrayEquals("root.users children", new String[]{"john", "joe"}, + csConfig.getQueues("root.users")); Set leafs = Sets.difference(ALL_QUEUES, Sets.newHashSet("root", @@ -171,7 +177,12 @@ public void testConvertQueueHierarchy() { "root.users", "root.misc")); - assertNoValueForQueues(leafs, ".queues", csConfig); + for (String queue : leafs) { + key = PREFIX + queue + ".queues"; + assertNull("Key " + key + " has value, but it should be null", + csConfig.getQueues(queue)); + } + } @Test @@ -181,18 +192,24 @@ public void testQueueMaxAMShare() { converter.convertQueueHierarchy(rootQueue); // root.admins.bob - assertEquals("root.admins.bob AM share", "1.0", - csConfig.get(PREFIX + "root.admins.bob.maximum-am-resource-percent")); + assertEquals("root.admins.bob AM share", 1.0f, + csConfig.getMaximumApplicationMasterResourcePerQueuePercent( + "root.admins.bob"), 0.0f); // root.admins.alice - assertEquals("root.admins.alice AM share", "0.15", - csConfig.get(PREFIX + - "root.admins.alice.maximum-am-resource-percent")); + assertEquals("root.admins.alice AM share", 0.15f, + csConfig.getMaximumApplicationMasterResourcePerQueuePercent( + "root.admins.alice"), 0.0f); Set remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins.bob", "root.admins.alice")); - assertNoValueForQueues(remaining, ".maximum-am-resource-percent", - csConfig); + + for (String queue : remaining) { + key = PREFIX + queue + ".maximum-am-resource-percent"; + assertEquals("Key " + key + " has different value", + 0.1f, csConfig + .getMaximumApplicationMasterResourcePerQueuePercent(queue), 0.0f); + } } @Test @@ -202,12 +219,17 @@ public void testQueueMaxParallelApps() { converter.convertQueueHierarchy(rootQueue); assertEquals("root.admins.alice max apps", 2, - csConfig.getInt(PREFIX + "root.admins.alice.max-parallel-apps", - -1)); + csConfig.getMaxParallelAppsForQueue("root.admins.alice"), 0); Set remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins.alice")); - assertNoValueForQueues(remaining, ".max-parallel-apps", csConfig); + + for (String queue : remaining) { + key = PREFIX + queue + ".max-parallel-apps"; + assertEquals("Key " + key + " has different value", + DEFAULT_MAX_PARALLEL_APPLICATIONS, csConfig + .getMaxParallelAppsForQueue(queue), 0); + } } @Test @@ -218,21 +240,30 @@ public void testQueueMaxAllocations() { // root.admins vcores + mb assertEquals("root.admins max vcores", 3, - csConfig.getInt(PREFIX + "root.admins.maximum-allocation-vcores", -1)); + csConfig.getQueueMaximumAllocationVcores("root.admins")); assertEquals("root.admins max memory", 4096, - csConfig.getInt(PREFIX + "root.admins.maximum-allocation-mb", -1)); + csConfig.getQueueMaximumAllocationMb("root.admins")); // root.users.john max vcores + mb assertEquals("root.users.john max vcores", 2, - csConfig.getInt(PREFIX + "root.users.john.maximum-allocation-vcores", - -1)); + csConfig.getQueueMaximumAllocationVcores("root.users.john")); assertEquals("root.users.john max memory", 8192, - csConfig.getInt(PREFIX + "root.users.john.maximum-allocation-mb", -1)); + csConfig.getQueueMaximumAllocationMb("root.users.john")); Set remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins", "root.users.john")); - assertNoValueForQueues(remaining, ".maximum-allocation-vcores", csConfig); - assertNoValueForQueues(remaining, ".maximum-allocation-mb", csConfig); + + for (String queue : remaining) { + key = PREFIX + queue + ".maximum-allocation-vcores"; + assertEquals("Key " + key + " has different value", + -1.0, csConfig + .getQueueMaximumAllocationVcores(queue), 0.0f); + + key = PREFIX + queue + ".maximum-allocation-mb"; + assertEquals("Key " + key + " has different value", + -1.0, csConfig + .getQueueMaximumAllocationMb(queue), 0.0f); + } } @Test @@ -242,15 +273,20 @@ public void testQueuePreemptionDisabled() { converter.convertQueueHierarchy(rootQueue); assertTrue("root.admins.alice preemption setting", - csConfig.getBoolean(PREFIX + "root.admins.alice.disable_preemption", - false)); + csConfig.getPreemptionDisabled( + "root.admins.alice", false)); assertTrue("root.users.joe preemption setting", - csConfig.getBoolean(PREFIX + "root.users.joe.disable_preemption", - false)); + csConfig.getPreemptionDisabled( + "root.users.joe", false)); Set remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins.alice", "root.users.joe")); - assertNoValueForQueues(remaining, ".disable_preemption", csConfig); + + for (String queue : remaining) { + key = PREFIX + queue + ".disable_preemption"; + assertEquals("Key " + key + " has different value", + false, csConfig.getPreemptionDisabled(queue, false)); + } } @Test @@ -259,7 +295,11 @@ public void testQueuePreemptionDisabledWhenGlobalPreemptionDisabled() { converter.convertQueueHierarchy(rootQueue); - assertNoValueForQueues(ALL_QUEUES, ".disable_preemption", csConfig); + for (String queue : ALL_QUEUES) { + key = PREFIX + queue + ".disable_preemption"; + assertEquals("Key " + key + " has different value", + false, csConfig.getPreemptionDisabled(queue, false)); + } } @Test @@ -269,32 +309,42 @@ public void testChildCapacityInCapacityMode() { converter.convertQueueHierarchy(rootQueue); // root - assertEquals("root.default capacity", "33.333", - csConfig.get(PREFIX + "root.default.capacity")); - assertEquals("root.admins capacity", "33.333", - csConfig.get(PREFIX + "root.admins.capacity")); - assertEquals("root.users capacity", "33.334", - csConfig.get(PREFIX + "root.users.capacity")); + assertEquals("root.default capacity", 33.333f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.default")), 0.0f); + assertEquals("root.admins capacity", 33.333f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.admins")), 0.0f); + assertEquals("root.users capacity", 33.334f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.users")), 0.0f); // root.users - assertEquals("root.users.john capacity", "25.000", - csConfig.get(PREFIX + "root.users.john.capacity")); - assertEquals("root.users.joe capacity", "75.000", - csConfig.get(PREFIX + "root.users.joe.capacity")); + assertEquals("root.users.john capacity", 25.000f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.users.john")), 0.0f); + assertEquals("root.users.joe capacity", 75.000f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.users.joe")), 0.0f); // root.admins - assertEquals("root.admins.alice capacity", "75.000", - csConfig.get(PREFIX + "root.admins.alice.capacity")); - assertEquals("root.admins.bob capacity", "25.000", - csConfig.get(PREFIX + "root.admins.bob.capacity")); + assertEquals("root.admins.alice capacity", 75.000f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.admins.alice")), 0.0f); + assertEquals("root.admins.bob capacity", 25.000f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.admins.bob")), 0.0f); // root.misc - assertEquals("root.misc capacity", "0.000", - csConfig.get(PREFIX + "root.misc.capacity")); - assertEquals("root.misc.a capacity", "0.000", - csConfig.get(PREFIX + "root.misc.a.capacity")); - assertEquals("root.misc.b capacity", "0.000", - csConfig.get(PREFIX + "root.misc.b.capacity")); + assertEquals("root.misc capacity", 0.000f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.misc")), 0.000f); + assertEquals("root.misc.a capacity", 0.000f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.misc.a")), 0.000f); + assertEquals("root.misc.b capacity", 0.000f, + csConfig.getNonLabeledQueueCapacity( + new QueuePath("root.misc.b")), 0.000f); } @Test @@ -304,32 +354,32 @@ public void testChildCapacityInWeightMode() { converter.convertQueueHierarchy(rootQueue); // root - assertEquals("root.default weight", "1.0w", - csConfig.get(PREFIX + "root.default.capacity")); - assertEquals("root.admins weight", "1.0w", - csConfig.get(PREFIX + "root.admins.capacity")); - assertEquals("root.users weight", "1.0w", - csConfig.get(PREFIX + "root.users.capacity")); + assertEquals("root.default weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root.default"), 0.01f); + assertEquals("root.admins weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root.admins"), 0.01f); + assertEquals("root.users weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root.users"), 0.01f); // root.users - assertEquals("root.users.john weight", "1.0w", - csConfig.get(PREFIX + "root.users.john.capacity")); - assertEquals("root.users.joe weight", "3.0w", - csConfig.get(PREFIX + "root.users.joe.capacity")); + assertEquals("root.users.john weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root.users.john"), 0.01f); + assertEquals("root.users.joe weight", 3.0f, + csConfig.getNonLabeledQueueWeight("root.users.joe"), 0.01f); // root.admins - assertEquals("root.admins.alice weight", "3.0w", - csConfig.get(PREFIX + "root.admins.alice.capacity")); - assertEquals("root.admins.bob weight", "1.0w", - csConfig.get(PREFIX + "root.admins.bob.capacity")); + assertEquals("root.admins.alice weight", 3.0f, + csConfig.getNonLabeledQueueWeight("root.admins.alice"), 0.01f); + assertEquals("root.admins.bob weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root.admins.bob"), 0.01f); // root.misc - assertEquals("root.misc weight", "0.0w", - csConfig.get(PREFIX + "root.misc.capacity")); - assertEquals("root.misc.a weight", "0.0w", - csConfig.get(PREFIX + "root.misc.a.capacity")); - assertEquals("root.misc.b weight", "0.0w", - csConfig.get(PREFIX + "root.misc.b.capacity")); + assertEquals("root.misc weight", 0.0f, + csConfig.getNonLabeledQueueWeight("root.misc"), 0.00f); + assertEquals("root.misc.a weight", 0.0f, + csConfig.getNonLabeledQueueWeight("root.misc.a"), 0.00f); + assertEquals("root.misc.b weight", 0.0f, + csConfig.getNonLabeledQueueWeight("root.misc.b"), 0.00f); } @Test @@ -339,21 +389,15 @@ public void testAutoCreateV2FlagsInWeightMode() { converter.convertQueueHierarchy(rootQueue); assertTrue("root autocreate v2 flag", - csConfig.getBoolean( - PREFIX + "root.auto-queue-creation-v2.enabled", false)); + csConfig.isAutoQueueCreationV2Enabled("root")); assertTrue("root.admins autocreate v2 flag", - csConfig.getBoolean( - PREFIX + "root.admins.auto-queue-creation-v2.enabled", false)); + csConfig.isAutoQueueCreationV2Enabled("root.admins")); assertTrue("root.admins.alice autocreate v2 flag", - csConfig.getBoolean( - PREFIX + "root.admins.alice.auto-queue-creation-v2.enabled", - false)); + csConfig.isAutoQueueCreationV2Enabled("root.admins.alice")); assertTrue("root.users autocreate v2 flag", - csConfig.getBoolean( - PREFIX + "root.users.auto-queue-creation-v2.enabled", false)); + csConfig.isAutoQueueCreationV2Enabled("root.users")); assertTrue("root.misc autocreate v2 flag", - csConfig.getBoolean( - PREFIX + "root.misc.auto-queue-creation-v2.enabled", false)); + csConfig.isAutoQueueCreationV2Enabled("root.misc")); //leaf queue root.admins.alice is removed from the below list //adding reservation to a leaf, it's queueType changes to FSParentQueue @@ -363,8 +407,14 @@ public void testAutoCreateV2FlagsInWeightMode() { "root.users", "root.misc", "root.admins.alice")); - assertNoValueForQueues(leafs, ".auto-queue-creation-v2.enabled", - csConfig); + + for (String queue : leafs) { + key = PREFIX + queue + ".auto-queue-creation-v2.enabled"; + assertEquals("Key " + key + " has different value", + false, csConfig + .isAutoQueueCreationV2Enabled(queue)); + } + } @Test @@ -375,11 +425,16 @@ public void testZeroSumCapacityValidation() { Set noZeroSumAllowedQueues = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.misc")); - assertNoValueForQueues(noZeroSumAllowedQueues, ".allow-zero-capacity-sum", - csConfig); - assertTrue("root.misc allow zero capacities", csConfig.getBoolean( - PREFIX + "root.misc.allow-zero-capacity-sum", false)); + for (String queue : noZeroSumAllowedQueues) { + key = PREFIX + queue + ".allow-zero-capacity-sum"; + assertEquals("Key " + key + " has different value", + false, csConfig + .getAllowZeroCapacitySum(queue)); + } + + assertTrue("root.misc allow zero capacities", + csConfig.getAllowZeroCapacitySum("root.misc")); } @Test @@ -388,7 +443,12 @@ public void testQueueMaximumCapacity() { converter.convertQueueHierarchy(rootQueue); - assertValueForQueues(ALL_QUEUES, ".maximum-capacity", csConfig, "100"); + for (String queue : ALL_QUEUES) { + key = PREFIX + queue + ".maximum-capacity"; + assertEquals("Key " + key + " has different value", + 100.0, csConfig + .getNonLabeledQueueMaximumCapacity(new QueuePath(queue)), 0.0f); + } verify(ruleHandler, times(3)).handleMaxResources(); } @@ -409,8 +469,11 @@ public void testQueueWithNoAutoCreateChildQueue() { converter.convertQueueHierarchy(rootQueue); - assertNoValueForQueues(ALL_QUEUES, ".auto-create-child-queue.enabled", - csConfig); + for (String queue : ALL_QUEUES) { + key = PREFIX + queue + ".auto-create-child-queue.enabled"; + assertEquals("Key " + key + " has different value", + false, csConfig.isAutoCreateChildQueueEnabled(queue)); + } } @Test @@ -419,8 +482,11 @@ public void testQueueSizeBasedWeightEnabled() { converter.convertQueueHierarchy(rootQueue); - assertTrueForQueues(ALL_QUEUES, - ".ordering-policy.fair.enable-size-based-weight", csConfig); + for (String queue : ALL_QUEUES) { + key = PREFIX + queue + ".ordering-policy.fair.enable-size-based-weight"; + assertTrue("Key " + key + " has different value", + csConfig.getBoolean(key, false)); + } } @Test @@ -429,8 +495,11 @@ public void testQueueSizeBasedWeightDisabled() { converter.convertQueueHierarchy(rootQueue); - assertNoValueForQueues(ALL_QUEUES, - ".ordering-policy.fair.enable-size-based-weight", csConfig); + for (String queue : ALL_QUEUES) { + key = PREFIX + queue + ".ordering-policy.fair.enable-size-based-weight"; + assertNull("Key " + key + " has different value", + csConfig.get(key)); + } } @Test @@ -446,28 +515,27 @@ public void testQueueOrderingPolicy() throws Exception { rootQueue = fs.getQueueManager().getRootQueue(); converter.convertQueueHierarchy(rootQueue); - // root - assertEquals("root ordering policy", null, - csConfig.get(PREFIX + "root.ordering-policy")); + assertEquals("root ordering policy", "fifo", + csConfig.getAppOrderingPolicy("root").getConfigName()); assertEquals("root.default ordering policy", "fair", - csConfig.get(PREFIX + "root.default.ordering-policy")); - assertEquals("root.admins ordering policy", null, - csConfig.get(PREFIX + "root.admins.ordering-policy")); - assertEquals("root.users ordering policy", null, - csConfig.get(PREFIX + "root.users.ordering-policy")); + csConfig.getAppOrderingPolicy("root.default").getConfigName()); + assertEquals("root.admins ordering policy", "fifo", + csConfig.getAppOrderingPolicy("root.admins").getConfigName()); + assertEquals("root.users ordering policy", "fifo", + csConfig.getAppOrderingPolicy("root.users").getConfigName()); // root.users assertEquals("root.users.joe ordering policy", "fair", - csConfig.get(PREFIX + "root.users.joe.ordering-policy")); + csConfig.getAppOrderingPolicy("root.users.joe").getConfigName()); assertEquals("root.users.john ordering policy", "fifo", - csConfig.get(PREFIX + "root.users.john.ordering-policy")); + csConfig.getAppOrderingPolicy("root.users.john").getConfigName()); // root.admins assertEquals("root.admins.alice ordering policy", "fifo", - csConfig.get(PREFIX + "root.admins.alice.ordering-policy")); + csConfig.getAppOrderingPolicy("root.admins.alice.").getConfigName()); assertEquals("root.admins.bob ordering policy", "fair", - csConfig.get(PREFIX + "root.admins.bob.ordering-policy")); + csConfig.getAppOrderingPolicy("root.admins.bob").getConfigName()); } @Test @@ -512,31 +580,4 @@ public void testReservationSystemNotSupported() { converter.convertQueueHierarchy(rootQueue); } - - private void assertNoValueForQueues(Set queues, String postfix, - Configuration config) { - for (String queue : queues) { - String key = PREFIX + queue + postfix; - assertNull("Key " + key + " has value, but it should be null", - config.get(key)); - } - } - - private void assertValueForQueues(Set queues, String postfix, - Configuration config, String expectedValue) { - for (String queue : queues) { - String key = PREFIX + queue + postfix; - assertEquals("Key " + key + " has different value", - expectedValue, config.get(key)); - } - } - - private void assertTrueForQueues(Set queues, String postfix, - Configuration config) { - for (String queue : queues) { - String key = PREFIX + queue + postfix; - assertTrue("Key " + key + " is false, should be true", - config.getBoolean(key, false)); - } - } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToPercentageConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToPercentageConverter.java index 9cfa494f00e7a..54fadf4571f43 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToPercentageConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToPercentageConverter.java @@ -28,6 +28,8 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.junit.Before; import org.junit.Test; @@ -35,104 +37,102 @@ public class TestWeightToPercentageConverter extends WeightConverterTestBase { private WeightToPercentConverter converter; - private Configuration config; + private CapacitySchedulerConfiguration csConfig; @Before public void setup() { converter = new WeightToPercentConverter(); - config = new Configuration(false); + csConfig = new CapacitySchedulerConfiguration( + new Configuration(false)); } @Test public void testSingleWeightConversion() { FSQueue root = createFSQueues(1); - converter.convertWeightsForChildQueues(root, config); + converter.convertWeightsForChildQueues(root, csConfig); assertFalse("Capacity zerosum allowed", - config.getBoolean(PREFIX + "root.allow-zero-capacity-sum", - false)); - assertEquals("root.a capacity", "100.000", - config.get(PREFIX + "root.a.capacity")); + csConfig.getAllowZeroCapacitySum("root")); + assertEquals("root.a capacity", 100.000f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); } @Test public void testNoChildQueueConversion() { FSQueue root = createFSQueues(); - converter.convertWeightsForChildQueues(root, config); + converter.convertWeightsForChildQueues(root, csConfig); - assertEquals("Converted items", 0, - config.getPropsWithPrefix(PREFIX).size()); + assertEquals("Converted items", 19, + csConfig.getPropsWithPrefix(PREFIX).size()); } @Test public void testMultiWeightConversion() { FSQueue root = createFSQueues(1, 2, 3); - converter.convertWeightsForChildQueues(root, config); + converter.convertWeightsForChildQueues(root, csConfig); - assertEquals("Number of properties", 3, - config.getPropsWithPrefix(PREFIX).size()); + assertEquals("Number of properties", 22, + csConfig.getPropsWithPrefix(PREFIX).size()); // this is no fixing - it's the result of BigDecimal rounding - assertEquals("root.a capacity", "16.667", - config.get(PREFIX + "root.a.capacity")); - assertEquals("root.b capacity", "33.333", - config.get(PREFIX + "root.b.capacity")); - assertEquals("root.c capacity", "50.000", - config.get(PREFIX + "root.c.capacity")); + assertEquals("root.a capacity", 16.667f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); + assertEquals("root.b capacity", 33.333f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.0f); + assertEquals("root.c capacity", 50.000f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.c")), 0.0f); } @Test public void testMultiWeightConversionWhenOfThemIsZero() { FSQueue root = createFSQueues(0, 1, 1); - converter.convertWeightsForChildQueues(root, config); + converter.convertWeightsForChildQueues(root, csConfig); assertFalse("Capacity zerosum allowed", - config.getBoolean(PREFIX + "root.allow-zero-capacity-sum", - false)); - assertEquals("Number of properties", 3, - config.getPropsWithPrefix(PREFIX).size()); - assertEquals("root.a capacity", "0.000", - config.get(PREFIX + "root.a.capacity")); - assertEquals("root.b capacity", "50.000", - config.get(PREFIX + "root.b.capacity")); - assertEquals("root.c capacity", "50.000", - config.get(PREFIX + "root.c.capacity")); + csConfig.getAllowZeroCapacitySum("root")); + assertEquals("Number of properties", 22, + csConfig.getPropsWithPrefix(PREFIX).size()); + assertEquals("root.a capacity", 0.000f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); + assertEquals("root.b capacity", 50.000f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.0f); + assertEquals("root.c capacity", 50.000f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.c")), 0.0f); } @Test public void testMultiWeightConversionWhenAllOfThemAreZero() { FSQueue root = createFSQueues(0, 0, 0); - converter.convertWeightsForChildQueues(root, config); + converter.convertWeightsForChildQueues(root, csConfig); - assertEquals("Number of properties", 4, - config.getPropsWithPrefix(PREFIX).size()); + assertEquals("Number of properties", 23, + csConfig.getPropsWithPrefix(PREFIX).size()); assertTrue("Capacity zerosum allowed", - config.getBoolean(PREFIX + "root.allow-zero-capacity-sum", - false)); - assertEquals("root.a capacity", "0.000", - config.get(PREFIX + "root.a.capacity")); - assertEquals("root.b capacity", "0.000", - config.get(PREFIX + "root.b.capacity")); - assertEquals("root.c capacity", "0.000", - config.get(PREFIX + "root.c.capacity")); + csConfig.getAllowZeroCapacitySum("root")); + assertEquals("root.a capacity", 0.000f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); + assertEquals("root.b capacity", 0.000f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.0f); + assertEquals("root.c capacity", 0.000f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.c")), 0.0f); } @Test public void testCapacityFixingWithThreeQueues() { FSQueue root = createFSQueues(1, 1, 1); - converter.convertWeightsForChildQueues(root, config); + converter.convertWeightsForChildQueues(root, csConfig); - assertEquals("Number of properties", 3, - config.getPropsWithPrefix(PREFIX).size()); - assertEquals("root.a capacity", "33.334", - config.get(PREFIX + "root.a.capacity")); - assertEquals("root.b capacity", "33.333", - config.get(PREFIX + "root.b.capacity")); - assertEquals("root.c capacity", "33.333", - config.get(PREFIX + "root.c.capacity")); + assertEquals("Number of properties", 22, + csConfig.getPropsWithPrefix(PREFIX).size()); + assertEquals("root.a capacity", 33.334f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); + assertEquals("root.b capacity", 33.333f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.0f); + assertEquals("root.c capacity", 33.333f, + csConfig.getNonLabeledQueueCapacity(new QueuePath("root.c")), 0.0f); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java index 28eb2b662c918..904dfe8b1f3d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java @@ -25,81 +25,81 @@ import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.junit.Before; import org.junit.Test; public class TestWeightToWeightConverter extends WeightConverterTestBase { private WeightToWeightConverter converter; - private Configuration config; + private CapacitySchedulerConfiguration csConfig; @Before public void setup() { converter = new WeightToWeightConverter(); - config = new Configuration(false); + csConfig = new CapacitySchedulerConfiguration( + new Configuration(false)); } @Test public void testNoChildQueueConversion() { FSQueue root = createFSQueues(); - converter.convertWeightsForChildQueues(root, config); + converter.convertWeightsForChildQueues(root, csConfig); - assertEquals("root weight", "1.0w", - config.get(PREFIX + "root.capacity")); - assertEquals("Converted items", 2, - config.getPropsWithPrefix(PREFIX).size()); + assertEquals("root weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root"), 0.0f); + assertEquals("Converted items", 21, + csConfig.getPropsWithPrefix(PREFIX).size()); } @Test public void testSingleWeightConversion() { FSQueue root = createFSQueues(1); - converter.convertWeightsForChildQueues(root, config); - - assertEquals("root weight", "1.0w", - config.get(PREFIX + "root.capacity")); - assertEquals("root.a weight", "1.0w", - config.get(PREFIX + "root.a.capacity")); - assertEquals("Number of properties", 3, - config.getPropsWithPrefix(PREFIX).size()); + converter.convertWeightsForChildQueues(root, csConfig); + + assertEquals("root weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root"), 0.0f); + assertEquals("root.a weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root.a"), 0.0f); + assertEquals("Number of properties", 22, + csConfig.getPropsWithPrefix(PREFIX).size()); } @Test public void testMultiWeightConversion() { FSQueue root = createFSQueues(1, 2, 3); - converter.convertWeightsForChildQueues(root, config); - - assertEquals("Number of properties", 5, - config.getPropsWithPrefix(PREFIX).size()); - assertEquals("root weight", "1.0w", - config.get(PREFIX + "root.capacity")); - assertEquals("root.a weight", "1.0w", - config.get(PREFIX + "root.a.capacity")); - assertEquals("root.b weight", "2.0w", - config.get(PREFIX + "root.b.capacity")); - assertEquals("root.c weight", "3.0w", - config.get(PREFIX + "root.c.capacity")); + converter.convertWeightsForChildQueues(root, csConfig); + + assertEquals("Number of properties", 24, + csConfig.getPropsWithPrefix(PREFIX).size()); + assertEquals("root weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root"), 0.0f); + assertEquals("root.a weight", 1.0f, + csConfig.getNonLabeledQueueWeight("root.a"), 0.0f); + assertEquals("root.b weight", 2.0f, + csConfig.getNonLabeledQueueWeight("root.b"), 0.0f); + assertEquals("root.c weight", 3.0f, + csConfig.getNonLabeledQueueWeight("root.c"), 0.0f); } @Test public void testAutoCreateV2FlagOnParent() { FSQueue root = createFSQueues(1); - converter.convertWeightsForChildQueues(root, config); + converter.convertWeightsForChildQueues(root, csConfig); assertTrue("root autocreate v2 enabled", - config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled", - false)); + csConfig.isAutoQueueCreationV2Enabled("root")); } @Test public void testAutoCreateV2FlagOnParentWithoutChildren() { FSQueue root = createParent(new ArrayList<>()); - converter.convertWeightsForChildQueues(root, config); + converter.convertWeightsForChildQueues(root, csConfig); - assertEquals("Number of properties", 2, - config.getPropsWithPrefix(PREFIX).size()); + assertEquals("Number of properties", 21, + csConfig.getPropsWithPrefix(PREFIX).size()); assertTrue("root autocreate v2 enabled", - config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled", - false)); + csConfig.isAutoQueueCreationV2Enabled("root")); } } From 6d3bcaa674d5e8b76a75c1cfeef690d2ea48b800 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sat, 5 Aug 2023 16:29:21 +0800 Subject: [PATCH 06/48] YARN-7402. BackPort [GPG] Fix potential connection leak in GPGUtils. (#5901) --- .../globalpolicygenerator/GPGUtils.java | 32 +++++++---- .../policygenerator/TestPolicyGenerator.java | 53 +++++++++++++++++-- 2 files changed, 71 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java index 636ce92500a12..6d2e1d4142122 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java @@ -18,20 +18,21 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator; +import static javax.servlet.http.HttpServletResponse.SC_OK; + import java.util.HashMap; import java.util.Map; import java.util.Set; -import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.MediaType; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; /** * GPGUtils contains utility functions for the GPG. @@ -57,15 +58,24 @@ public static T invokeRMWebService(String webAddr, String path, final Class< T obj = null; WebResource webResource = client.resource(webAddr); - ClientResponse response = webResource.path("ws/v1/cluster").path(path) - .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); - if (response.getStatus() == HttpServletResponse.SC_OK) { - obj = response.getEntity(returnType); - } else { - throw new YarnRuntimeException("Bad response from remote web service: " - + response.getStatus()); + ClientResponse response = null; + try { + response = webResource.path("ws/v1/cluster").path(path) + .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + if (response.getStatus() == SC_OK) { + obj = response.getEntity(returnType); + } else { + throw new YarnRuntimeException( + "Bad response from remote web service: " + response.getStatus()); + } + return obj; + } finally { + if (response != null) { + response.close(); + response = null; + } + client.destroy(); } - return obj; } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java index 0fe475e3fdd96..9893e85e56f29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java @@ -48,6 +48,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; import org.junit.Assert; @@ -292,11 +294,56 @@ public void testCallRM() { resourceManager.start(); String rmAddress = WebAppUtils.getRMWebAppURLWithScheme(this.conf); - SchedulerTypeInfo sti = GPGUtils - .invokeRMWebService(rmAddress, RMWSConsts.SCHEDULER, - SchedulerTypeInfo.class); + SchedulerTypeInfo sti = GPGUtils.invokeRMWebService(rmAddress, RMWSConsts.SCHEDULER, + SchedulerTypeInfo.class); Assert.assertNotNull(sti); + SchedulerInfo schedulerInfo = sti.getSchedulerInfo(); + Assert.assertTrue(schedulerInfo instanceof CapacitySchedulerInfo); + + CapacitySchedulerInfo capacitySchedulerInfo = (CapacitySchedulerInfo) schedulerInfo; + Assert.assertNotNull(capacitySchedulerInfo); + + CapacitySchedulerQueueInfoList queues = capacitySchedulerInfo.getQueues(); + Assert.assertNotNull(queues); + ArrayList queueInfoList = queues.getQueueInfoList(); + Assert.assertNotNull(queueInfoList); + Assert.assertEquals(2, queueInfoList.size()); + + CapacitySchedulerQueueInfo queueA = queueInfoList.get(0); + Assert.assertNotNull(queueA); + Assert.assertEquals("root.a", queueA.getQueuePath()); + Assert.assertEquals(10.5f, queueA.getCapacity(), 0.00001); + CapacitySchedulerQueueInfoList queueAQueues = queueA.getQueues(); + Assert.assertNotNull(queueAQueues); + ArrayList queueInfoAList = queueAQueues.getQueueInfoList(); + Assert.assertNotNull(queueInfoAList); + Assert.assertEquals(2, queueInfoAList.size()); + CapacitySchedulerQueueInfo queueA1 = queueInfoAList.get(0); + Assert.assertNotNull(queueA1); + Assert.assertEquals(30f, queueA1.getCapacity(), 0.00001); + CapacitySchedulerQueueInfo queueA2 = queueInfoAList.get(1); + Assert.assertNotNull(queueA2); + Assert.assertEquals(70f, queueA2.getCapacity(), 0.00001); + + CapacitySchedulerQueueInfo queueB = queueInfoList.get(1); + Assert.assertNotNull(queueB); + Assert.assertEquals("root.b", queueB.getQueuePath()); + Assert.assertEquals(89.5f, queueB.getCapacity(), 0.00001); + CapacitySchedulerQueueInfoList queueBQueues = queueB.getQueues(); + Assert.assertNotNull(queueBQueues); + ArrayList queueInfoBList = queueBQueues.getQueueInfoList(); + Assert.assertNotNull(queueInfoBList); + Assert.assertEquals(3, queueInfoBList.size()); + CapacitySchedulerQueueInfo queueB1 = queueInfoBList.get(0); + Assert.assertNotNull(queueB1); + Assert.assertEquals(79.2f, queueB1.getCapacity(), 0.00001); + CapacitySchedulerQueueInfo queueB2 = queueInfoBList.get(1); + Assert.assertNotNull(queueB2); + Assert.assertEquals(0.8f, queueB2.getCapacity(), 0.00001); + CapacitySchedulerQueueInfo queueB3 = queueInfoBList.get(2); + Assert.assertNotNull(queueB3); + Assert.assertEquals(20f, queueB3.getCapacity(), 0.00001); } /** From 001d353cc596039124c7fbb66441bb6b72408618 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sat, 5 Aug 2023 16:33:35 +0800 Subject: [PATCH 07/48] YARN-7708. BackPort [GPG] Load based policy generator. (#5902) Contributed by Young Chen. --- .../dev-support/findbugs-exclude.xml | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 + .../src/main/resources/yarn-default.xml | 62 ++++ .../globalpolicygenerator/GPGUtils.java | 5 +- .../policygenerator/GlobalPolicy.java | 2 +- .../LoadBasedGlobalPolicy.java | 329 ++++++++++++++++++ .../policygenerator/PolicyGenerator.java | 6 +- .../TestLoadBasedGlobalPolicy.java | 206 +++++++++++ 8 files changed, 625 insertions(+), 7 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestLoadBasedGlobalPolicy.java diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index cf457c23eb1d3..309c028580081 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -209,6 +209,10 @@ + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index faa5c82d7e9a2..ae7ea196d4783 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4427,6 +4427,24 @@ public static boolean isAclEnabled(Configuration conf) { public static final String GPG_POLICY_GENERATOR_BLACKLIST = FEDERATION_GPG_POLICY_PREFIX + "blacklist"; + private static final String FEDERATION_GPG_LOAD_BASED_PREFIX = + YarnConfiguration.FEDERATION_GPG_PREFIX + "policy.generator.load-based."; + public static final String FEDERATION_GPG_LOAD_BASED_MIN_PENDING = + FEDERATION_GPG_LOAD_BASED_PREFIX + "pending.minimum"; + public static final int DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_PENDING = 100; + public static final String FEDERATION_GPG_LOAD_BASED_MAX_PENDING = + FEDERATION_GPG_LOAD_BASED_PREFIX + "pending.maximum"; + public static final int DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_PENDING = 1000; + public static final String FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT = + FEDERATION_GPG_LOAD_BASED_PREFIX + "weight.minimum"; + public static final float DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT = 0.0f; + public static final String FEDERATION_GPG_LOAD_BASED_MAX_EDIT = + FEDERATION_GPG_LOAD_BASED_PREFIX + "edit.maximum"; + public static final int DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_EDIT = 3; + public static final String FEDERATION_GPG_LOAD_BASED_SCALING = + FEDERATION_GPG_LOAD_BASED_PREFIX + "scaling"; + public static final String DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING = "LINEAR"; + /** * Connection and Read timeout from the Router to RM. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index b643bd8d08d49..6cab018e5c41d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5468,4 +5468,66 @@ + + + GPG load policy, the minimum number of pending applications in the subCluster. + + yarn.federation.gpg.policy.generator.load-based.pending.minimum + 100 + + + + + GPG load policy, the maximum number of pending applications in the subCluster. + + yarn.federation.gpg.policy.generator.load-based.pending.maximum + 1000 + + + + + GPG load policy, the subCluster minimum weight, + If a subCluster has a very high load, we will assign this value to the subCluster. + The default value is 0, which means that we no longer assign appliaction to this subCluster. + + yarn.federation.gpg.policy.generator.load-based.weight.minimum + 0 + + + + + GPG load policy, We choose the subCluster computing load of TopN. + This value represents the number of subClusters we want to calculate. + + yarn.federation.gpg.policy.generator.load-based.edit.maximum + 3 + + + + + GPG load policy, We provide 4 calculation methods: NONE, LINEAR, QUADRATIC, LOG. + + Note, this calculation method is when the number of Pending Applications in + the subCluster is less than yarn.federation.gpg.policy.generator.load-based.pending.maximum. + + maxPendingVal = yarn.federation.gpg.policy.generator.load-based.pending.maximum - + yarn.federation.gpg.policy.generator.load-based.pending.minimum + curPendingVal = Pending Applications in the subCluster - + yarn.federation.gpg.policy.generator.load-based.pending.minimum + + 1. NONE: No calculation is required, and the weight is 1 at this time. + 2. LINEAR: For linear computation, we will use (maxPendingVal - curPendingVal) / (maxPendingVal). + 3. QUADRATIC: Calculated using quadratic, + We will calculate quadratic for maxPendingVal, curPendingVal, + then use this formula = (maxPendingVal - curPendingVal) / (maxPendingVal). + 4. LOG(LOGARITHM): Calculated using logarithm, + We will calculate logarithm for maxPendingVal, curPendingVal, + then use this formula = (maxPendingVal - curPendingVal) / (maxPendingVal). + + LINEAR is used by default. + + yarn.federation.gpg.policy.generator.load-based.scaling + LINEAR + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java index 6d2e1d4142122..2bb56caeffba2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java @@ -55,7 +55,7 @@ private GPGUtils() { */ public static T invokeRMWebService(String webAddr, String path, final Class returnType) { Client client = Client.create(); - T obj = null; + T obj; WebResource webResource = client.resource(webAddr); ClientResponse response = null; @@ -86,8 +86,7 @@ public static T invokeRMWebService(String webAddr, String path, final Class< */ public static Map createUniformWeights( Set ids) { - Map weights = - new HashMap<>(); + Map weights = new HashMap<>(); for(SubClusterId id : ids) { weights.put(new SubClusterIdInfo(id), 1.0f); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java index c6d6558dbec1e..ab60a48434e0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java @@ -53,7 +53,7 @@ public Configuration getConf() { * * @return a map of the object type and RM path. */ - protected Map registerPaths() { + protected Map, String> registerPaths() { // Default register nothing return Collections.emptyMap(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java new file mode 100644 index 0000000000000..f728b92d71f53 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java @@ -0,0 +1,329 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator; + +import org.apache.commons.collections.MapUtils; +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager; +import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; +import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGUtils; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_MIN_PENDING; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_PENDING; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_MAX_PENDING; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_PENDING; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_MAX_EDIT; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_EDIT; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_SCALING; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING; + +/** + * Load based policy that generates weighted policies by scaling + * the cluster load (based on pending) to a weight from 0.0 to 1.0. + */ +public class LoadBasedGlobalPolicy extends GlobalPolicy { + + private static final Logger LOG = LoggerFactory.getLogger(LoadBasedGlobalPolicy.class); + + public enum Scaling { + LINEAR, + QUADRATIC, + LOG, + NONE + } + + // Minimum pending count before the policy starts scaling down the weights + private int minPending; + // Maximum pending count before policy stops scaling down the weights + // (they'll be set to min weight) + private int maxPending; + // Minimum weight that a sub cluster will be assigned + private float minWeight; + // Maximum number of weights that can be scaled down simultaneously + private int maxEdit; + // Scaling type + private Scaling scaling = Scaling.NONE; + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + minPending = conf.getInt(FEDERATION_GPG_LOAD_BASED_MIN_PENDING, + DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_PENDING); + maxPending = conf.getInt(FEDERATION_GPG_LOAD_BASED_MAX_PENDING, + DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_PENDING); + minWeight = conf.getFloat(FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT, + DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT); + maxEdit = conf.getInt(FEDERATION_GPG_LOAD_BASED_MAX_EDIT, + DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_EDIT); + + try { + scaling = Scaling.valueOf(conf.get(FEDERATION_GPG_LOAD_BASED_SCALING, + DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING)); + } catch (IllegalArgumentException e) { + LOG.warn("Invalid scaling mode provided", e); + } + + // Check that all configuration values are valid + if (!(minPending <= maxPending)) { + throw new YarnRuntimeException("minPending = " + minPending + + " must be less than or equal to maxPending=" + maxPending); + } + if (!(minWeight >= 0 && minWeight < 1)) { + throw new YarnRuntimeException( + "minWeight = " + minWeight + " must be within range [0,1)"); + } + } + + @Override + protected Map, String> registerPaths() { + // Register for the endpoints we want to receive information on + Map, String> map = new HashMap<>(); + map.put(ClusterMetricsInfo.class, RMWSConsts.METRICS); + return map; + } + + /** + * Update the policy of the queue. + * + * @param queueName name of the queue + * @param clusterInfo subClusterId map to cluster information about the + * SubCluster used to make policy decisions + * @param currentManager the FederationPolicyManager for the queue's existing + * policy the manager may be null, in which case the policy + * will need to be created. + * + * @return FederationPolicyManager. + */ + @Override + protected FederationPolicyManager updatePolicy(String queueName, + Map> clusterInfo, + FederationPolicyManager currentManager) { + if (currentManager == null) { + LOG.info("Creating load based weighted policy queue {}.", queueName); + currentManager = getWeightedLocalityPolicyManager(queueName, clusterInfo); + } else if (currentManager instanceof WeightedLocalityPolicyManager) { + LOG.info("Updating load based weighted policy queue {}.", queueName); + currentManager = getWeightedLocalityPolicyManager(queueName, clusterInfo); + } else { + LOG.warn("Policy for queue {} is of type {}, expected {}.", queueName, + currentManager.getClass(), WeightedLocalityPolicyManager.class); + } + return currentManager; + } + + /** + * GPG can help update the policy of the queue. + * + * We automatically generate the weight of the subCluster + * according to the clusterMetrics of the subCluster. + * + * @param queue queueName. + * @param subClusterMetricInfos Metric information of the subCluster. + * @return WeightedLocalityPolicyManager. + */ + protected WeightedLocalityPolicyManager getWeightedLocalityPolicyManager(String queue, + Map> subClusterMetricInfos) { + + // Parse the metric information of the subCluster. + Map clusterMetrics = + getSubClustersMetricsInfo(subClusterMetricInfos); + + if (MapUtils.isEmpty(clusterMetrics)) { + return null; + } + + // Get the new weight of the subCluster. + WeightedLocalityPolicyManager manager = new WeightedLocalityPolicyManager(); + Map weights = getTargetWeights(clusterMetrics); + manager.setQueue(queue); + manager.getWeightedPolicyInfo().setAMRMPolicyWeights(weights); + manager.getWeightedPolicyInfo().setRouterPolicyWeights(weights); + return manager; + } + + /** + * Get the ClusterMetric information of the subCluster. + * + * @param subClusterMetricsInfo subCluster Metric Information. + * @return Mapping relationship between subCluster and Metric. + */ + protected Map getSubClustersMetricsInfo( + Map> subClusterMetricsInfo) { + + // Check whether the Metric information of the sub-cluster is empty, + // if it is empty, we will directly return null. + if(MapUtils.isEmpty(subClusterMetricsInfo)) { + LOG.warn("The metric info of the subCluster is empty."); + return null; + } + + Map clusterMetrics = new HashMap<>(); + for (Map.Entry> entry : subClusterMetricsInfo.entrySet()) { + SubClusterId subClusterId = entry.getKey(); + Map subClusterMetrics = entry.getValue(); + ClusterMetricsInfo clusterMetricsInfo = (ClusterMetricsInfo) + subClusterMetrics.getOrDefault(ClusterMetricsInfo.class, null); + clusterMetrics.put(subClusterId, clusterMetricsInfo); + } + + // return subCluster Metric Information. + return clusterMetrics; + } + + /** + * Get subCluster target weight. + * + * @param clusterMetrics Metric of the subCluster. + * @return subCluster Weights. + */ + @VisibleForTesting + protected Map getTargetWeights( + Map clusterMetrics) { + Map weights = GPGUtils.createUniformWeights(clusterMetrics.keySet()); + + List scs = new ArrayList<>(clusterMetrics.keySet()); + // Sort the sub clusters into descending order based on pending load + scs.sort(new SortByDescendingLoad(clusterMetrics)); + + // Keep the top N loaded sub clusters + scs = scs.subList(0, Math.min(maxEdit, scs.size())); + + for (SubClusterId sc : scs) { + LOG.info("Updating weight for sub cluster {}", sc.toString()); + int pending = clusterMetrics.get(sc).getAppsPending(); + if (pending <= minPending) { + LOG.info("Load ({}) is lower than minimum ({}), skipping", pending, minPending); + } else if (pending < maxPending) { + // The different scaling strategies should all map values from the + // range min_pending+1 to max_pending to the range min_weight to 1.0f + // so we pre-process and simplify the domain to some value [1, MAX-MIN) + int val = pending - minPending; + int maxVal = maxPending - minPending; + + // Scale the weights to respect the config minimum + float weight = getWeightByScaling(maxVal, val); + weight = weight * (1.0f - minWeight); + weight += minWeight; + weights.put(new SubClusterIdInfo(sc), weight); + LOG.info("Load ({}) is within maximum ({}), setting weights via {} " + + "scale to {}", pending, maxPending, scaling, weight); + } else { + weights.put(new SubClusterIdInfo(sc), minWeight); + LOG.info("Load ({}) exceeded maximum ({}), setting weight to minimum: {}", + pending, maxPending, minWeight); + } + } + validateWeights(weights); + return weights; + } + + /** + * Get weight information. + * We will calculate the weight information according to different Scaling. + * + * NONE: No calculation is required, and the weight is 1 at this time. + * + * LINEAR: For linear computation, we will use (maxPendingVal - curPendingVal) / (maxPendingVal). + * + * QUADRATIC: Calculated using quadratic, + * We will calculate quadratic for maxPendingVal, curPendingVal, + * then use this formula = (maxPendingVal - curPendingVal) / (maxPendingVal). + * + * LOG(LOGARITHM): Calculated using logarithm, + * We will calculate logarithm for maxPendingVal, curPendingVal, + * then use this formula = (maxPendingVal - curPendingVal) / (maxPendingVal). + * + * @param maxPendingVal maxPending - minPending + * @param curPendingVal pending - minPending + * @return Calculated weight information. + */ + protected float getWeightByScaling(int maxPendingVal, int curPendingVal) { + float weight = 1.0f; + switch (scaling) { + case NONE: + break; + case LINEAR: + weight = (float) (maxPendingVal - curPendingVal) / (float) (maxPendingVal); + break; + case QUADRATIC: + double maxValQuad = Math.pow(maxPendingVal, 2); + double valQuad = Math.pow(curPendingVal, 2); + weight = (float) (maxValQuad - valQuad) / (float) (maxValQuad); + break; + case LOG: + double maxValLog = Math.log(maxPendingVal); + double valLog = Math.log(curPendingVal); + weight = (float) (maxValLog - valLog) / (float) (maxValLog); + break; + default: + LOG.warn("No suitable scaling found, Skip."); + break; + } + return weight; + } + + /** + * Helper to avoid all zero weights. If weights are all zero, they're reset + * to one + * @param weights weights to validate + */ + private void validateWeights(Map weights) { + for(Float w : weights.values()) { + // If we find a nonzero weight, we're validated + if(w > 0.0f) { + return; + } + } + LOG.warn("All {} generated weights were 0.0f. Resetting to 1.0f.", weights.size()); + // All weights were zero. Reset all back to 1.0 + weights.replaceAll((i, v) -> 1.0f); + } + + private static final class SortByDescendingLoad + implements Comparator { + + private Map clusterMetrics; + + private SortByDescendingLoad( + Map clusterMetrics) { + this.clusterMetrics = clusterMetrics; + } + + public int compare(SubClusterId a, SubClusterId b) { + // Sort by pending load + return clusterMetrics.get(b).getAppsPending() - clusterMetrics.get(a) + .getAppsPending(); + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java index 33501fb1e3e73..3c94d6576e738 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java @@ -129,15 +129,15 @@ public final void run() { try { manager = this.gpgContext.getPolicyFacade().getPolicyManager(queueName); } catch (YarnException e) { - LOG.error("GetPolicy for queue {} failed", queueName, e); + LOG.error("GetPolicy for queue {} failed.", queueName, e); continue; } - LOG.info("Updating policy for queue {}", queueName); + LOG.info("Updating policy for queue {}.", queueName); manager = policy.updatePolicy(queueName, clusterInfo, manager); try { this.gpgContext.getPolicyFacade().setPolicyManager(manager); } catch (YarnException e) { - LOG.error("SetPolicy for queue {} failed", queueName, e); + LOG.error("SetPolicy for queue {} failed.", queueName, e); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestLoadBasedGlobalPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestLoadBasedGlobalPolicy.java new file mode 100644 index 0000000000000..df58b30aaaa37 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestLoadBasedGlobalPolicy.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_MAX_EDIT; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_MIN_PENDING; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_MAX_PENDING; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_GPG_LOAD_BASED_SCALING; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Unit test for the Load Based Global Policy. + */ +public class TestLoadBasedGlobalPolicy { + private static final Logger LOG = + LoggerFactory.getLogger(TestLoadBasedGlobalPolicy.class); + + private static final int NUM_SC = 3; + private static final float DELTA = 0.00001f; + + private static final int MIN_PENDING = 100; + private static final int MAX_PENDING = 500; + + private List subClusterIds; + private Map clusterMetricsInfos; + private Map weights; + + private final Configuration conf; + private final LoadBasedGlobalPolicy policyGenerator; + + public TestLoadBasedGlobalPolicy() { + conf = new Configuration(); + policyGenerator = new LoadBasedGlobalPolicy(); + } + + @Before + public void setUp() { + + conf.setInt(FEDERATION_GPG_LOAD_BASED_MAX_EDIT, 2); + conf.setInt(FEDERATION_GPG_LOAD_BASED_MIN_PENDING, MIN_PENDING); + conf.setInt(FEDERATION_GPG_LOAD_BASED_MAX_PENDING, MAX_PENDING); + conf.setFloat(FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT, 0.0f); + conf.set(FEDERATION_GPG_LOAD_BASED_SCALING, LoadBasedGlobalPolicy.Scaling.LINEAR.name()); + policyGenerator.setConf(conf); + + subClusterIds = new ArrayList<>(); + clusterMetricsInfos = new HashMap<>(); + // Set up sub clusters + for (int i = 0; i < NUM_SC; ++i) { + // subClusterId + SubClusterId id = SubClusterId.newInstance("sc" + i); + subClusterIds.add(id); + + // Cluster metrics info + ClusterMetricsInfo metricsInfo = new ClusterMetricsInfo(); + metricsInfo.setAppsPending(50); + clusterMetricsInfos.put(id, metricsInfo); + } + } + + @Test + public void testSimpleTargetWeights() { + weights = policyGenerator.getTargetWeights(clusterMetricsInfos); + assertEquals(weights.size(), 3); + assertEquals(1.0, getWeight(0), DELTA); + assertEquals(1.0, getWeight(1), DELTA); + assertEquals(1.0, getWeight(2), DELTA); + } + + @Test + public void testLoadTargetWeights() { + getMetric(0).setAppsPending(100); + weights = policyGenerator.getTargetWeights(clusterMetricsInfos); + assertEquals(weights.size(), 3); + assertEquals(1.0, getWeight(0), DELTA); + assertEquals(1.0, getWeight(1), DELTA); + assertEquals(1.0, getWeight(2), DELTA); + getMetric(0).setAppsPending(500); + weights = policyGenerator.getTargetWeights(clusterMetricsInfos); + assertEquals(weights.size(), 3); + assertEquals(0.0, getWeight(0), DELTA); + assertEquals(1.0, getWeight(1), DELTA); + assertEquals(1.0, getWeight(2), DELTA); + } + + @Test + public void testMaxEdit() { + // The policy should be able to edit 2 weights + getMetric(0).setAppsPending(MAX_PENDING + 200); + getMetric(1).setAppsPending(MAX_PENDING + 100); + weights = policyGenerator.getTargetWeights(clusterMetricsInfos); + assertEquals(weights.size(), 3); + assertEquals(0.0, getWeight(0), DELTA); + assertEquals(0.0, getWeight(1), DELTA); + assertEquals(1.0, getWeight(2), DELTA); + // After updating the config, it should only edit the most loaded + conf.setInt(FEDERATION_GPG_LOAD_BASED_MAX_EDIT, 1); + policyGenerator.setConf(conf); + weights = policyGenerator.getTargetWeights(clusterMetricsInfos); + assertEquals(weights.size(), 3); + assertEquals(0.0, getWeight(0), DELTA); + assertEquals(1.0, getWeight(1), DELTA); + assertEquals(1.0, getWeight(2), DELTA); + } + + @Test + public void testMinWeight() { + // If a minimum weight is set, the generator should not go below it + conf.setFloat(FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT, 0.5f); + policyGenerator.setConf(conf); + getMetric(0).setAppsPending(Integer.MAX_VALUE); + weights = policyGenerator.getTargetWeights(clusterMetricsInfos); + assertEquals(weights.size(), 3); + assertEquals(0.5, getWeight(0), DELTA); + assertEquals(1.0, getWeight(1), DELTA); + assertEquals(1.0, getWeight(2), DELTA); + } + + @Test + public void testScaling() { + LOG.info("Testing that the generator weights are monotonically" + + " decreasing regardless of scaling method"); + for (LoadBasedGlobalPolicy.Scaling scaling : + new LoadBasedGlobalPolicy.Scaling[] {LoadBasedGlobalPolicy.Scaling.LINEAR, + LoadBasedGlobalPolicy.Scaling.QUADRATIC, LoadBasedGlobalPolicy.Scaling.LOG }) { + LOG.info("Testing {} scaling...", scaling); + conf.set(DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING, scaling.name()); + policyGenerator.setConf(conf); + // Test a continuous range for scaling + float prevWeight = 1.01f; + for (int load = 0; load < MAX_PENDING * 2; ++load) { + getMetric(0).setAppsPending(load); + weights = policyGenerator.getTargetWeights(clusterMetricsInfos); + if (load < MIN_PENDING) { + // Below the minimum load, it should stay 1.0f + assertEquals(1.0f, getWeight(0), DELTA); + } else if (load < MAX_PENDING) { + // In the specified range, the weight should consistently decrease + float weight = getWeight(0); + assertTrue(weight < prevWeight); + prevWeight = weight; + } else { + // Above the maximum load, it should stay 0.0f + assertEquals(0.0f, getWeight(0), DELTA); + } + } + } + } + + @Test + public void testNonZero() { + // If all generated weights are zero, they should be set back to one + conf.setFloat(FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT, 0.0f); + conf.setInt(FEDERATION_GPG_LOAD_BASED_MAX_EDIT, 3); + policyGenerator.setConf(conf); + getMetric(0).setAppsPending(Integer.MAX_VALUE); + getMetric(1).setAppsPending(Integer.MAX_VALUE); + getMetric(2).setAppsPending(Integer.MAX_VALUE); + weights = policyGenerator.getTargetWeights(clusterMetricsInfos); + assertEquals(weights.size(), 3); + assertEquals(1.0, getWeight(0), DELTA); + assertEquals(1.0, getWeight(1), DELTA); + assertEquals(1.0, getWeight(2), DELTA); + } + + private float getWeight(int sc) { + return weights.get(new SubClusterIdInfo(subClusterIds.get(sc))); + } + + private ClusterMetricsInfo getMetric(int sc) { + return clusterMetricsInfos.get(subClusterIds.get(sc)); + } +} From 55e8301470e2b8bed8ac2cc66f01398a51e0fa23 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Sat, 5 Aug 2023 12:27:52 -0700 Subject: [PATCH 08/48] HDFS-17118 Fixed a couple checkstyle warnings in TestObserverReadProxyProvider (#5880) --- .../ha/TestObserverReadProxyProvider.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverReadProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverReadProxyProvider.java index e3c1a0388b00c..2c69a102caf62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverReadProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverReadProxyProvider.java @@ -46,7 +46,6 @@ import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.event.Level; @@ -59,8 +58,8 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -367,8 +366,10 @@ public void testGetHAServiceStateWithTimeout() throws Exception { setupProxyProvider(1, NAMENODE_HA_STATE_PROBE_TIMEOUT_SHORT); final HAServiceState state = HAServiceState.STANDBY; + @SuppressWarnings("unchecked") NNProxyInfo dummyNNProxyInfo = (NNProxyInfo) mock(NNProxyInfo.class); + @SuppressWarnings("unchecked") Future task = mock(Future.class); when(task.get(anyLong(), any(TimeUnit.class))).thenReturn(state); @@ -389,8 +390,10 @@ public void testTimeoutExceptionGetHAServiceStateWithTimeout() throws Exception proxyLog.clearOutput(); setupProxyProvider(1, NAMENODE_HA_STATE_PROBE_TIMEOUT_SHORT); + @SuppressWarnings("unchecked") NNProxyInfo dummyNNProxyInfo = (NNProxyInfo) Mockito.mock(NNProxyInfo.class); + @SuppressWarnings("unchecked") Future task = mock(Future.class); TimeoutException e = new TimeoutException("Timeout"); when(task.get(anyLong(), any(TimeUnit.class))).thenThrow(e); @@ -413,8 +416,10 @@ public void testInterruptedExceptionGetHAServiceStateWithTimeout() throws Except proxyLog.clearOutput(); setupProxyProvider(1, NAMENODE_HA_STATE_PROBE_TIMEOUT_SHORT); + @SuppressWarnings("unchecked") NNProxyInfo dummyNNProxyInfo = (NNProxyInfo) Mockito.mock(NNProxyInfo.class); + @SuppressWarnings("unchecked") Future task = mock(Future.class); InterruptedException e = new InterruptedException("Interrupted"); when(task.get(anyLong(), any(TimeUnit.class))).thenThrow(e); @@ -436,8 +441,10 @@ public void testExecutionExceptionGetHAServiceStateWithTimeout() throws Exceptio proxyLog.clearOutput(); setupProxyProvider(1, NAMENODE_HA_STATE_PROBE_TIMEOUT_SHORT); + @SuppressWarnings("unchecked") NNProxyInfo dummyNNProxyInfo = (NNProxyInfo) Mockito.mock(NNProxyInfo.class); + @SuppressWarnings("unchecked") Future task = mock(Future.class); Exception e = new ExecutionException(new InterruptedException("Interrupted")); when(task.get(anyLong(), any(TimeUnit.class))).thenThrow(e); @@ -452,7 +459,7 @@ public void testExecutionExceptionGetHAServiceStateWithTimeout() throws Exceptio } /** - * Test GetHAServiceState when timeout is disabled (test the else { task.get() } code path) + * Test GetHAServiceState when timeout is disabled (test the else { task.get() } code path). */ @Test public void testGetHAServiceStateWithoutTimeout() throws Exception { @@ -460,8 +467,10 @@ public void testGetHAServiceStateWithoutTimeout() throws Exception { setupProxyProvider(1, 0); final HAServiceState state = HAServiceState.STANDBY; + @SuppressWarnings("unchecked") NNProxyInfo dummyNNProxyInfo = (NNProxyInfo) mock(NNProxyInfo.class); + @SuppressWarnings("unchecked") Future task = mock(Future.class); when(task.get()).thenReturn(state); From 440698eb0706636ab5955518bc2bc4b2c743981f Mon Sep 17 00:00:00 2001 From: WangYuanben <48795318+YuanbenWang@users.noreply.github.com> Date: Mon, 7 Aug 2023 20:03:23 +0800 Subject: [PATCH 09/48] HADOOP-18836. Some properties are missing from hadoop-policy.xml (#5922) --- .../src/main/conf/hadoop-policy.xml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml index e1640f97546ac..ecda924a54db7 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml @@ -63,6 +63,16 @@ A special value of "*" means all users are allowed. + + security.datanode.lifeline.protocol.acl + * + ACL for DatanodeLifelineProtocol, which is used by a + DataNode to send lifeline messages to the NameNode. + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + security.namenode.protocol.acl * @@ -82,6 +92,16 @@ A special value of "*" means all users are allowed. + + security.get.user.mappings.protocol.acl + * + ACL for GetUserMappingsProtocol, implemented by the NameNode + and Job Tracker which maps users to groups. + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + security.refresh.user.mappings.protocol.acl * @@ -92,6 +112,16 @@ users are allowed. + + security.reconfiguration.protocol.acl + * + ACL for ReconfigurationProtocol, used by HDFS admin to + reload configuration for NameNode/DataNode without restarting them. + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + security.refresh.policy.protocol.acl * @@ -102,6 +132,26 @@ A special value of "*" means all users are allowed. + + security.refresh.callqueue.protocol.acl + * + ACL for RefreshCallQueueProtocol, which is used to refresh + the call queue in use currently. + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.refresh.generic.protocol.acl + * + ACL for GenericRefreshProtocol, which is used to refresh + arbitrary things at runtime. + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + security.ha.service.protocol.acl * From 1e3e246934b274d28bbfebdb5306e7a71b555be9 Mon Sep 17 00:00:00 2001 From: WangYuanben <48795318+YuanbenWang@users.noreply.github.com> Date: Tue, 8 Aug 2023 07:37:26 +0800 Subject: [PATCH 10/48] HADOOP-18810. Document missing a lot of properties in core-default.xml. (#5912) Contributed by WangYuanben. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../src/main/resources/core-default.xml | 316 +++++++++++++++++- .../conf/TestCommonConfigurationFields.java | 1 + 2 files changed, 315 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 5f841bd233d34..14ffe3d9dec0e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -73,6 +73,27 @@ Is service-level authorization enabled? + + security.service.authorization.default.acl + + + Define the default acl for the Hadoop service if the acl of Hadoop + service is not defined in hadoop-policy.xml. If not set, `*` is applied + meaning that all users are allowed to access the service. The list of + users and groups are both comma-separated list of names separated by + a space. Example: `user1,user2 group1,group2`. + + + + + security.service.authorization.default.acl.blocked + + + This property specifies the list of users and groups who are not + authorized to access Hadoop service. + + + hadoop.security.instrumentation.requires.admin false @@ -225,6 +246,17 @@ + + hadoop.security.group.mapping.ldap.ctx.factory.class + + + Used to specify the fully qualified class name of the initial context + factory when connecting to an LDAP server. The default value is + "com.sun.jndi.ldap.LdapCtxFactory", but set to null now to avoid + LifecycleExecutionException with JDK 11(see HADOOP-15941). + + + hadoop.security.group.mapping.ldap.connection.timeout.ms 60000 @@ -803,7 +835,19 @@ hadoop.token.files - List of token cache files that have delegation tokens for hadoop service + + A comma-separated list of token cache files that have delegation tokens + for hadoop service + + + + + hadoop.tokens + + + A comma-separated list of delegation tokens from base64 encoding + for hadoop service. + @@ -855,6 +899,65 @@ operate entirely in Java, specify "java-builtin". + + io.compression.codec.lz4.buffersize + 262144 + + Internal buffer size for Lz4 compressor/decompressors. + + + + + io.compression.codec.lz4.use.lz4hc + false + + Enable lz4hc(slow but with high compression ratio) for lz4 compression. + + + + + io.compression.codec.lzo.buffersize + 65536 + + Internal buffer size for Lzo compressor/decompressors. + + + + + io.compression.codec.lzo.class + org.apache.hadoop.io.compress.LzoCodec + + Codec class that implements Lzo compression algorithm. + + + + + io.compression.codec.snappy.buffersize + 262144 + + Internal buffer size for Snappy compressor/decompressors. + + + + + io.compression.codec.zstd.buffersize + 0 + + Indicate ZStandard buffer size. The default value 0 means use the + recommended zstd buffer size that the library recommends. + + + + + io.compression.codec.zstd.level + 3 + + Indicate ZStandard compression level. The higher the compression level, + the higher the compression ratio and memory usage, but the slower the + compression and decompression speed. + + + io.serializations org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization @@ -1145,6 +1248,33 @@ + + fs.file.impl + + + Specify the implementation class used for accessing the file system. It + is a fully qualified class name, including both the package name and the + class name. + + + + + fs.creation.parallel.count + 64 + + This property sets a a semaphore to throttle the number of FileSystem + instances which can be created simultaneously. This is designed to reduce + the impact of many threads in an application calling FileSystem#get() on + a filesystem which takes time to instantiate -for example to an object + where HTTPS connections are set up during initialization. Many threads + trying to do this may create spurious delays by conflicting for access + to synchronized blocks, when simply limiting the parallelism diminishes + the conflict, so speeds up all threads trying to access the store. If a + service appears to be blocking on all threads initializing connections to + abfs, s3a or store, try a smaller (possibly significantly smaller) value. + + + fs.AbstractFileSystem.ftp.impl org.apache.hadoop.fs.ftp.FtpFs @@ -1231,6 +1361,22 @@ + + fs.iostatistics.logging.level + debug + + Logging level for IOStatistics. + + + + + fs.iostatistics.thread.level.enabled + true + + Enable IOStatisticsContext support for thread level. + + + fs.s3a.access.key AWS access key ID used by S3A file system. Omit for IAM role-based or provider-based authentication. @@ -2230,6 +2376,13 @@ The switch to turn S3A auditing on or off. + + ipc.client.async.calls.max + 100 + + Define the maximum number of outstanding async calls. + + ipc.client.idlethreshold @@ -2239,6 +2392,14 @@ The switch to turn S3A auditing on or off. + + ipc.client.connection.idle-scan-interval.ms + 10000 + + Indicate how often the server scans for idle connections. + + + ipc.client.kill.max 10 @@ -2286,6 +2447,14 @@ The switch to turn S3A auditing on or off. + + ipc.client.connect.max.retries.on.sasl + 5 + + The maximum retries on SASL connection failures in RPC client. + + + ipc.client.tcpnodelay true @@ -2329,6 +2498,14 @@ The switch to turn S3A auditing on or off. + + ipc.server.tcpnodelay + true + + If true then disable Nagle's Algorithm. + + + ipc.server.handler.queue.size 100 @@ -2338,6 +2515,24 @@ The switch to turn S3A auditing on or off. + + ipc.server.max.response.size + 1048576 + + The maximum size when large IPC handler response buffer is reset. + + + + + ipc.server.metrics.update.runner.interval + 5000 + + To configure scheduling of server metrics update thread. This config is + used to indicate initial delay and delay between each execution of the + metric update runnable thread. + + + ipc.server.listen.queue.size 256 @@ -2363,6 +2558,22 @@ The switch to turn S3A auditing on or off. + + ipc.server.read.connection-queue.size + 100 + + Number of pending connections that may be queued per socket reader. + + + + + ipc.server.read.threadpool.size + 1 + + Indicates the number of threads in RPC server reading from the socket. + + + ipc.maximum.data.length 134217728 @@ -2392,6 +2603,14 @@ The switch to turn S3A auditing on or off. + + callqueue.overflow.trigger.failover + false + + Enable callqueue overflow trigger failover for stateless servers. + + + @@ -2476,6 +2695,20 @@ The switch to turn S3A auditing on or off. + + ipc.[port_number].callqueue.capacity.weights + + + When FairCallQueue is enabled, user can specify capacity allocation + among all sub-queues via this property. The value of this config is + a comma-separated list of positive integers, each of which specifies + the weight associated with the sub-queue at that index. This list + length should be IPC scheduler priority levels, defined by + "scheduler.priority.levels". By default, each sub-queue is associated + with weight 1, i.e., all sub-queues are allocated with the same capacity. + + + ipc.[port_number].scheduler.priority.levels 4 @@ -2744,6 +2977,24 @@ The switch to turn S3A auditing on or off. + + net.topology.configured.node.mapping + + + Key to define the node mapping as a comma-delimited list of host=rack + mappings. e.g. host1=r1,host2=r1,host3=r2. Important: spaces not trimmed + and are considered significant. + + + + + net.topology.dependency.script.file.name + + + Key to the dependency script filename. + + + file.stream-buffer-size @@ -3274,6 +3525,17 @@ The switch to turn S3A auditing on or off. + + hadoop.user.group.metrics.percentiles.intervals + + + A comma-delimited list of integers denoting the desired rollover + intervals (in seconds) for percentile latency metrics on the Namenode + and Datanode for each user in the group. By default, percentile + latency metrics are disabled. + + + rpc.metrics.quantile.enable false @@ -3532,6 +3794,24 @@ The switch to turn S3A auditing on or off. + + hadoop.security.kms.client.failover.max.retries + + + Default value is the number of providers specified. + + + + + hadoop.security.kerberos.ticket.cache.path + + + Path to the Kerberos ticket cache. Setting this will force + UserGroupInformation to use only this ticket cache file when + creating a FileSystem instance. + + + ipc.server.max.connections 0 @@ -3944,6 +4224,30 @@ The switch to turn S3A auditing on or off. + + hadoop.zk.server.principal + + + Principal name for zookeeper servers. + + + + + hadoop.zk.kerberos.principal + + + Kerberos principal name for zookeeper connection. + + + + + hadoop.zk.kerberos.keytab + + + Kerberos keytab for zookeeper connection. + + + hadoop.zk.ssl.keystore.location @@ -3986,7 +4290,15 @@ The switch to turn S3A auditing on or off. YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL - System tags to group related properties together. + A comma-separated list of system tags to group related properties together. + + + + + hadoop.tags.custom + + + A comma-separated list of custom tags to group related properties together. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index 74b2f55065dec..f7303fb0f5e1a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -157,6 +157,7 @@ public void initializeMemberVariables() { xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.impl"); xmlPropsToSkipCompare.add("ipc.scheduler.impl"); xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.priority.levels"); + xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.capacity.weights"); xmlPropsToSkipCompare.add( "ipc.[port_number].faircallqueue.multiplexer.weights"); xmlPropsToSkipCompare.add("ipc.[port_number].identity-provider.impl"); From bc48e5cbe8b6823975d66c35290a92d0642027df Mon Sep 17 00:00:00 2001 From: hchaverri <55413673+hchaverri@users.noreply.github.com> Date: Mon, 7 Aug 2023 16:45:14 -0700 Subject: [PATCH 11/48] HDFS-17128. Updating SQLDelegationTokenSecretManager to use LoadingCache so tokens are updated frequently. (#5897) Contributed by Hector Sandoval Chaverri. Reviewed-by: Simbarashe Dzinamarira Reviewed-by: Inigo Goiri Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../AbstractDelegationTokenSecretManager.java | 10 +- .../DelegationTokenLoadingCache.java | 118 ++++++++++++++++++ .../SQLDelegationTokenSecretManager.java | 96 +++++++++----- ...stSQLDelegationTokenSecretManagerImpl.java | 118 +++++++++++++++++- 4 files changed, 308 insertions(+), 34 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationTokenLoadingCache.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index cde4cf4841383..cf5e9a66311aa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -88,8 +88,7 @@ private String formatTokenId(TokenIdent id) { * Cache of currently valid tokens, mapping from DelegationTokenIdentifier * to DelegationTokenInformation. Protected by this object lock. */ - protected final Map currentTokens - = new ConcurrentHashMap<>(); + protected Map currentTokens; /** * Map of token real owners to its token count. This is used to generate @@ -155,6 +154,7 @@ public AbstractDelegationTokenSecretManager(long delegationKeyUpdateInterval, this.tokenRenewInterval = delegationTokenRenewInterval; this.tokenRemoverScanInterval = delegationTokenRemoverScanInterval; this.storeTokenTrackingId = false; + this.currentTokens = new ConcurrentHashMap<>(); } /** @@ -771,10 +771,14 @@ protected void logExpireTokens( for (TokenIdent ident : expiredTokens) { logExpireToken(ident); LOG.info("Removing expired token " + formatTokenId(ident)); - removeStoredToken(ident); + removeExpiredStoredToken(ident); } } + protected void removeExpiredStoredToken(TokenIdent ident) throws IOException { + removeStoredToken(ident); + } + public void stopThreads() { if (LOG.isDebugEnabled()) LOG.debug("Stopping expired delegation token remover thread"); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationTokenLoadingCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationTokenLoadingCache.java new file mode 100644 index 0000000000000..cc21f45a1deb5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationTokenLoadingCache.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security.token.delegation; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; + + +/** + * Cache for delegation tokens that can handle high volume of tokens. A + * loading cache will prevent all active tokens from being in memory at the + * same time. It will also trigger more requests from the persistent token storage. + */ +public class DelegationTokenLoadingCache implements Map { + private LoadingCache internalLoadingCache; + + public DelegationTokenLoadingCache(long cacheExpirationMs, long maximumCacheSize, + Function singleEntryFunction) { + this.internalLoadingCache = CacheBuilder.newBuilder() + .expireAfterWrite(cacheExpirationMs, TimeUnit.MILLISECONDS) + .maximumSize(maximumCacheSize) + .build(new CacheLoader() { + @Override + public V load(K k) throws Exception { + return singleEntryFunction.apply(k); + } + }); + } + + @Override + public int size() { + return (int) this.internalLoadingCache.size(); + } + + @Override + public boolean isEmpty() { + return size() == 0; + } + + @Override + public boolean containsKey(Object key) { + return this.internalLoadingCache.getIfPresent(key) != null; + } + + @Override + public boolean containsValue(Object value) { + throw new UnsupportedOperationException(); + } + + @Override + public V get(Object key) { + try { + return this.internalLoadingCache.get((K) key); + } catch (Exception e) { + return null; + } + } + + @Override + public V put(K key, V value) { + this.internalLoadingCache.put(key, value); + return this.internalLoadingCache.getIfPresent(key); + } + + @Override + public V remove(Object key) { + V value = this.internalLoadingCache.getIfPresent(key); + this.internalLoadingCache.invalidate(key); + return value; + } + + @Override + public void putAll(Map m) { + this.internalLoadingCache.putAll(m); + } + + @Override + public void clear() { + this.internalLoadingCache.invalidateAll(); + } + + @Override + public Set keySet() { + return this.internalLoadingCache.asMap().keySet(); + } + + @Override + public Collection values() { + return this.internalLoadingCache.asMap().values(); + } + + @Override + public Set> entrySet() { + return this.internalLoadingCache.asMap().entrySet(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java index 4b6ae21d7a95b..75f00d3f924a8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java @@ -24,9 +24,13 @@ import java.io.DataOutputStream; import java.io.IOException; import java.sql.SQLException; +import java.util.NoSuchElementException; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager; +import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,6 +50,13 @@ public abstract class SQLDelegationTokenSecretManager(cacheExpirationMs, maximumCacheSize, + this::getTokenInfoFromSQL); } /** @@ -126,15 +144,11 @@ protected void updateToken(TokenIdent ident, @Override public synchronized TokenIdent cancelToken(Token token, String canceller) throws IOException { - try (ByteArrayInputStream bis = new ByteArrayInputStream(token.getIdentifier()); - DataInputStream din = new DataInputStream(bis)) { - TokenIdent id = createIdentifier(); - id.readFields(din); + TokenIdent id = createTokenIdent(token.getIdentifier()); - // Calling getTokenInfo to load token into local cache if not present. - // super.cancelToken() requires token to be present in local cache. - getTokenInfo(id); - } + // Calling getTokenInfo to load token into local cache if not present. + // super.cancelToken() requires token to be present in local cache. + getTokenInfo(id); return super.cancelToken(token, canceller); } @@ -153,6 +167,24 @@ protected void removeStoredToken(TokenIdent ident) throws IOException { } } + @Override + protected void removeExpiredStoredToken(TokenIdent ident) { + try { + // Ensure that the token has not been renewed in SQL by + // another secret manager + DelegationTokenInformation tokenInfo = getTokenInfoFromSQL(ident); + if (tokenInfo.getRenewDate() >= Time.now()) { + LOG.info("Token was renewed by a different router and has not been deleted: {}", ident); + return; + } + removeStoredToken(ident); + } catch (NoSuchElementException e) { + LOG.info("Token has already been deleted by a different router: {}", ident); + } catch (Exception e) { + LOG.warn("Could not remove token {}", ident, e); + } + } + /** * Obtains the DelegationTokenInformation associated with the given * TokenIdentifier in the SQL database. @@ -160,29 +192,35 @@ protected void removeStoredToken(TokenIdent ident) throws IOException { * @return DelegationTokenInformation that matches the given TokenIdentifier or * null if it doesn't exist in the database. */ - @Override - protected DelegationTokenInformation getTokenInfo(TokenIdent ident) { - // Look for token in local cache - DelegationTokenInformation tokenInfo = super.getTokenInfo(ident); - - if (tokenInfo == null) { - try { - // Look for token in SQL database - byte[] tokenInfoBytes = selectTokenInfo(ident.getSequenceNumber(), ident.getBytes()); + @VisibleForTesting + protected DelegationTokenInformation getTokenInfoFromSQL(TokenIdent ident) { + try { + byte[] tokenInfoBytes = selectTokenInfo(ident.getSequenceNumber(), ident.getBytes()); + if (tokenInfoBytes == null) { + // Throw exception so value is not added to cache + throw new NoSuchElementException("Token not found in SQL secret manager: " + ident); + } + return createTokenInfo(tokenInfoBytes); + } catch (SQLException | IOException e) { + LOG.error("Failed to get token in SQL secret manager", e); + throw new RuntimeException(e); + } + } - if (tokenInfoBytes != null) { - tokenInfo = new DelegationTokenInformation(); - try (ByteArrayInputStream bis = new ByteArrayInputStream(tokenInfoBytes)) { - try (DataInputStream dis = new DataInputStream(bis)) { - tokenInfo.readFields(dis); - } - } + private TokenIdent createTokenIdent(byte[] tokenIdentBytes) throws IOException { + try (ByteArrayInputStream bis = new ByteArrayInputStream(tokenIdentBytes); + DataInputStream din = new DataInputStream(bis)) { + TokenIdent id = createIdentifier(); + id.readFields(din); + return id; + } + } - // Update token in local cache - currentTokens.put(ident, tokenInfo); - } - } catch (IOException | SQLException e) { - LOG.error("Failed to get token in SQL secret manager", e); + private DelegationTokenInformation createTokenInfo(byte[] tokenInfoBytes) throws IOException { + DelegationTokenInformation tokenInfo = new DelegationTokenInformation(); + try (ByteArrayInputStream bis = new ByteArrayInputStream(tokenInfoBytes)) { + try (DataInputStream dis = new DataInputStream(bis)) { + tokenInfo.readFields(dis); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/security/token/TestSQLDelegationTokenSecretManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/security/token/TestSQLDelegationTokenSecretManagerImpl.java index 569a274042bfa..dbbb85662cf47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/security/token/TestSQLDelegationTokenSecretManagerImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/security/token/TestSQLDelegationTokenSecretManagerImpl.java @@ -25,13 +25,21 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.SQLDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -43,6 +51,7 @@ public class TestSQLDelegationTokenSecretManagerImpl { private static final String CONNECTION_URL = "jdbc:derby:memory:TokenStore"; private static final int TEST_MAX_RETRIES = 3; + private static final int TOKEN_EXPIRATION_SECONDS = 1; private static Configuration conf; @Before @@ -111,6 +120,96 @@ public void testMultipleSecretManagers() throws Exception { } } + @Test + public void testCancelToken() throws Exception { + DelegationTokenManager tokenManager1 = createTokenManager(getShortLivedTokenConf()); + DelegationTokenManager tokenManager2 = createTokenManager(getShortLivedTokenConf()); + + TestDelegationTokenSecretManager secretManager2 = + (TestDelegationTokenSecretManager) tokenManager2.getDelegationTokenSecretManager(); + + try { + // Create token on token manager 1 + Token token1 = tokenManager1.createToken(UserGroupInformation.getCurrentUser(), "foo"); + + // Load token on token manager 2 to test it doesn't get stale + tokenManager2.verifyToken(token1); + + // Cancel token on token manager 1 + tokenManager1.cancelToken(token1, "foo"); + + // Validate that token cancellation is propagated to token manager 2 + secretManager2.waitForTokenEviction(token1.decodeIdentifier()); + LambdaTestUtils.intercept(SecretManager.InvalidToken.class, + () -> tokenManager2.verifyToken(token1)); + } finally { + stopTokenManager(tokenManager1); + stopTokenManager(tokenManager2); + } + } + + @Test + public void testRenewToken() throws Exception { + DelegationTokenManager tokenManager1 = createTokenManager(getShortLivedTokenConf()); + DelegationTokenManager tokenManager2 = createTokenManager(getShortLivedTokenConf()); + + TestDelegationTokenSecretManager secretManager2 = + (TestDelegationTokenSecretManager) tokenManager2.getDelegationTokenSecretManager(); + + try { + // Create token on token manager 1 + Token token1 = tokenManager1.createToken(UserGroupInformation.getCurrentUser(), "foo"); + long expirationTime = Time.monotonicNow() + + TimeUnit.SECONDS.toMillis(TOKEN_EXPIRATION_SECONDS) * 2; + + // Load token on token manager 2 to test it doesn't get stale + tokenManager2.verifyToken(token1); + + // Renew token on token manager 1 and verify token is updated on token manager 2 + // Do this for long enough that the token should be expired if not renewed + AbstractDelegationTokenIdentifier token1Id = + (AbstractDelegationTokenIdentifier) token1.decodeIdentifier(); + while (Time.monotonicNow() < expirationTime) { + tokenManager1.renewToken(token1, "foo"); + callRemoveExpiredTokensAndValidateSQL(secretManager2, token1Id, true); + secretManager2.waitForTokenEviction(token1Id); + tokenManager2.verifyToken(token1); + } + + // Stop renewing token and validate it's no longer valid and removed + // from SQL + Thread.sleep(TimeUnit.SECONDS.toMillis(TOKEN_EXPIRATION_SECONDS) * 2); + LambdaTestUtils.intercept(SecretManager.InvalidToken.class, + () -> tokenManager2.verifyToken(token1)); + callRemoveExpiredTokensAndValidateSQL(secretManager2, token1Id, false); + } finally { + stopTokenManager(tokenManager1); + stopTokenManager(tokenManager2); + } + } + + private Configuration getShortLivedTokenConf() { + Configuration shortLivedConf = new Configuration(conf); + shortLivedConf.setTimeDuration( + SQLDelegationTokenSecretManager.SQL_DTSM_TOKEN_LOADING_CACHE_EXPIRATION, + 200, TimeUnit.MILLISECONDS); + shortLivedConf.setInt(DelegationTokenManager.RENEW_INTERVAL, TOKEN_EXPIRATION_SECONDS); + return shortLivedConf; + } + + private void callRemoveExpiredTokensAndValidateSQL( + TestDelegationTokenSecretManager secretManager, AbstractDelegationTokenIdentifier tokenId, + boolean expectedInSQL) throws SQLException { + secretManager.removeExpiredStoredToken(tokenId); + byte[] tokenInfo = secretManager.selectTokenInfo(tokenId.getSequenceNumber(), + tokenId.getBytes()); + if (expectedInSQL) { + Assert.assertNotNull("Verify token exists in database", tokenInfo); + } else { + Assert.assertNull("Verify token was removed from database", tokenInfo); + } + } + @Test public void testSequenceNumAllocation() throws Exception { int tokensPerManager = SQLDelegationTokenSecretManagerImpl.DEFAULT_SEQ_NUM_BATCH_SIZE * 5; @@ -292,8 +391,13 @@ public void testRetries() throws Exception { } private DelegationTokenManager createTokenManager() { + return createTokenManager(conf); + } + + private DelegationTokenManager createTokenManager(Configuration config) { DelegationTokenManager tokenManager = new DelegationTokenManager(new Configuration(), null); - tokenManager.setExternalDelegationTokenSecretManager(new TestDelegationTokenSecretManager()); + tokenManager.setExternalDelegationTokenSecretManager( + new TestDelegationTokenSecretManager(config)); return tokenManager; } @@ -401,7 +505,7 @@ private synchronized ReentrantLock getKeyRollLock() { return keyRollLock; } - TestDelegationTokenSecretManager() { + TestDelegationTokenSecretManager(Configuration conf) { super(conf, new TestConnectionFactory(conf), SQLSecretManagerRetriableHandlerImpl.getInstance(conf, new TestRetryHandler())); } @@ -428,6 +532,16 @@ protected void rollMasterKey() throws IOException { } } + public void waitForTokenEviction(TokenIdentifier tokenId) + throws InterruptedException, TimeoutException { + // Wait until token is not found on cache + GenericTestUtils.waitFor(() -> !this.currentTokens.containsKey(tokenId), 100, 5000); + } + + public void removeExpiredStoredToken(TokenIdentifier tokenId) { + super.removeExpiredStoredToken((AbstractDelegationTokenIdentifier) tokenId); + } + public void setReadOnly(boolean readOnly) { ((TestConnectionFactory) getConnectionFactory()).readOnly = readOnly; } From b5b9a81538723528e6187b6a98f4f9f0e2b40d41 Mon Sep 17 00:00:00 2001 From: hfutatzhanghb Date: Tue, 8 Aug 2023 09:26:51 +0800 Subject: [PATCH 12/48] HDFS-17144. Remove incorrect comment in method storeAllocatedBlock. (#5932) Signed-off-by: Tao Li --- .../org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 0d3cb89ac01be..339873efadc63 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -251,7 +251,6 @@ static LocatedBlock storeAllocatedBlock(FSNamesystem fsn, String src, fsn.commitOrCompleteLastBlock(pendingFile, fileState.iip, ExtendedBlock.getLocalBlock(previous)); - // allocate new block, record block locations in INode. final BlockType blockType = pendingFile.getBlockType(); // allocate new block, record block locations in INode. Block newBlock = fsn.createNewBlock(blockType); From 5b81caf0cf44a532b24387aad2630d92155ce7c4 Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Tue, 8 Aug 2023 15:42:23 +0800 Subject: [PATCH 13/48] HDFS-17137. Standby/Observer NameNode skip to handle redundant replica block logic when set decrease replication. (#5913). Contributed by Haiyang Hu. Reviewed-by: Tao Li Signed-off-by: He Xiaoqiao --- .../server/blockmanagement/BlockManager.java | 5 ++ .../ha/TestStandbyBlockManagement.java | 54 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index d00bface655a6..eb960e62e36e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -4030,6 +4030,11 @@ public void setReplication( // update neededReconstruction priority queues b.setReplication(newRepl); + + // Process the block only when active NN is out of safe mode. + if (!isPopulatingReplQueues()) { + return; + } NumberReplicas num = countNodes(b); updateNeededReconstructions(b, 0, newRepl - oldRepl); if (shouldProcessExtraRedundancy(num, newRepl)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java index 74c6f21240812..4ddbbaa10f9b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java @@ -97,4 +97,58 @@ public void testInvalidateBlock() throws Exception { } } + /** + * Test Standby/Observer NameNode should not handle redundant replica block logic + * when set decrease replication. + * @throws Exception + */ + @Test(timeout = 60000) + public void testNotHandleRedundantReplica() throws Exception { + Configuration conf = new Configuration(); + HAUtil.setAllowStandbyReads(conf, true); + conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); + + // Create HA Cluster. + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(4).build()) { + cluster.waitActive(); + cluster.transitionToActive(0); + + NameNode nn1 = cluster.getNameNode(0); + assertEquals("ACTIVE", nn1.getNamesystem().getState().name()); + NameNode nn2 = cluster.getNameNode(1); + assertEquals("STANDBY", nn2.getNamesystem().getState().name()); + + cluster.triggerHeartbeats(); + // Sending the FBR. + cluster.triggerBlockReports(); + + // Default excessRedundancyMap size as 0. + assertEquals(0, nn1.getNamesystem().getBlockManager().getExcessBlocksCount()); + assertEquals(0, nn2.getNamesystem().getBlockManager().getExcessBlocksCount()); + + FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); + + // Create test file. + Path file = new Path("/test"); + long fileLength = 512; + DFSTestUtil.createFile(fs, file, fileLength, (short) 4, 0L); + DFSTestUtil.waitReplication(fs, file, (short) 4); + + // Set decrease 3 replication. + fs.setReplication(file, (short) 3); + HATestUtil.waitForStandbyToCatchUp(nn1, nn2); + + // Make sure the DN has deleted the block and report to NNs. + cluster.triggerHeartbeats(); + HATestUtil.waitForDNDeletions(cluster); + cluster.triggerDeletionReports(); + + DFSTestUtil.waitReplication(fs, file, (short) 3); + + // Delete excess replica, active and standby nn excessRedundancyMap size as 0. + assertEquals(0, nn1.getNamesystem().getBlockManager().getExcessBlocksCount()); + assertEquals(0, nn2.getNamesystem().getBlockManager().getExcessBlocksCount()); + } + } } From b1ed23654c01052074ea81fadb685d2ea7bb4bfa Mon Sep 17 00:00:00 2001 From: rohit-kb <115476286+rohit-kb@users.noreply.github.com> Date: Tue, 8 Aug 2023 18:07:20 +0530 Subject: [PATCH 14/48] HADOOP-18837. Upgrade okio to 3.4.0 due to CVE-2023-3635. (#5914) Contributed by Rohit Kumar --- LICENSE-binary | 2 +- hadoop-project/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 72318d9bf0a94..b1a24c2bd57cb 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -242,7 +242,7 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.microsoft.azure:azure-storage:7.0.0 com.nimbusds:nimbus-jose-jwt:9.31 com.squareup.okhttp3:okhttp:4.10.0 -com.squareup.okio:okio:3.2.0 +com.squareup.okio:okio:3.4.0 com.zaxxer:HikariCP:4.0.3 commons-beanutils:commons-beanutils:1.9.4 commons-cli:commons-cli:1.5.0 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 02feba3d1e685..63c2a741c9e76 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -133,7 +133,7 @@ 10.14.2.0 6.2.1.jre7 4.10.0 - 3.2.0 + 3.4.0 1.6.20 1.6.20 2.0.6.1 From ba32ea70fdd3587a542d95f9ad670e397b2ae5f9 Mon Sep 17 00:00:00 2001 From: Anuj Modi <128447756+anujmodi2021@users.noreply.github.com> Date: Tue, 8 Aug 2023 11:00:02 -0700 Subject: [PATCH 15/48] HADOOP-18826. [ABFS] Fix for GetFileStatus("/") failure. (#5909) Contributed by Anmol Asrani --- .../fs/azurebfs/AzureBlobFileSystemStore.java | 7 +++++- .../ITestAzureBlobFileSystemFileStatus.java | 22 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 5c06270fa5bb8..49dff3360899e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1661,7 +1661,12 @@ private String getOctalNotation(FsPermission fsPermission) { private String getRelativePath(final Path path) { Preconditions.checkNotNull(path, "path"); - return path.toUri().getPath(); + String relPath = path.toUri().getPath(); + if (relPath.isEmpty()) { + // This means that path passed by user is absolute path of root without "/" at end. + relPath = ROOT_PATH; + } + return relPath; } private long parseContentLength(final String contentLength) { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java index 4fa7a0fca68ae..dfaf203c590a1 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java @@ -18,7 +18,9 @@ package org.apache.hadoop.fs.azurebfs; +import java.io.File; import java.io.IOException; +import java.io.OutputStream; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.junit.Test; @@ -83,9 +85,11 @@ private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name, if (isDir) { assertEquals(errorInStatus + ": permission", new FsPermission(DEFAULT_DIR_PERMISSION_VALUE), fileStatus.getPermission()); + assertTrue(errorInStatus + "not a directory", fileStatus.isDirectory()); } else { assertEquals(errorInStatus + ": permission", new FsPermission(DEFAULT_FILE_PERMISSION_VALUE), fileStatus.getPermission()); + assertTrue(errorInStatus + "not a file", fileStatus.isFile()); } } @@ -144,4 +148,22 @@ public void testLastModifiedTime() throws IOException { assertTrue("lastModifiedTime should be before createEndTime", createEndTime > lastModifiedTime); } + + @Test + public void testFileStatusOnRoot() throws IOException { + AzureBlobFileSystem fs = getFileSystem(); + + // Assert that passing relative root path works + Path testPath = new Path("/"); + validateStatus(fs, testPath, true); + + // Assert that passing absolute root path works + String testPathStr = makeQualified(testPath).toString(); + validateStatus(fs, new Path(testPathStr), true); + + // Assert that passing absolute root path without "/" works + testPathStr = testPathStr.substring(0, testPathStr.length() - 1); + validateStatus(fs, new Path(testPathStr), true); + + } } From df0381f236789c6fc63a3dbbd50ce3b159fa2ff3 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Thu, 10 Aug 2023 10:27:36 +0800 Subject: [PATCH 16/48] YARN-11536. [Federation] Router CLI Supports Batch Save the SubClusterPolicyConfiguration Of Queues. (#5862) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- ...ResourceManagerAdministrationProtocol.java | 16 +- ...tchSaveFederationQueuePoliciesRequest.java | 53 ++++ ...chSaveFederationQueuePoliciesResponse.java | 47 ++++ .../FederationQueueWeight.java | 41 +++ ...ourcemanager_administration_protocol.proto | 1 + ...erver_resourcemanager_service_protos.proto | 8 + .../src/main/proto/yarn_protos.proto | 2 + .../hadoop/yarn/client/cli/RouterCLI.java | 251 +++++++++++++++++- .../yarn/client/util/MemoryPageUtils.java | 58 ++++ .../yarn/client/TestMemoryPageUtils.java | 59 ++++ .../hadoop/yarn/client/cli/TestRouterCLI.java | 23 ++ .../src/test/resources/federation-weights.xml | 74 ++++++ ...gerAdministrationProtocolPBClientImpl.java | 19 ++ ...erAdministrationProtocolPBServiceImpl.java | 23 ++ ...eFederationQueuePoliciesRequestPBImpl.java | 149 +++++++++++ ...FederationQueuePoliciesResponsePBImpl.java | 99 +++++++ .../impl/pb/FederationQueueWeightPBImpl.java | 40 +++ .../server/MockResourceManagerFacade.java | 8 + .../server/resourcemanager/AdminService.java | 34 ++- .../yarn/server/router/RouterMetrics.java | 34 ++- .../DefaultRMAdminRequestInterceptor.java | 8 + .../rmadmin/FederationRMAdminInterceptor.java | 99 ++++++- .../router/rmadmin/RouterRMAdminService.java | 9 + .../yarn/server/router/TestRouterMetrics.java | 36 +++ .../PassThroughRMAdminRequestInterceptor.java | 8 + .../TestFederationRMAdminInterceptor.java | 144 ++++++++++ 26 files changed, 1330 insertions(+), 13 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/MemoryPageUtils.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestMemoryPageUtils.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/federation-weights.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesResponsePBImpl.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java index a4967960a0331..1ad77e0b30ecc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java @@ -60,7 +60,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; - +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; @Private public interface ResourceManagerAdministrationProtocol extends GetUserMappingsProtocol { @@ -189,4 +190,17 @@ DeregisterSubClusterResponse deregisterSubCluster(DeregisterSubClusterRequest re @Idempotent SaveFederationQueuePolicyResponse saveFederationQueuePolicy( SaveFederationQueuePolicyRequest request) throws YarnException, IOException; + + /** + * In YARN-Federation mode, this method provides a way to save queue policies in batches. + * + * @param request BatchSaveFederationQueuePolicies Request + * @return Response from batchSaveFederationQueuePolicies. + * @throws YarnException exceptions from yarn servers. + * @throws IOException if an IO error occurred. + */ + @Private + @Idempotent + BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( + BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesRequest.java new file mode 100644 index 0000000000000..c71c61aca9f1a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesRequest.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +import java.util.List; + +/** + * In Federation mode, + * we will support batch save queues policies to FederationStateStore. + */ +@Private +@Unstable +public abstract class BatchSaveFederationQueuePoliciesRequest { + + @Private + @Unstable + public static BatchSaveFederationQueuePoliciesRequest newInstance( + List federationQueueWeights) { + BatchSaveFederationQueuePoliciesRequest request = + Records.newRecord(BatchSaveFederationQueuePoliciesRequest.class); + request.setFederationQueueWeights(federationQueueWeights); + return request; + } + + @Public + @Unstable + public abstract List getFederationQueueWeights(); + + @Private + @Unstable + public abstract void setFederationQueueWeights( + List federationQueueWeights); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesResponse.java new file mode 100644 index 0000000000000..b9d1e69c4a832 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/BatchSaveFederationQueuePoliciesResponse.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +@Private +@Unstable +public abstract class BatchSaveFederationQueuePoliciesResponse { + + public static BatchSaveFederationQueuePoliciesResponse newInstance() { + return Records.newRecord(BatchSaveFederationQueuePoliciesResponse.class); + } + + public static BatchSaveFederationQueuePoliciesResponse newInstance(String msg) { + BatchSaveFederationQueuePoliciesResponse response = + Records.newRecord(BatchSaveFederationQueuePoliciesResponse.class); + response.setMessage(msg); + return response; + } + + @Public + @Unstable + public abstract String getMessage(); + + @Public + @Unstable + public abstract void setMessage(String msg); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/FederationQueueWeight.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/FederationQueueWeight.java index c63ee1b713d0d..aa9d114527410 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/FederationQueueWeight.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/FederationQueueWeight.java @@ -75,6 +75,19 @@ public static FederationQueueWeight newInstance(String routerWeight, return federationQueueWeight; } + @Private + @Unstable + public static FederationQueueWeight newInstance(String routerWeight, + String amrmWeight, String headRoomAlpha, String queue, String policyManagerClassName) { + FederationQueueWeight federationQueueWeight = Records.newRecord(FederationQueueWeight.class); + federationQueueWeight.setRouterWeight(routerWeight); + federationQueueWeight.setAmrmWeight(amrmWeight); + federationQueueWeight.setHeadRoomAlpha(headRoomAlpha); + federationQueueWeight.setQueue(queue); + federationQueueWeight.setPolicyManagerClassName(policyManagerClassName); + return federationQueueWeight; + } + @Public @Unstable public abstract String getRouterWeight(); @@ -166,4 +179,32 @@ public static void checkHeadRoomAlphaValid(String headRoomAlpha) throws YarnExce protected static boolean isNumeric(String value) { return NumberUtils.isCreatable(value); } + + @Public + @Unstable + public abstract String getQueue(); + + @Public + @Unstable + public abstract void setQueue(String queue); + + @Public + @Unstable + public abstract String getPolicyManagerClassName(); + + @Public + @Unstable + public abstract void setPolicyManagerClassName(String policyManagerClassName); + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("FederationQueueWeight { "); + builder.append("Queue: ").append(getQueue()).append(", "); + builder.append("RouterWeight: ").append(getRouterWeight()).append(", "); + builder.append("AmrmWeight: ").append(getAmrmWeight()).append(", "); + builder.append("PolicyManagerClassName: ").append(getPolicyManagerClassName()); + builder.append(" }"); + return builder.toString(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto index 809817fa9f9d8..aca7a4c0b8320 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto @@ -49,4 +49,5 @@ service ResourceManagerAdministrationProtocolService { rpc mapAttributesToNodes(NodesToAttributesMappingRequestProto) returns (NodesToAttributesMappingResponseProto); rpc deregisterSubCluster(DeregisterSubClusterRequestProto) returns (DeregisterSubClusterResponseProto); rpc saveFederationQueuePolicy(SaveFederationQueuePolicyRequestProto) returns (SaveFederationQueuePolicyResponseProto); + rpc batchSaveFederationQueuePolicies(BatchSaveFederationQueuePoliciesRequestProto) returns (BatchSaveFederationQueuePoliciesResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index 4e330fb1e632f..06e11f913b6b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -180,6 +180,14 @@ message SaveFederationQueuePolicyResponseProto { required string message = 1; } +message BatchSaveFederationQueuePoliciesRequestProto { + repeated FederationQueueWeightProto federationQueueWeights = 1; +} + +message BatchSaveFederationQueuePoliciesResponseProto { + required string message = 1; +} + ////////////////////////////////////////////////////////////////// ///////////// RM Failover related records //////////////////////// ////////////////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 847919091cfbe..71c102f4f8577 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -444,6 +444,8 @@ message FederationQueueWeightProto { optional string routerWeight = 1; optional string amrmWeight = 2; optional string headRoomAlpha = 3; + optional string queue = 4; + optional string policyManagerClassName = 5; } //////////////////////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 1c5873f28bd66..788ef8feb1ea5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -32,6 +32,7 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.util.FormattingCLIUtils; +import org.apache.hadoop.yarn.client.util.MemoryPageUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; @@ -40,16 +41,28 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusters; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.File; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -71,7 +84,8 @@ public class RouterCLI extends Configured implements Tool { "set the state of the subCluster to SC_LOST.")) // Command2: policy .put("-policy", new UsageInfo( - "[-s|--save [queue;router weight;amrm weight;headroomalpha]]", + "[-s|--save [queue;router weight;amrm weight;headroomalpha]] " + + "[-bs|--batch-save [--format xml] [-f|--input-file fileName]]", "We provide a set of commands for Policy:" + " Include list policies, save policies, batch save policies. " + " (Note: The policy type will be directly read from the" + @@ -102,8 +116,23 @@ public class RouterCLI extends Configured implements Tool { // Command2: policy // save policy private static final String OPTION_S = "s"; + private static final String OPTION_BATCH_S = "bs"; private static final String OPTION_SAVE = "save"; + private static final String OPTION_BATCH_SAVE = "batch-save"; + private static final String OPTION_FORMAT = "format"; + private static final String OPTION_FILE = "f"; + private static final String OPTION_INPUT_FILE = "input-file"; + private static final String CMD_POLICY = "-policy"; + private static final String FORMAT_XML = "xml"; + private static final String FORMAT_JSON = "json"; + private static final String XML_TAG_SUBCLUSTERIDINFO = "subClusterIdInfo"; + private static final String XML_TAG_AMRMPOLICYWEIGHTS = "amrmPolicyWeights"; + private static final String XML_TAG_ROUTERPOLICYWEIGHTS = "routerPolicyWeights"; + private static final String XML_TAG_HEADROOMALPHA = "headroomAlpha"; + private static final String XML_TAG_FEDERATION_WEIGHTS = "federationWeights"; + private static final String XML_TAG_QUEUE = "queue"; + private static final String XML_TAG_NAME = "name"; public RouterCLI() { super(); @@ -161,7 +190,8 @@ private static void printHelp() { .append("The full syntax is: \n\n") .append("routeradmin\n") .append(" [-deregisterSubCluster [-sc|--subClusterId [subCluster Id]]\n") - .append(" [-policy [-s|--save [queue;router weight;amrm weight;headroomalpha]]\n") + .append(" [-policy [-s|--save [queue;router weight;amrm weight;headroomalpha] " + + "[-bs|--batch-save [--format xml,json] [-f|--input-file fileName]]]\n") .append(" [-help [cmd]]").append("\n"); StringBuilder helpBuilder = new StringBuilder(); System.out.println(summary); @@ -304,7 +334,23 @@ private int handlePolicy(String[] args) "We will save the policy information of the queue, " + "including queue and weight information"); saveOpt.setOptionalArg(true); + Option batchSaveOpt = new Option(OPTION_BATCH_S, OPTION_BATCH_SAVE, false, + "We will save queue policies in bulk, " + + "where users can provide XML or JSON files containing the policies. " + + "This command will parse the file contents and store the results " + + "in the FederationStateStore."); + Option formatOpt = new Option(null, "format", true, + "Users can specify the file format using this option. " + + "Currently, there are one supported file formats: XML." + + "These files contain the policy information for storing queue policies."); + Option fileOpt = new Option("f", "input-file", true, + "The location of the input configuration file. "); + formatOpt.setOptionalArg(true); + opts.addOption(saveOpt); + opts.addOption(batchSaveOpt); + opts.addOption(formatOpt); + opts.addOption(fileOpt); // Parse command line arguments. CommandLine cliParser; @@ -317,12 +363,42 @@ private int handlePolicy(String[] args) } // Try to parse the cmd save. + // Save a single queue policy if (cliParser.hasOption(OPTION_S) || cliParser.hasOption(OPTION_SAVE)) { String policy = cliParser.getOptionValue(OPTION_S); if (StringUtils.isBlank(policy)) { policy = cliParser.getOptionValue(OPTION_SAVE); } return handleSavePolicy(policy); + } else if (cliParser.hasOption(OPTION_BATCH_S) || cliParser.hasOption(OPTION_BATCH_SAVE)) { + // Save Queue Policies in Batches + // Determine whether the file format is accurate, XML or JSON format. + // If it is not XML or JSON, we will directly prompt the user with an error message. + String format = null; + if (cliParser.hasOption(OPTION_FORMAT)) { + format = cliParser.getOptionValue(OPTION_FORMAT); + if (StringUtils.isBlank(format) || + !StringUtils.equalsAnyIgnoreCase(format, FORMAT_XML)) { + System.out.println("We currently only support policy configuration files " + + "in XML formats."); + return EXIT_ERROR; + } + } + + // Parse configuration file path. + String filePath = null; + if (cliParser.hasOption(OPTION_FILE) || cliParser.hasOption(OPTION_INPUT_FILE)) { + filePath = cliParser.getOptionValue(OPTION_FILE); + if (StringUtils.isBlank(filePath)) { + filePath = cliParser.getOptionValue(OPTION_INPUT_FILE); + } + } + + // Batch SavePolicies. + return handBatchSavePolicies(format, filePath); + } else { + // printUsage + printUsage(args[0]); } return EXIT_ERROR; @@ -342,6 +418,30 @@ private int handleSavePolicy(String policy) { } } + private int handBatchSavePolicies(String format, String policyFile) { + + if(StringUtils.isBlank(format)) { + LOG.error("Batch Save Federation Policies. Format is Empty."); + return EXIT_ERROR; + } + + if(StringUtils.isBlank(policyFile)) { + LOG.error("Batch Save Federation Policies. policyFile is Empty."); + return EXIT_ERROR; + } + + LOG.info("Batch Save Federation Policies. Format = {}, PolicyFile = {}.", + format, policyFile); + + switch (format) { + case FORMAT_XML: + return parseXml2PoliciesAndBatchSavePolicies(policyFile); + default: + System.out.println("We currently only support XML formats."); + return EXIT_ERROR; + } + } + /** * We will parse the policy, and it has specific formatting requirements. * @@ -384,6 +484,140 @@ protected SaveFederationQueuePolicyRequest parsePolicy(String policy) throws Yar return request; } + /** + * Parse Policies from XML and save them in batches to FederationStateStore. + * + * We save 20 policies in one batch. + * If the user needs to save 1000 policies, it will cycle 50 times. + * + * Every time a page is saved, we will print whether a page + * has been saved successfully or failed. + * + * @param policiesXml Policies Xml Path. + * @return 0, success; 1, failed. + */ + protected int parseXml2PoliciesAndBatchSavePolicies(String policiesXml) { + try { + List federationQueueWeightsList = parsePoliciesByXml(policiesXml); + MemoryPageUtils memoryPageUtils = new MemoryPageUtils<>(20); + federationQueueWeightsList.forEach(federationQueueWeight -> + memoryPageUtils.addToMemory(federationQueueWeight)); + int pages = memoryPageUtils.getPages(); + for (int i = 0; i < pages; i++) { + List federationQueueWeights = + memoryPageUtils.readFromMemory(i); + BatchSaveFederationQueuePoliciesRequest request = + BatchSaveFederationQueuePoliciesRequest.newInstance(federationQueueWeights); + ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); + BatchSaveFederationQueuePoliciesResponse response = + adminProtocol.batchSaveFederationQueuePolicies(request); + System.out.println("page <" + (i + 1) + "> : " + response.getMessage()); + } + } catch (Exception e) { + LOG.error("BatchSaveFederationQueuePolicies error", e); + } + return EXIT_ERROR; + } + + /** + * Parse FederationQueueWeight from the xml configuration file. + *

+ * We allow users to provide an xml configuration file, + * which stores the weight information of the queue. + * + * @param policiesXml Policies Xml Path. + * @return FederationQueueWeight List. + * @throws IOException an I/O exception of some sort has occurred. + * @throws SAXException Encapsulate a general SAX error or warning. + * @throws ParserConfigurationException a serious configuration error.. + */ + protected List parsePoliciesByXml(String policiesXml) + throws IOException, SAXException, ParserConfigurationException { + + List weights = new ArrayList<>(); + + File xmlFile = new File(policiesXml); + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(xmlFile); + + NodeList federationsList = document.getElementsByTagName(XML_TAG_FEDERATION_WEIGHTS); + + for (int i = 0; i < federationsList.getLength(); i++) { + + Node federationNode = federationsList.item(i); + + if (federationNode.getNodeType() == Node.ELEMENT_NODE) { + Element federationElement = (Element) federationNode; + NodeList queueList = federationElement.getElementsByTagName(XML_TAG_QUEUE); + + for (int j = 0; j < queueList.getLength(); j++) { + + Node queueNode = queueList.item(j); + if (queueNode.getNodeType() == Node.ELEMENT_NODE) { + Element queueElement = (Element) queueNode; + // parse queueName. + String queueName = queueElement.getElementsByTagName(XML_TAG_NAME) + .item(0).getTextContent(); + + // parse amrmPolicyWeights / routerPolicyWeights. + String amrmWeight = parsePolicyWeightsNode(queueElement, XML_TAG_AMRMPOLICYWEIGHTS); + String routerWeight = parsePolicyWeightsNode(queueElement, XML_TAG_ROUTERPOLICYWEIGHTS); + + // parse headroomAlpha. + String headroomAlpha = queueElement.getElementsByTagName(XML_TAG_HEADROOMALPHA) + .item(0).getTextContent(); + + String policyManager = getConf().get(YarnConfiguration.FEDERATION_POLICY_MANAGER, + YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER); + + LOG.debug("Queue: {}, AmrmPolicyWeights: {}, RouterWeight: {}, HeadroomAlpha: {}.", + queueName, amrmWeight, routerWeight, headroomAlpha); + + FederationQueueWeight weight = FederationQueueWeight.newInstance(routerWeight, + amrmWeight, headroomAlpha, queueName, policyManager); + + weights.add(weight); + } + } + } + } + + return weights; + } + + /** + * We will parse the policyWeight information. + * + * @param queueElement xml Element. + * @param weightType weightType, including 2 types, AmrmPolicyWeight and RouterPolicyWeight. + * @return concatenated string of sub-cluster weights. + */ + private String parsePolicyWeightsNode(Element queueElement, String weightType) { + NodeList amrmPolicyWeightsList = queueElement.getElementsByTagName(weightType); + Node amrmPolicyWeightsNode = amrmPolicyWeightsList.item(0); + List amRmPolicyWeights = new ArrayList<>(); + if (amrmPolicyWeightsNode.getNodeType() == Node.ELEMENT_NODE) { + Element amrmPolicyWeightsElement = (Element) amrmPolicyWeightsNode; + NodeList subClusterIdInfoList = + amrmPolicyWeightsElement.getElementsByTagName(XML_TAG_SUBCLUSTERIDINFO); + for (int i = 0; i < subClusterIdInfoList.getLength(); i++) { + Node subClusterIdInfoNode = subClusterIdInfoList.item(i); + if (subClusterIdInfoNode.getNodeType() == Node.ELEMENT_NODE) { + Element subClusterIdInfoElement = (Element) subClusterIdInfoNode; + String subClusterId = + subClusterIdInfoElement.getElementsByTagName("id").item(0).getTextContent(); + String weight = + subClusterIdInfoElement.getElementsByTagName("weight").item(0).getTextContent(); + LOG.debug("WeightType[{}] - SubCluster ID: {}, Weight: {}.", + weightType, subClusterId, weight); + amRmPolicyWeights.add(subClusterId + ":" + weight); + } + } + } + return StringUtils.join(amRmPolicyWeights, ","); + } + @Override public int run(String[] args) throws Exception { YarnConfiguration yarnConf = getConf() == null ? @@ -405,14 +639,13 @@ public int run(String[] args) throws Exception { printHelp(); } return EXIT_SUCCESS; - } - - if (CMD_DEREGISTERSUBCLUSTER.equals(cmd)) { + } else if (CMD_DEREGISTERSUBCLUSTER.equals(cmd)) { return handleDeregisterSubCluster(args); - } - - if (CMD_POLICY.equals(cmd)) { + } else if (CMD_POLICY.equals(cmd)) { return handlePolicy(args); + } else { + System.out.println("No related commands found."); + printHelp(); } return EXIT_SUCCESS; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/MemoryPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/MemoryPageUtils.java new file mode 100644 index 0000000000000..24e688779aebf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/MemoryPageUtils.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.client.util; + +import java.util.ArrayList; +import java.util.List; + +/** + * This is a memory paging utility that is used to paginate a dataset. + * + * This class is designed to support batch entry queue policies. + */ +public class MemoryPageUtils { + private List dataList; + private int pageSize; + + /** + * MemoryPageUtils constructor. + * + * @param pageSize Number of records returned per page. + */ + public MemoryPageUtils(int pageSize) { + this.pageSize = pageSize; + this.dataList = new ArrayList<>(); + } + + public void addToMemory(T data) { + dataList.add(data); + } + + public List readFromMemory(int pageNumber) { + int startIndex = pageNumber * pageSize; + int endIndex = Math.min(startIndex + pageSize, dataList.size()); + if (startIndex >= dataList.size()) { + return null; + } + return dataList.subList(startIndex, endIndex); + } + + public int getPages() { + return (dataList.size() / pageSize + 1); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestMemoryPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestMemoryPageUtils.java new file mode 100644 index 0000000000000..4783c73d669e0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestMemoryPageUtils.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.client; + +import org.apache.hadoop.yarn.client.util.MemoryPageUtils; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * The purpose of this class is to test + * whether the memory paging function is as expected. + */ +public class TestMemoryPageUtils { + + @Test + public void testMemoryPage() { + // We design such a unit test for testing pagination, and we prepare 6 pieces of policy data. + // If 1 page is followed by 5 pieces of data, we will get 2 pages. + // Page 1 will contain 5 records and page 2 will contain 1 record. + MemoryPageUtils policies = new MemoryPageUtils<>(5); + policies.addToMemory("policy-1"); + policies.addToMemory("policy-2"); + policies.addToMemory("policy-3"); + policies.addToMemory("policy-4"); + policies.addToMemory("policy-5"); + policies.addToMemory("policy-6"); + + // Page 1 will return 5 records. + List firstPage = policies.readFromMemory(0); + assertEquals(5, firstPage.size()); + + // Page 2 will return 1 records + List secondPage = policies.readFromMemory(1); + assertEquals(1, secondPage.size()); + + // Page 10, This is a wrong number of pages, we will get null. + List tenPage = policies.readFromMemory(10); + assertNull(tenPage); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java index c7989e11af561..476ba75263dae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java @@ -217,4 +217,27 @@ public void testSavePolicy() throws Exception { args = new String[]{"-policy", "-save", "root.a;SC-1:0.1,SC-2:0.9;SC-1:0.7,SC-2:0.3;1.0"}; assertEquals(0, rmAdminCLI.run(args)); } + + @Test + public void testParsePoliciesByXml() throws Exception { + String filePath = + TestRouterCLI.class.getClassLoader().getResource("federation-weights.xml").getFile(); + List federationQueueWeights = rmAdminCLI.parsePoliciesByXml(filePath); + assertNotNull(federationQueueWeights); + assertEquals(2, federationQueueWeights.size()); + + // Queue1: root.a + FederationQueueWeight queueWeight1 = federationQueueWeights.get(0); + assertNotNull(queueWeight1); + assertEquals("root.a", queueWeight1.getQueue()); + assertEquals("SC-1:0.7,SC-2:0.3", queueWeight1.getAmrmWeight()); + assertEquals("SC-1:0.6,SC-2:0.4", queueWeight1.getRouterWeight()); + + // Queue2: root.b + FederationQueueWeight queueWeight2 = federationQueueWeights.get(1); + assertNotNull(queueWeight2); + assertEquals("root.b", queueWeight2.getQueue()); + assertEquals("SC-1:0.8,SC-2:0.2", queueWeight2.getAmrmWeight()); + assertEquals("SC-1:0.6,SC-2:0.4", queueWeight2.getRouterWeight()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/federation-weights.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/federation-weights.xml new file mode 100644 index 0000000000000..d5fceaf35fcd0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/federation-weights.xml @@ -0,0 +1,74 @@ + + + + + + + + + root.a + + + SC-1 + 0.7 + + + SC-2 + 0.3 + + + + + SC-1 + 0.6 + + + SC-2 + 0.4 + + + 1.0 + + + + + root.b + + + SC-1 + 0.8 + + + SC-2 + 0.2 + + + + + SC-1 + 0.6 + + + SC-2 + 0.4 + + + 1.0 + + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java index 3242fe1400bdd..7107f8014e46f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeregisterSubClusterRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; @@ -81,6 +82,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl; @@ -113,6 +116,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeregisterSubClusterResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl; import org.apache.hadoop.thirdparty.protobuf.ServiceException; @@ -381,4 +386,18 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy( return null; } } + + @Override + public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( + BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException { + BatchSaveFederationQueuePoliciesRequestProto requestProto = + ((BatchSaveFederationQueuePoliciesRequestPBImpl) request).getProto(); + try { + return new BatchSaveFederationQueuePoliciesResponsePBImpl( + proxy.batchSaveFederationQueuePolicies(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java index e393523ed2e33..01feef9d9feae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java @@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto; @@ -79,6 +81,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl; @@ -111,6 +115,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeregisterSubClusterResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl; import org.apache.hadoop.thirdparty.protobuf.RpcController; import org.apache.hadoop.thirdparty.protobuf.ServiceException; @@ -399,4 +405,21 @@ public SaveFederationQueuePolicyResponseProto saveFederationQueuePolicy(RpcContr throw new ServiceException(e); } } + + @Override + public BatchSaveFederationQueuePoliciesResponseProto batchSaveFederationQueuePolicies( + RpcController controller, BatchSaveFederationQueuePoliciesRequestProto proto) + throws ServiceException { + BatchSaveFederationQueuePoliciesRequest request = + new BatchSaveFederationQueuePoliciesRequestPBImpl(proto); + try { + BatchSaveFederationQueuePoliciesResponse response = + real.batchSaveFederationQueuePolicies(request); + return ((BatchSaveFederationQueuePoliciesResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesRequestPBImpl.java new file mode 100644 index 0000000000000..a52e47372bda3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesRequestPBImpl.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.proto.YarnProtos.FederationQueueWeightProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight; + +import java.util.ArrayList; +import java.util.List; + +/** + * The class is responsible for batch-saving queue policies requests. + */ +@Private +@Unstable +public class BatchSaveFederationQueuePoliciesRequestPBImpl + extends BatchSaveFederationQueuePoliciesRequest { + + private BatchSaveFederationQueuePoliciesRequestProto proto = + BatchSaveFederationQueuePoliciesRequestProto.getDefaultInstance(); + private BatchSaveFederationQueuePoliciesRequestProto.Builder builder = null; + private boolean viaProto = false; + private List federationQueueWeights = null; + + public BatchSaveFederationQueuePoliciesRequestPBImpl() { + this.builder = BatchSaveFederationQueuePoliciesRequestProto.newBuilder(); + } + + public BatchSaveFederationQueuePoliciesRequestPBImpl( + BatchSaveFederationQueuePoliciesRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = BatchSaveFederationQueuePoliciesRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + if (this.federationQueueWeights != null) { + for (FederationQueueWeight federationQueueWeight : federationQueueWeights) { + FederationQueueWeightPBImpl federationQueueWeightPBImpl = + (FederationQueueWeightPBImpl) federationQueueWeight; + builder.addFederationQueueWeights(federationQueueWeightPBImpl.getProto()); + } + } + proto = builder.build(); + viaProto = true; + } + + public BatchSaveFederationQueuePoliciesRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void initDeregisterSubClustersMapping() { + if (this.federationQueueWeights != null) { + return; + } + + BatchSaveFederationQueuePoliciesRequestProtoOrBuilder p = viaProto ? proto : builder; + List batchSaveFederationQueuePoliciesProtoList = + p.getFederationQueueWeightsList(); + + List attributes = new ArrayList<>(); + if (batchSaveFederationQueuePoliciesProtoList == null || + batchSaveFederationQueuePoliciesProtoList.size() == 0) { + this.federationQueueWeights = attributes; + return; + } + + for (FederationQueueWeightProto federationQueueWeightProto : + batchSaveFederationQueuePoliciesProtoList) { + attributes.add(new FederationQueueWeightPBImpl(federationQueueWeightProto)); + } + + this.federationQueueWeights = attributes; + } + + @Override + public List getFederationQueueWeights() { + initDeregisterSubClustersMapping(); + return this.federationQueueWeights; + } + + @Override + public void setFederationQueueWeights(List pFederationQueueWeights) { + if (federationQueueWeights == null) { + federationQueueWeights = new ArrayList<>(); + } + if(federationQueueWeights == null) { + throw new IllegalArgumentException("federationQueueWeights cannot be null"); + } + federationQueueWeights.clear(); + federationQueueWeights.addAll(pFederationQueueWeights); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof BatchSaveFederationQueuePoliciesRequest)) { + return false; + } + + BatchSaveFederationQueuePoliciesRequestPBImpl otherImpl = this.getClass().cast(other); + return new EqualsBuilder() + .append(this.getProto(), otherImpl.getProto()) + .isEquals(); + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesResponsePBImpl.java new file mode 100644 index 0000000000000..80653d4177b8f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesResponsePBImpl.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; + +/** + * The class is responsible for batch-saving queue policies responses. + */ +public class BatchSaveFederationQueuePoliciesResponsePBImpl + extends BatchSaveFederationQueuePoliciesResponse { + + private BatchSaveFederationQueuePoliciesResponseProto proto = + BatchSaveFederationQueuePoliciesResponseProto.getDefaultInstance(); + private BatchSaveFederationQueuePoliciesResponseProto.Builder builder = null; + private boolean viaProto = false; + + public BatchSaveFederationQueuePoliciesResponsePBImpl() { + builder = BatchSaveFederationQueuePoliciesResponseProto.newBuilder(); + } + + public BatchSaveFederationQueuePoliciesResponsePBImpl( + BatchSaveFederationQueuePoliciesResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + public BatchSaveFederationQueuePoliciesResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public String getMessage() { + BatchSaveFederationQueuePoliciesResponseProtoOrBuilder p = viaProto ? proto : builder; + boolean hasMessage = p.hasMessage(); + if (hasMessage) { + return p.getMessage(); + } + return null; + } + + @Override + public void setMessage(String msg) { + maybeInitBuilder(); + if (msg == null) { + builder.clearMessage(); + return; + } + builder.setMessage(msg); + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = BatchSaveFederationQueuePoliciesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/FederationQueueWeightPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/FederationQueueWeightPBImpl.java index 4ca7f783bd97e..0c9aa711ba747 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/FederationQueueWeightPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/FederationQueueWeightPBImpl.java @@ -114,6 +114,46 @@ public void setHeadRoomAlpha(String headRoomAlpha) { builder.setHeadRoomAlpha(headRoomAlpha); } + @Override + public String getQueue() { + FederationQueueWeightProtoOrBuilder p = this.viaProto ? this.proto : this.builder; + boolean hasQueue = p.hasQueue(); + if (hasQueue) { + return p.getQueue(); + } + return null; + } + + @Override + public void setQueue(String queue) { + maybeInitBuilder(); + if (queue == null) { + builder.clearQueue(); + return; + } + builder.setQueue(queue); + } + + @Override + public String getPolicyManagerClassName() { + FederationQueueWeightProtoOrBuilder p = this.viaProto ? this.proto : this.builder; + boolean hasPolicyManagerClassName = p.hasPolicyManagerClassName(); + if (hasPolicyManagerClassName) { + return p.getPolicyManagerClassName(); + } + return null; + } + + @Override + public void setPolicyManagerClassName(String policyManagerClassName) { + maybeInitBuilder(); + if (policyManagerClassName == null) { + builder.clearPolicyManagerClassName(); + return; + } + builder.setPolicyManagerClassName(policyManagerClassName); + } + @Override public int hashCode() { return getProto().hashCode(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index 14bb941c9d61e..0fc52898434d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -177,6 +177,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; /** @@ -972,6 +974,12 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy( return null; } + @Override + public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( + BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException { + return null; + } + @VisibleForTesting public HashMap> getApplicationContainerIdMap() { return applicationContainerIdMap; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index d9eade8fe2d69..7f1c7b93e946b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -99,6 +99,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; @@ -1056,11 +1058,41 @@ public DeregisterSubClusterResponse deregisterSubCluster( "Please call Router's deregisterSubCluster to set."); } + /** + * In YARN-Federation mode, We will be storing the Policy information for Queues. + * + * RM does not support saveFederationQueuePolicy, + * saveFederationQueuePolicy is supported by Router. + * + * @param request saveFederationQueuePolicy Request + * @return Response from saveFederationQueuePolicy. + * @throws YarnException exceptions from yarn servers. + * @throws IOException if an IO error occurred. + */ @Override public SaveFederationQueuePolicyResponse saveFederationQueuePolicy( SaveFederationQueuePolicyRequest request) throws YarnException, IOException { throw new YarnException("It is not allowed to call the RM's saveFederationQueuePolicy. " + - " Please call Router's deregisterSubCluster to set Policy."); + " Please call Router's saveFederationQueuePolicy to set Policy."); + } + + /** + * In YARN-Federation mode, this method provides a way to save queue policies in batches. + * + * RM does not support batchSaveFederationQueuePolicies, + * batchSaveFederationQueuePolicies is supported by Router. + * + * @param request BatchSaveFederationQueuePolicies Request + * @return Response from batchSaveFederationQueuePolicies. + * @throws YarnException exceptions from yarn servers. + * @throws IOException if an IO error occurred. + */ + @Override + public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( + BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException { + throw new YarnException("It is not allowed to call the RM's " + + " batchSaveFederationQueuePolicies. " + + " Please call Router's batchSaveFederationQueuePolicies to set Policies."); } private void validateAttributesExists( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java index f9769f7026c3d..6503765ede118 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java @@ -151,6 +151,8 @@ public final class RouterMetrics { private MutableGaugeInt numDeregisterSubClusterFailedRetrieved; @Metric("# of saveFederationQueuePolicy failed to be retrieved") private MutableGaugeInt numSaveFederationQueuePolicyFailedRetrieved; + @Metric("# of batchSaveFederationQueuePolicies failed to be retrieved") + private MutableGaugeInt numBatchSaveFederationQueuePoliciesFailedRetrieved; @Metric("# of refreshAdminAcls failed to be retrieved") private MutableGaugeInt numRefreshAdminAclsFailedRetrieved; @Metric("# of refreshServiceAcls failed to be retrieved") @@ -299,6 +301,8 @@ public final class RouterMetrics { private MutableRate totalSucceededDeregisterSubClusterRetrieved; @Metric("Total number of successful Retrieved SaveFederationQueuePolicy and latency(ms)") private MutableRate totalSucceededSaveFederationQueuePolicyRetrieved; + @Metric("Total number of successful Retrieved BatchSaveFederationQueuePolicies and latency(ms)") + private MutableRate totalSucceededBatchSaveFederationQueuePoliciesRetrieved; @Metric("Total number of successful Retrieved RefreshAdminAcls and latency(ms)") private MutableRate totalSucceededRefreshAdminAclsRetrieved; @Metric("Total number of successful Retrieved RefreshServiceAcls and latency(ms)") @@ -386,6 +390,7 @@ public final class RouterMetrics { private MutableQuantiles refreshUserToGroupsMappingsLatency; private MutableQuantiles refreshDeregisterSubClusterLatency; private MutableQuantiles saveFederationQueuePolicyLatency; + private MutableQuantiles batchSaveFederationQueuePoliciesLatency; private MutableQuantiles refreshAdminAclsLatency; private MutableQuantiles refreshServiceAclsLatency; private MutableQuantiles replaceLabelsOnNodesLatency; @@ -598,7 +603,11 @@ private RouterMetrics() { "latency of deregister subcluster timeouts", "ops", "latency", 10); saveFederationQueuePolicyLatency = registry.newQuantiles("saveFederationQueuePolicyLatency", - "latency of refresh subcluster timeouts", "ops", "latency", 10); + "latency of save federation queue policy timeouts", "ops", "latency", 10); + + batchSaveFederationQueuePoliciesLatency = registry.newQuantiles( + "batchSaveFederationQueuePoliciesLatency", + "latency of batch save federationqueuepolicies timeouts", "ops", "latency", 10); refreshAdminAclsLatency = registry.newQuantiles("refreshAdminAclsLatency", "latency of refresh admin acls timeouts", "ops", "latency", 10); @@ -934,6 +943,11 @@ public long getNumSucceededSaveFederationQueuePolicyRetrieved() { return totalSucceededSaveFederationQueuePolicyRetrieved.lastStat().numSamples(); } + @VisibleForTesting + public long getNumSucceededBatchSaveFederationQueuePoliciesRetrieved() { + return totalSucceededBatchSaveFederationQueuePoliciesRetrieved.lastStat().numSamples(); + } + @VisibleForTesting public long getNumSucceededRefreshAdminAclsRetrieved() { return totalSucceededRefreshAdminAclsRetrieved.lastStat().numSamples(); @@ -1284,6 +1298,11 @@ public double getLatencySucceededSaveFederationQueuePolicyRetrieved() { return totalSucceededSaveFederationQueuePolicyRetrieved.lastStat().mean(); } + @VisibleForTesting + public double getLatencySucceededBatchSaveFederationQueuePoliciesRetrieved() { + return totalSucceededBatchSaveFederationQueuePoliciesRetrieved.lastStat().mean(); + } + @VisibleForTesting public double getLatencySucceededRefreshAdminAclsRetrieved() { return totalSucceededRefreshAdminAclsRetrieved.lastStat().mean(); @@ -1583,6 +1602,10 @@ public int getSaveFederationQueuePolicyFailedRetrieved() { return numSaveFederationQueuePolicyFailedRetrieved.value(); } + public int getBatchSaveFederationQueuePoliciesFailedRetrieved() { + return numBatchSaveFederationQueuePoliciesFailedRetrieved.value(); + } + public int getNumRefreshAdminAclsFailedRetrieved() { return numRefreshAdminAclsFailedRetrieved.value(); } @@ -1940,6 +1963,11 @@ public void succeededSaveFederationQueuePolicyRetrieved(long duration) { saveFederationQueuePolicyLatency.add(duration); } + public void succeededBatchSaveFederationQueuePoliciesRetrieved(long duration) { + totalSucceededBatchSaveFederationQueuePoliciesRetrieved.add(duration); + batchSaveFederationQueuePoliciesLatency.add(duration); + } + public void succeededRefreshAdminAclsRetrieved(long duration) { totalSucceededRefreshAdminAclsRetrieved.add(duration); refreshAdminAclsLatency.add(duration); @@ -2222,6 +2250,10 @@ public void incrSaveFederationQueuePolicyFailedRetrieved() { numSaveFederationQueuePolicyFailedRetrieved.incr(); } + public void incrBatchSaveFederationQueuePoliciesFailedRetrieved() { + numBatchSaveFederationQueuePoliciesFailedRetrieved.incr(); + } + public void incrRefreshAdminAclsFailedRetrieved() { numRefreshAdminAclsFailedRetrieved.incr(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java index 20aa0dda5fdb4..ed0698b1a812c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java @@ -60,6 +60,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -217,4 +219,10 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy( SaveFederationQueuePolicyRequest request) throws YarnException, IOException { return rmAdminProxy.saveFederationQueuePolicy(request); } + + @Override + public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( + BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException { + return rmAdminProxy.batchSaveFederationQueuePolicies(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java index 8125d4eb2b99c..5c8726083421c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java @@ -66,6 +66,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; @@ -950,6 +952,100 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy( throw new YarnException("Unable to saveFederationQueuePolicy."); } + /** + * Batch Save the Queue Policies for the Federation. + * + * @param request BatchSaveFederationQueuePolicies Request + * @return Response from batchSaveFederationQueuePolicies. + * @throws YarnException exceptions from yarn servers. + * @throws IOException if an IO error occurred. + */ + @Override + public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( + BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException { + + // Parameter validation. + if (request == null) { + routerMetrics.incrBatchSaveFederationQueuePoliciesFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "Missing BatchSaveFederationQueuePoliciesRequest request.", null); + } + + List federationQueueWeights = request.getFederationQueueWeights(); + if (federationQueueWeights == null) { + routerMetrics.incrBatchSaveFederationQueuePoliciesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Missing FederationQueueWeights information.", null); + } + + try { + long startTime = clock.getTime(); + for (FederationQueueWeight federationQueueWeight : federationQueueWeights) { + saveFederationQueuePolicy(federationQueueWeight); + } + long stopTime = clock.getTime(); + routerMetrics.succeededBatchSaveFederationQueuePoliciesRetrieved(stopTime - startTime); + return BatchSaveFederationQueuePoliciesResponse.newInstance("batch save policies success."); + } catch (Exception e) { + routerMetrics.incrBatchSaveFederationQueuePoliciesFailedRetrieved(); + RouterServerUtil.logAndThrowException(e, + "Unable to batchSaveFederationQueuePolicies due to exception. " + e.getMessage()); + } + + routerMetrics.incrBatchSaveFederationQueuePoliciesFailedRetrieved(); + throw new YarnException("Unable to batchSaveFederationQueuePolicies."); + } + + /** + * Save FederationQueuePolicy. + * + * @param federationQueueWeight queue weight. + * @throws YarnException exceptions from yarn servers. + */ + private void saveFederationQueuePolicy(FederationQueueWeight federationQueueWeight) + throws YarnException { + + // Step1, Check whether the weight setting of the queue is as expected. + String queue = federationQueueWeight.getQueue(); + String policyManagerClassName = federationQueueWeight.getPolicyManagerClassName(); + + if (StringUtils.isBlank(queue)) { + RouterServerUtil.logAndThrowException("Missing Queue information.", null); + } + + if (StringUtils.isBlank(policyManagerClassName)) { + RouterServerUtil.logAndThrowException("Missing PolicyManagerClassName information.", null); + } + + String amRmWeight = federationQueueWeight.getAmrmWeight(); + FederationQueueWeight.checkSubClusterQueueWeightRatioValid(amRmWeight); + + String routerWeight = federationQueueWeight.getRouterWeight(); + FederationQueueWeight.checkSubClusterQueueWeightRatioValid(routerWeight); + + String headRoomAlpha = federationQueueWeight.getHeadRoomAlpha(); + FederationQueueWeight.checkHeadRoomAlphaValid(headRoomAlpha); + + // Step2, parse amRMPolicyWeights. + Map amRMPolicyWeights = getSubClusterWeightMap(amRmWeight); + LOG.debug("amRMPolicyWeights = {}.", amRMPolicyWeights); + + // Step3, parse routerPolicyWeights. + Map routerPolicyWeights = getSubClusterWeightMap(routerWeight); + LOG.debug("routerWeights = {}.", amRMPolicyWeights); + + // Step4, Initialize WeightedPolicyInfo. + WeightedPolicyInfo weightedPolicyInfo = new WeightedPolicyInfo(); + weightedPolicyInfo.setHeadroomAlpha(Float.parseFloat(headRoomAlpha)); + weightedPolicyInfo.setAMRMPolicyWeights(amRMPolicyWeights); + weightedPolicyInfo.setRouterPolicyWeights(routerPolicyWeights); + + // Step5, Set SubClusterPolicyConfiguration. + SubClusterPolicyConfiguration policyConfiguration = + SubClusterPolicyConfiguration.newInstance(queue, policyManagerClassName, + weightedPolicyInfo.toByteBuffer()); + federationFacade.setPolicyConfiguration(policyConfiguration); + } + /** * Get the Map of SubClusterWeight. * @@ -961,8 +1057,9 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy( * * @param policyWeight policyWeight. * @return Map of SubClusterWeight. + * @throws YarnException exceptions from yarn servers. */ - private Map getSubClusterWeightMap(String policyWeight) + protected Map getSubClusterWeightMap(String policyWeight) throws YarnException { FederationQueueWeight.checkSubClusterQueueWeightRatioValid(policyWeight); Map result = new HashMap<>(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index 3c63ee9bd7f69..3b760e28f7889 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -67,6 +67,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; @@ -401,4 +403,11 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy( RequestInterceptorChainWrapper pipeline = getInterceptorChain(); return pipeline.getRootInterceptor().saveFederationQueuePolicy(request); } + + @Override + public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( + BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException { + RequestInterceptorChainWrapper pipeline = getInterceptorChain(); + return pipeline.getRootInterceptor().batchSaveFederationQueuePolicies(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java index c53ecdd472e2a..f62ddb27962e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java @@ -628,6 +628,11 @@ public void getSaveFederationQueuePolicyFailedRetrieved() { LOG.info("Mocked: failed refreshClusterMaxPriority call"); metrics.incrSaveFederationQueuePolicyFailedRetrieved(); } + + public void getBatchSaveFederationQueuePoliciesFailedRetrieved() { + LOG.info("Mocked: failed BatchSaveFederationQueuePolicies call"); + metrics.incrBatchSaveFederationQueuePoliciesFailedRetrieved(); + } } // Records successes for all calls @@ -963,6 +968,12 @@ public void getSaveFederationQueuePolicyRetrieved(long duration) { duration); metrics.succeededSaveFederationQueuePolicyRetrieved(duration); } + + public void getBatchSaveFederationQueuePoliciesRetrieved(long duration) { + LOG.info("Mocked: successful BatchSaveFederationQueuePoliciesRetrieved " + + " call with duration {}", duration); + metrics.succeededBatchSaveFederationQueuePoliciesRetrieved(duration); + } } @Test @@ -2241,4 +2252,29 @@ public void testSaveFederationQueuePolicyRetrieved() { Assert.assertEquals(225, metrics.getLatencySucceededSaveFederationQueuePolicyRetrieved(), ASSERT_DOUBLE_DELTA); } + + @Test + public void testGetBatchSaveFederationQueuePoliciesFailedRetrieved() { + long totalBadBefore = metrics.getBatchSaveFederationQueuePoliciesFailedRetrieved(); + badSubCluster.getBatchSaveFederationQueuePoliciesFailedRetrieved(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getBatchSaveFederationQueuePoliciesFailedRetrieved()); + } + + @Test + public void testGetBatchSaveFederationQueuePoliciesRetrieved() { + long totalGoodBefore = metrics.getNumSucceededBatchSaveFederationQueuePoliciesRetrieved(); + goodSubCluster.getBatchSaveFederationQueuePoliciesRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededBatchSaveFederationQueuePoliciesRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededBatchSaveFederationQueuePoliciesRetrieved(), + ASSERT_DOUBLE_DELTA); + goodSubCluster.getBatchSaveFederationQueuePoliciesRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededBatchSaveFederationQueuePoliciesRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededBatchSaveFederationQueuePoliciesRetrieved(), + ASSERT_DOUBLE_DELTA); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java index da44923128af9..263926e728b26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java @@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; /** * Mock interceptor that does not do anything other than forwarding it to the @@ -169,4 +171,10 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy( SaveFederationQueuePolicyRequest request) throws YarnException, IOException { return getNextInterceptor().saveFederationQueuePolicy(request); } + + @Override + public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( + BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException { + return getNextInterceptor().batchSaveFederationQueuePolicies(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java index 2c4cda2d2a9e6..6bdf6009a2e89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.router.rmadmin; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.test.LambdaTestUtils; @@ -59,6 +60,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; @@ -74,6 +77,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -81,6 +85,7 @@ import java.util.Map; import java.util.Set; import java.util.HashSet; +import java.util.Random; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -700,4 +705,143 @@ public void testSaveFederationQueuePolicyRequest() throws IOException, YarnExcep assertNotNull(sc2AMRMWeight); assertEquals(0.4f, sc2AMRMWeight.floatValue(), 0.00001); } + + @Test + public void testBatchSaveFederationQueuePoliciesRequest() throws IOException, YarnException { + + // subClusters + List subClusterLists = new ArrayList<>(); + subClusterLists.add("SC-1"); + subClusterLists.add("SC-2"); + + // generate queue A, queue B, queue C + FederationQueueWeight rootA = generateFederationQueueWeight("root.a", subClusterLists); + FederationQueueWeight rootB = generateFederationQueueWeight("root.b", subClusterLists); + FederationQueueWeight rootC = generateFederationQueueWeight("root.b", subClusterLists); + + List federationQueueWeights = new ArrayList<>(); + federationQueueWeights.add(rootA); + federationQueueWeights.add(rootB); + federationQueueWeights.add(rootC); + + // Step1. Save Queue Policies in Batches + BatchSaveFederationQueuePoliciesRequest request = + BatchSaveFederationQueuePoliciesRequest.newInstance(federationQueueWeights); + + BatchSaveFederationQueuePoliciesResponse policiesResponse = + interceptor.batchSaveFederationQueuePolicies(request); + + assertNotNull(policiesResponse); + assertNotNull(policiesResponse.getMessage()); + assertEquals("batch save policies success.", policiesResponse.getMessage()); + + // Step2. We query Policy information from FederationStateStore. + FederationStateStoreFacade federationFacade = interceptor.getFederationFacade(); + SubClusterPolicyConfiguration policyConfiguration = + federationFacade.getPolicyConfiguration("root.a"); + assertNotNull(policyConfiguration); + assertEquals("root.a", policyConfiguration.getQueue()); + + ByteBuffer params = policyConfiguration.getParams(); + assertNotNull(params); + WeightedPolicyInfo weightedPolicyInfo = WeightedPolicyInfo.fromByteBuffer(params); + assertNotNull(weightedPolicyInfo); + Map amrmPolicyWeights = weightedPolicyInfo.getAMRMPolicyWeights(); + Map routerPolicyWeights = weightedPolicyInfo.getRouterPolicyWeights(); + + SubClusterIdInfo sc1 = new SubClusterIdInfo("SC-1"); + SubClusterIdInfo sc2 = new SubClusterIdInfo("SC-2"); + + // Check whether the AMRMWeight of SC-1 and SC-2 of root.a meet expectations + FederationQueueWeight queueWeight = federationQueueWeights.get(0); + Map subClusterAmrmWeightMap = + interceptor.getSubClusterWeightMap(queueWeight.getAmrmWeight()); + Float sc1ExpectedAmrmWeightFloat = amrmPolicyWeights.get(sc1); + Float sc1AmrmWeightFloat = subClusterAmrmWeightMap.get(sc1); + assertNotNull(sc1AmrmWeightFloat); + assertEquals(sc1ExpectedAmrmWeightFloat, sc1AmrmWeightFloat, 0.00001); + + Float sc2ExpectedAmrmWeightFloat = amrmPolicyWeights.get(sc2); + Float sc2AmrmWeightFloat = subClusterAmrmWeightMap.get(sc2); + assertNotNull(sc2ExpectedAmrmWeightFloat); + assertEquals(sc2ExpectedAmrmWeightFloat, sc2AmrmWeightFloat, 0.00001); + + // Check whether the RouterPolicyWeight of SC-1 and SC-2 of root.a meet expectations + Map subClusterRouterWeightMap = + interceptor.getSubClusterWeightMap(queueWeight.getRouterWeight()); + Float sc1ExpectedRouterWeightFloat = routerPolicyWeights.get(sc1); + Float sc1RouterWeightFloat = subClusterRouterWeightMap.get(sc1); + assertNotNull(sc1RouterWeightFloat); + assertEquals(sc1ExpectedRouterWeightFloat, sc1RouterWeightFloat, 0.00001); + + Float sc2ExpectedRouterWeightFloat = routerPolicyWeights.get(sc2); + Float sc2RouterWeightFloat = subClusterRouterWeightMap.get(sc2); + assertNotNull(sc2ExpectedRouterWeightFloat); + assertEquals(sc2ExpectedRouterWeightFloat, sc2RouterWeightFloat, 0.00001); + } + + /** + * Generate FederationQueueWeight. + * We will generate the weight information of the queue. + * + * @param queue queue name + * @param pSubClusters subClusters + * @return subCluster FederationQueueWeight + */ + private FederationQueueWeight generateFederationQueueWeight( + String queue, List pSubClusters) { + String routerWeight = generatePolicyWeight(pSubClusters); + String amrmWeight = generatePolicyWeight(pSubClusters); + String policyTypeName = WeightedLocalityPolicyManager.class.getCanonicalName(); + String headRoomAlpha = "1.0"; + return FederationQueueWeight.newInstance(routerWeight, amrmWeight, headRoomAlpha, + queue, policyTypeName); + } + + /** + * Generating Policy Weight Data. + * + * @param pSubClusters set of sub-clusters. + * @return policy Weight String, like SC-1:0.7,SC-2:0. + */ + private String generatePolicyWeight(List pSubClusters) { + List weights = generateWeights(subClusters.size()); + List subClusterWeight = new ArrayList<>(); + for (int i = 0; i < pSubClusters.size(); i++) { + String subCluster = pSubClusters.get(i); + String weight = weights.get(i); + subClusterWeight.add(subCluster + ":" + weight); + } + return StringUtils.join(subClusterWeight, ","); + } + + /** + * Generate a set of random numbers, and the sum of the numbers is 1. + * + * @param n number of random numbers generated. + * @return a set of random numbers + */ + private List generateWeights(int n) { + List randomNumbers = new ArrayList<>(); + float total = 0.0f; + + Random random = new Random(); + for (int i = 0; i < n - 1; i++) { + float randNum = random.nextFloat(); + randomNumbers.add(randNum); + total += randNum; + } + + float lastNumber = 1 - total; + randomNumbers.add(lastNumber); + + DecimalFormat decimalFormat = new DecimalFormat("#.##"); + List formattedRandomNumbers = new ArrayList<>(); + for (double number : randomNumbers) { + String formattedNumber = decimalFormat.format(number); + formattedRandomNumbers.add(formattedNumber); + } + + return formattedRandomNumbers; + } } From b6edcb9a84ceac340c79cd692637b3e11c997cc5 Mon Sep 17 00:00:00 2001 From: Liangjun He <2005hithlj@163.com> Date: Thu, 10 Aug 2023 10:38:48 +0800 Subject: [PATCH 17/48] HADOOP-18840. Add enQueue time to RpcMetrics (#5926). Contributed by Liangjun He. Reviewed-by: Shilun Fan Reviewed-by: Xing Lin Signed-off-by: He Xiaoqiao --- .../java/org/apache/hadoop/ipc/Server.java | 3 +++ .../apache/hadoop/ipc/metrics/RpcMetrics.java | 24 +++++++++++++++++++ .../java/org/apache/hadoop/ipc/TestRPC.java | 11 ++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index b46a78553ec48..a594d2be01ccb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -616,6 +616,9 @@ void updateMetrics(Call call, long processingStartTimeNanos, boolean connDropped deltaNanos -= details.get(Timing.RESPONSE); details.set(Timing.HANDLER, deltaNanos); + long enQueueTime = details.get(Timing.ENQUEUE, rpcMetrics.getMetricsTimeUnit()); + rpcMetrics.addRpcEnQueueTime(enQueueTime); + long queueTime = details.get(Timing.QUEUE, rpcMetrics.getMetricsTimeUnit()); rpcMetrics.addRpcQueueTime(queueTime); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java index ad7aa88d60b30..5e0ea6228c27c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java @@ -69,6 +69,8 @@ public class RpcMetrics { CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT); metricsTimeUnit = getMetricsTimeUnit(conf); if (rpcQuantileEnable) { + rpcEnQueueTimeQuantiles = + new MutableQuantiles[intervals.length]; rpcQueueTimeQuantiles = new MutableQuantiles[intervals.length]; rpcLockWaitTimeQuantiles = @@ -81,6 +83,9 @@ public class RpcMetrics { new MutableQuantiles[intervals.length]; for (int i = 0; i < intervals.length; i++) { int interval = intervals[i]; + rpcEnQueueTimeQuantiles[i] = registry.newQuantiles("rpcEnQueueTime" + + interval + "s", "rpc enqueue time in " + metricsTimeUnit, "ops", + "latency", interval); rpcQueueTimeQuantiles[i] = registry.newQuantiles("rpcQueueTime" + interval + "s", "rpc queue time in " + metricsTimeUnit, "ops", "latency", interval); @@ -114,6 +119,8 @@ public static RpcMetrics create(Server server, Configuration conf) { @Metric("Number of received bytes") MutableCounterLong receivedBytes; @Metric("Number of sent bytes") MutableCounterLong sentBytes; + @Metric("EQueue time") MutableRate rpcEnQueueTime; + MutableQuantiles[] rpcEnQueueTimeQuantiles; @Metric("Queue time") MutableRate rpcQueueTime; MutableQuantiles[] rpcQueueTimeQuantiles; @Metric("Lock wait time") MutableRate rpcLockWaitTime; @@ -257,6 +264,23 @@ public void incrReceivedBytes(int count) { receivedBytes.incr(count); } + /** + * Sometimes, the request time observed by the client is much longer than + * the queue + process time on the RPC server.Perhaps the RPC request + * 'waiting enQueue' took too long on the RPC server, so we should add + * enQueue time to RpcMetrics. See HADOOP-18840 for details. + * Add an RPC enqueue time sample + * @param enQTime the queue time + */ + public void addRpcEnQueueTime(long enQTime) { + rpcEnQueueTime.add(enQTime); + if (rpcQuantileEnable) { + for (MutableQuantiles q : rpcEnQueueTimeQuantiles) { + q.add(enQTime); + } + } + } + /** * Add an RPC queue time sample * @param qTime the queue time diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index 1373a8a40e8e3..88d9204f69078 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -1334,6 +1334,8 @@ public TestRpcService run() { } MetricsRecordBuilder rpcMetrics = getMetrics(server.getRpcMetrics().name()); + assertEquals("Expected correct rpc en queue count", + 3000, getLongCounter("RpcEnQueueTimeNumOps", rpcMetrics)); assertEquals("Expected correct rpc queue count", 3000, getLongCounter("RpcQueueTimeNumOps", rpcMetrics)); assertEquals("Expected correct rpc processing count", @@ -1344,6 +1346,8 @@ public TestRpcService run() { 3000, getLongCounter("RpcResponseTimeNumOps", rpcMetrics)); assertEquals("Expected zero rpc lock wait time", 0, getDoubleGauge("RpcLockWaitTimeAvgTime", rpcMetrics), 0.001); + MetricsAsserts.assertQuantileGauges("RpcEnQueueTime" + interval + "s", + rpcMetrics); MetricsAsserts.assertQuantileGauges("RpcQueueTime" + interval + "s", rpcMetrics); MetricsAsserts.assertQuantileGauges("RpcProcessingTime" + interval + "s", @@ -2007,6 +2011,8 @@ public void testRpcMetricsInNanos() throws Exception { getMetrics(server.getRpcMetrics().name()); assertEquals("Expected zero rpc lock wait time", 0, getDoubleGauge("RpcLockWaitTimeAvgTime", rpcMetrics), 0.001); + MetricsAsserts.assertQuantileGauges("RpcEnQueueTime" + interval + "s", + rpcMetrics); MetricsAsserts.assertQuantileGauges("RpcQueueTime" + interval + "s", rpcMetrics); MetricsAsserts.assertQuantileGauges("RpcProcessingTime" + interval + "s", @@ -2017,12 +2023,15 @@ public void testRpcMetricsInNanos() throws Exception { assertGauge("RpcLockWaitTimeAvgTime", (double)(server.getRpcMetrics().getMetricsTimeUnit().convert(10L, TimeUnit.SECONDS)), rpcMetrics); - LOG.info("RpcProcessingTimeAvgTime: {} , RpcQueueTimeAvgTime: {}", + LOG.info("RpcProcessingTimeAvgTime: {} , RpcEnQueueTimeAvgTime: {} , RpcQueueTimeAvgTime: {}", getDoubleGauge("RpcProcessingTimeAvgTime", rpcMetrics), + getDoubleGauge("RpcEnQueueTimeAvgTime", rpcMetrics), getDoubleGauge("RpcQueueTimeAvgTime", rpcMetrics)); assertTrue(getDoubleGauge("RpcProcessingTimeAvgTime", rpcMetrics) > 4000000D); + assertTrue(getDoubleGauge("RpcEnQueueTimeAvgTime", rpcMetrics) + > 4000D); assertTrue(getDoubleGauge("RpcQueueTimeAvgTime", rpcMetrics) > 4000D); } finally { From 85b3ea6f50235e70fea6a39152e57fbb2010179e Mon Sep 17 00:00:00 2001 From: Benjamin Teke Date: Thu, 10 Aug 2023 22:47:17 +0200 Subject: [PATCH 18/48] MAPREDUCE-7449: Add add-opens flag to container launch commands on JDK17 nodes (#5935) --- .../v2/app/job/impl/TestMapReduceChildJVM.java | 5 +++-- .../main/java/org/apache/hadoop/mapred/JobConf.java | 8 ++++++++ .../org/apache/hadoop/mapreduce/MRJobConfig.java | 9 +++++++++ .../src/main/resources/mapred-default.xml | 12 ++++++++++++ .../java/org/apache/hadoop/mapred/YARNRunner.java | 7 +++++++ .../apache/hadoop/yarn/api/ApplicationConstants.java | 7 +++++++ .../yarn/applications/distributedshell/Client.java | 2 ++ .../containermanager/launcher/ContainerLaunch.java | 7 +++++++ .../launcher/TestContainerLaunch.java | 9 ++++++--- 9 files changed, 61 insertions(+), 5 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java index f00ff281f3056..e22bd44ee905b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java @@ -52,6 +52,7 @@ public void testCommandLine() throws Exception { MyMRApp app = new MyMRApp(1, 0, true, this.getClass().getName(), true); Configuration conf = new Configuration(); conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true); + conf.setBoolean(MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT, false); Job job = app.submit(conf); app.waitForState(job, JobState.SUCCEEDED); app.verifyCompleted(); @@ -123,7 +124,7 @@ private void testReduceCommandLine(Configuration conf) "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN " + - " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + + " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties" + " -Dyarn.app.container.log.dir=" + " -Dyarn.app.container.log.filesize=0" + @@ -165,7 +166,7 @@ public void testCommandLineWithLog4JConifg() throws Exception { "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN " + - " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + + " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=" + testLogPropertieFile + " -Dyarn.app.container.log.dir=" + " -Dyarn.app.container.log.filesize=0" + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index 895c9f022efab..db398e8dbdc65 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -51,6 +51,7 @@ import org.apache.hadoop.util.ClassUtil; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -2207,6 +2208,13 @@ public String getTaskJavaOpts(TaskType taskType) { javaOpts += " " + xmxArg; } + // JDK17 support: automatically add --add-opens=java.base/java.lang=ALL-UNNAMED + // so the tasks can launch on a JDK17 node. + if (getBoolean(MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT, + MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT_DEFAULT)) { + javaOpts += " " + ApplicationConstants.JVM_ADD_OPENS_VAR; + } + return javaOpts; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 02fa4eca6366f..8ec984e777bb6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -305,6 +305,15 @@ public interface MRJobConfig { "os.name,os.version,java.home,java.runtime.version,java.vendor," + "java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name"; + /* + * Flag to indicate whether JDK17's required add-opens flags should be added to MR AM and + * map/reduce containers regardless of the user specified java opts. + */ + public static final String MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT = + "mapreduce.jvm.add-opens-as-default"; + + public static final boolean MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT_DEFAULT = true; + public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor"; public static final int DEFAULT_IO_SORT_FACTOR = 10; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 2921b31caf910..a6d68acda34ab 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1804,6 +1804,18 @@ Comma-delimited list of system properties to log on mapreduce JVM start + + mapreduce.jvm.add-opens-as-default + true + Since on JDK17 it's no longer possible to use the reflection API to + access non-public fields and methods add-opens flags should be added to MR AM + and map/reduce containers regardless of the user specified java opts. Setting + this to true will add the flags to the container launch commands on nodes with + JDK17 or higher. Defaults to true, but the setting has no effect on nodes using + JDK16 and before. + + + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 084d373f1e24f..12e8c7896e306 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -485,6 +485,13 @@ private List setupAMCommand(Configuration jobConf) { MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV); vargs.add(mrAppMasterUserOptions); + // JDK17 support: automatically add --add-opens=java.base/java.lang=ALL-UNNAMED + // so the tasks can launch on a JDK17 node. + if (conf.getBoolean(MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT, + MRJobConfig.MAPREDUCE_JVM_ADD_OPENS_JAVA_OPT_DEFAULT)) { + vargs.add(ApplicationConstants.JVM_ADD_OPENS_VAR); + } + if (jobConf.getBoolean(MRJobConfig.MR_AM_PROFILE, MRJobConfig.DEFAULT_MR_AM_PROFILE)) { final String profileParams = jobConf.get(MRJobConfig.MR_AM_PROFILE_PARAMS, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 533eaddabb4f6..31ba5a9200c85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -77,6 +77,13 @@ public interface ApplicationConstants { String APPLICATION_WEB_PROXY_BASE_ENV = "APPLICATION_WEB_PROXY_BASE"; + /** + * The environmental variable for JDK17's add-opens workaround. This + * should be replaced either a correctly formatted add-opens option if JDK17 is used + * or an empty string if not on container launch. + */ + String JVM_ADD_OPENS_VAR = ""; + /** * The temporary environmental variable for container log directory. This * should be replaced by real container log directory on container launch. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 098f3981cfd4e..2987165486e63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -952,6 +952,8 @@ public boolean run() throws IOException, YarnException { vargs.add("\"" + Environment.JAVA_HOME.$$() + "/bin/java\""); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); + // JDK17 support + vargs.add(ApplicationConstants.JVM_ADD_OPENS_VAR); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 7d91e5d395f86..143086db2aec1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -170,6 +170,13 @@ public static String expandEnvironment(String var, var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR, File.pathSeparator); + if (Shell.isJavaVersionAtLeast(17)) { + var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR, + "--add-opens=java.base/java.lang=ALL-UNNAMED"); + } else { + var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR, ""); + } + // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced // as %VAR% and on Linux replaced as "$VAR" if (Shell.WINDOWS) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 15dac3daa54ac..bd135ff519382 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -575,18 +575,21 @@ public void testEnvExpansion() throws IOException { + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*" + ApplicationConstants.CLASS_PATH_SEPARATOR + Apps.crossPlatformify("HADOOP_LOG_HOME") - + ApplicationConstants.LOG_DIR_EXPANSION_VAR; + + ApplicationConstants.LOG_DIR_EXPANSION_VAR + + " " + ApplicationConstants.JVM_ADD_OPENS_VAR; String res = ContainerLaunch.expandEnvironment(input, logPath); + String expectedAddOpens = Shell.isJavaVersionAtLeast(17) ? + "--add-opens=java.base/java.lang=ALL-UNNAMED" : ""; if (Shell.WINDOWS) { Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;" + "%HADOOP_HOME%/share/hadoop/common/lib/*;" - + "%HADOOP_LOG_HOME%/nm/container/logs", res); + + "%HADOOP_LOG_HOME%/nm/container/logs" + " " + expectedAddOpens, res); } else { Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:" + "$HADOOP_HOME/share/hadoop/common/lib/*:" - + "$HADOOP_LOG_HOME/nm/container/logs", res); + + "$HADOOP_LOG_HOME/nm/container/logs" + " " + expectedAddOpens, res); } System.out.println(res); } From 58314cbbf6f076c04135083ac840dd7db59e3bb9 Mon Sep 17 00:00:00 2001 From: hfutatzhanghb Date: Fri, 11 Aug 2023 13:35:41 +0800 Subject: [PATCH 19/48] HDFS-17145. Fix description of property dfs.namenode.file.close.num-committed-allowed. (#5933) Reviewed-by: Shilun Fan Signed-off-by: Tao Li --- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 8e6ef99040a6e..9a75ec24d2eda 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5247,7 +5247,7 @@ dfs.namenode.file.close.num-committed-allowed 0 - Normally a file can only be closed with all its blocks are committed. + Normally a file can only be closed with all its blocks are complete. When this value is set to a positive integer N, a file can be closed when N blocks are committed and the rest complete. In case of Erasure Coded blocks, the committed block shall be allowed only when the block group is From 59f3a168199d47bf28777cb6eead6a5de8d073ae Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sat, 12 Aug 2023 04:02:58 +0800 Subject: [PATCH 20/48] YARN-11153. Make proxy server support YARN federation. (#4314) --- .../resourcemanager/ResourceManager.java | 5 +- .../server/webproxy/AppReportFetcher.java | 115 +++--- .../webproxy/DefaultAppReportFetcher.java | 95 +++++ .../server/webproxy/FedAppReportFetcher.java | 110 ++++++ .../yarn/server/webproxy/WebAppProxy.java | 12 +- .../server/webproxy/WebAppProxyServlet.java | 32 +- .../server/webproxy/TestAppReportFetcher.java | 2 +- .../webproxy/TestFedAppReportFetcher.java | 169 ++++++++ .../webproxy/TestWebAppProxyServlet.java | 27 +- .../webproxy/TestWebAppProxyServletFed.java | 366 ++++++++++++++++++ 10 files changed, 854 insertions(+), 79 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/DefaultAppReportFetcher.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/FedAppReportFetcher.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestFedAppReportFetcher.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletFed.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 52aa466b3f67e..2730dde72fc17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -24,6 +24,7 @@ import com.sun.jersey.spi.container.servlet.ServletContainer; import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics; +import org.apache.hadoop.yarn.server.webproxy.DefaultAppReportFetcher; import org.apache.hadoop.yarn.webapp.WebAppException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1426,9 +1427,9 @@ protected void startWepApp() { if(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf). equals(proxyHostAndPort)) { if (HAUtil.isHAEnabled(conf)) { - fetcher = new AppReportFetcher(conf); + fetcher = new DefaultAppReportFetcher(conf); } else { - fetcher = new AppReportFetcher(conf, getClientRMService()); + fetcher = new DefaultAppReportFetcher(conf, getClientRMService()); } builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME, ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java index 94c34da031c68..245e471cdf78b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java @@ -19,6 +19,9 @@ package org.apache.hadoop.yarn.server.webproxy; import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; @@ -27,41 +30,45 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.client.AHSProxy; -import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.util.StringHelper; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; /** * This class abstracts away how ApplicationReports are fetched. */ -public class AppReportFetcher { - enum AppReportSource { RM, AHS } +public abstract class AppReportFetcher { + + protected enum AppReportSource {RM, AHS} + private final Configuration conf; - private final ApplicationClientProtocol applicationsManager; - private final ApplicationHistoryProtocol historyManager; + private ApplicationHistoryProtocol historyManager; + private String ahsAppPageUrlBase; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); private boolean isAHSEnabled; /** - * Create a new Connection to the RM/Application History Server - * to fetch Application reports. + * Create a new Connection to the RM/Application History Server to fetch Application reports. + * * @param conf the conf to use to know where the RM is. */ public AppReportFetcher(Configuration conf) { + this.conf = conf; if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { - isAHSEnabled = true; + this.isAHSEnabled = true; + String scheme = WebAppUtils.getHttpSchemePrefix(conf); + String historyUrl = WebAppUtils.getAHSWebAppURLWithoutScheme(conf); + this.ahsAppPageUrlBase = StringHelper.pjoin(scheme + historyUrl, "applicationhistory", "app"); } - this.conf = conf; try { - applicationsManager = ClientRMProxy.createRMProxy(conf, - ApplicationClientProtocol.class); - if (isAHSEnabled) { - historyManager = getAHSProxy(conf); + if (this.isAHSEnabled) { + this.historyManager = getAHSProxy(conf); } else { this.historyManager = null; } @@ -69,39 +76,13 @@ public AppReportFetcher(Configuration conf) { throw new YarnRuntimeException(e); } } - - /** - * Create a direct connection to RM instead of a remote connection when - * the proxy is running as part of the RM. Also create a remote connection to - * Application History Server if it is enabled. - * @param conf the configuration to use - * @param applicationsManager what to use to get the RM reports. - */ - public AppReportFetcher(Configuration conf, ApplicationClientProtocol applicationsManager) { - if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, - YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { - isAHSEnabled = true; - } - this.conf = conf; - this.applicationsManager = applicationsManager; - if (isAHSEnabled) { - try { - historyManager = getAHSProxy(conf); - } catch (IOException e) { - throw new YarnRuntimeException(e); - } - } else { - this.historyManager = null; - } - } protected ApplicationHistoryProtocol getAHSProxy(Configuration configuration) throws IOException { - return AHSProxy.createAHSProxy(configuration, - ApplicationHistoryProtocol.class, - configuration.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + InetSocketAddress addr = configuration.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT)); + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); + return AHSProxy.createAHSProxy(configuration, ApplicationHistoryProtocol.class, addr); } /** @@ -112,17 +93,29 @@ protected ApplicationHistoryProtocol getAHSProxy(Configuration configuration) * @throws YarnException on any error. * @throws IOException */ - public FetchedAppReport getApplicationReport(ApplicationId appId) - throws YarnException, IOException { - GetApplicationReportRequest request = recordFactory - .newRecordInstance(GetApplicationReportRequest.class); + public abstract FetchedAppReport getApplicationReport(ApplicationId appId) + throws YarnException, IOException; + + /** + * Get an application report for the specified application id from the RM and + * fall back to the Application History Server if not found in RM. + * + * @param applicationsManager what to use to get the RM reports. + * @param appId id of the application to get. + * @return the ApplicationReport for the appId. + * @throws YarnException on any error. + * @throws IOException connection exception. + */ + protected FetchedAppReport getApplicationReport(ApplicationClientProtocol applicationsManager, + ApplicationId appId) throws YarnException, IOException { + GetApplicationReportRequest request = + this.recordFactory.newRecordInstance(GetApplicationReportRequest.class); request.setApplicationId(appId); ApplicationReport appReport; FetchedAppReport fetchedAppReport; try { - appReport = applicationsManager. - getApplicationReport(request).getApplicationReport(); + appReport = applicationsManager.getApplicationReport(request).getApplicationReport(); fetchedAppReport = new FetchedAppReport(appReport, AppReportSource.RM); } catch (ApplicationNotFoundException e) { if (!isAHSEnabled) { @@ -130,33 +123,43 @@ public FetchedAppReport getApplicationReport(ApplicationId appId) throw e; } //Fetch the application report from AHS - appReport = historyManager. - getApplicationReport(request).getApplicationReport(); + appReport = historyManager.getApplicationReport(request).getApplicationReport(); fetchedAppReport = new FetchedAppReport(appReport, AppReportSource.AHS); } return fetchedAppReport; } + public abstract String getRmAppPageUrlBase(ApplicationId appId) throws IOException, YarnException; + + public String getAhsAppPageUrlBase() { + return this.ahsAppPageUrlBase; + } + + protected Configuration getConf() { + return this.conf; + } + public void stop() { - if (this.applicationsManager != null) { - RPC.stopProxy(this.applicationsManager); - } if (this.historyManager != null) { RPC.stopProxy(this.historyManager); } } + @VisibleForTesting + public void setHistoryManager(ApplicationHistoryProtocol historyManager) { + this.historyManager = historyManager; + } + /* * This class creates a bundle of the application report and the source from * where the report was fetched. This allows the WebAppProxyServlet * to make decisions for the application report based on the source. */ - static class FetchedAppReport { + protected static class FetchedAppReport { private ApplicationReport appReport; private AppReportSource appReportSource; - public FetchedAppReport(ApplicationReport appReport, - AppReportSource appReportSource) { + public FetchedAppReport(ApplicationReport appReport, AppReportSource appReportSource) { this.appReport = appReport; this.appReportSource = appReportSource; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/DefaultAppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/DefaultAppReportFetcher.java new file mode 100644 index 0000000000000..1c2b0c0c34f89 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/DefaultAppReportFetcher.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.webproxy; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.util.StringHelper; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; + +public class DefaultAppReportFetcher extends AppReportFetcher { + + private final ApplicationClientProtocol applicationsManager; + private String rmAppPageUrlBase; + + /** + * Create a new Connection to the RM/Application History Server + * to fetch Application reports. + * + * @param conf the conf to use to know where the RM is. + */ + public DefaultAppReportFetcher(Configuration conf) { + super(conf); + this.rmAppPageUrlBase = + StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "app"); + try { + this.applicationsManager = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); + } catch (IOException e) { + throw new YarnRuntimeException(e); + } + } + + /** + * Create a direct connection to RM instead of a remote connection when + * the proxy is running as part of the RM. Also create a remote connection to + * Application History Server if it is enabled. + * + * @param conf the configuration to use + * @param applicationsManager what to use to get the RM reports. + */ + public DefaultAppReportFetcher(Configuration conf, + ApplicationClientProtocol applicationsManager) { + super(conf); + this.rmAppPageUrlBase = + StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "app"); + this.applicationsManager = applicationsManager; + } + + /** + * Get an application report for the specified application id from the RM and + * fall back to the Application History Server if not found in RM. + * + * @param appId id of the application to get. + * @return the ApplicationReport for the appId. + * @throws YarnException on any error. + * @throws IOException connection exception. + */ + @Override + public FetchedAppReport getApplicationReport(ApplicationId appId) + throws YarnException, IOException { + return super.getApplicationReport(applicationsManager, appId); + } + + public String getRmAppPageUrlBase(ApplicationId appId) throws YarnException, IOException { + return this.rmAppPageUrlBase; + } + + public void stop() { + super.stop(); + if (this.applicationsManager != null) { + RPC.stopProxy(this.applicationsManager); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/FedAppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/FedAppReportFetcher.java new file mode 100644 index 0000000000000..24e675160829e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/FedAppReportFetcher.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.webproxy; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.util.StringHelper; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; + +public class FedAppReportFetcher extends AppReportFetcher { + + private final Map> subClusters; + private FederationStateStoreFacade federationFacade; + + /** + * Create a new Connection to the RM/Application History Server to fetch + * Application reports. + * + * @param conf the conf to use to know where the RM is. + */ + public FedAppReportFetcher(Configuration conf) { + super(conf); + subClusters = new ConcurrentHashMap<>(); + federationFacade = FederationStateStoreFacade.getInstance(); + } + + /** + * Get an application report for the specified application id from the RM and + * fall back to the Application History Server if not found in RM. + * + * @param appId id of the application to get. + * @return the ApplicationReport for the appId. + * @throws YarnException on any error. + * @throws IOException connection exception. + */ + @Override + public FetchedAppReport getApplicationReport(ApplicationId appId) + throws YarnException, IOException { + SubClusterId scid = federationFacade.getApplicationHomeSubCluster(appId); + createSubclusterIfAbsent(scid); + ApplicationClientProtocol applicationsManager = subClusters.get(scid).getRight(); + + return super.getApplicationReport(applicationsManager, appId); + } + + @Override + public String getRmAppPageUrlBase(ApplicationId appId) + throws IOException, YarnException { + SubClusterId scid = federationFacade.getApplicationHomeSubCluster(appId); + createSubclusterIfAbsent(scid); + + SubClusterInfo subClusterInfo = subClusters.get(scid).getLeft(); + String scheme = WebAppUtils.getHttpSchemePrefix(getConf()); + return StringHelper.pjoin(scheme + subClusterInfo.getRMWebServiceAddress(), "cluster", "app"); + } + + private void createSubclusterIfAbsent(SubClusterId scId) throws YarnException, IOException { + if (subClusters.containsKey(scId)) { + return; + } + SubClusterInfo subClusterInfo = federationFacade.getSubCluster(scId); + Configuration subClusterConf = new Configuration(getConf()); + FederationProxyProviderUtil + .updateConfForFederation(subClusterConf, subClusterInfo.getSubClusterId().toString()); + ApplicationClientProtocol proxy = + ClientRMProxy.createRMProxy(subClusterConf, ApplicationClientProtocol.class); + subClusters.put(scId, Pair.of(subClusterInfo, proxy)); + } + + public void stop() { + super.stop(); + for (Pair pair : this.subClusters.values()) { + RPC.stopProxy(pair.getRight()); + } + } + + @VisibleForTesting + public void registerSubCluster(SubClusterInfo info, ApplicationClientProtocol proxy) { + subClusters.put(info.getSubClusterId(), Pair.of(info, proxy)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java index 0a65c6d34a8c4..bfe469be8a768 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java @@ -26,6 +26,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -71,7 +72,11 @@ protected void serviceInit(Configuration conf) throws Exception { String[] proxyParts = proxy.split(":"); proxyHost = proxyParts[0]; - fetcher = new AppReportFetcher(conf); + if (HAUtil.isFederationEnabled(conf)) { + fetcher = new FedAppReportFetcher(conf); + } else { + fetcher = new DefaultAppReportFetcher(conf); + } bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); if(bindAddress == null || bindAddress.isEmpty()) { throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS + @@ -157,4 +162,9 @@ public void join() { String getBindAddress() { return bindAddress + ":" + port; } + + @VisibleForTesting + public AppReportFetcher getFetcher() { + return fetcher; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index 75b891b720603..56adabe8f30c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Set; +import javax.servlet.ServletContext; import javax.servlet.ServletException; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServlet; @@ -95,8 +96,6 @@ public class WebAppProxyServlet extends HttpServlet { public static final String PROXY_USER_COOKIE_NAME = "proxy-user"; private transient List trackingUriPlugins; - private final String rmAppPageUrlBase; - private final String ahsAppPageUrlBase; private final String failurePageUrlBase; private transient YarnConfiguration conf; @@ -134,16 +133,21 @@ public WebAppProxyServlet() { this.trackingUriPlugins = conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR, TrackingUriPlugin.class); - this.rmAppPageUrlBase = - StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), - "cluster", "app"); this.failurePageUrlBase = StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "failure"); - this.ahsAppPageUrlBase = - StringHelper.pjoin(WebAppUtils.getHttpSchemePrefix(conf) - + WebAppUtils.getAHSWebAppURLWithoutScheme(conf), - "applicationhistory", "app"); + } + + private String getRmAppPageUrlBase(ApplicationId id) throws YarnException, IOException { + ServletContext context = getServletContext(); + AppReportFetcher af = (AppReportFetcher) context.getAttribute(WebAppProxy.FETCHER_ATTRIBUTE); + return af.getRmAppPageUrlBase(id); + } + + private String getAhsAppPageUrlBase() { + ServletContext context = getServletContext(); + AppReportFetcher af = (AppReportFetcher) context.getAttribute(WebAppProxy.FETCHER_ATTRIBUTE); + return af.getAhsAppPageUrlBase(); } /** @@ -578,7 +582,7 @@ private URI buildTrackingUrl(URI trackingUri, final HttpServletRequest req, */ private URI getTrackingUri(HttpServletRequest req, HttpServletResponse resp, ApplicationId id, String originalUri, AppReportSource appReportSource) - throws IOException, URISyntaxException { + throws IOException, URISyntaxException, YarnException { URI trackingUri = null; if ((originalUri == null) || @@ -589,15 +593,15 @@ private URI getTrackingUri(HttpServletRequest req, HttpServletResponse resp, // and Application Report was fetched from RM LOG.debug("Original tracking url is '{}'. Redirecting to RM app page", originalUri == null ? "NULL" : originalUri); - ProxyUtils.sendRedirect(req, resp, - StringHelper.pjoin(rmAppPageUrlBase, id.toString())); + ProxyUtils.sendRedirect(req, resp, StringHelper.pjoin(getRmAppPageUrlBase(id), + id.toString())); } else if (appReportSource == AppReportSource.AHS) { // fallback to Application History Server app page if the application // report was fetched from AHS LOG.debug("Original tracking url is '{}'. Redirecting to AHS app page", originalUri == null ? "NULL" : originalUri); - ProxyUtils.sendRedirect(req, resp, - StringHelper.pjoin(ahsAppPageUrlBase, id.toString())); + ProxyUtils.sendRedirect(req, resp, StringHelper.pjoin(getAhsAppPageUrlBase(), + id.toString())); } } else if (ProxyUriUtils.getSchemeFromUrl(originalUri).isEmpty()) { trackingUri = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java index cc7542f3e596b..fdd077910945a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java @@ -92,7 +92,7 @@ void testFetchReportAHSDisabled() throws YarnException, IOException { } } - static class AppReportFetcherForTest extends AppReportFetcher { + static class AppReportFetcherForTest extends DefaultAppReportFetcher { public AppReportFetcherForTest(Configuration conf, ApplicationClientProtocol acp) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestFedAppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestFedAppReportFetcher.java new file mode 100644 index 0000000000000..e1aaf7ae693ec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestFedAppReportFetcher.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.webproxy; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; +import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; +import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.util.StringHelper; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.junit.Assert.fail; + +public class TestFedAppReportFetcher { + + private Configuration conf; + private static ApplicationHistoryProtocol history; + + private SubClusterId subClusterId1 = SubClusterId.newInstance("subCluster1"); + private SubClusterId subClusterId2 = SubClusterId.newInstance("subCluster2"); + private SubClusterInfo clusterInfo1 = SubClusterInfo.newInstance(subClusterId1, "10.0.0.1:1000", + "10.0.0.1:1000", "10.0.0.1:1000", "10.0.0.1:1000", SubClusterState.SC_RUNNING, 0L, ""); + private SubClusterInfo clusterInfo2 = SubClusterInfo.newInstance(subClusterId2, "10.0.0.2:1000", + "10.0.0.2:1000", "10.0.0.2:1000", "10.0.0.2:1000", SubClusterState.SC_RUNNING, 0L, ""); + private ApplicationClientProtocol appManager1; + private ApplicationClientProtocol appManager2; + private ApplicationId appId1 = ApplicationId.newInstance(0, 1); + private ApplicationId appId2 = ApplicationId.newInstance(0, 2); + + private static FedAppReportFetcher fetcher; + private final String appNotFoundExceptionMsg = "APP NOT FOUND"; + + @After + public void cleanUp() { + history = null; + fetcher = null; + } + + private void testHelper(boolean isAHSEnabled) + throws YarnException, IOException { + conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, isAHSEnabled); + + FederationStateStoreFacade fedFacade = FederationStateStoreFacade.getInstance(); + FederationStateStore fss = new MemoryFederationStateStore(); + fss.init(conf); + fedFacade.reinitialize(fss, conf); + + fss.registerSubCluster(SubClusterRegisterRequest.newInstance(clusterInfo1)); + fss.registerSubCluster(SubClusterRegisterRequest.newInstance(clusterInfo2)); + fss.addApplicationHomeSubCluster(AddApplicationHomeSubClusterRequest + .newInstance(ApplicationHomeSubCluster.newInstance(appId1, subClusterId1))); + fss.addApplicationHomeSubCluster(AddApplicationHomeSubClusterRequest + .newInstance(ApplicationHomeSubCluster.newInstance(appId2, subClusterId2))); + + appManager1 = Mockito.mock(ApplicationClientProtocol.class); + Mockito.when(appManager1.getApplicationReport(Mockito.any(GetApplicationReportRequest.class))) + .thenThrow(new ApplicationNotFoundException(appNotFoundExceptionMsg)); + + appManager2 = Mockito.mock(ApplicationClientProtocol.class); + Mockito.when(appManager2.getApplicationReport(Mockito.any(GetApplicationReportRequest.class))) + .thenThrow(new ApplicationNotFoundException(appNotFoundExceptionMsg)); + + fetcher = new TestFedAppReportFetcher.FedAppReportFetcherForTest(conf); + fetcher.registerSubCluster(clusterInfo1, appManager1); + fetcher.registerSubCluster(clusterInfo2, appManager2); + } + + @Test + public void testFetchReportAHSEnabled() throws YarnException, IOException { + testHelper(true); + fetcher.getApplicationReport(appId1); + fetcher.getApplicationReport(appId2); + Mockito.verify(history, Mockito.times(2)) + .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); + Mockito.verify(appManager1, Mockito.times(1)) + .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); + Mockito.verify(appManager2, Mockito.times(1)) + .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); + } + + @Test + public void testFetchReportAHSDisabled() throws Exception { + testHelper(false); + + /* RM will not know of the app and Application History Service is disabled + * So we will not try to get the report from AHS and RM will throw + * ApplicationNotFoundException + */ + LambdaTestUtils.intercept(ApplicationNotFoundException.class, appNotFoundExceptionMsg, + () -> fetcher.getApplicationReport(appId1)); + LambdaTestUtils.intercept(ApplicationNotFoundException.class, appNotFoundExceptionMsg, + () -> fetcher.getApplicationReport(appId2)); + + Mockito.verify(appManager1, Mockito.times(1)) + .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); + Mockito.verify(appManager2, Mockito.times(1)) + .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); + Assert.assertNull("HistoryManager should be null as AHS is disabled", history); + } + + @Test + public void testGetRmAppPageUrlBase() throws IOException, YarnException { + testHelper(true); + String scheme = WebAppUtils.getHttpSchemePrefix(conf); + Assert.assertEquals(fetcher.getRmAppPageUrlBase(appId1), + StringHelper.pjoin(scheme + clusterInfo1.getRMWebServiceAddress(), "cluster", "app")); + Assert.assertEquals(fetcher.getRmAppPageUrlBase(appId2), + StringHelper.pjoin(scheme + clusterInfo2.getRMWebServiceAddress(), "cluster", "app")); + } + + static class FedAppReportFetcherForTest extends FedAppReportFetcher { + + FedAppReportFetcherForTest(Configuration conf) { + super(conf); + } + + @Override + protected ApplicationHistoryProtocol getAHSProxy(Configuration conf) + throws IOException { + GetApplicationReportResponse resp = Mockito.mock(GetApplicationReportResponse.class); + history = Mockito.mock(ApplicationHistoryProtocol.class); + try { + Mockito.when(history.getApplicationReport(Mockito.any(GetApplicationReportRequest.class))) + .thenReturn(resp); + } catch (YarnException e) { + // This should never happen + fail("Found exception when getApplicationReport!"); + } + return history; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index 39699ae7d906e..1d0ca00e7eff6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -54,6 +54,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.http.HttpServer2; @@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.webapp.MimeType; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -110,8 +112,7 @@ public static void start() throws Exception { ((ServerConnector)server.getConnectors()[0]).setHost("localhost"); server.start(); originalPort = ((ServerConnector)server.getConnectors()[0]).getLocalPort(); - LOG.info("Running embedded servlet container at: http://localhost:" - + originalPort); + LOG.info("Running embedded servlet container at: http://localhost:{}", originalPort); // This property needs to be set otherwise CORS Headers will be dropped // by HttpUrlConnection System.setProperty("sun.net.http.allowRestrictedHeaders", "true"); @@ -364,11 +365,13 @@ void testAppReportForEmptyTrackingUrl() throws Exception { String appAddressInRm = WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration) + "/cluster" + "/app/" + app.toString(); - assertEquals(proxyConn.getURL().toString(), appAddressInRm); + assertEquals(proxyConn.getURL().toString(), appAddressInRm, + "Webapp proxy servlet should have redirected to RM"); //set AHS_ENABLED = true to simulate getting the app report from AHS configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); + proxy.proxy.appReportFetcher.setAhsAppPageUrlBase(configuration); proxyConn = (HttpURLConnection) url.openConnection(); proxyConn.connect(); try { @@ -381,7 +384,8 @@ void testAppReportForEmptyTrackingUrl() throws Exception { String appAddressInAhs = WebAppUtils.getHttpSchemePrefix(configuration) + WebAppUtils.getAHSWebAppURLWithoutScheme( configuration) + "/applicationhistory" + "/app/" + app.toString(); - assertEquals(proxyConn.getURL().toString(), appAddressInAhs); + assertEquals(proxyConn.getURL().toString(), appAddressInAhs, + "Webapp proxy servlet should have redirected to AHS"); } finally { proxy.close(); } @@ -607,8 +611,9 @@ protected void serviceStart() throws Exception { } - private class AppReportFetcherForTest extends AppReportFetcher { + private class AppReportFetcherForTest extends DefaultAppReportFetcher { int answer = 0; + private String ahsAppPageUrlBase = null; public AppReportFetcherForTest(Configuration conf) { super(conf); @@ -679,5 +684,17 @@ private FetchedAppReport getDefaultApplicationReport(ApplicationId appId, private FetchedAppReport getDefaultApplicationReport(ApplicationId appId) { return getDefaultApplicationReport(appId, true); } + + @VisibleForTesting + public String getAhsAppPageUrlBase() { + return ahsAppPageUrlBase != null ? ahsAppPageUrlBase : super.getAhsAppPageUrlBase(); + } + + @VisibleForTesting + public void setAhsAppPageUrlBase(Configuration conf) { + this.ahsAppPageUrlBase = StringHelper.pjoin( + WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils.getAHSWebAppURLWithoutScheme(conf), + "applicationhistory", "app"); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletFed.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletFed.java new file mode 100644 index 0000000000000..8931bc577a224 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletFed.java @@ -0,0 +1,366 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.webproxy; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test the WebAppProxyServlet and WebAppProxy. For back end use simple web server. + */ +public class TestWebAppProxyServletFed { + + private static final Logger LOG = LoggerFactory.getLogger(TestWebAppProxyServletFed.class); + + public static final String AM_PREFIX = "AM"; + public static final String RM_PREFIX = "RM"; + public static final String AHS_PREFIX = "AHS"; + + /* + * Mocked Server is used for simulating the web of AppMaster, ResourceMamanger or TimelineServer. + * */ + private static Server mockServer; + private static int mockServerPort = 0; + + /** + * Simple http server. Server should send answer with status 200 + */ + @BeforeClass + public static void setUp() throws Exception { + mockServer = new Server(0); + ((QueuedThreadPool) mockServer.getThreadPool()).setMaxThreads(20); + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/"); + context.addServlet(new ServletHolder(new MockWebServlet(AM_PREFIX)), "/amweb/*"); + context.addServlet(new ServletHolder(new MockWebServlet(RM_PREFIX)), "/cluster/app/*"); + context.addServlet(new ServletHolder(new MockWebServlet(AHS_PREFIX)), + "/applicationhistory/app/*"); + mockServer.setHandler(context); + + ((ServerConnector) mockServer.getConnectors()[0]).setHost("localhost"); + mockServer.start(); + mockServerPort = ((ServerConnector) mockServer.getConnectors()[0]).getLocalPort(); + LOG.info("Running embedded servlet container at: http://localhost:" + mockServerPort); + } + + @AfterClass + public static void tearDown() throws Exception { + if (mockServer != null) { + mockServer.stop(); + mockServer.destroy(); + mockServer = null; + } + } + + @Test + public void testWebServlet() throws IOException { + HttpURLConnection conn; + // 1. Mocked AppMaster web Test + URL url = new URL("http", "localhost", mockServerPort, "/amweb/apptest"); + conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(AM_PREFIX + "/apptest", readResponse(conn)); + conn.disconnect(); + + // 2. Mocked RM web Test + url = new URL("http", "localhost", mockServerPort, "/cluster/app/apptest"); + conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(RM_PREFIX + "/apptest", readResponse(conn)); + conn.disconnect(); + + // 3. Mocked AHS web Test + url = new URL("http", "localhost", mockServerPort, "/applicationhistory/app/apptest"); + conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(AHS_PREFIX + "/apptest", readResponse(conn)); + conn.disconnect(); + } + + @Test(timeout=5000) + public void testWebAppProxyServletFed() throws Exception { + + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); + conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:" + mockServerPort); + // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS + conf.setInt("hadoop.http.max.threads", 10); + + // Create sub cluster information. + SubClusterId subClusterId1 = SubClusterId.newInstance("scid1"); + SubClusterId subClusterId2 = SubClusterId.newInstance("scid2"); + SubClusterInfo subClusterInfo1 = SubClusterInfo.newInstance(subClusterId1, "10.0.0.1:1", + "10.0.0.1:1", "10.0.0.1:1", "localhost:" + mockServerPort, SubClusterState.SC_RUNNING, 0, + ""); + SubClusterInfo subClusterInfo2 = SubClusterInfo.newInstance(subClusterId2, "10.0.0.2:1", + "10.0.0.2:1", "10.0.0.2:1", "10.0.0.2:1", SubClusterState.SC_RUNNING, 0, ""); + + // App1 and App2 is running applications. + ApplicationId appId1 = ApplicationId.newInstance(0, 1); + ApplicationId appId2 = ApplicationId.newInstance(0, 2); + String appUrl1 = "http://localhost:" + mockServerPort + "/amweb/" + appId1; + String appUrl2 = "http://localhost:" + mockServerPort + "/amweb/" + appId2; + // App3 is accepted application, has not registered original url to am. + ApplicationId appId3 = ApplicationId.newInstance(0, 3); + // App4 is finished application, has remove from rm, but not remove from timeline server. + ApplicationId appId4 = ApplicationId.newInstance(0, 4); + + // Mock for application + ApplicationClientProtocol appManager1 = Mockito.mock(ApplicationClientProtocol.class); + Mockito.when(appManager1.getApplicationReport(GetApplicationReportRequest.newInstance(appId1))) + .thenReturn(GetApplicationReportResponse + .newInstance(newApplicationReport(appId1, YarnApplicationState.RUNNING, appUrl1))); + Mockito.when(appManager1.getApplicationReport(GetApplicationReportRequest.newInstance(appId3))) + .thenReturn(GetApplicationReportResponse + .newInstance(newApplicationReport(appId3, YarnApplicationState.ACCEPTED, null))); + + ApplicationClientProtocol appManager2 = Mockito.mock(ApplicationClientProtocol.class); + Mockito.when(appManager2.getApplicationReport(GetApplicationReportRequest.newInstance(appId2))) + .thenReturn(GetApplicationReportResponse + .newInstance(newApplicationReport(appId2, YarnApplicationState.RUNNING, appUrl2))); + Mockito.when(appManager2.getApplicationReport(GetApplicationReportRequest.newInstance(appId4))) + .thenThrow(new ApplicationNotFoundException("APP NOT FOUND")); + + ApplicationHistoryProtocol historyManager = Mockito.mock(ApplicationHistoryProtocol.class); + Mockito + .when(historyManager.getApplicationReport(GetApplicationReportRequest.newInstance(appId4))) + .thenReturn(GetApplicationReportResponse + .newInstance(newApplicationReport(appId4, YarnApplicationState.FINISHED, null))); + + // Initial federation store. + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + facade.getStateStore() + .registerSubCluster(SubClusterRegisterRequest.newInstance(subClusterInfo1)); + facade.getStateStore() + .registerSubCluster(SubClusterRegisterRequest.newInstance(subClusterInfo2)); + facade.addApplicationHomeSubCluster( + ApplicationHomeSubCluster.newInstance(appId1, subClusterId1)); + facade.addApplicationHomeSubCluster( + ApplicationHomeSubCluster.newInstance(appId2, subClusterId2)); + facade.addApplicationHomeSubCluster( + ApplicationHomeSubCluster.newInstance(appId3, subClusterId1)); + facade.addApplicationHomeSubCluster( + ApplicationHomeSubCluster.newInstance(appId4, subClusterId2)); + + // Start proxy server + WebAppProxyServerForTest proxy = new WebAppProxyServerForTest(); + proxy.init(conf); + proxy.start(); + + try { + // set Mocked rm and timeline + int proxyPort = proxy.proxy.proxyServer.getConnectorAddress(0).getPort(); + FedAppReportFetcher appReportFetcher = proxy.proxy.appReportFetcher; + appReportFetcher.registerSubCluster(subClusterInfo1, appManager1); + appReportFetcher.registerSubCluster(subClusterInfo2, appManager2); + appReportFetcher.setHistoryManager(historyManager); + + // App1 is running in subcluster1, and original url is registered + // in rm of subCluster1. So proxy server will get original url from rm by + // getApplicationReport. Then proxy server will fetch the webapp directly. + URL url = new URL("http", "localhost", proxyPort, "/proxy/" + appId1.toString()); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(AM_PREFIX + "/" + appId1.toString(), readResponse(conn)); + conn.disconnect(); + + // App2 is running in subcluster2, and original url is registered + // in rm of subCluster2. So proxy server will get original url from rm by + // getApplicationReport. Then proxy server will fetch the webapp directly. + url = new URL("http", "localhost", proxyPort, "/proxy/" + appId2.toString()); + conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(AM_PREFIX + "/" + appId2.toString(), readResponse(conn)); + conn.disconnect(); + + // App3 is accepted in subcluster1, and original url is not registered + // yet. So proxy server will fetch the application web from rm. + url = new URL("http", "localhost", proxyPort, "/proxy/" + appId3.toString()); + conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(RM_PREFIX + "/" + appId3.toString(), readResponse(conn)); + conn.disconnect(); + + // App4 is finished in subcluster2, and have removed from rm, but not + // removed from timeline server. So proxy server will fetch the + // application web from timeline server. + url = new URL("http", "localhost", proxyPort, "/proxy/" + appId4.toString()); + conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(AHS_PREFIX + "/" + appId4.toString(), readResponse(conn)); + conn.disconnect(); + } finally { + proxy.close(); + } + } + + private ApplicationReport newApplicationReport(ApplicationId appId, + YarnApplicationState state, String origTrackingUrl) { + return ApplicationReport.newInstance(appId, null, "testuser", null, null, null, 0, null, state, + null, null, 0, 0, 0, null, null, origTrackingUrl, 0f, null, null); + } + + private String readResponse(HttpURLConnection conn) throws IOException { + InputStream input = conn.getInputStream(); + byte[] bytes = new byte[input.available()]; + input.read(bytes); + return new String(bytes); + } + + private class WebAppProxyServerForTest extends CompositeService { + + private WebAppProxyForTest proxy = null; + + WebAppProxyServerForTest() { + super(WebAppProxyServer.class.getName()); + } + + @Override + protected synchronized void serviceInit(Configuration conf) throws Exception { + proxy = new WebAppProxyForTest(); + addService(proxy); + super.serviceInit(conf); + } + } + + /* + * This servlet is used for simulate the web of AppMaster, ResourceManager, + * TimelineServer and so on. + * */ + public static class MockWebServlet extends HttpServlet { + + private String role; + + public MockWebServlet(String role) { + this.role = role; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + if (req.getPathInfo() != null) { + resp.getWriter().write(role + req.getPathInfo()); + } + resp.setStatus(HttpServletResponse.SC_OK); + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + InputStream is = req.getInputStream(); + OutputStream os = resp.getOutputStream(); + int c = is.read(); + while (c > -1) { + os.write(c); + c = is.read(); + } + is.close(); + os.close(); + resp.setStatus(HttpServletResponse.SC_OK); + } + } + + private class WebAppProxyForTest extends WebAppProxy { + + private HttpServer2 proxyServer; + private FedAppReportFetcher appReportFetcher; + + @Override + protected void serviceStart() throws Exception { + Configuration conf = getConfig(); + String bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); + bindAddress = StringUtils.split(bindAddress, ':')[0]; + AccessControlList acl = new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL, + YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); + proxyServer = new HttpServer2.Builder() + .setName("proxy") + .addEndpoint(URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress + ":0")) + .setFindPort(true) + .setConf(conf) + .setACL(acl) + .build(); + proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME, ProxyUriUtils.PROXY_PATH_SPEC, + WebAppProxyServlet.class); + + appReportFetcher = new FedAppReportFetcher(conf); + proxyServer.setAttribute(FETCHER_ATTRIBUTE, appReportFetcher); + proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.FALSE); + + String proxy = WebAppUtils.getProxyHostAndPort(conf); + String[] proxyParts = proxy.split(":"); + String proxyHost = proxyParts[0]; + + proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost); + proxyServer.start(); + LOG.info("Proxy server is started at port {}", proxyServer.getConnectorAddress(0).getPort()); + } + } +} From ad2f45c64f01a520a01f6876d8493140b9f89b03 Mon Sep 17 00:00:00 2001 From: hchaverri <55413673+hchaverri@users.noreply.github.com> Date: Fri, 11 Aug 2023 13:04:32 -0700 Subject: [PATCH 21/48] HDFS-17148. RBF: SQLDelegationTokenSecretManager must cleanup expired tokens in SQL (#5936) --- .../AbstractDelegationTokenSecretManager.java | 14 +++- .../SQLDelegationTokenSecretManager.java | 45 +++++++++++ .../SQLDelegationTokenSecretManagerImpl.java | 24 ++++++ ...stSQLDelegationTokenSecretManagerImpl.java | 79 +++++++++++++++++-- 4 files changed, 154 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index cf5e9a66311aa..283e773c81795 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -190,6 +190,14 @@ public long getCurrentTokensSize() { return currentTokens.size(); } + /** + * Interval for tokens to be renewed. + * @return Renew interval in milliseconds. + */ + protected long getTokenRenewInterval() { + return this.tokenRenewInterval; + } + /** * Add a previously used master key to cache (when NN restarts), * should be called before activate(). @@ -751,7 +759,7 @@ private void removeExpiredToken() throws IOException { Set expiredTokens = new HashSet<>(); synchronized (this) { Iterator> i = - currentTokens.entrySet().iterator(); + getCandidateTokensForCleanup().entrySet().iterator(); while (i.hasNext()) { Map.Entry entry = i.next(); long renewDate = entry.getValue().getRenewDate(); @@ -766,6 +774,10 @@ private void removeExpiredToken() throws IOException { logExpireTokens(expiredTokens); } + protected Map getCandidateTokensForCleanup() { + return this.currentTokens; + } + protected void logExpireTokens( Collection expiredTokens) throws IOException { for (TokenIdent ident : expiredTokens) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java index 75f00d3f924a8..d2c41f31d1d94 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java @@ -24,6 +24,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; import java.util.NoSuchElementException; import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.VisibleForTesting; @@ -50,6 +52,9 @@ public abstract class SQLDelegationTokenSecretManager token, return super.cancelToken(token, canceller); } + /** + * Obtain a list of tokens that will be considered for cleanup, based on the last + * time the token was updated in SQL. This list may include tokens that are not + * expired and should not be deleted (e.g. if the token was last renewed using a + * higher renewal interval). + * The number of results is limited to reduce performance impact. Some level of + * contention is expected when multiple routers run cleanup simultaneously. + * @return Map of tokens that have not been updated in SQL after the token renewal + * period. + */ + @Override + protected Map getCandidateTokensForCleanup() { + Map tokens = new HashMap<>(); + try { + // Query SQL for tokens that haven't been updated after + // the last token renewal period. + long maxModifiedTime = Time.now() - getTokenRenewInterval(); + Map tokenInfoBytesList = selectStaleTokenInfos(maxModifiedTime, + this.maxTokenCleanupResults); + + LOG.info("Found {} tokens for cleanup", tokenInfoBytesList.size()); + for (Map.Entry tokenInfoBytes : tokenInfoBytesList.entrySet()) { + TokenIdent tokenIdent = createTokenIdent(tokenInfoBytes.getKey()); + DelegationTokenInformation tokenInfo = createTokenInfo(tokenInfoBytes.getValue()); + tokens.put(tokenIdent, tokenInfo); + } + } catch (IOException | SQLException e) { + LOG.error("Failed to get candidate tokens for cleanup in SQL secret manager", e); + } + + return tokens; + } + /** * Removes the existing TokenInformation from the SQL database to * invalidate it. @@ -415,6 +458,8 @@ public int incrementCurrentKeyId() { // Token operations in SQL database protected abstract byte[] selectTokenInfo(int sequenceNum, byte[] tokenIdentifier) throws SQLException; + protected abstract Map selectStaleTokenInfos(long maxModifiedTime, + int maxResults) throws SQLException; protected abstract void insertToken(int sequenceNum, byte[] tokenIdentifier, byte[] tokenInfo) throws SQLException; protected abstract void updateToken(int sequenceNum, byte[] tokenIdentifier, byte[] tokenInfo) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/SQLDelegationTokenSecretManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/SQLDelegationTokenSecretManagerImpl.java index 7da54778f3127..e85baae0c3ad3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/SQLDelegationTokenSecretManagerImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/SQLDelegationTokenSecretManagerImpl.java @@ -23,6 +23,9 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -150,6 +153,27 @@ protected byte[] selectTokenInfo(int sequenceNum, byte[] tokenIdentifier) throws }); } + @Override + protected Map selectStaleTokenInfos(long maxModifiedTime, int maxResults) + throws SQLException { + return retryHandler.execute(() -> { + try (Connection connection = connectionFactory.getConnection(); + PreparedStatement statement = connection.prepareStatement( + "SELECT tokenIdentifier, tokenInfo FROM Tokens WHERE modifiedTime < ?")) { + statement.setTimestamp(1, new Timestamp(maxModifiedTime)); + statement.setMaxRows(maxResults); + try (ResultSet result = statement.executeQuery()) { + Map results = new HashMap<>(); + while (result.next()) { + results.put(result.getBytes("tokenIdentifier"), + result.getBytes("tokenInfo")); + } + return results; + } + } + }); + } + @Override protected void insertDelegationKey(int keyId, byte[] delegationKey) throws SQLException { retryHandler.execute(() -> { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/security/token/TestSQLDelegationTokenSecretManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/security/token/TestSQLDelegationTokenSecretManagerImpl.java index dbbb85662cf47..d82be8f5cb29b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/security/token/TestSQLDelegationTokenSecretManagerImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/security/token/TestSQLDelegationTokenSecretManagerImpl.java @@ -30,12 +30,15 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.SQLDelegationTokenSecretManager; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; @@ -52,6 +55,7 @@ public class TestSQLDelegationTokenSecretManagerImpl { private static final String CONNECTION_URL = "jdbc:derby:memory:TokenStore"; private static final int TEST_MAX_RETRIES = 3; private static final int TOKEN_EXPIRATION_SECONDS = 1; + private static final int TOKEN_EXPIRATION_SCAN_SECONDS = 1; private static Configuration conf; @Before @@ -75,6 +79,7 @@ public static void initDatabase() throws SQLException { conf.set(SQLConnectionFactory.CONNECTION_DRIVER, "org.apache.derby.jdbc.EmbeddedDriver"); conf.setInt(SQLSecretManagerRetriableHandlerImpl.MAX_RETRIES, TEST_MAX_RETRIES); conf.setInt(SQLSecretManagerRetriableHandlerImpl.RETRY_SLEEP_TIME_MS, 10); + conf.setInt(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, TOKEN_EXPIRATION_SCAN_SECONDS); } @AfterClass @@ -188,6 +193,62 @@ public void testRenewToken() throws Exception { } } + @Test + public void testRemoveExpiredTokens() throws Exception { + DelegationTokenManager tokenManager = createTokenManager(getShortLivedTokenConf()); + + try { + TestDelegationTokenSecretManager secretManager = + (TestDelegationTokenSecretManager) tokenManager.getDelegationTokenSecretManager(); + + // Create token to be constantly renewed. + Token token1 = tokenManager.createToken(UserGroupInformation.getCurrentUser(), "foo"); + AbstractDelegationTokenIdentifier tokenId1 = + (AbstractDelegationTokenIdentifier) token1.decodeIdentifier(); + + // Create token expected to expire soon. + long expirationTime2 = Time.now(); + AbstractDelegationTokenIdentifier tokenId2 = storeToken(secretManager, 2, expirationTime2); + + // Create token not expected to expire soon. + long expirationTime3 = Time.now() + TimeUnit.SECONDS.toMillis(TOKEN_EXPIRATION_SECONDS) * 10; + AbstractDelegationTokenIdentifier tokenId3 = storeToken(secretManager, 3, expirationTime3); + + GenericTestUtils.waitFor(() -> { + try { + // Constantly renew token so it doesn't expire. + tokenManager.renewToken(token1, "foo"); + + // Wait for cleanup to happen so expired token is deleted from SQL. + return !isTokenInSQL(secretManager, tokenId2); + } catch (IOException | SQLException e) { + throw new RuntimeException(e); + } + }, 100, 6000); + + Assert.assertTrue("Renewed token must not be cleaned up", + isTokenInSQL(secretManager, tokenId1)); + Assert.assertTrue("Token with future expiration must not be cleaned up", + isTokenInSQL(secretManager, tokenId3)); + } finally { + stopTokenManager(tokenManager); + } + } + + private AbstractDelegationTokenIdentifier storeToken( + TestDelegationTokenSecretManager secretManager, int sequenceNum, long expirationTime) + throws IOException { + AbstractDelegationTokenIdentifier tokenId = new DelegationTokenIdentifier(new Text("Test")); + tokenId.setOwner(new Text("foo")); + tokenId.setSequenceNumber(sequenceNum); + + AbstractDelegationTokenSecretManager.DelegationTokenInformation tokenInfo = + new AbstractDelegationTokenSecretManager.DelegationTokenInformation(expirationTime, null); + secretManager.storeToken(tokenId, tokenInfo); + + return tokenId; + } + private Configuration getShortLivedTokenConf() { Configuration shortLivedConf = new Configuration(conf); shortLivedConf.setTimeDuration( @@ -201,13 +262,12 @@ private void callRemoveExpiredTokensAndValidateSQL( TestDelegationTokenSecretManager secretManager, AbstractDelegationTokenIdentifier tokenId, boolean expectedInSQL) throws SQLException { secretManager.removeExpiredStoredToken(tokenId); - byte[] tokenInfo = secretManager.selectTokenInfo(tokenId.getSequenceNumber(), - tokenId.getBytes()); - if (expectedInSQL) { - Assert.assertNotNull("Verify token exists in database", tokenInfo); - } else { - Assert.assertNull("Verify token was removed from database", tokenInfo); - } + Assert.assertEquals(expectedInSQL, isTokenInSQL(secretManager, tokenId)); + } + + private boolean isTokenInSQL(TestDelegationTokenSecretManager secretManager, + AbstractDelegationTokenIdentifier tokenId) throws SQLException { + return secretManager.selectTokenInfo(tokenId.getSequenceNumber(), tokenId.getBytes()) != null; } @Test @@ -542,6 +602,11 @@ public void removeExpiredStoredToken(TokenIdentifier tokenId) { super.removeExpiredStoredToken((AbstractDelegationTokenIdentifier) tokenId); } + public void storeToken(AbstractDelegationTokenIdentifier ident, + DelegationTokenInformation tokenInfo) throws IOException { + super.storeToken(ident, tokenInfo); + } + public void setReadOnly(boolean readOnly) { ((TestConnectionFactory) getConnectionFactory()).readOnly = readOnly; } From 8d95c588d2df0048b0d3eb711d74bf34bf4ae3c4 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sat, 12 Aug 2023 09:15:34 +0800 Subject: [PATCH 22/48] YARN-8934. [BackPort][GPG] Add JvmMetricsInfo and pause monitor. (#5929) Contributed by Bilwa S T. --- .../hadoop/yarn/conf/YarnConfiguration.java | 21 ++++++ .../hadoop-yarn/hadoop-yarn-common/pom.xml | 1 + .../hadoop/yarn/webapp/util/WebAppUtils.java | 10 +++ .../src/main/resources/webapps/gpg/.keep | 0 .../src/main/resources/yarn-default.xml | 28 ++++++++ .../GlobalPolicyGenerator.java | 70 +++++++++++++++++++ .../webapp/GPGController.java | 44 ++++++++++++ .../webapp/GPGOverviewBlock.java | 49 +++++++++++++ .../webapp/GPGOverviewPage.java | 52 ++++++++++++++ .../webapp/GPGWebApp.java | 45 ++++++++++++ .../webapp/NavBlock.java | 42 +++++++++++ .../webapp/package-info.java | 24 +++++++ .../TestGlobalPolicyGenerator.java | 20 ++++++ 13 files changed, 406 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/gpg/.keep create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewBlock.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewPage.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/package-info.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ae7ea196d4783..de76369b01444 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4445,6 +4445,27 @@ public static boolean isAclEnabled(Configuration conf) { FEDERATION_GPG_LOAD_BASED_PREFIX + "scaling"; public static final String DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING = "LINEAR"; + public static final String GPG_WEBAPP_PREFIX = FEDERATION_GPG_PREFIX + "webapp."; + + /** Enable/disable CORS filter. */ + public static final String GPG_WEBAPP_ENABLE_CORS_FILTER = + GPG_WEBAPP_PREFIX + "cross-origin.enabled"; + public static final boolean DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER = false; + + /** The address of the GPG web application. */ + public static final String GPG_WEBAPP_ADDRESS = GPG_WEBAPP_PREFIX + "address"; + + public static final int DEFAULT_GPG_WEBAPP_PORT = 8069; + public static final String DEFAULT_GPG_WEBAPP_ADDRESS = + "0.0.0.0:" + DEFAULT_GPG_WEBAPP_PORT; + + /** The https address of the GPG web application. */ + public static final String GPG_WEBAPP_HTTPS_ADDRESS = GPG_WEBAPP_PREFIX + "https.address"; + + public static final int DEFAULT_GPG_WEBAPP_HTTPS_PORT = 8070; + public static final String DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS = + "0.0.0.0:" + DEFAULT_GPG_WEBAPP_HTTPS_PORT; + /** * Connection and Read timeout from the Router to RM. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 9b4f4fd47bf95..f85d875bf490b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -271,6 +271,7 @@ src/main/resources/webapps/test/.keep src/main/resources/webapps/proxy/.keep src/main/resources/webapps/node/.keep + src/main/resources/webapps/gpg/.keep src/main/resources/webapps/static/dt-1.11.5/css/jquery.dataTables.css src/main/resources/webapps/static/dt-1.11.5/css/custom_datatable.css src/main/resources/webapps/static/dt-1.11.5/css/jui-dt.css diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 02fec11505990..ff04ffe47a2ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -183,6 +183,16 @@ public static String getRouterWebAppURLWithoutScheme(Configuration conf) { } } + public static String getGPGWebAppURLWithoutScheme(Configuration conf) { + if (YarnConfiguration.useHttps(conf)) { + return conf.get(YarnConfiguration.GPG_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS); + } else { + return conf.get(YarnConfiguration.GPG_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_GPG_WEBAPP_ADDRESS); + } + } + public static List getProxyHostsAndPortsForAmFilter( Configuration conf) { List addrs = new ArrayList(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/gpg/.keep b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/gpg/.keep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 6cab018e5c41d..bec4393061411 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5530,4 +5530,32 @@ LINEAR + + Flag to enable cross-origin (CORS) support in the GPG. This flag + requires the CORS filter initializer to be added to the filter initializers + list in core-site.xml. + yarn.federation.gpg.webapp.cross-origin.enabled + false + + + + + The http address of the GPG web application. + If only a host is provided as the value, + the webapp will be served on a random port. + + yarn.federation.gpg.webapp.address + 0.0.0.0:8069 + + + + + The https address of the GPG web application. + If only a host is provided as the value, + the webapp will be served on a random port. + + yarn.federation.gpg.webapp.https.address + 0.0.0.0:8070 + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index 518f69d5630f6..c24cedf95f62e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -21,15 +21,22 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang.time.DurationFormatUtils; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.source.JvmMetrics; +import org.apache.hadoop.security.AuthenticationFilterInitializer; +import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; @@ -38,6 +45,10 @@ import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.PolicyGenerator; import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner; +import org.apache.hadoop.yarn.server.globalpolicygenerator.webapp.GPGWebApp; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebApps; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,6 +72,7 @@ public class GlobalPolicyGenerator extends CompositeService { public static final int SHUTDOWN_HOOK_PRIORITY = 30; private AtomicBoolean isStopping = new AtomicBoolean(false); private static final String METRICS_NAME = "Global Policy Generator"; + private static long gpgStartupTime = System.currentTimeMillis(); // Federation Variables private GPGContext gpgContext; @@ -69,6 +81,9 @@ public class GlobalPolicyGenerator extends CompositeService { private ScheduledThreadPoolExecutor scheduledExecutorService; private SubClusterCleaner subClusterCleaner; private PolicyGenerator policyGenerator; + private String webAppAddress; + private JvmPauseMonitor pauseMonitor; + private WebApp webApp; public GlobalPolicyGenerator() { super(GlobalPolicyGenerator.class.getName()); @@ -107,7 +122,12 @@ protected void serviceInit(Configuration conf) throws Exception { this.subClusterCleaner = new SubClusterCleaner(conf, this.gpgContext); this.policyGenerator = new PolicyGenerator(conf, this.gpgContext); + this.webAppAddress = WebAppUtils.getGPGWebAppURLWithoutScheme(conf); DefaultMetricsSystem.initialize(METRICS_NAME); + JvmMetrics jm = JvmMetrics.initSingleton("GPG", null); + pauseMonitor = new JvmPauseMonitor(); + addService(pauseMonitor); + jm.setPauseMonitor(pauseMonitor); // super.serviceInit after all services are added super.serviceInit(conf); @@ -163,6 +183,7 @@ protected void serviceStart() throws Exception { LOG.info("Scheduled policy-generator with interval: {}", DurationFormatUtils.formatDurationISO(policyGeneratorIntervalMillis)); } + startWepApp(); } @Override @@ -181,6 +202,9 @@ protected void serviceStop() throws Exception { if (this.isStopping.getAndSet(true)) { return; } + if (webApp != null) { + webApp.stop(); + } DefaultMetricsSystem.shutdown(); super.serviceStop(); } @@ -193,6 +217,43 @@ public GPGContext getGPGContext() { return this.gpgContext; } + @VisibleForTesting + public void startWepApp() { + Configuration configuration = getConfig(); + + boolean enableCors = configuration.getBoolean(YarnConfiguration.GPG_WEBAPP_ENABLE_CORS_FILTER, + YarnConfiguration.DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER); + + if (enableCors) { + configuration.setBoolean(HttpCrossOriginFilterInitializer.PREFIX + + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true); + } + + // Always load pseudo authentication filter to parse "user.name" in an URL + // to identify a HTTP request's user. + boolean hasHadoopAuthFilterInitializer = false; + String filterInitializerConfKey = "hadoop.http.filter.initializers"; + Class[] initializersClasses = configuration.getClasses(filterInitializerConfKey); + + List targets = new ArrayList<>(); + if (initializersClasses != null) { + for (Class initializer : initializersClasses) { + if (initializer.getName().equals(AuthenticationFilterInitializer.class.getName())) { + hasHadoopAuthFilterInitializer = true; + break; + } + targets.add(initializer.getName()); + } + } + if (!hasHadoopAuthFilterInitializer) { + targets.add(AuthenticationFilterInitializer.class.getName()); + configuration.set(filterInitializerConfKey, StringUtils.join(",", targets)); + } + LOG.info("Instantiating GPGWebApp at {}.", webAppAddress); + GPGWebApp gpgWebApp = new GPGWebApp(this); + webApp = WebApps.$for("gpg").at(webAppAddress).start(gpgWebApp); + } + @SuppressWarnings("resource") public static void startGPG(String[] argv, Configuration conf) { boolean federationEnabled = conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED, @@ -232,4 +293,13 @@ public static void main(String[] argv) { System.exit(-1); } } + + public static long getGPGStartupTime() { + return gpgStartupTime; + } + + @VisibleForTesting + public WebApp getWebApp() { + return webApp; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java new file mode 100644 index 0000000000000..c3955ea3fb379 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java @@ -0,0 +1,44 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import org.apache.hadoop.yarn.webapp.Controller; +import com.google.inject.Inject; + +/** + * Controller for the GPG Web UI. + */ +public class GPGController extends Controller { + + @Inject + GPGController(RequestContext ctx) { + super(ctx); + } + + @Override + public void index() { + setTitle("GPG"); + render(GPGOverviewPage.class); + } + + public void overview() { + setTitle("GPG Details"); + render(GPGOverviewPage.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewBlock.java new file mode 100644 index 0000000000000..6e5e463d5f067 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewBlock.java @@ -0,0 +1,49 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import java.util.Date; + +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator; +import org.apache.hadoop.yarn.util.YarnVersionInfo; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.InfoBlock; + +import com.google.inject.Inject; + +/** + * Overview block for the GPG Web UI. + */ +public class GPGOverviewBlock extends HtmlBlock { + + @Inject + GPGOverviewBlock(GlobalPolicyGenerator gpg, ViewContext ctx) { + super(ctx); + } + + @Override + protected void render(Block html) { + info("GPG Details") + .__("GPG started on", new Date(GlobalPolicyGenerator.getGPGStartupTime())) + .__("GPG Version", YarnVersionInfo.getVersion()) + .__("Hadoop Version", VersionInfo.getVersion()); + + html.__(InfoBlock.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewPage.java new file mode 100644 index 0000000000000..b7086ea71ae5e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewPage.java @@ -0,0 +1,52 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; + +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout; + +/** + * Overview page for the GPG Web UI. + */ +public class GPGOverviewPage extends TwoColumnLayout { + + @Override + protected void preHead(Page.HTML<__> html) { + commonPreHead(html); + setTitle("GPG"); + } + + protected void commonPreHead(Page.HTML<__> html) { + set(ACCORDION_ID, "nav"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); + } + + @Override + protected Class nav() { + return NavBlock.class; + } + + @Override + protected Class content() { + return GPGOverviewBlock.class; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java new file mode 100644 index 0000000000000..eedb1790019eb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java @@ -0,0 +1,45 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.WebApp; + +/** + * The GPG webapp. + */ +public class GPGWebApp extends WebApp{ + private GlobalPolicyGenerator gpg; + + public GPGWebApp(GlobalPolicyGenerator gpg) { + this.gpg = gpg; + } + + @Override + public void setup() { + bind(JAXBContextResolver.class); + bind(GPGWebApp.class).toInstance(this); + bind(GenericExceptionHandler.class); + if (gpg != null) { + bind(GlobalPolicyGenerator.class).toInstance(gpg); + } + route("/", GPGController.class, "overview"); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java new file mode 100644 index 0000000000000..1fee9d08e8c1e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java @@ -0,0 +1,42 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; + +/** + * Navigation block for the GPG Web UI. + */ +public class NavBlock extends HtmlBlock { + + @Override + public void render(Block html) { + html. + div("#nav"). + h3("GPG"). + ul(). + li().a(url(""), "Overview").__(). + __(). + h3("Tools"). + ul(). + li().a("/conf", "Configuration").__(). + li().a("/logs", "Local logs").__(). + li().a("/stacks", "Server stacks").__(). + li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/package-info.java new file mode 100644 index 0000000000000..762212b49a124 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/package-info.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Classes comprising the policy generator for the GPG. Responsibilities include + * generating and updating policies based on the cluster status. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java index f657b86c83628..bc9a325c47ebd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java @@ -19,9 +19,14 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.Test; +import java.util.List; +import java.util.concurrent.TimeoutException; + /** * Unit test for GlobalPolicyGenerator. */ @@ -35,4 +40,19 @@ public void testNonFederation() { // If GPG starts running, this call will not return GlobalPolicyGenerator.startGPG(new String[0], conf); } + + @Test + public void testGpgWithFederation() throws InterruptedException, TimeoutException { + // In this test case, we hope that gpg can start normally in federation mode. + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + + GlobalPolicyGenerator gpg = new GlobalPolicyGenerator(); + gpg.initAndStart(conf, false); + + GenericTestUtils.waitFor(() -> { + List services = gpg.getServices(); + return (services.size() == 1 && gpg.getWebApp() != null); + }, 100, 5000); + } } From 655c3df0508ca4f3a89b57eddc61454bf2d92ccf Mon Sep 17 00:00:00 2001 From: zhangshuyan <81411509+zhangshuyan0@users.noreply.github.com> Date: Tue, 15 Aug 2023 19:54:15 +0800 Subject: [PATCH 23/48] HDFS-17150. EC: Fix the bug of failed lease recovery. (#5937). Contributed by Shuyan Zhang. Reviewed-by: hfutatzhanghb <1036798979@qq.com> Reviewed-by: Haiyang Hu Signed-off-by: He Xiaoqiao --- .../hdfs/server/namenode/FSNamesystem.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 13a9809627e73..b41120aebfec2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -111,6 +111,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotDeletionGc; import org.apache.hadoop.thirdparty.protobuf.ByteString; @@ -3802,7 +3803,12 @@ boolean internalReleaseLease(Lease lease, String src, INodesInPath iip, lastBlock.getBlockType()); } - if (uc.getNumExpectedLocations() == 0 && lastBlock.getNumBytes() == 0) { + int minLocationsNum = 1; + if (lastBlock.isStriped()) { + minLocationsNum = ((BlockInfoStriped) lastBlock).getRealDataBlockNum(); + } + if (uc.getNumExpectedLocations() < minLocationsNum && + lastBlock.getNumBytes() == 0) { // There is no datanode reported to this block. // may be client have crashed before writing data to pipeline. // This blocks doesn't need any recovery. @@ -3810,8 +3816,18 @@ boolean internalReleaseLease(Lease lease, String src, INodesInPath iip, pendingFile.removeLastBlock(lastBlock); finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false); - NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: " - + "Removed empty last block and closed file " + src); + if (uc.getNumExpectedLocations() == 0) { + // If uc.getNumExpectedLocations() is 0, regardless of whether it + // is a striped block or not, we should consider it as an empty block. + NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: " + + "Removed empty last block and closed file " + src); + } else { + // If uc.getNumExpectedLocations() is greater than 0, it means that + // minLocationsNum must be greater than 1, so this must be a striped + // block. + NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: " + + "Removed last unrecoverable block group and closed file " + src); + } return true; } // Start recovery of the last block for this file From ea87aa2f5b2c4571677369e2ed96499d7a96b0d9 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 16 Aug 2023 04:28:29 +0800 Subject: [PATCH 24/48] YARN-11037. Add configurable logic to split resource request to the least loaded SC. (#5515) --- .../yarn/api/records/EnhancedHeadroom.java | 5 + .../hadoop/yarn/conf/YarnConfiguration.java | 39 +++ .../src/main/resources/yarn-default.xml | 80 ++++++ .../hadoop/yarn/server/AMRMClientRelayer.java | 55 ++++- .../server/ContainerAllocationHistory.java | 69 ++++++ .../LocalityMulticastAMRMProxyPolicy.java | 228 +++++++++++++++++- .../server/scheduler/ResourceRequestSet.java | 14 ++ .../scheduler/ResourceRequestSetKey.java | 48 ++++ .../uam/UnmanagedApplicationManager.java | 2 +- .../yarn/server/TestAMRMClientRelayer.java | 2 +- .../TestLocalityMulticastAMRMProxyPolicy.java | 96 ++++++++ .../metrics/TestAMRMClientRelayerMetrics.java | 4 +- .../amrmproxy/FederationInterceptor.java | 2 +- 13 files changed, 623 insertions(+), 21 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/ContainerAllocationHistory.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/EnhancedHeadroom.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/EnhancedHeadroom.java index 7a5ff6adcc5ec..3824d2cd8e811 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/EnhancedHeadroom.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/EnhancedHeadroom.java @@ -69,4 +69,9 @@ public String toString() { sb.append(">"); return sb.toString(); } + + public double getNormalizedPendingCount(long multiplier) { + int totalPendingCount = getTotalPendingCount(); + return (double) totalPendingCount * multiplier; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index de76369b01444..a753cda7908de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4058,6 +4058,45 @@ public static boolean isAclEnabled(Configuration conf) { public static final long DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT = 60000; // one minute + // Prefix for configs related to selecting SC based on load + public static final String LOAD_BASED_SC_SELECTOR_PREFIX = + NM_PREFIX + "least-load-policy-selector."; + + // Config to enable re-rerouting node requests base on SC load + public static final String LOAD_BASED_SC_SELECTOR_ENABLED = + LOAD_BASED_SC_SELECTOR_PREFIX + "enabled"; + public static final boolean DEFAULT_LOAD_BASED_SC_SELECTOR_ENABLED = false; + + // Pending container threshold for selecting SC + public static final String LOAD_BASED_SC_SELECTOR_THRESHOLD = + LOAD_BASED_SC_SELECTOR_PREFIX + "pending-container.threshold"; + public static final int DEFAULT_LOAD_BASED_SC_SELECTOR_THRESHOLD = 10000; + + // Whether to consider total number of active cores in the subcluster for load + public static final String LOAD_BASED_SC_SELECTOR_USE_ACTIVE_CORE = + LOAD_BASED_SC_SELECTOR_PREFIX + "use-active-core"; + public static final boolean DEFAULT_LOAD_BASED_SC_SELECTOR_USE_ACTIVE_CORE = false; + + // multiplier to normalize pending container to active cores + public static final String LOAD_BASED_SC_SELECTOR_MULTIPLIER = + LOAD_BASED_SC_SELECTOR_PREFIX + "multiplier"; + public static final int DEFAULT_LOAD_BASED_SC_SELECTOR_MULTIPLIER = 50000; + + // max count to maintain for container allocation history + public static final String FEDERATION_ALLOCATION_HISTORY_MAX_ENTRY = + FEDERATION_PREFIX + "amrmproxy.allocation.history.max.entry"; + public static final int DEFAULT_FEDERATION_ALLOCATION_HISTORY_MAX_ENTRY = 100; + + // Whether to fail directly if activeSubCluster is less than 1. + public static final String LOAD_BASED_SC_SELECTOR_FAIL_ON_ERROR = + LOAD_BASED_SC_SELECTOR_PREFIX + "fail-on-error"; + public static final boolean DEFAULT_LOAD_BASED_SC_SELECTOR_FAIL_ON_ERROR = true; + + // Blacklisted subClusters. + public static final String FEDERATION_BLACKLIST_SUBCLUSTERS = + LOAD_BASED_SC_SELECTOR_PREFIX + "blacklist-subclusters"; + public static final String DEFAULT_FEDERATION_BLACKLIST_SUBCLUSTERS = ""; + // AMRMProxy Register UAM Retry-Num public static final String FEDERATION_AMRMPROXY_REGISTER_UAM_RETRY_COUNT = FEDERATION_PREFIX + "amrmproxy.register.uam.retry-count"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index bec4393061411..a201f4b2345be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5558,4 +5558,84 @@ 0.0.0.0:8070 + + + This configuration will enable request rerouting according to the load of the subCluster. + If it is true, it will reroute the request according to the load of the subCluster. + The default configuration is false. + + yarn.nodemanager.least-load-policy-selector.enabled + false + + + + + SubCluster pending container threshold. The default value is 10000. + This configuration will determine the load weight of a subCluster. + For SC with pending containers count bigger than container threshold / 2, + use threshold / pending as weight. + For SC with pending containers count less than threshold / 2, we cap the weight at 2. + + yarn.nodemanager.least-load-policy-selector.pending-container.threshold + 10000 + + + + + Whether to consider the configured vcores when calculating the subCluster load. + The default value is false, we only consider the number of cluster pending containers. + If this configuration item is set to true, This configuration item needs to be used together + with yarn.nodemanager.least-load-policy-selector.multiplier. We will use the following formula + when calculating subCluster pending. + pendingContainersCountNormalize = (totalPendingContainersCount * multiplier) / totalActiveCores. + + yarn.nodemanager.least-load-policy-selector.use-active-core + false + + + + + Max count to maintain for container allocation history. + + yarn.federation.amrmproxy.allocation.history.max.entry + 100 + + + + + Whether to fail directly if activeSubCluster is less than 1. + The default is true. + If We set to false, We will try to re-fetch activeSubCluster list. + + yarn.nodemanager.least-load-policy-selector.fail-on-error + true + + + + + The subCluster configured in the blacklist will not be forwarded requests. + The default value is empty. + + yarn.nodemanager.least-load-policy-selector.blacklist-subclusters + + + + + + Max count to maintain for container allocation history. + + yarn.federation.amrmproxy.allocation.history.max.entry + 100 + + + + + This configuration will be used + when yarn.nodemanager.least-load-policy-selector.use-active-core is set to true. + The purpose of this value is to help normalize the pendingContainersCount. + + yarn.nodemanager.least-load-policy-selector.multiplier + 50000 + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java index 83977245757c9..fdee94d2f9da4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java @@ -28,6 +28,7 @@ import java.util.TreeSet; import org.apache.hadoop.HadoopIllegalArgumentException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; @@ -132,16 +133,23 @@ public class AMRMClientRelayer implements ApplicationMasterProtocol { private AMRMClientRelayerMetrics metrics; + private ContainerAllocationHistory allocationHistory; + public AMRMClientRelayer(ApplicationMasterProtocol rmClient, ApplicationId appId, String rmId) { this.resetResponseId = -1; this.metrics = AMRMClientRelayerMetrics.getInstance(); - this.rmId = ""; this.rmClient = rmClient; this.appId = appId; this.rmId = rmId; } + public AMRMClientRelayer(ApplicationMasterProtocol rmClient, + ApplicationId appId, String rmId, Configuration conf) { + this(rmClient, appId, rmId); + this.allocationHistory = new ContainerAllocationHistory(conf); + } + public void setAMRegistrationRequest( RegisterApplicationMasterRequest registerRequest) { this.amRegistrationRequest = registerRequest; @@ -444,6 +452,8 @@ private void updateMetrics(AllocateResponse allocateResponse, if (this.knownContainers.add(container.getId())) { this.metrics.addFulfilledQPS(this.rmId, AMRMClientRelayerMetrics .getRequestType(container.getExecutionType()), 1); + long currentTime = System.currentTimeMillis(); + long fulfillLatency = -1; if (container.getAllocationRequestId() != 0) { Integer count = this.pendingCountForMetrics .get(container.getAllocationRequestId()); @@ -453,13 +463,14 @@ private void updateMetrics(AllocateResponse allocateResponse, this.metrics.decrClientPending(this.rmId, AMRMClientRelayerMetrics .getRequestType(container.getExecutionType()), 1); - this.metrics.addFulfillLatency(this.rmId, - AMRMClientRelayerMetrics - .getRequestType(container.getExecutionType()), - System.currentTimeMillis() - this.askTimeStamp - .get(container.getAllocationRequestId())); + fulfillLatency = currentTime - this.askTimeStamp.get( + container.getAllocationRequestId()); + AMRMClientRelayerMetrics.RequestType requestType = AMRMClientRelayerMetrics + .getRequestType(container.getExecutionType()); + this.metrics.addFulfillLatency(this.rmId, requestType, fulfillLatency); } } + addAllocationHistoryEntry(container, currentTime, fulfillLatency); } } } @@ -576,6 +587,38 @@ private void addResourceRequestToAsk(ResourceRequest remoteRequest) { this.ask.add(remoteRequest); } + public ContainerAllocationHistory getAllocationHistory() { + return this.allocationHistory; + } + + private void addAllocationHistoryEntry(Container container, long fulfillTimeStamp, + long fulfillLatency) { + ResourceRequestSetKey key = ResourceRequestSetKey.extractMatchingKey(container, + this.remotePendingAsks.keySet()); + if (key == null) { + LOG.info("allocation history ignoring {}, no matching request key found.", container); + return; + } + this.allocationHistory.addAllocationEntry(container, this.remotePendingAsks.get(key), + fulfillTimeStamp, fulfillLatency); + } + + public void gatherReadOnlyPendingAsksInfo(Map pendingAsks, Map pendingTime) { + pendingAsks.clear(); + pendingTime.clear(); + synchronized (this) { + pendingAsks.putAll(this.remotePendingAsks); + for (ResourceRequestSetKey key : pendingAsks.keySet()) { + Long startTime = this.askTimeStamp.get(key.getAllocationRequestId()); + if (startTime != null) { + long elapsedMs = System.currentTimeMillis() - startTime; + pendingTime.put(key, elapsedMs); + } + } + } + } + @VisibleForTesting protected Map getRemotePendingAsks() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/ContainerAllocationHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/ContainerAllocationHistory.java new file mode 100644 index 0000000000000..77cb8faf82f11 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/ContainerAllocationHistory.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server; + +import java.util.AbstractMap; +import java.util.LinkedList; +import java.util.Map.Entry; +import java.util.Queue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.scheduler.ResourceRequestSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Records the allocation history from YarnRM and provide aggregated insights. + */ +public class ContainerAllocationHistory { + private static final Logger LOG = LoggerFactory.getLogger(AMRMClientRelayer.class); + + private int maxEntryCount; + + // Allocate timing history + private Queue> relaxableG = new LinkedList<>(); + + public ContainerAllocationHistory(Configuration conf) { + this.maxEntryCount = conf.getInt( + YarnConfiguration.FEDERATION_ALLOCATION_HISTORY_MAX_ENTRY, + YarnConfiguration.DEFAULT_FEDERATION_ALLOCATION_HISTORY_MAX_ENTRY); + } + + /** + * Record the allocation history for the container. + * + * @param container to add record for + * @param requestSet resource request ask set + * @param fulfillTimeStamp time at which allocation happened + * @param fulfillLatency time elapsed in allocating since asked + */ + public synchronized void addAllocationEntry(Container container, + ResourceRequestSet requestSet, long fulfillTimeStamp, long fulfillLatency){ + if (!requestSet.isANYRelaxable()) { + LOG.info("allocation history ignoring {}, relax locality is false", container); + return; + } + this.relaxableG.add(new AbstractMap.SimpleEntry<>( + fulfillTimeStamp, fulfillLatency)); + if (this.relaxableG.size() > this.maxEntryCount) { + this.relaxableG.remove(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java index 17cc8390a11a5..a98ec138f604a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.federation.policies.amrmproxy; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -31,6 +32,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.collections.MapUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.EnhancedHeadroom; import org.apache.hadoop.yarn.api.records.Resource; @@ -54,6 +58,19 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.util.Preconditions; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOAD_BASED_SC_SELECTOR_ENABLED; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_LOAD_BASED_SC_SELECTOR_ENABLED; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOAD_BASED_SC_SELECTOR_THRESHOLD; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_LOAD_BASED_SC_SELECTOR_THRESHOLD; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOAD_BASED_SC_SELECTOR_USE_ACTIVE_CORE; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_LOAD_BASED_SC_SELECTOR_USE_ACTIVE_CORE; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOAD_BASED_SC_SELECTOR_MULTIPLIER; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_LOAD_BASED_SC_SELECTOR_MULTIPLIER; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOAD_BASED_SC_SELECTOR_FAIL_ON_ERROR; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_LOAD_BASED_SC_SELECTOR_FAIL_ON_ERROR; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_BLACKLIST_SUBCLUSTERS; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_FEDERATION_BLACKLIST_SUBCLUSTERS; + /** * An implementation of the {@link FederationAMRMProxyPolicy} interface that * carefully multicasts the requests with the following behavior: @@ -131,11 +148,44 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy { private Map weights; private SubClusterResolver resolver; + private Configuration conf; private Map headroom; private Map enhancedHeadroom; private float hrAlpha; private FederationStateStoreFacade federationFacade; private SubClusterId homeSubcluster; + private int printRRMax; + public static final String PRINT_RR_MAX = + "yarn.nodemanager.amrmproxy.address.splitmerge.printmaxrrcount"; + public static final int DEFAULT_PRINT_RR_MAX = 1000; + private boolean failOnError = DEFAULT_LOAD_BASED_SC_SELECTOR_FAIL_ON_ERROR; + + /** + * Print a list of Resource Requests into a one line string. + * + * @param response list of ResourceRequest + * @param max number of ResourceRequest to print + * @return the printed one line string + */ + public static String prettyPrintRequests(List response, int max) { + StringBuilder builder = new StringBuilder(); + for (ResourceRequest rr : response) { + builder.append("[id:").append(rr.getAllocationRequestId()) + .append(" loc:") + .append(rr.getResourceName()) + .append(" num:") + .append(rr.getNumContainers()) + .append(" pri:") + .append(((rr.getPriority() != null) ? rr.getPriority().getPriority() : -1)) + .append("], "); + if (max != -1) { + if (max-- <= 0) { + break; + } + } + } + return builder.toString(); + } @Override public void reinitialize( @@ -182,6 +232,7 @@ public void reinitialize( weights = newWeightsConverted; resolver = policyContext.getFederationSubclusterResolver(); + // Data structures that only need to initialize once if (headroom == null) { headroom = new ConcurrentHashMap<>(); enhancedHeadroom = new ConcurrentHashMap<>(); @@ -191,6 +242,11 @@ public void reinitialize( this.federationFacade = policyContext.getFederationStateStoreFacade(); this.homeSubcluster = policyContext.getHomeSubcluster(); + + this.conf = this.federationFacade.getConf(); + this.printRRMax = this.conf.getInt(PRINT_RR_MAX, DEFAULT_PRINT_RR_MAX); + this.failOnError = this.conf.getBoolean(LOAD_BASED_SC_SELECTOR_FAIL_ON_ERROR, + DEFAULT_LOAD_BASED_SC_SELECTOR_FAIL_ON_ERROR); } @Override @@ -217,10 +273,9 @@ public Map> splitResourceRequests( // active subclusters. Create a new instance per call because this method // can be called concurrently. AllocationBookkeeper bookkeeper = new AllocationBookkeeper(); - bookkeeper.reinitialize(getActiveSubclusters(), timedOutSubClusters); + bookkeeper.reinitialize(getActiveSubclusters(), timedOutSubClusters, conf); - List nonLocalizedRequests = - new ArrayList(); + List nonLocalizedRequests = new ArrayList<>(); SubClusterId targetId = null; Set targetIds = null; @@ -240,6 +295,17 @@ public Map> splitResourceRequests( // Handle "node" requests try { targetId = resolver.getSubClusterForNode(rr.getResourceName()); + + // If needed, re-reroute node requests base on SC load + boolean loadBasedSCSelectorEnabled = + conf.getBoolean(LOAD_BASED_SC_SELECTOR_ENABLED, DEFAULT_LOAD_BASED_SC_SELECTOR_ENABLED); + if (loadBasedSCSelectorEnabled) { + int maxPendingThreshold = conf.getInt(LOAD_BASED_SC_SELECTOR_THRESHOLD, + DEFAULT_LOAD_BASED_SC_SELECTOR_THRESHOLD); + targetId = routeNodeRequestIfNeeded(targetId, maxPendingThreshold, + bookkeeper.getActiveAndEnabledSC()); + } + LOG.debug("Node request {}", rr.getResourceName()); } catch (YarnException e) { // this might happen as we can't differentiate node from rack names // we log altogether later @@ -285,7 +351,16 @@ public Map> splitResourceRequests( // handle all non-localized requests (ANY) splitAnyRequests(nonLocalizedRequests, bookkeeper); - return bookkeeper.getAnswer(); + // Take the split result, feed into the askBalancer + Map> answer = bookkeeper.getAnswer(); + LOG.info("Before split {} RRs: {}", resourceRequests.size(), + prettyPrintRequests(resourceRequests, this.printRRMax)); + + for (Map.Entry> entry : bookkeeper.getAnswer().entrySet()) { + LOG.info("After split {} has {} RRs: {}", entry.getKey(), entry.getValue().size(), + prettyPrintRequests(entry.getValue(), this.printRRMax)); + } + return answer; } /** @@ -495,6 +570,120 @@ private float getHeadroomWeighting(SubClusterId targetId, return headroomWeighting; } + /** + * When certain subcluster is too loaded, reroute Node requests going there. + * + * @param targetId current subClusterId where request is sent + * @param maxThreshold threshold for Pending count + * @param activeAndEnabledSCs list of active sc + * @return subClusterId target sc id + */ + protected SubClusterId routeNodeRequestIfNeeded(SubClusterId targetId, + int maxThreshold, Set activeAndEnabledSCs) { + // If targetId is not in the active and enabled SC list, reroute the traffic + if (activeAndEnabledSCs.contains(targetId)) { + int targetPendingCount = getSubClusterLoad(targetId); + if (targetPendingCount == -1 || targetPendingCount < maxThreshold) { + return targetId; + } + } + SubClusterId scId = chooseSubClusterIdForMaxLoadSC(targetId, maxThreshold, activeAndEnabledSCs); + return scId; + } + + /** + * Check if the current target subcluster is over max load, and if it is + * reroute it. + * + * @param targetId the original target subcluster id + * @param maxThreshold the max load threshold to reroute + * @param activeAndEnabledSCs the list of active and enabled subclusters + * @return targetId if it is within maxThreshold, otherwise a new id + */ + private SubClusterId chooseSubClusterIdForMaxLoadSC(SubClusterId targetId, + int maxThreshold, Set activeAndEnabledSCs) { + ArrayList weight = new ArrayList<>(); + ArrayList scIds = new ArrayList<>(); + int targetLoad = getSubClusterLoad(targetId); + if (targetLoad == -1 || !activeAndEnabledSCs.contains(targetId)) { + // Probably a SC that's not active and enabled. Forcing a reroute + targetLoad = Integer.MAX_VALUE; + } + + /* + * Prepare the weight for a random draw among all known SCs. + * + * For SC with pending bigger than maxThreshold / 2, use maxThreshold / + * pending as weight. We multiplied by maxThreshold so that the weight + * won't be too small in value. + * + * For SC with pending less than maxThreshold / 2, we cap the weight at 2 + * = (maxThreshold / (maxThreshold / 2)) so that SC with small pending + * will not get a huge weight and thus get swamped. + */ + for (SubClusterId sc : activeAndEnabledSCs) { + int scLoad = getSubClusterLoad(sc); + if (scLoad > targetLoad) { + // Never mind if it is not the most loaded SC + return targetId; + } + if (scLoad <= maxThreshold / 2) { + weight.add(2f); + } else { + weight.add((float) maxThreshold / scLoad); + } + scIds.add(sc); + } + if (weights.size() == 0) { + return targetId; + } + return scIds.get(FederationPolicyUtils.getWeightedRandom(weight)); + } + + /** + * get the Load data of the subCluster. + * + * @param subClusterId subClusterId. + * @return The number of pending containers for the subCluster. + */ + private int getSubClusterLoad(SubClusterId subClusterId) { + EnhancedHeadroom headroomData = this.enhancedHeadroom.get(subClusterId); + if (headroomData == null) { + return -1; + } + + // Use new data from enhanced headroom + boolean useActiveCoreEnabled = conf.getBoolean(LOAD_BASED_SC_SELECTOR_USE_ACTIVE_CORE, + DEFAULT_LOAD_BASED_SC_SELECTOR_USE_ACTIVE_CORE); + + // If we consider the number of vCores in the subCluster + if (useActiveCoreEnabled) { + + // If the vcore of the subCluster is less than or equal to 0, + // it means that containers cannot be scheduled to this subCluster, + // and we will return a very large number, indicating that the subCluster is unavailable. + if (headroomData.getTotalActiveCores() <= 0) { + return Integer.MAX_VALUE; + } + + // Multiply by a constant factor, to ensure the numerator > denominator. + // We will normalize the PendingCount, using PendingCount * multiplier / TotalActiveCores. + long multiplier = conf.getLong(LOAD_BASED_SC_SELECTOR_MULTIPLIER, + DEFAULT_LOAD_BASED_SC_SELECTOR_MULTIPLIER); + double value = + headroomData.getNormalizedPendingCount(multiplier) / headroomData.getTotalActiveCores(); + if (value > Integer.MAX_VALUE) { + return Integer.MAX_VALUE; + } else { + return (int) value; + } + } else { + // If the number of vcores in the subCluster is not considered, + // we directly return the number of pending containers in the subCluster. + return headroomData.getTotalPendingCount(); + } + } + /** * This helper class is used to book-keep the requests made to each * subcluster, and maintain useful statistics to split ANY requests. @@ -523,8 +712,9 @@ protected final class AllocationBookkeeper { private void reinitialize( Map activeSubclusters, - Set timedOutSubClusters) throws YarnException { - if (activeSubclusters == null) { + Set timedOutSubClusters, Configuration pConf) throws YarnException { + + if (MapUtils.isEmpty(activeSubclusters)) { throw new YarnRuntimeException("null activeSubclusters received"); } @@ -548,10 +738,28 @@ private void reinitialize( } } + // subCluster blacklisting from configuration + String blacklistedSubClusters = pConf.get(FEDERATION_BLACKLIST_SUBCLUSTERS, + DEFAULT_FEDERATION_BLACKLIST_SUBCLUSTERS); + if (blacklistedSubClusters != null) { + Collection tempList = StringUtils.getStringCollection(blacklistedSubClusters); + for (String item : tempList) { + activeAndEnabledSC.remove(SubClusterId.newInstance(item.trim())); + } + } + if (activeAndEnabledSC.size() < 1) { - throw new NoActiveSubclustersException( - "None of the subclusters enabled in this policy (weight>0) are " - + "currently active we cannot forward the ResourceRequest(s)"); + String errorMsg = "None of the subClusters enabled in this Policy (weight > 0) are " + + "currently active we cannot forward the ResourceRequest(s)"; + if (failOnError) { + throw new NoActiveSubclustersException(errorMsg); + } else { + LOG.error(errorMsg + ", continuing by enabling all active subClusters."); + activeAndEnabledSC.addAll(activeSubclusters.keySet()); + for (SubClusterId sc : activeSubclusters.keySet()) { + policyWeights.put(sc, 1.0f); + } + } } Set tmpSCSet = new HashSet<>(activeAndEnabledSC); @@ -559,7 +767,7 @@ private void reinitialize( if (tmpSCSet.size() < 1) { LOG.warn("All active and enabled subclusters have expired last " - + "heartbeat time. Ignore the expiry check for this request"); + + "heartbeat time. Ignore the expiry check for this request."); } else { activeAndEnabledSC = tmpSCSet; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSet.java index ed615e85c9ca4..9f21077aa2e94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSet.java @@ -36,6 +36,8 @@ public class ResourceRequestSet { private ResourceRequestSetKey key; private int numContainers; + // Whether the ANY RR is relaxable + private boolean relaxable; // ResourceName -> RR private Map asks; @@ -49,6 +51,7 @@ public ResourceRequestSet(ResourceRequestSetKey key) throws YarnException { this.key = key; // leave it zero for now, as if it is a cancel this.numContainers = 0; + this.relaxable = true; this.asks = new HashMap<>(); } @@ -61,6 +64,7 @@ public ResourceRequestSet(ResourceRequestSet other) { this.key = other.key; this.numContainers = other.numContainers; this.asks = new HashMap<>(); + this.relaxable = other.relaxable; // The assumption is that the RR objects should not be modified without // making a copy this.asks.putAll(other.asks); @@ -86,6 +90,7 @@ public void addAndOverrideRR(ResourceRequest ask) throws YarnException { // For G requestSet, update the numContainers only for ANY RR if (ask.getResourceName().equals(ResourceRequest.ANY)) { this.numContainers = ask.getNumContainers(); + this.relaxable = ask.getRelaxLocality(); } } else { // The assumption we made about O asks is that all RR in a requestSet has @@ -182,6 +187,15 @@ public void setNumContainers(int newValue) throws YarnException { } } + /** + * Whether the request set is relaxable at ANY level. + * + * @return whether the request set is relaxable at ANY level + */ + public boolean isANYRelaxable() { + return this.relaxable; + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSetKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSetKey.java index 38162619c85f4..8e76f3f1c8922 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSetKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSetKey.java @@ -18,11 +18,16 @@ package org.apache.hadoop.yarn.server.scheduler; +import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Set; /** * The scheduler key for a group of {@link ResourceRequest}. @@ -32,6 +37,9 @@ */ public class ResourceRequestSetKey extends SchedulerRequestKey { + private static final Logger LOG = + LoggerFactory.getLogger(ResourceRequestSetKey.class); + // More ResourceRequest key fields on top of SchedulerRequestKey private final Resource resource; private final ExecutionType execType; @@ -123,6 +131,46 @@ public int compareTo(SchedulerRequestKey other) { return this.execType.compareTo(otherKey.execType); } + /** + * Extract the corresponding ResourceRequestSetKey for an allocated container + * from a given set. Return null if not found. + * + * @param container the allocated container + * @param keys the set of keys to look from + * @return ResourceRequestSetKey + */ + public static ResourceRequestSetKey extractMatchingKey(Container container, + Set keys) { + ResourceRequestSetKey resourceRequestSetKey = new ResourceRequestSetKey( + container.getAllocationRequestId(), container.getPriority(), + container.getResource(), container.getExecutionType()); + if (keys.contains(resourceRequestSetKey)) { + return resourceRequestSetKey; + } + + if (container.getAllocationRequestId() > 0) { + // If no exact match, look for the one with the same (non-zero) + // allocationRequestId + for (ResourceRequestSetKey candidate : keys) { + if (candidate.getAllocationRequestId() == container.getAllocationRequestId()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Using possible match for {} : {}", resourceRequestSetKey, candidate); + } + return candidate; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("not match found for container {}.", container.getId()); + for (ResourceRequestSetKey candidate : keys) { + LOG.debug("candidate set keys: {}.", candidate.toString()); + } + } + + return null; + } + @Override public String toString() { return "[id:" + getAllocationRequestId() + " p:" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java index 56be136bda789..66cc8f02790bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java @@ -140,7 +140,7 @@ public UnmanagedApplicationManager(Configuration conf, ApplicationId appId, this.userUgi = null; // Relayer's rmClient will be set after the RM connection is created this.rmProxyRelayer = - new AMRMClientRelayer(null, this.applicationId, rmName); + new AMRMClientRelayer(null, this.applicationId, rmName, this.conf); this.heartbeatHandler = createAMHeartbeatRequestHandler(this.conf, this.applicationId, this.rmProxyRelayer); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java index 46570a1465d8d..3f78e70b6c8bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java @@ -155,7 +155,7 @@ public void setup() throws YarnException, IOException { this.conf = new Configuration(); this.mockAMS = new MockApplicationMasterService(); - this.relayer = new AMRMClientRelayer(this.mockAMS, null, "TEST"); + this.relayer = new AMRMClientRelayer(this.mockAMS, null, "TEST", conf); this.relayer.registerApplicationMaster( RegisterApplicationMasterRequest.newInstance("", 0, "")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java index 2806ac0e61d59..d7102e03e738f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.records.EnhancedHeadroom; import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Resource; @@ -820,4 +821,99 @@ protected SubClusterId getSubClusterForUnResolvedRequest( return getHomeSubCluster(); } } + + /** + * Test the rerouting behavior when some subclusters are loaded. Make sure + * that the AMRMProxy rerouting decisions attempt to redirect requests + * to the least loaded subcluster when load thresholds are exceeded + */ + @Test + public void testLoadBasedSubClusterReroute() throws YarnException { + int pendingThreshold = 1000; + + LocalityMulticastAMRMProxyPolicy policy = (LocalityMulticastAMRMProxyPolicy) getPolicy(); + initializePolicy(); + + SubClusterId sc0 = SubClusterId.newInstance("0"); + SubClusterId sc1 = SubClusterId.newInstance("1"); + SubClusterId sc2 = SubClusterId.newInstance("2"); + SubClusterId sc3 = SubClusterId.newInstance("3"); + SubClusterId sc4 = SubClusterId.newInstance("4"); + + Set scList = new HashSet<>(); + scList.add(sc0); + scList.add(sc1); + scList.add(sc2); + scList.add(sc3); + scList.add(sc4); + + // This cluster is the most overloaded - 4 times the threshold. + policy.notifyOfResponse(sc0, + getAllocateResponseWithEnhancedHeadroom(4 * pendingThreshold, 0)); + + // This cluster is the most overloaded - 4 times the threshold. + policy.notifyOfResponse(sc1, + getAllocateResponseWithEnhancedHeadroom(4 * pendingThreshold, 0)); + + // This cluster is 2 times the threshold, but not the most loaded. + policy.notifyOfResponse(sc2, + getAllocateResponseWithEnhancedHeadroom(2 * pendingThreshold, 0)); + + // This cluster is at the threshold, but not the most loaded. + policy.notifyOfResponse(sc3, + getAllocateResponseWithEnhancedHeadroom(pendingThreshold, 0)); + + // This cluster has zero pending. + policy.notifyOfResponse(sc4, getAllocateResponseWithEnhancedHeadroom(0, 0)); + + // sc2, sc3 and sc4 should just return the original subcluster. + Assert.assertEquals( + policy.routeNodeRequestIfNeeded(sc2, pendingThreshold, scList), sc2); + Assert.assertEquals( + policy.routeNodeRequestIfNeeded(sc3, pendingThreshold, scList), sc3); + Assert.assertEquals( + policy.routeNodeRequestIfNeeded(sc4, pendingThreshold, scList), sc4); + + // sc0 and sc1 must select from sc0/sc1/sc2/sc3/sc4 according to weights + // 1/4, 1/4, 1/2, 1, 2. Let's run tons of random of samples, and verify that + // the proportion approximately holds. + Map counts = new HashMap<>(); + counts.put(sc0, 0); + counts.put(sc1, 0); + counts.put(sc2, 0); + counts.put(sc3, 0); + counts.put(sc4, 0); + + int n = 100000; + for (int i = 0; i < n; i++) { + SubClusterId selectedId = policy.routeNodeRequestIfNeeded(sc0, pendingThreshold, scList); + counts.put(selectedId, counts.get(selectedId) + 1); + + selectedId = policy.routeNodeRequestIfNeeded(sc1, pendingThreshold, scList); + counts.put(selectedId, counts.get(selectedId) + 1); + + // Also try a new SCId that's not active and enabled. Should be rerouted + // to sc0-4 with the same distribution as above + selectedId = policy.routeNodeRequestIfNeeded(SubClusterId.newInstance("10"), + pendingThreshold, scList); + counts.put(selectedId, counts.get(selectedId) + 1); + } + + // The probability should be 1/16, 1/16, 1/8, 1/4, 1/2R + Assert.assertEquals((double) counts.get(sc0) / n / 3, 1 / 16.0, 0.01); + Assert.assertEquals((double) counts.get(sc1) / n / 3, 1 / 16.0, 0.01); + Assert.assertEquals((double) counts.get(sc2) / n / 3, 1 / 8.0, 0.01); + Assert.assertEquals((double) counts.get(sc3) / n / 3, 1 / 4.0, 0.01); + Assert.assertEquals((double) counts.get(sc4) / n / 3, 1 / 2.0, 0.01); + + // Everything should be routed to these five active and enabled SCs + Assert.assertEquals(5, counts.size()); + } + + private AllocateResponse getAllocateResponseWithEnhancedHeadroom(int pending, int activeCores) { + return AllocateResponse.newInstance(0, null, null, + Collections.emptyList(), Resource.newInstance(0, 0), null, 10, null, + Collections.emptyList(), null, null, null, + EnhancedHeadroom.newInstance(pending, activeCores)); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java index 3e7c57a1dae76..65f928ae034c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java @@ -140,12 +140,12 @@ public void setup() throws YarnException, IOException { this.mockAMS = new MockApplicationMasterService(); this.homeRelayer = new AMRMClientRelayer(this.mockAMS, - ApplicationId.newInstance(0, 0), this.homeID); + ApplicationId.newInstance(0, 0), this.homeID, conf); this.homeRelayer.registerApplicationMaster( RegisterApplicationMasterRequest.newInstance("", 0, "")); this.uamRelayer = new AMRMClientRelayer(this.mockAMS, - ApplicationId.newInstance(0, 0), this.uamID); + ApplicationId.newInstance(0, 0), this.uamID, conf); this.uamRelayer.registerApplicationMaster( RegisterApplicationMasterRequest.newInstance("", 0, "")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java index a53efebb7f6e5..251e5fac64b7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java @@ -321,7 +321,7 @@ public void init(AMRMProxyApplicationContext appContext) { SubClusterId.newInstance(YarnConfiguration.getClusterId(conf)); this.homeRMRelayer = new AMRMClientRelayer(createHomeRMProxy(appContext, ApplicationMasterProtocol.class, appOwner), appId, - this.homeSubClusterId.toString()); + this.homeSubClusterId.toString(), conf); this.homeHeartbeatHandler = createHomeHeartbeatHandler(conf, appId, this.homeRMRelayer); From 10b1d7340b7567849605e283585d6a292a53b127 Mon Sep 17 00:00:00 2001 From: zhangshuyan <81411509+zhangshuyan0@users.noreply.github.com> Date: Wed, 16 Aug 2023 20:29:19 +0800 Subject: [PATCH 25/48] HDFS-17154. EC: Fix bug in updateBlockForPipeline after failover. (#5941). Contributed by Shuyan Zhang. Reviewed-by: Haiyang Hu Signed-off-by: He Xiaoqiao --- .../hdfs/protocol/LocatedStripedBlock.java | 6 ++++- .../hdfs/server/namenode/FSNamesystem.java | 23 +++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java index ad67e64fb65cd..a683a2b53ebd5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java @@ -36,7 +36,7 @@ public class LocatedStripedBlock extends LocatedBlock { private static final byte[] EMPTY_INDICES = {}; private static final Token EMPTY_TOKEN = new Token<>(); - private final byte[] blockIndices; + private byte[] blockIndices; private Token[] blockTokens; @SuppressWarnings({"unchecked"}) @@ -72,6 +72,10 @@ public byte[] getBlockIndices() { return this.blockIndices; } + public void setBlockIndices(byte[] blockIndices) { + this.blockIndices = blockIndices; + } + @Override public boolean isStriped() { return true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index b41120aebfec2..e08002a8dde9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -102,6 +102,7 @@ import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; +import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; @@ -5966,8 +5967,26 @@ LocatedBlock bumpBlockGenerationStamp(ExtendedBlock block, block.setGenerationStamp(nextGenerationStamp( blockManager.isLegacyBlock(block.getLocalBlock()))); - locatedBlock = BlockManager.newLocatedBlock( - block, file.getLastBlock(), null, -1); + BlockInfo lastBlockInfo = file.getLastBlock(); + locatedBlock = BlockManager.newLocatedBlock(block, lastBlockInfo, + null, -1); + if (lastBlockInfo.isStriped() && + ((BlockInfoStriped) lastBlockInfo).getTotalBlockNum() > + ((LocatedStripedBlock) locatedBlock).getBlockIndices().length) { + // The location info in BlockUnderConstructionFeature may not be + // complete after a failover, so we just return all block tokens for a + // striped block. This will disrupt the correspondence between + // LocatedStripedBlock.blockIndices and LocatedStripedBlock.locs, + // which is not used in client side. The correspondence between + // LocatedStripedBlock.blockIndices and LocatedBlock.blockToken is + // ensured. + byte[] indices = + new byte[((BlockInfoStriped) lastBlockInfo).getTotalBlockNum()]; + for (int i = 0; i < indices.length; ++i) { + indices[i] = (byte) i; + } + ((LocatedStripedBlock) locatedBlock).setBlockIndices(indices); + } blockManager.setBlockToken(locatedBlock, BlockTokenIdentifier.AccessMode.WRITE); } finally { From 911e9e0c012a9b0753a36777a38cb0fd8080c73f Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 16 Aug 2023 06:34:36 -0700 Subject: [PATCH 26/48] HADOOP-18832. Upgrade aws-java-sdk to 1.12.499 (#5908) Contributed by Viraj Jasani --- LICENSE-binary | 2 +- hadoop-project/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index b1a24c2bd57cb..e7f9edbdf76d0 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -215,7 +215,7 @@ com.aliyun:aliyun-java-sdk-kms:2.11.0 com.aliyun:aliyun-java-sdk-ram:3.1.0 com.aliyun:aliyun-java-sdk-sts:3.0.0 com.aliyun.oss:aliyun-sdk-oss:3.13.2 -com.amazonaws:aws-java-sdk-bundle:1.12.367 +com.amazonaws:aws-java-sdk-bundle:1.12.499 com.cedarsoftware:java-util:1.9.0 com.cedarsoftware:json-io:2.5.1 com.fasterxml.jackson.core:jackson-annotations:2.12.7 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 63c2a741c9e76..91cf8e013fd61 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -183,7 +183,7 @@ 1.3.1 1.0-beta-1 900 - 1.12.367 + 1.12.499 2.7.1 1.11.2 2.1 From 65e4a66e257a7a04ba36a6c3c0cf3139dffaee9f Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Thu, 17 Aug 2023 09:33:50 +0800 Subject: [PATCH 27/48] HDFS-17087. Add Throttler for datanode reading block (#5845) Reviewed-by: He Xiaoqiao Reviewed-by: Ayush Saxena Signed-off-by: Tao Li --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 + .../hadoop/hdfs/server/datanode/DataNode.java | 18 +++- .../hdfs/server/datanode/DataXceiver.java | 3 +- .../server/datanode/DataXceiverServer.java | 19 +++++ .../src/main/resources/hdfs-default.xml | 9 ++ .../datanode/TestDataNodeReconfiguration.java | 22 +++++ .../datanode/TestDataTransferThrottler.java | 83 +++++++++++++++++++ .../hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- 8 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataTransferThrottler.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 262729c3f836d..7c6683dfc176b 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -123,6 +123,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.datanode.data.write.bandwidthPerSec"; // A value of zero indicates no limit public static final long DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_DEFAULT = 0; + public static final String DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY = + "dfs.datanode.data.read.bandwidthPerSec"; + // A value of zero indicates no limit + public static final long DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_DEFAULT = 0; public static final String DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_KEY = "dfs.datanode.ec.reconstruct.read.bandwidthPerSec"; public static final long DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_DEFAULT = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index bb61e8037e9f5..42082019bad20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -57,6 +57,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT; @@ -368,7 +370,8 @@ public class DataNode extends ReconfigurableBase DFS_DISK_BALANCER_ENABLED, DFS_DISK_BALANCER_PLAN_VALID_INTERVAL, DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY, - DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY)); + DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY, + DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY)); public static final String METRICS_LOG_NAME = "DataNodeMetricsLog"; @@ -702,6 +705,7 @@ public String reconfigurePropertyImpl(String property, String newVal) case DFS_DATANODE_MAX_RECEIVER_THREADS_KEY: case DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY: case DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY: + case DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY: return reconfDataXceiverParameters(property, newVal); case DFS_CACHEREPORT_INTERVAL_MSEC_KEY: return reconfCacheReportParameters(property, newVal); @@ -765,6 +769,18 @@ private String reconfDataXceiverParameters(String property, String newVal) } result = Long.toString(bandwidthPerSec); getXferServer().setWriteThrottler(writeThrottler); + } else if (property.equals(DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY)) { + Preconditions.checkNotNull(getXferServer(), "DataXceiverServer has not been initialized."); + long bandwidthPerSec = (newVal == null ? DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_DEFAULT : + Long.parseLong(newVal)); + DataTransferThrottler readThrottler = null; + if (bandwidthPerSec > 0) { + readThrottler = new DataTransferThrottler(bandwidthPerSec); + } else { + bandwidthPerSec = 0; + } + result = Long.toString(bandwidthPerSec); + getXferServer().setReadThrottler(readThrottler); } LOG.info("RECONFIGURE* changed {} to {}", property, newVal); return result; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index d8c55a54d4ce7..e97e179702970 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -608,7 +608,8 @@ public void readBlock(final ExtendedBlock block, writeSuccessWithChecksumInfo(blockSender, new DataOutputStream(getOutputStream())); long beginRead = Time.monotonicNow(); - read = blockSender.sendBlock(out, baseStream, null); // send data + // send data + read = blockSender.sendBlock(out, baseStream, dataXceiverServer.getReadThrottler()); long duration = Time.monotonicNow() - beginRead; if (blockSender.didSendEntireByteRange()) { // If we sent the entire range, then we should expect the client diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java index 9b31dd3b21855..ad61f94b9252a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java @@ -172,6 +172,8 @@ void release() { private volatile DataTransferThrottler writeThrottler; + private volatile DataTransferThrottler readThrottler; + /** * Stores an estimate for block size to check if the disk partition has enough * space. Newer clients pass the expected block size to the DataNode. For @@ -221,6 +223,15 @@ private void initBandwidthPerSec(Configuration conf) { } else { this.writeThrottler = null; } + + bandwidthPerSec = conf.getLongBytes( + DFSConfigKeys.DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY, + DFSConfigKeys.DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_DEFAULT); + if (bandwidthPerSec > 0) { + this.readThrottler = new DataTransferThrottler(bandwidthPerSec); + } else { + this.readThrottler = null; + } } @Override @@ -478,6 +489,10 @@ public DataTransferThrottler getWriteThrottler() { return writeThrottler; } + public DataTransferThrottler getReadThrottler() { + return readThrottler; + } + /** * Release a peer. * @@ -535,4 +550,8 @@ public void setTransferThrottler(DataTransferThrottler transferThrottler) { public void setWriteThrottler(DataTransferThrottler writeThrottler) { this.writeThrottler = writeThrottler; } + + public void setReadThrottler(DataTransferThrottler readThrottler) { + this.readThrottler = readThrottler; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 9a75ec24d2eda..8d85d7cc3c0af 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4721,6 +4721,15 @@ + + dfs.datanode.data.read.bandwidthPerSec + 0 + + Specifies the maximum amount of bandwidth that can utilize for reading block. + When the bandwidth value is zero, there is no limit. + + + dfs.datanode.ec.reconstruct.read.bandwidthPerSec 0 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java index 0ca5bedff8bbe..14c1c301b1bdd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java @@ -31,6 +31,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT; @@ -459,6 +460,13 @@ public void testDataXceiverReconfiguration() assertTrue("expecting NumberFormatException", expected.getCause() instanceof NumberFormatException); } + try { + dn.reconfigureProperty(DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY, "text"); + fail("ReconfigurationException expected"); + } catch (ReconfigurationException expected) { + assertTrue("expecting NumberFormatException", + expected.getCause() instanceof NumberFormatException); + } // Change properties and verify change. dn.reconfigureProperty(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, String.valueOf(123)); @@ -477,6 +485,12 @@ public void testDataXceiverReconfiguration() DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY), 1000, dn.getXferServer().getWriteThrottler().getBandwidth()); + dn.reconfigureProperty(DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY, + String.valueOf(1000)); + assertEquals(String.format("%s has wrong value", + DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY), + 1000, dn.getXferServer().getReadThrottler().getBandwidth()); + // Revert to default. dn.reconfigureProperty(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, null); assertEquals(String.format("%s has wrong value", DFS_DATANODE_MAX_RECEIVER_THREADS_KEY), @@ -500,6 +514,14 @@ public void testDataXceiverReconfiguration() assertNull(String.format("expect %s is not configured", DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY), dn.getConf().get(DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY)); + + dn.reconfigureProperty(DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY, null); + assertEquals(String.format("%s has wrong value", + DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY), + null, dn.getXferServer().getReadThrottler()); + assertNull(String.format("expect %s is not configured", + DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY), + dn.getConf().get(DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataTransferThrottler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataTransferThrottler.java new file mode 100644 index 0000000000000..a3e4222d75b66 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataTransferThrottler.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.hadoop.util.Time.monotonicNow; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY; +import static org.junit.Assert.assertTrue; + +/** + * Tests throttle the data transfers related functions. + */ +public class TestDataTransferThrottler { + + /** + * Test read data transfer throttler. + */ + @Test + public void testReadDataTransferThrottler() throws Exception { + final HdfsConfiguration conf = new HdfsConfiguration(); + + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) { + cluster.waitActive(); + final DistributedFileSystem fs = cluster.getFileSystem(); + + // Create file. + Path file = new Path("/test"); + long fileLength = 1024 * 1024 * 10 * 8; + DFSTestUtil.createFile(fs, file, fileLength, (short) 1, 0L); + DFSTestUtil.waitReplication(fs, file, (short) 1); + + DataNode dataNode = cluster.getDataNodes().get(0); + // DataXceiverServer#readThrottler is null if + // dfs.datanode.data.read.bandwidthPerSec default value is 0. + Assert.assertNull(dataNode.xserver.getReadThrottler()); + + // Read file. + Assert.assertEquals(fileLength, DFSTestUtil.readFileAsBytes(fs, file).length); + + // Set dfs.datanode.data.read.bandwidthPerSec. + long bandwidthPerSec = 1024 * 1024 * 8; + conf.setLong(DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY, bandwidthPerSec); + + // Restart the first datanode. + cluster.stopDataNode(0); + cluster.startDataNodes(conf, 1, true, null, null); + dataNode = cluster.getDataNodes().get(0); + Assert.assertEquals(bandwidthPerSec, dataNode.xserver.getReadThrottler().getBandwidth()); + + // Read file with throttler. + long start = monotonicNow(); + Assert.assertEquals(fileLength, DFSTestUtil.readFileAsBytes(fs, file).length); + long elapsedTime = monotonicNow() - start; + // Ensure throttler is effective, read 1024 * 1024 * 10 * 8 bytes, + // should take approximately 10 seconds (1024 * 1024 * 8 bytes per second). + long expectedElapsedTime = fileLength / bandwidthPerSec * 1000; // in milliseconds. + long acceptableError = 1000; // 1 milliseconds, allowing for a small margin of error. + assertTrue(elapsedTime >= expectedElapsedTime - acceptableError); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index d7fee5f1f809e..682e033bf298e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -346,7 +346,7 @@ public void testDataNodeGetReconfigurableProperties() throws IOException, Interr final List outs = Lists.newArrayList(); final List errs = Lists.newArrayList(); getReconfigurableProperties("datanode", address, outs, errs); - assertEquals(24, outs.size()); + assertEquals(25, outs.size()); assertEquals(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, outs.get(1)); } From 42b4525f75b828bf58170187f030b08622e238ab Mon Sep 17 00:00:00 2001 From: Chunyi Yang <32279893+chunyiyang@users.noreply.github.com> Date: Fri, 18 Aug 2023 01:56:34 +0900 Subject: [PATCH 28/48] HDFS-17156. Client may receive old state ID which will lead to inconsistent reads. (#5951) Reviewed-by: Simbarashe Dzinamarira Signed-off-by: Takanobu Asanuma --- .../java/org/apache/hadoop/ipc/Client.java | 2 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 40 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 0fb1fd7abffd6..4ccb4254c710f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -1214,10 +1214,10 @@ private void receiveRpcResponse() { if (status == RpcStatusProto.SUCCESS) { Writable value = packet.newInstance(valueClass, conf); final Call call = calls.remove(callId); - call.setRpcResponse(value); if (call.alignmentContext != null) { call.alignmentContext.receiveResponseState(header); } + call.setRpcResponse(value); } // verify that packet length was correct if (packet.remaining() > 0) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 394bcfc89690c..7cfd65d482129 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -82,6 +82,7 @@ import org.apache.hadoop.ipc.RPC.RpcKind; import org.apache.hadoop.ipc.Server.Call; import org.apache.hadoop.ipc.Server.Connection; +import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos; import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto; import org.apache.hadoop.net.ConnectTimeoutException; import org.apache.hadoop.net.NetUtils; @@ -162,9 +163,15 @@ static LongWritable call(Client client, long param, InetSocketAddress addr, static LongWritable call(Client client, LongWritable param, InetSocketAddress addr, int rpcTimeout, Configuration conf) throws IOException { + return call(client, param, addr, rpcTimeout, conf, null); + } + + static LongWritable call(Client client, LongWritable param, + InetSocketAddress addr, int rpcTimeout, Configuration conf, AlignmentContext alignmentContext) + throws IOException { final ConnectionId remoteId = getConnectionId(addr, rpcTimeout, conf); return (LongWritable)client.call(RPC.RpcKind.RPC_BUILTIN, param, remoteId, - RPC.RPC_SERVICE_CLASS_DEFAULT, null); + RPC.RPC_SERVICE_CLASS_DEFAULT, null, alignmentContext); } static class TestServer extends Server { @@ -1330,6 +1337,37 @@ public void run() { server.stop(); } } + + /** + * Verify that stateID is received into call before + * caller is notified. + * @throws IOException + */ + @Test(timeout=60000) + public void testReceiveStateBeforeCallerNotification() throws IOException { + AtomicBoolean stateReceived = new AtomicBoolean(false); + AlignmentContext alignmentContext = Mockito.mock(AlignmentContext.class); + Mockito.doAnswer((Answer) invocation -> { + Thread.sleep(1000); + stateReceived.set(true); + return null; + }).when(alignmentContext) + .receiveResponseState(any(RpcHeaderProtos.RpcResponseHeaderProto.class)); + + final Client client = new Client(LongWritable.class, conf); + final TestServer server = new TestServer(1, false); + + try { + InetSocketAddress addr = NetUtils.getConnectAddress(server); + server.start(); + call(client, new LongWritable(RANDOM.nextLong()), addr, + 0, conf, alignmentContext); + Assert.assertTrue(stateReceived.get()); + } finally { + client.stop(); + server.stop(); + } + } /** A dummy protocol */ interface DummyProtocol { From ebc32fbcaf6ae18622f44e1b6088253a6a6e4f9e Mon Sep 17 00:00:00 2001 From: Benjamin Teke Date: Fri, 18 Aug 2023 12:01:56 +0200 Subject: [PATCH 29/48] YARN-11551. RM format-conf-store should delete all the content of ZKConfigStore (#5958) --- .../scheduler/capacity/conf/ZKConfigurationStore.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index 492ff43dce72b..ad0a4a4c6293b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -64,6 +64,7 @@ public class ZKConfigurationStore extends YarnConfigurationStore { private static final String CONF_VERSION_PATH = "CONF_VERSION"; private static final String NODEEXISTS_MSG = "Encountered NodeExists error." + " Skipping znode creation since another RM has already created it"; + private String znodeParentPath; private String zkVersionPath; private String logsPath; private String confStorePath; @@ -78,7 +79,7 @@ public void initialize(Configuration config, Configuration schedConf, RMContext rmContext) throws Exception { this.conf = config; - String znodeParentPath = conf.get( + this.znodeParentPath = conf.get( YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH, YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH); @@ -144,7 +145,7 @@ public Version getConfStoreVersion() throws Exception { @Override public void format() throws Exception { - zkManager.delete(confStorePath); + zkManager.delete(znodeParentPath); } @Override From 136111314deb0b14d77f25ca4ece62191aac8c89 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sat, 19 Aug 2023 10:29:26 +0800 Subject: [PATCH 30/48] YARN-11154. Make router support proxy server. (#5946) --- .../hadoop/yarn/conf/YarnConfiguration.java | 3 + .../src/main/resources/yarn-default.xml | 8 + .../hadoop/yarn/server/router/Router.java | 31 ++ .../yarn/server/router/RouterServerUtil.java | 5 + .../clientrm/RouterClientRMService.java | 43 ++- .../router/webapp/TestRouterWebAppProxy.java | 298 ++++++++++++++++++ 6 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebAppProxy.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a753cda7908de..e252590ea3188 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4401,6 +4401,9 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + public static final String ROUTER_WEBAPP_PROXY_ENABLE = ROUTER_WEBAPP_PREFIX + "proxy.enable"; + public static final boolean DEFAULT_ROUTER_WEBAPP_PROXY_ENABLE = true; + private static final String FEDERATION_GPG_PREFIX = FEDERATION_PREFIX + "gpg."; // The number of threads to use for the GPG scheduled executor service diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a201f4b2345be..408295b8f4844 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5264,6 +5264,14 @@ false + + + Whether to enable proxy service in router. Default is true. + + yarn.router.webapp.proxy.enable + true + + The number of threads to use for the GPG scheduled executor service. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index 77abf18bd5fe4..601e4595558cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -44,6 +45,10 @@ import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService; import org.apache.hadoop.yarn.server.router.rmadmin.RouterRMAdminService; import org.apache.hadoop.yarn.server.router.webapp.RouterWebApp; +import org.apache.hadoop.yarn.server.webproxy.FedAppReportFetcher; +import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; +import org.apache.hadoop.yarn.server.webproxy.WebAppProxy; +import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.WebApps.Builder; @@ -91,6 +96,7 @@ public class Router extends CompositeService { @VisibleForTesting protected String webAppAddress; private static long clusterTimeStamp = System.currentTimeMillis(); + private FedAppReportFetcher fetcher = null; /** * Priority of the Router shutdown hook. @@ -209,9 +215,29 @@ public void startWepApp() { Builder builder = WebApps.$for("cluster", null, null, "ws").with(conf).at(webAppAddress); + if (RouterServerUtil.isRouterWebProxyEnable(conf)) { + fetcher = new FedAppReportFetcher(conf); + builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME, ProxyUriUtils.PROXY_PATH_SPEC, + WebAppProxyServlet.class); + builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher); + String proxyHostAndPort = getProxyHostAndPort(conf); + String[] proxyParts = proxyHostAndPort.split(":"); + builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]); + } webApp = builder.start(new RouterWebApp(this)); } + public static String getProxyHostAndPort(Configuration conf) { + String addr = conf.get(YarnConfiguration.PROXY_ADDRESS); + if(addr == null || addr.isEmpty()) { + InetSocketAddress address = conf.getSocketAddr(YarnConfiguration.ROUTER_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_PORT); + addr = WebAppUtils.getResolvedAddress(address); + } + return addr; + } + public static void main(String[] argv) { Configuration conf = new YarnConfiguration(); Thread @@ -267,4 +293,9 @@ private String getHostName(Configuration config) public static long getClusterTimeStamp() { return clusterTimeStamp; } + + @VisibleForTesting + public FedAppReportFetcher getFetcher() { + return fetcher; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index dff135bdb05a5..94dbcdce1ae2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -801,4 +801,9 @@ public static ApplicationSubmissionContext getTrimmedAppSubmissionContext( return trimmedContext; } + + public static boolean isRouterWebProxyEnable(Configuration conf) { + return conf.getBoolean(YarnConfiguration.ROUTER_WEBAPP_PROXY_ENABLE, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_PROXY_ENABLE); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index e3e84079b7100..dd87bee0d3116 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -19,11 +19,14 @@ package org.apache.hadoop.yarn.server.router.clientrm; import java.io.IOException; +import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.URL; import java.util.Collections; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -104,11 +107,13 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,6 +143,7 @@ public class RouterClientRMService extends AbstractService // and remove the oldest used ones. private Map userPipelineMap; + private URL redirectURL; private RouterDelegationTokenSecretManager routerDTSecretManager; public RouterClientRMService() { @@ -157,6 +163,10 @@ protected void serviceStart() throws Exception { YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_ADDRESS, YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_PORT); + if (RouterServerUtil.isRouterWebProxyEnable(conf)) { + redirectURL = getRedirectURL(); + } + int maxCacheSize = conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); @@ -318,7 +328,22 @@ public GetClusterNodeLabelsResponse getClusterNodeLabels( public GetApplicationReportResponse getApplicationReport( GetApplicationReportRequest request) throws YarnException, IOException { RequestInterceptorChainWrapper pipeline = getInterceptorChain(); - return pipeline.getRootInterceptor().getApplicationReport(request); + GetApplicationReportResponse response = pipeline.getRootInterceptor() + .getApplicationReport(request); + if (RouterServerUtil.isRouterWebProxyEnable(getConfig())) { + // After redirect url, tracking url in application report will + // redirect to embeded proxy server of router + URL url = new URL(response.getApplicationReport().getTrackingUrl()); + String redirectUrl = new URL(redirectURL.getProtocol(), + redirectURL.getHost(), redirectURL.getPort(), url.getFile()) + .toString(); + if (LOG.isDebugEnabled()) { + LOG.debug("The tracking url of application {} is redirect from {} to {}", + response.getApplicationReport().getApplicationId(), url, redirectUrl); + } + response.getApplicationReport().setTrackingUrl(redirectUrl); + } + return response; } @Override @@ -623,4 +648,20 @@ public void initUserPipelineMap(Configuration conf) { YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true)); } + + private URL getRedirectURL() throws Exception { + Configuration conf = getConfig(); + String webAppAddress = WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.ROUTER_BIND_HOST, + WebAppUtils.getRouterWebAppURLWithoutScheme(conf)); + String[] hostPort = StringUtils.split(webAppAddress, ':'); + if (hostPort.length != 2) { + throw new YarnRuntimeException("Router can't get valid redirect proxy url"); + } + String host = hostPort[0]; + int port = Integer.parseInt(hostPort[1]); + if (StringUtils.isBlank(host) || host.equals("0.0.0.0")) { + host = InetAddress.getLocalHost().getCanonicalHostName(); + } + return new URL(YarnConfiguration.useHttps(this.getConfig()) ? "https" : "http", host, port, ""); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebAppProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebAppProxy.java new file mode 100644 index 0000000000000..244ed132410db --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebAppProxy.java @@ -0,0 +1,298 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.webapp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor; +import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService.RequestInterceptorChainWrapper; +import org.apache.hadoop.yarn.server.webproxy.FedAppReportFetcher; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +public class TestRouterWebAppProxy { + + private static final Logger LOG = LoggerFactory.getLogger(TestRouterWebAppProxy.class); + + public static final String AM_PREFIX = "AM"; + public static final String RM_PREFIX = "RM"; + public static final String AHS_PREFIX = "AHS"; + + /* + * Mocked Server is used for simulating the web of AppMaster, ResourceMamanger or TimelineServer. + * */ + private static Server mockServer; + private static int mockServerPort = 0; + + /** + * Simple http server. Server should send answer with status 200 + */ + @BeforeClass + public static void setUp() throws Exception { + mockServer = new Server(0); + ((QueuedThreadPool) mockServer.getThreadPool()).setMaxThreads(20); + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/"); + context.addServlet(new ServletHolder(new MockWebServlet(AM_PREFIX)), "/amweb/*"); + context.addServlet(new ServletHolder(new MockWebServlet(RM_PREFIX)), "/cluster/app/*"); + context.addServlet(new ServletHolder(new MockWebServlet(AHS_PREFIX)), + "/applicationhistory/app/*"); + mockServer.setHandler(context); + ((ServerConnector) mockServer.getConnectors()[0]).setHost("localhost"); + mockServer.start(); + mockServerPort = ((ServerConnector) mockServer.getConnectors()[0]).getLocalPort(); + LOG.info("Running embedded servlet container at: http://localhost:" + mockServerPort); + } + + @Test(timeout=10000) + public void testRouterWebAppProxyFed() throws Exception { + + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.ROUTER_WEBAPP_ADDRESS, "localhost:9090"); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); + conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:" + mockServerPort); + // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS + conf.setInt("hadoop.http.max.threads", 10); + + // Create sub cluster information. + SubClusterId subClusterId1 = SubClusterId.newInstance("scid1"); + SubClusterId subClusterId2 = SubClusterId.newInstance("scid2"); + SubClusterInfo subClusterInfo1 = SubClusterInfo.newInstance(subClusterId1, "10.0.0.1:1", + "10.0.0.1:1", "10.0.0.1:1", "localhost:" + mockServerPort, SubClusterState.SC_RUNNING, 0, + ""); + SubClusterInfo subClusterInfo2 = SubClusterInfo.newInstance(subClusterId2, "10.0.0.2:1", + "10.0.0.2:1", "10.0.0.2:1", "10.0.0.2:1", SubClusterState.SC_RUNNING, 0, ""); + + // App1 and App2 is running applications. + ApplicationId appId1 = ApplicationId.newInstance(0, 1); + ApplicationId appId2 = ApplicationId.newInstance(0, 2); + String appUrl1 = "http://localhost:" + mockServerPort + "/amweb/" + appId1; + String proxyAppUrl1 = "http://localhost:" + mockServerPort + "/proxy/" + appId1; + String appUrl2 = "http://localhost:" + mockServerPort + "/amweb/" + appId2; + String proxyAppUrl2 = "http://localhost:" + mockServerPort + "/proxy/" + appId2; + // App3 is accepted application, has not registered original url to am. + ApplicationId appId3 = ApplicationId.newInstance(0, 3); + String proxyAppUrl3 = "http://localhost:" + mockServerPort + "/proxy/" + appId3; + // App4 is finished application, has remove from rm, but not remove from timeline server. + ApplicationId appId4 = ApplicationId.newInstance(0, 4); + String proxyAppUrl4 = "http://localhost:" + mockServerPort + "/proxy/" + appId4; + + // Mock for application + ApplicationClientProtocol appManager1 = mock(ApplicationClientProtocol.class); + Mockito.when(appManager1.getApplicationReport(GetApplicationReportRequest.newInstance(appId1))) + .thenReturn(GetApplicationReportResponse.newInstance( + newApplicationReport(appId1, YarnApplicationState.RUNNING, proxyAppUrl1, appUrl1))); + Mockito.when(appManager1.getApplicationReport(GetApplicationReportRequest.newInstance(appId3))) + .thenReturn(GetApplicationReportResponse.newInstance( + newApplicationReport(appId3, YarnApplicationState.ACCEPTED, proxyAppUrl2, null))); + + ApplicationClientProtocol appManager2 = mock(ApplicationClientProtocol.class); + Mockito.when(appManager2.getApplicationReport(GetApplicationReportRequest.newInstance(appId2))) + .thenReturn(GetApplicationReportResponse.newInstance( + newApplicationReport(appId2, YarnApplicationState.RUNNING, proxyAppUrl3, appUrl2))); + Mockito.when(appManager2.getApplicationReport(GetApplicationReportRequest.newInstance(appId4))) + .thenThrow(new ApplicationNotFoundException("APP NOT FOUND")); + + ApplicationHistoryProtocol historyManager = mock(ApplicationHistoryProtocol.class); + Mockito.when( + historyManager.getApplicationReport(GetApplicationReportRequest.newInstance(appId4))) + .thenReturn(GetApplicationReportResponse.newInstance( + newApplicationReport(appId4, YarnApplicationState.FINISHED, proxyAppUrl4, null))); + + // Initial federation store. + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + facade.getStateStore() + .registerSubCluster(SubClusterRegisterRequest.newInstance(subClusterInfo1)); + facade.getStateStore() + .registerSubCluster(SubClusterRegisterRequest.newInstance(subClusterInfo2)); + facade.addApplicationHomeSubCluster( + ApplicationHomeSubCluster.newInstance(appId1, subClusterId1)); + facade.addApplicationHomeSubCluster( + ApplicationHomeSubCluster.newInstance(appId2, subClusterId2)); + facade.addApplicationHomeSubCluster( + ApplicationHomeSubCluster.newInstance(appId3, subClusterId1)); + facade.addApplicationHomeSubCluster( + ApplicationHomeSubCluster.newInstance(appId4, subClusterId2)); + + // Start router for test + Router router = new Router(); + router.init(conf); + router.start(); + String user = UserGroupInformation.getCurrentUser().getUserName(); + RequestInterceptorChainWrapper wrapper = mock(RequestInterceptorChainWrapper.class); + FederationClientInterceptor interceptor = mock(FederationClientInterceptor.class); + Mockito.when(interceptor.getApplicationReport(GetApplicationReportRequest.newInstance(appId1))) + .thenReturn(GetApplicationReportResponse.newInstance( + newApplicationReport(appId1, YarnApplicationState.RUNNING, proxyAppUrl1, appUrl1))); + Mockito.when(interceptor.getApplicationReport(GetApplicationReportRequest.newInstance(appId2))) + .thenReturn(GetApplicationReportResponse.newInstance( + newApplicationReport(appId2, YarnApplicationState.RUNNING, proxyAppUrl2, appUrl2))); + Mockito.when(interceptor.getApplicationReport(GetApplicationReportRequest.newInstance(appId3))) + .thenReturn(GetApplicationReportResponse.newInstance( + newApplicationReport(appId3, YarnApplicationState.ACCEPTED, proxyAppUrl3, null))); + Mockito.when(interceptor.getApplicationReport(GetApplicationReportRequest.newInstance(appId4))) + .thenReturn(GetApplicationReportResponse.newInstance( + newApplicationReport(appId4, YarnApplicationState.FINISHED, proxyAppUrl4, null))); + Mockito.when(wrapper.getRootInterceptor()).thenReturn(interceptor); + router.getClientRMProxyService().getUserPipelineMap().put(user, wrapper); + try { + // set Mocked rm and timeline + FedAppReportFetcher appReportFetcher = router.getFetcher(); + appReportFetcher.registerSubCluster(subClusterInfo1, appManager1); + appReportFetcher.registerSubCluster(subClusterInfo2, appManager2); + appReportFetcher.setHistoryManager(historyManager); + + // App1 is running in subcluster1, and original url is registered in rm of subCluster1. + // So router will get original url from rm by getApplicationReport. Then router + // will fetch the webapp directly. + GetApplicationReportResponse response = router.getClientRMProxyService() + .getApplicationReport(GetApplicationReportRequest.newInstance(appId1)); + URL url = new URL(response.getApplicationReport().getTrackingUrl()); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(AM_PREFIX + "/" + appId1, readResponse(conn)); + conn.disconnect(); + + // App2 is running in subcluster2, and original url is registered + // in rm of subCluster2. So router will get original url from rm by + // getApplicationReport. Then router will fetch the webapp directly. + response = router.getClientRMProxyService() + .getApplicationReport(GetApplicationReportRequest.newInstance(appId2)); + url = new URL(response.getApplicationReport().getTrackingUrl()); + conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(AM_PREFIX + "/" + appId2, readResponse(conn)); + conn.disconnect(); + + // App3 is accepted in subcluster1, and original url is not registered + // yet. So router will fetch the application web from rm. + response = router.getClientRMProxyService() + .getApplicationReport(GetApplicationReportRequest.newInstance(appId3)); + url = new URL(response.getApplicationReport().getTrackingUrl()); + conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(RM_PREFIX + "/" + appId3, readResponse(conn)); + conn.disconnect(); + + // App4 is finished in subcluster2, and have removed from rm, but not + // removed from timeline server. So rouer will fetch the + // application web from timeline server. + response = router.getClientRMProxyService() + .getApplicationReport(GetApplicationReportRequest.newInstance(appId4)); + url = new URL(response.getApplicationReport().getTrackingUrl()); + conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + assertEquals(AHS_PREFIX + "/" + appId4, readResponse(conn)); + conn.disconnect(); + } finally { + router.close(); + } + } + + private ApplicationReport newApplicationReport(ApplicationId appId, YarnApplicationState state, + String trackingUrl, String origTrackingUrl) { + return ApplicationReport.newInstance(appId, null, "testuser", null, null, null, 0, null, state, + null, trackingUrl, 0, 0, 0, null, null, origTrackingUrl, 0f, null, null); + } + + private String readResponse(HttpURLConnection conn) throws IOException { + InputStream input = conn.getInputStream(); + byte[] bytes = new byte[input.available()]; + input.read(bytes); + return new String(bytes); + } + + /* + * This servlet is used for simulate the web of AppMaster, ResourceManager, + * TimelineServer and so on. + * */ + public static class MockWebServlet extends HttpServlet { + + private String role; + + public MockWebServlet(String role) { + this.role = role; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + if (req.getPathInfo() != null) { + resp.getWriter().write(role + req.getPathInfo()); + } + resp.setStatus(HttpServletResponse.SC_OK); + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + InputStream is = req.getInputStream(); + OutputStream os = resp.getOutputStream(); + int c = is.read(); + while (c > -1) { + os.write(c); + c = is.read(); + } + is.close(); + os.close(); + resp.setStatus(HttpServletResponse.SC_OK); + } + } +} \ No newline at end of file From ff1570acfa567ac868fc2fcb1b515ecd73eda628 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:32:06 +0800 Subject: [PATCH 31/48] YARN-10218. [GPG] Support HTTPS in GPG. (#5945) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../globalpolicygenerator/GPGUtils.java | 20 ++++++++++++++++--- .../GlobalPolicyGenerator.java | 3 +++ .../policygenerator/PolicyGenerator.java | 7 +++---- .../policygenerator/TestPolicyGenerator.java | 13 ++++++++++-- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java index 2bb56caeffba2..a802e37979bb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java @@ -19,13 +19,20 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator; import static javax.servlet.http.HttpServletResponse.SC_OK; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.RM_WEB_SERVICE_PATH; +import static org.apache.hadoop.yarn.webapp.util.WebAppUtils.HTTPS_PREFIX; +import static org.apache.hadoop.yarn.webapp.util.WebAppUtils.HTTP_PREFIX; +import java.net.InetSocketAddress; import java.util.HashMap; import java.util.Map; import java.util.Set; import javax.ws.rs.core.MediaType; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; @@ -51,16 +58,23 @@ private GPGUtils() { * @param webAddr WebAddress. * @param path url path. * @param returnType return type. + * @param conf configuration. * @return response entity. */ - public static T invokeRMWebService(String webAddr, String path, final Class returnType) { + public static T invokeRMWebService(String webAddr, String path, final Class returnType, + Configuration conf) { Client client = Client.create(); T obj; - WebResource webResource = client.resource(webAddr); + // webAddr stores the form of host:port in subClusterInfo + InetSocketAddress socketAddress = NetUtils + .getConnectAddress(NetUtils.createSocketAddr(webAddr)); + String scheme = YarnConfiguration.useHttps(conf) ? HTTPS_PREFIX : HTTP_PREFIX; + String webAddress = scheme + socketAddress.getHostName() + ":" + socketAddress.getPort(); + WebResource webResource = client.resource(webAddress); ClientResponse response = null; try { - response = webResource.path("ws/v1/cluster").path(path) + response = webResource.path(RM_WEB_SERVICE_PATH).path(path) .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); if (response.getStatus() == SC_OK) { obj = response.getEntity(returnType); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index c24cedf95f62e..5f89052fd89eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.apache.hadoop.yarn.webapp.util.WebServiceClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -131,6 +132,7 @@ protected void serviceInit(Configuration conf) throws Exception { // super.serviceInit after all services are added super.serviceInit(conf); + WebServiceClient.initialize(conf); } @Override @@ -207,6 +209,7 @@ protected void serviceStop() throws Exception { } DefaultMetricsSystem.shutdown(); super.serviceStop(); + WebServiceClient.destroy(); } public String getName() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java index 3c94d6576e738..df28192a0c668 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java @@ -159,7 +159,7 @@ protected Map> getInfos( clusterInfo.put(sci.getSubClusterId(), new HashMap<>()); } Object ret = GPGUtils.invokeRMWebService(sci.getRMWebServiceAddress(), - e.getValue(), e.getKey()); + e.getValue(), e.getKey(), getConf()); clusterInfo.get(sci.getSubClusterId()).put(e.getKey(), ret); } } @@ -181,12 +181,11 @@ protected Map getSchedulerInfo( for (SubClusterInfo sci : activeSubClusters.values()) { SchedulerTypeInfo sti = GPGUtils .invokeRMWebService(sci.getRMWebServiceAddress(), - RMWSConsts.SCHEDULER, SchedulerTypeInfo.class); + RMWSConsts.SCHEDULER, SchedulerTypeInfo.class, getConf()); if(sti != null){ schedInfo.put(sci.getSubClusterId(), sti.getSchedulerInfo()); } else { - LOG.warn("Skipped null scheduler info from SubCluster " + sci - .getSubClusterId().toString()); + LOG.warn("Skipped null scheduler info from SubCluster {}.", sci.getSubClusterId()); } } return schedInfo; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java index 9893e85e56f29..5fcbea0760f2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java @@ -22,6 +22,7 @@ import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONUnmarshaller; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager; @@ -60,6 +61,7 @@ import javax.xml.bind.JAXBException; import java.io.IOException; import java.io.StringReader; +import java.net.InetSocketAddress; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -294,8 +296,10 @@ public void testCallRM() { resourceManager.start(); String rmAddress = WebAppUtils.getRMWebAppURLWithScheme(this.conf); - SchedulerTypeInfo sti = GPGUtils.invokeRMWebService(rmAddress, RMWSConsts.SCHEDULER, - SchedulerTypeInfo.class); + String webAppAddress = getServiceAddress(NetUtils.createSocketAddr(rmAddress)); + + SchedulerTypeInfo sti = GPGUtils.invokeRMWebService(webAppAddress, RMWSConsts.SCHEDULER, + SchedulerTypeInfo.class, this.conf); Assert.assertNotNull(sti); SchedulerInfo schedulerInfo = sti.getSchedulerInfo(); @@ -346,6 +350,11 @@ public void testCallRM() { Assert.assertEquals(20f, queueB3.getCapacity(), 0.00001); } + private String getServiceAddress(InetSocketAddress address) { + InetSocketAddress socketAddress = NetUtils.getConnectAddress(address); + return socketAddress.getAddress().getHostAddress() + ":" + socketAddress.getPort(); + } + /** * Testable policy generator overrides the methods that communicate * with the RM REST endpoint, allowing us to inject faked responses. From 7368226332709dbda98757f3588aaa9be83e0f1e Mon Sep 17 00:00:00 2001 From: WangYuanben <48795318+YuanbenWang@users.noreply.github.com> Date: Sun, 20 Aug 2023 10:12:21 +0800 Subject: [PATCH 32/48] YARN-11553. Change the time unit of scCleanerIntervalMs in Router. (#5965) --- .../main/java/org/apache/hadoop/yarn/server/router/Router.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index 601e4595558cc..993fb81f3a695 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -158,7 +158,7 @@ protected void serviceStart() throws Exception { ROUTER_DEREGISTER_SUBCLUSTER_ENABLED, DEFAULT_ROUTER_DEREGISTER_SUBCLUSTER_ENABLED); if (isDeregisterSubClusterEnabled) { long scCleanerIntervalMs = this.conf.getTimeDuration(ROUTER_SUBCLUSTER_CLEANER_INTERVAL_TIME, - DEFAULT_ROUTER_SUBCLUSTER_CLEANER_INTERVAL_TIME, TimeUnit.MINUTES); + DEFAULT_ROUTER_SUBCLUSTER_CLEANER_INTERVAL_TIME, TimeUnit.MILLISECONDS); this.scheduledExecutorService.scheduleAtFixedRate(this.subClusterCleaner, 0, scCleanerIntervalMs, TimeUnit.MILLISECONDS); LOG.info("Scheduled SubClusterCleaner With Interval: {}.", From 60867de422949be416948bd106419c771c7d13fd Mon Sep 17 00:00:00 2001 From: zhangshuyan <81411509+zhangshuyan0@users.noreply.github.com> Date: Mon, 21 Aug 2023 10:05:34 +0800 Subject: [PATCH 33/48] HDFS-17151. EC: Fix wrong metadata in BlockInfoStriped after recovery. (#5938). Contributed by Shuyan Zhang. Signed-off-by: He Xiaoqiao --- .../hadoop/hdfs/server/datanode/BlockRecoveryWorker.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java index e4861f9774870..249888c4642ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java @@ -468,9 +468,9 @@ protected void recover() throws IOException { // notify Namenode the new size and locations final DatanodeID[] newLocs = new DatanodeID[totalBlkNum]; final String[] newStorages = new String[totalBlkNum]; - for (int i = 0; i < blockIndices.length; i++) { - newLocs[blockIndices[i]] = DatanodeID.EMPTY_DATANODE_ID; - newStorages[blockIndices[i]] = ""; + for (int i = 0; i < newLocs.length; i++) { + newLocs[i] = DatanodeID.EMPTY_DATANODE_ID; + newStorages[i] = ""; } for (BlockRecord r : rurList) { int index = (int) (r.rInfo.getBlockId() & From 271b4b25cd04663edc043ad47a101b5f4238ab2c Mon Sep 17 00:00:00 2001 From: Susheel Gupta <38013283+susheelgupta7@users.noreply.github.com> Date: Mon, 21 Aug 2023 21:13:08 +0530 Subject: [PATCH 34/48] Revert "YARN-11535: Jackson-dataformat-yaml should be upgraded to 2.15.2 as it may cause transitive dependency issue with 2.12.7" (#5969) This reverts commit 35af8b9d021e240e5f0617fa332c00366d300be7. --- hadoop-project/pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 91cf8e013fd61..c837a6fc0d5df 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -71,7 +71,6 @@ 2.12.7 2.12.7.1 - 2.15.2 4.5.13 @@ -1304,7 +1303,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - ${jackson2.dataformat-yaml.version} + ${jackson2.version} org.mockito From 43c889636afe69260822522e592b5054fd02b42b Mon Sep 17 00:00:00 2001 From: Benjamin Teke Date: Tue, 22 Aug 2023 16:42:49 +0200 Subject: [PATCH 35/48] YARN-11535. Remove jackson-dataformat-yaml dependency. (#5970) --- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 ++++ hadoop-project/pom.xml | 5 ----- .../hadoop-yarn-server-resourcemanager/pom.xml | 4 ---- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index b362e001ea614..01ab5f41373f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -115,6 +115,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> io.netty netty-transport + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c837a6fc0d5df..9fca0fa159b87 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1300,11 +1300,6 @@ jackson-dataformat-cbor ${jackson2.version} - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson2.version} - org.mockito mockito-core diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 9d096d20c5fdd..d36ca02d7d099 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -330,10 +330,6 @@ com.fasterxml.jackson.core jackson-databind - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - From 70b6c155bcb9d802a0f4c1132c75bfef202d1829 Mon Sep 17 00:00:00 2001 From: suzu Date: Wed, 23 Aug 2023 19:38:07 +0900 Subject: [PATCH 36/48] HADOOP-18328. S3A to support S3 on Outposts (#4533) Contributed by Sotetsu Suzugamine --- .../org/apache/hadoop/fs/s3a/ArnResource.java | 10 ++++++-- .../apache/hadoop/fs/s3a/TestArnResource.java | 24 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java index 0294f7722905d..a85f26223ffcf 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java @@ -26,7 +26,8 @@ * Represents an Arn Resource, this can be an accesspoint or bucket. */ public final class ArnResource { - private final static String ACCESSPOINT_ENDPOINT_FORMAT = "s3-accesspoint.%s.amazonaws.com"; + private final static String S3_ACCESSPOINT_ENDPOINT_FORMAT = "s3-accesspoint.%s.amazonaws.com"; + private final static String S3_OUTPOSTS_ACCESSPOINT_ENDPOINT_FORMAT = "s3-outposts.%s.amazonaws.com"; /** * Resource name. @@ -69,6 +70,10 @@ private ArnResource(String name, String owner, String region, String partition, this.accessPointRegionKey = String.format("accesspoint-%s", region); } + private boolean isOutposts(){ + return fullArn.contains("s3-outposts"); + } + /** * Resource name. * @return resource name. @@ -106,7 +111,8 @@ public String getFullArn() { * @return resource endpoint. */ public String getEndpoint() { - return String.format(ACCESSPOINT_ENDPOINT_FORMAT, region); + String format = isOutposts() ? S3_OUTPOSTS_ACCESSPOINT_ENDPOINT_FORMAT : S3_ACCESSPOINT_ENDPOINT_FORMAT; + return String.format(format, region); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java index 36381bf14b169..c881aac35d94d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java @@ -56,7 +56,7 @@ public void parseAccessPointFromArn() throws IllegalArgumentException { String region = testPair[0]; String partition = testPair[1]; - ArnResource resource = getArnResourceFrom(partition, region, MOCK_ACCOUNT, accessPoint); + ArnResource resource = getArnResourceFrom(partition, "s3", region, MOCK_ACCOUNT, accessPoint); assertEquals("Access Point name does not match", accessPoint, resource.getName()); assertEquals("Account Id does not match", MOCK_ACCOUNT, resource.getOwnerAccountId()); assertEquals("Region does not match", region, resource.getRegion()); @@ -64,10 +64,10 @@ public void parseAccessPointFromArn() throws IllegalArgumentException { } @Test - public void makeSureEndpointHasTheCorrectFormat() { + public void makeSureS3EndpointHasTheCorrectFormat() { // Access point (AP) endpoints are different from S3 bucket endpoints, thus when using APs the // endpoints for the client are modified. This test makes sure endpoint is set up correctly. - ArnResource accessPoint = getArnResourceFrom("aws", "eu-west-1", MOCK_ACCOUNT, + ArnResource accessPoint = getArnResourceFrom("aws", "s3", "eu-west-1", MOCK_ACCOUNT, "test"); String expected = "s3-accesspoint.eu-west-1.amazonaws.com"; @@ -76,6 +76,19 @@ public void makeSureEndpointHasTheCorrectFormat() { .isEqualTo(expected); } + @Test + public void makeSureS3OutpostsEndpointHasTheCorrectFormat() { + // Access point (AP) endpoints are different from S3 bucket endpoints, thus when using APs the + // endpoints for the client are modified. This test makes sure endpoint is set up correctly. + ArnResource accessPoint = getArnResourceFrom("aws", "s3-outposts", "eu-west-1", MOCK_ACCOUNT, + "test"); + String expected = "s3-outposts.eu-west-1.amazonaws.com"; + + Assertions.assertThat(accessPoint.getEndpoint()) + .describedAs("Endpoint has invalid format. Access Point requests will not work") + .isEqualTo(expected); + } + @Test public void invalidARNsMustThrow() throws Exception { describe("Using an invalid ARN format must throw when initializing an ArnResource."); @@ -87,15 +100,16 @@ public void invalidARNsMustThrow() throws Exception { /** * Create an {@link ArnResource} from string components * @param partition - partition for ARN + * @param service - service for ARN * @param region - region for ARN * @param accountId - accountId for ARN * @param resourceName - ARN resource name * @return ArnResource described by its properties */ - private ArnResource getArnResourceFrom(String partition, String region, String accountId, + private ArnResource getArnResourceFrom(String partition, String service, String region, String accountId, String resourceName) { // arn:partition:service:region:account-id:resource-type/resource-id - String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId, + String arn = String.format("arn:%s:%s:%s:%s:accesspoint/%s", partition, service, region, accountId, resourceName); return ArnResource.accessPointFromArn(arn); From 439c8653cf7af7ae63be8ca84b1ac9b0113b330e Mon Sep 17 00:00:00 2001 From: tian bao <2011xuesong@gmail.com> Date: Thu, 24 Aug 2023 09:30:09 +0800 Subject: [PATCH 37/48] Add missing comments in StateStoreService (#5959) --- .../hdfs/server/federation/store/StateStoreService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java index ce460a87c8ee3..ff0ea486e216c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java @@ -64,6 +64,12 @@ *

  • File {@link * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl * StateStoreFileImpl} + *
  • FileSystem {@link + * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl + * StateStoreFileSystemImpl} + *
  • MySQL {@link + * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreMySQLImpl + * StateStoreMySQLImpl} *
  • ZooKeeper {@link * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl * StateStoreZooKeeperImpl} From ce5bc4891f1309915533fce5d64889bca703f538 Mon Sep 17 00:00:00 2001 From: Yuting Chen Date: Thu, 24 Aug 2023 02:16:10 -0700 Subject: [PATCH 38/48] HADOOP-18328. Add documentation for S3A support on S3 Outposts (#5976) Contributed by Yuting Chen --- .../site/markdown/tools/hadoop-aws/index.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index f0f6e926321eb..052d52e4a8904 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -1709,6 +1709,24 @@ the storage class you want. Please note that S3A does not support reading from archive storage classes at the moment. `AccessDeniedException` with InvalidObjectState will be thrown if you're trying to do so. +## Configuring S3A for S3 on Outposts + +S3A now supports [S3 on Outposts](https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html). +Accessing data through an access point is done by using its Amazon Resource Name (ARN), as opposed to just the bucket name. +The only supported storage class on Outposts is **OUTPOSTS**, and by default objects are encrypted with [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-outposts-data-encryption.html). +You can set the Access Point ARN property using the following per bucket configuration property: + +```xml + + fs.s3a.bucket.sample-outpost-bucket.accesspoint.arn + arn:aws:s3-outposts:region:account-id:outpost/outpost-id/accesspoint/accesspoint-name + Configure S3a traffic to use this S3 on Outposts Access Point ARN + +``` + +This configures access to the `sample-outpost-bucket` for S3A to go through the new Access Point ARN. So, for example `s3a://sample-outpost-bucket/key` will now use your configured ARN when getting data from S3 on Outpost instead of your bucket. + + ## How S3A writes data to S3 The original S3A client implemented file writes by From 2dfe947f202db7cd321b152d9c202a7c677c3293 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 25 Aug 2023 14:02:10 +0100 Subject: [PATCH 39/48] Revert "Add missing comments in StateStoreService (#5959)" This reverts commit 439c8653cf7af7ae63be8ca84b1ac9b0113b330e. --- .../hdfs/server/federation/store/StateStoreService.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java index ff0ea486e216c..ce460a87c8ee3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java @@ -64,12 +64,6 @@ *
  • File {@link * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl * StateStoreFileImpl} - *
  • FileSystem {@link - * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl - * StateStoreFileSystemImpl} - *
  • MySQL {@link - * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreMySQLImpl - * StateStoreMySQLImpl} *
  • ZooKeeper {@link * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl * StateStoreZooKeeperImpl} From 11404c57cbe4cd941548610bd532cfc0f86753cc Mon Sep 17 00:00:00 2001 From: tian bao <2011xuesong@gmail.com> Date: Thu, 24 Aug 2023 09:30:09 +0800 Subject: [PATCH 40/48] HDFS-17162. RBF: Add missing comments in StateStoreService #5959 Contributed by tian bao --- .../hdfs/server/federation/store/StateStoreService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java index ce460a87c8ee3..ff0ea486e216c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java @@ -64,6 +64,12 @@ *
  • File {@link * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl * StateStoreFileImpl} + *
  • FileSystem {@link + * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl + * StateStoreFileSystemImpl} + *
  • MySQL {@link + * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreMySQLImpl + * StateStoreMySQLImpl} *
  • ZooKeeper {@link * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl * StateStoreZooKeeperImpl} From 28d190b90416e9c4be402e7ef3d3c361a4f4a5a0 Mon Sep 17 00:00:00 2001 From: Mukund Thakur Date: Fri, 25 Aug 2023 12:23:17 -0500 Subject: [PATCH 41/48] HADOOP-18845. Add ability to configure s3 connection ttl using fs.s3a.connection.ttl (#5948) Contributed By: Mukund Thakur --- .../org/apache/hadoop/fs/s3a/Constants.java | 11 +++++++ .../org/apache/hadoop/fs/s3a/Invoker.java | 5 ++-- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 2 ++ .../tools/hadoop-aws/troubleshooting_s3a.md | 20 +++++++++++++ .../hadoop/fs/s3a/ITestS3AConfiguration.java | 30 +++++++++++++++++++ 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 6c1d6371d179d..4e35dc17317f6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -154,6 +154,17 @@ private Constants() { public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum"; public static final int DEFAULT_MAXIMUM_CONNECTIONS = 96; + /** + * Configuration option to configure expiration time of + * s3 http connection from the connection pool in milliseconds: {@value}. + */ + public static final String CONNECTION_TTL = "fs.s3a.connection.ttl"; + + /** + * Default value for {@code CONNECTION_TTL}: {@value}. + */ + public static final long DEFAULT_CONNECTION_TTL = 5 * 60_000; + // connect to s3 over ssl? public static final String SECURE_CONNECTIONS = "fs.s3a.connection.ssl.enabled"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java index 279bfeba98769..67c8e7d809cc8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java @@ -462,7 +462,7 @@ public T retryUntranslated( do { try { if (retryCount > 0) { - LOG.debug("retry #{}", retryCount); + LOG.debug("{} retry #{}", text, retryCount); } // execute the operation, returning if successful return operation.apply(); @@ -471,7 +471,8 @@ public T retryUntranslated( } // you only get here if the operation didn't complete // normally, hence caught != null - + LOG.debug("{} ; {}, ", text, caught.toString()); + LOG.trace("", caught); // translate the exception into an IOE for the retry logic IOException translated; if (caught instanceof IOException) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index e22433322c9c5..27f061482ca98 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -1299,6 +1299,8 @@ public static void initConnectionSettings(Configuration conf, ClientConfiguration awsConf) throws IOException { awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS, 1)); + awsConf.setConnectionTTL(longOption(conf, CONNECTION_TTL, + DEFAULT_CONNECTION_TTL, -1)); initProtocolSettings(conf, awsConf); awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES, 0)); diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 3cd3bb43c5daa..1ead08081f158 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -1782,6 +1782,26 @@ will attempt to retry the operation; it may just be a transient event. If there are many such exceptions in logs, it may be a symptom of connectivity or network problems. +The above error could be because of a stale http connections. The default value in AWS +SDK is set to -1 (infinite) which means the connection will be reused indefinitely. +We have introduced a new config `fs.s3a.connection.ttl` to configure this. +Tuning this setting down (together with an appropriately-low setting for Java's DNS cache TTL) +ensures that your application will quickly rotate over to new IP addresses when the +service begins announcing them through DNS, at the cost of having to re-establish new +connections more frequently. + +```xml + + fs.s3a.connection.ttl + 300000 + + Expiration time for a connection in the connection pool in milliseconds. + When a connection is retrieved from the connection pool, + this parameter is checked to see if the connection can be reused. + Default value is 5 minutes. + + +``` ### `AWSBadRequestException` IllegalLocationConstraintException/The unspecified location constraint is incompatible ``` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 26d00bc7d359a..ff75f6e26138d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -31,6 +31,8 @@ import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3native.S3xLoginHelper; import org.apache.hadoop.test.GenericTestUtils; + +import org.assertj.core.api.Assertions; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -511,6 +513,34 @@ public void testConfOptionPropagationToFS() throws Exception { assertOptionEquals(updated, "fs.s3a.propagation", "propagated"); } + @Test(timeout = 10_000L) + public void testConnectTtlPropagation() throws Exception { + Configuration config = new Configuration(false); + ClientConfiguration awsConf = new ClientConfiguration(); + initConnectionSettings(config, awsConf); + Assertions.assertThat(awsConf.getConnectionTTL()) + .describedAs("connection ttl should be set to default value as" + + " %s is not set", CONNECTION_TTL) + .isEqualTo(DEFAULT_CONNECTION_TTL); + long connectionTtlTestVal = 1000; + config.setLong(CONNECTION_TTL, connectionTtlTestVal); + initConnectionSettings(config, awsConf); + Assertions.assertThat(awsConf.getConnectionTTL()) + .describedAs("%s not propagated to aws conf", CONNECTION_TTL) + .isEqualTo(connectionTtlTestVal); + + long connectionTtlTestVal1 = -1; + config.setLong(CONNECTION_TTL, connectionTtlTestVal1); + initConnectionSettings(config, awsConf); + Assertions.assertThat(awsConf.getConnectionTTL()) + .describedAs("%s not propagated to aws conf", CONNECTION_TTL) + .isEqualTo(connectionTtlTestVal1); + + long connectionTtlTestVal2 = -100; + config.setLong(CONNECTION_TTL, connectionTtlTestVal2); + intercept(IllegalArgumentException.class, () -> initConnectionSettings(config, awsConf)); + } + @Test(timeout = 10_000L) public void testS3SpecificSignerOverride() throws IOException { ClientConfiguration clientConfiguration = null; From b58885624bdbad3a6940a63fc7537fc0beda6784 Mon Sep 17 00:00:00 2001 From: yuyanlei <33611279+Tre2878@users.noreply.github.com> Date: Mon, 28 Aug 2023 14:52:38 +0800 Subject: [PATCH 42/48] HDFS-17093. Fix block report lease issue to avoid missing some storages report. (#5855). Contributed by Yanlei Yu. Reviewed-by: Shuyan Zhang Reviewed-by: Xing Lin Signed-off-by: He Xiaoqiao --- .../server/blockmanagement/BlockManager.java | 19 ++++- .../blockmanagement/TestBlockReportLease.java | 81 +++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index eb960e62e36e8..30c48c4878b14 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2909,7 +2909,7 @@ public boolean processReport(final DatanodeID nodeID, + "discarded non-initial block report from {}" + " because namenode still in startup phase", strBlockReportId, fullBrLeaseId, nodeID); - blockReportLeaseManager.removeLease(node); + removeDNLeaseIfNeeded(node); return !node.hasStaleStorages(); } @@ -2957,6 +2957,23 @@ public boolean processReport(final DatanodeID nodeID, return !node.hasStaleStorages(); } + /** + * Remove the DN lease only when we have received block reports, + * for all storages for a particular DN. + */ + void removeDNLeaseIfNeeded(DatanodeDescriptor node) { + boolean needRemoveLease = true; + for (DatanodeStorageInfo sInfo : node.getStorageInfos()) { + if (sInfo.getBlockReportCount() == 0) { + needRemoveLease = false; + break; + } + } + if (needRemoveLease) { + blockReportLeaseManager.removeLease(node); + } + } + public void removeBRLeaseIfNeeded(final DatanodeID nodeID, final BlockReportContext context) throws IOException { namesystem.writeLock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java index 225f7fc96c458..0c7a8af3898ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java @@ -23,6 +23,8 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; @@ -269,4 +271,83 @@ private StorageBlockReport[] createReports(DatanodeStorage[] dnStorages, } return storageBlockReports; } + + @Test(timeout = 360000) + public void testFirstIncompleteBlockReport() throws Exception { + HdfsConfiguration conf = new HdfsConfiguration(); + Random rand = new Random(); + + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(1).build()) { + cluster.waitActive(); + + FSNamesystem fsn = cluster.getNamesystem(); + + NameNode nameNode = cluster.getNameNode(); + // Pretend to be in safemode. + NameNodeAdapter.enterSafeMode(nameNode, false); + + BlockManager blockManager = fsn.getBlockManager(); + BlockManager spyBlockManager = spy(blockManager); + fsn.setBlockManagerForTesting(spyBlockManager); + String poolId = cluster.getNamesystem().getBlockPoolId(); + + NamenodeProtocols rpcServer = cluster.getNameNodeRpc(); + + // Test based on one DataNode report to Namenode. + DataNode dn = cluster.getDataNodes().get(0); + DatanodeDescriptor datanodeDescriptor = spyBlockManager + .getDatanodeManager().getDatanode(dn.getDatanodeId()); + + DatanodeRegistration dnRegistration = dn.getDNRegistrationForBP(poolId); + StorageReport[] storages = dn.getFSDataset().getStorageReports(poolId); + + // Send heartbeat and request full block report lease. + HeartbeatResponse hbResponse = rpcServer.sendHeartbeat( + dnRegistration, storages, 0, 0, 0, 0, 0, null, true, + SlowPeerReports.EMPTY_REPORT, SlowDiskReports.EMPTY_REPORT); + + DelayAnswer delayer = new DelayAnswer(BlockManager.LOG); + doAnswer(delayer).when(spyBlockManager).processReport( + any(DatanodeStorageInfo.class), + any(BlockListAsLongs.class)); + + // Trigger sendBlockReport. + BlockReportContext brContext = new BlockReportContext(1, 0, + rand.nextLong(), hbResponse.getFullBlockReportLeaseId()); + // Build every storage with 100 blocks for sending report. + DatanodeStorage[] datanodeStorages + = new DatanodeStorage[storages.length]; + for (int i = 0; i < storages.length; i++) { + datanodeStorages[i] = storages[i].getStorage(); + StorageBlockReport[] reports = createReports(datanodeStorages, 100); + + // The first multiple send once, simulating the failure of the first report, + // only send successfully once. + if(i == 0){ + rpcServer.blockReport(dnRegistration, poolId, reports, brContext); + } + + // Send blockReport. + DatanodeCommand datanodeCommand = rpcServer.blockReport(dnRegistration, poolId, reports, + brContext); + + // Wait until BlockManager calls processReport. + delayer.waitForCall(); + + // Allow blockreport to proceed. + delayer.proceed(); + + // Get result, it will not null if process successfully. + assertTrue(datanodeCommand instanceof FinalizeCommand); + assertEquals(poolId, ((FinalizeCommand)datanodeCommand) + .getBlockPoolId()); + if(i == 0){ + assertEquals(2, datanodeDescriptor.getStorageInfos()[i].getBlockReportCount()); + }else{ + assertEquals(1, datanodeDescriptor.getStorageInfos()[i].getBlockReportCount()); + } + } + } + } } From 39eaf8d14ebddb2b9d9ec0db0ac4419ec715c88c Mon Sep 17 00:00:00 2001 From: WangYuanben <48795318+YuanbenWang@users.noreply.github.com> Date: Tue, 29 Aug 2023 12:16:10 +0800 Subject: [PATCH 43/48] YARN-11556. Let Federation.md more standardized. (#5983) --- .../hadoop-yarn-site/src/site/markdown/Federation.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index bdd9d82859a90..abe2384a39283 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -235,7 +235,7 @@ SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**. |`yarn.federation.subcluster-resolver.class` | `org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl` | The class used to resolve which subcluster a node belongs to, and which subcluster(s) a rack belongs to. | |`yarn.federation.machine-list` | `` | Path of machine-list file used by `SubClusterResolver`. Each line of the file is a node with sub-cluster and rack information. Below is the example:

    node1, subcluster1, rack1
    node2, subcluster2, rack1
    node3, subcluster3, rack2
    node4, subcluster3, rack2 | -How to configure the policy-manager? +How to configure the policy-manager -------------------- Router Policy @@ -317,7 +317,7 @@ Policy Manager - WeightedLocalityPolicyManager - Policy that allows operator to configure "weights" for routing. This picks a LocalityRouterPolicy for the router and a LocalityMulticastAMRMProxyPolicy for the amrmproxy as they are designed to work together. -How to configure the queue policy? +How to configure the queue policy -------------------- We will provide a set of commands to view and save queue policies. @@ -433,14 +433,14 @@ Cache: Cache is not enabled by default. When we set the `yarn.federation.cache-ttl.secs` parameter and its value is greater than 0, Cache will be enabled. We currently provide two Cache implementations: `JCache` and `GuavaCache`. -> JCache +- JCache We used `geronimo-jcache`,`geronimo-jcache` is an implementation of the Java Caching API (JSR-107) specification provided by the Apache Geronimo project. It defines a standardized implementation of the JCache API, allowing developers to use the same API to access different caching implementations. In YARN Federation we use a combination of `geronimo-jcache` and `Ehcache`. If we want to use JCache, we can configure `yarn.federation.cache.class` to `org.apache.hadoop.yarn.server.federation.cache.FederationJCache`. -> GuavaCache +- GuavaCache This is a Cache implemented based on the Guava framework. If we want to use it, we can configure `yarn.federation.cache.class` to `org.apache.hadoop.yarn.server.federation.cache.FederationGuavaCache`. @@ -506,6 +506,7 @@ How to build a Test Federation Cluster The purpose of this document is to help users quickly set up a testing environment for YARN Federation. With this testing environment, users can utilize the core functionality of YARN Federation. This is the simplest test cluster setup (based on Linux) with only essential configurations (YARN non-HA mode). We require 3 machines, and each machine should have at least <4C, 8GB> of resources. We only cover YARN configuration in this document. For information on configuring HDFS and ZooKeeper, please refer to other documentation sources. Test Environment Description: + - We need to build a HDFS test environment, this part can refer to HDFS documentation. [HDFS SingleCluster](../../hadoop-project-dist/hadoop-common/SingleCluster.html) - We need two YARN clusters, each YARN cluster has one RM and one NM, The RM and NM on the same node. - We need one ZK cluster(We only need one ZooKeeper node.), this part can refer to Zookeeper documentation. [ZookeeperStarted](https://zookeeper.apache.org/doc/current/zookeeperStarted.html) From 1046f9cf9888155c27923f3f56efa107d908ad5b Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 29 Aug 2023 16:42:27 +0530 Subject: [PATCH 44/48] HADOOP-18860. Upgrade mockito version to 4.11.0 (#5977) As well as the POM update, this patch moves to the (renamed) verify methods. Backporting mockito test changes may now require cherrypicking this patch, otherwise use the old method names. Contributed by Anmol Asrani --- .../org/apache/hadoop/ipc/TestServer.java | 2 +- .../security/http/TestCrossOriginFilter.java | 9 +++---- .../http/TestRestCsrfPreventionFilter.java | 10 +++---- .../federation/router/TestRouterAdmin.java | 16 +++++++----- .../router/TestRouterRpcMultiDestination.java | 2 +- .../server/namenode/TestCacheDirectives.java | 2 +- .../namenode/TestSnapshotPathINodes.java | 2 +- hadoop-project/pom.xml | 18 ++++++++++++- .../containerlaunch/TestAbstractLauncher.java | 4 +-- .../resources/TestCGroupsHandlerImpl.java | 8 +++--- .../gpu/TestGpuResourceAllocator.java | 4 +-- .../runtime/TestDockerContainerRuntime.java | 4 +-- .../com/nec/TestNECVEPlugin.java | 6 ++--- ...TestFSConfigToCSConfigArgumentHandler.java | 6 ++--- .../TestFSConfigToCSConfigConverter.java | 4 +-- .../TestQueuePlacementConverter.java | 26 +++++++++---------- hadoop-yarn-project/pom.xml | 5 ++++ 17 files changed, 75 insertions(+), 53 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java index 748d99e2a0d34..b6e696fddeefa 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java @@ -151,7 +151,7 @@ public Writable call( // Nothing should be logged for a suppressed exception. server.logException(logger, new TestException1(), dummyCall); - verifyZeroInteractions(logger); + verifyNoInteractions(logger); // No stack trace should be logged for a terse exception. server.logException(logger, new TestException2(), dummyCall); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java index 0b396be48f983..9f7a6114b0ac3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java @@ -58,8 +58,7 @@ public void testSameOrigin() throws ServletException, IOException { CrossOriginFilter filter = new CrossOriginFilter(); filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - - Mockito.verifyZeroInteractions(mockRes); + Mockito.verifyNoInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -224,7 +223,7 @@ public void testDisallowedOrigin() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockRes); + Mockito.verifyNoInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -252,7 +251,7 @@ public void testDisallowedMethod() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockRes); + Mockito.verifyNoInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -283,7 +282,7 @@ public void testDisallowedHeader() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockRes); + Mockito.verifyNoInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java index 6052ef059a732..f125ffac1e191 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java @@ -75,7 +75,7 @@ public void testNoHeaderDefaultConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } @Test @@ -110,7 +110,7 @@ public void testNoHeaderCustomAgentConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } @Test @@ -228,7 +228,7 @@ public void testMissingHeaderWithCustomHeaderConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } @Test @@ -260,7 +260,7 @@ public void testMissingHeaderNoMethodsToIgnoreConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } @Test @@ -356,6 +356,6 @@ public void testMissingHeaderMultipleIgnoreMethodsConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index c2eaddc17a2a0..dac5be209ab11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; import java.util.Collections; import java.util.HashMap; @@ -68,7 +69,6 @@ import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; -import org.mockito.internal.util.reflection.FieldSetter; /** * The administrator interface of the {@link Router} implemented by @@ -118,18 +118,20 @@ public static void globalSetUp() throws Exception { * @throws IOException * @throws NoSuchFieldException */ - private static void setUpMocks() throws IOException, NoSuchFieldException { + private static void setUpMocks() throws IOException, NoSuchFieldException, + IllegalAccessException { RouterRpcServer spyRpcServer = Mockito.spy(routerContext.getRouter().createRpcServer()); - FieldSetter.setField(routerContext.getRouter(), - Router.class.getDeclaredField("rpcServer"), spyRpcServer); + Field rpcServerField = Router.class.getDeclaredField("rpcServer"); + rpcServerField.setAccessible(true); + rpcServerField.set(routerContext.getRouter(), spyRpcServer); Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString()); // mock rpc client for destination check when editing mount tables. mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient()); - FieldSetter.setField(spyRpcServer, - RouterRpcServer.class.getDeclaredField("rpcClient"), - mockRpcClient); + Field rpcClientField = RouterRpcServer.class.getDeclaredField("rpcClient"); + rpcClientField.setAccessible(true); + rpcClientField.set(spyRpcServer, mockRpcClient); RemoteLocation remoteLocation0 = new RemoteLocation("ns0", "/testdir", null); RemoteLocation remoteLocation1 = diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java index 336ea3913859e..ab51a8224271a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.apache.hadoop.test.Whitebox.getInternalState; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index 1331c50e80b3a..48c1527de1031 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -1575,7 +1575,7 @@ public void testNoLookupsWhenNotUsed() throws Exception { CacheManager cm = cluster.getNamesystem().getCacheManager(); LocatedBlocks locations = Mockito.mock(LocatedBlocks.class); cm.setCachedLocations(locations); - Mockito.verifyZeroInteractions(locations); + Mockito.verifyNoInteractions(locations); } @Test(timeout=120000) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java index b62a4180d43ba..cda5b39f1d03e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java @@ -447,6 +447,6 @@ public void testShortCircuitSnapshotSearch() throws SnapshotException { INodesInPath iip = Mockito.mock(INodesInPath.class); List snapDirs = new ArrayList<>(); FSDirSnapshotOp.checkSnapshot(fsn.getFSDirectory(), iip, snapDirs); - Mockito.verifyZeroInteractions(iip); + Mockito.verifyNoInteractions(iip); } } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 9fca0fa159b87..b2176ccaa90d7 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -84,6 +84,9 @@ 1.1 + + 4.11.0 + 2.5.0 @@ -1303,7 +1306,20 @@ org.mockito mockito-core - 2.28.2 + ${mockito.version} + test + + + org.mockito + mockito-inline + ${mockito.version} + test + + + org.mockito + mockito-core + + org.mockito diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java index 31ca38297c856..6b3da713964cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java @@ -43,7 +43,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; /** @@ -90,7 +90,7 @@ public void testContainerRetries() throws Exception { providerService.buildContainerRetry(mockLauncher, getConfig(), componentLaunchContext, componentInstance); - verifyZeroInteractions(mockLauncher); + verifyNoInteractions(mockLauncher); //OnFailure restart policy diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java index b1e8989213ba1..ff5b9be6ddc08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java @@ -51,8 +51,8 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; /** * Tests for the CGroups handler implementation. @@ -192,7 +192,7 @@ public void testMountController() throws IOException { assertTrue("cgroup dir should be cerated", cgroup.mkdirs()); //Since we enabled (deferred) cgroup controller mounting, no interactions //should have occurred, with this mock - verifyZeroInteractions(privilegedOperationExecutorMock); + verifyNoInteractions(privilegedOperationExecutorMock); File emptyMtab = createEmptyCgroups(); try { @@ -238,7 +238,7 @@ public void testMountController() throws IOException { public void testCGroupPaths() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - verifyZeroInteractions(privilegedOperationExecutorMock); + verifyNoInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); @@ -281,7 +281,7 @@ public void testCGroupPaths() throws IOException { public void testCGroupOperations() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - verifyZeroInteractions(privilegedOperationExecutorMock); + verifyNoInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java index ba8a9309d02e9..07ce7b256d4c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java @@ -27,7 +27,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.io.IOException; @@ -210,7 +210,7 @@ private void assertAllocatedGpus(int gpus, int deniedGpus, private void assertNoAllocation(GpuAllocation allocation) { assertEquals(1, allocation.getDeniedGPUs().size()); assertEquals(0, allocation.getAllowedGPUs().size()); - verifyZeroInteractions(nmStateStore); + verifyNoInteractions(nmStateStore); } private void assertAssignmentInStateStore(GpuDevice expectedGpu, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index ea7c213809330..f13d8388f9a32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -1272,7 +1272,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added here - Mockito.verifyZeroInteractions(command); + Mockito.verifyNoMoreInteractions(command); String resourceOptionsCpu = "/sys/fs/cgroup/cpu/" + hierarchy + containerIdStr; @@ -1296,7 +1296,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added in either case - Mockito.verifyZeroInteractions(command); + Mockito.verifyNoMoreInteractions(command); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java index 86ef9058f26fb..a9b01bca6fac7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java @@ -25,7 +25,7 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.anyString; @@ -394,7 +394,7 @@ public void testFindDevicesWithUdev() assertEquals("No. of devices", 1, devices.size()); Device device = devices.iterator().next(); assertSame("Device", device, testDevice); - verifyZeroInteractions(mockCommandExecutor); + verifyNoInteractions(mockCommandExecutor); verify(mockEnvProvider).apply(eq("NEC_USE_UDEV")); verifyNoMoreInteractions(mockEnvProvider); } @@ -442,6 +442,6 @@ private Device getTestDevice(int id) { private void verifyBinaryPathSet(Path expectedPath) { assertEquals("Binary path", expectedPath.toString(), plugin.getBinaryPath()); - verifyZeroInteractions(udevUtil); + verifyNoInteractions(udevUtil); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java index cb8cc587f68f7..33cc3f0dba266 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -23,7 +23,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import java.io.File; import java.io.IOException; @@ -666,7 +666,7 @@ public void testValidationSkippedWhenCmdLineSwitchIsDefined() FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s"); argumentHandler.parseAndConvert(args); - verifyZeroInteractions(mockValidator); + verifyNoInteractions(mockValidator); } @Test @@ -681,7 +681,7 @@ public void testValidationSkippedWhenOutputIsConsole() throws Exception { FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s", "-p"); argumentHandler.parseAndConvert(args); - verifyZeroInteractions(mockValidator); + verifyNoInteractions(mockValidator); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 55c43666cdb5b..530c6ddc55301 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -36,7 +36,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import java.io.ByteArrayInputStream; import java.io.File; @@ -673,7 +673,7 @@ public void testPlacementRulesConversionDisabled() throws Exception { converter.setPlacementConverter(placementConverter); converter.convert(params); - verifyZeroInteractions(placementConverter); + verifyNoInteractions(placementConverter); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java index 6599080aab59c..6871444baa03c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java @@ -21,7 +21,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.util.List; @@ -85,7 +85,7 @@ public void testConvertUserRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.USER); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -96,7 +96,7 @@ public void testConvertSpecifiedRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SPECIFIED); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -108,7 +108,7 @@ public void testConvertPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -120,7 +120,7 @@ public void testConvertSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -134,7 +134,7 @@ public void testConvertDefaultRuleWithQueueName() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.CUSTOM); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -147,7 +147,7 @@ public void testConvertDefaultRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.DEFAULT_QUEUE); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -168,7 +168,7 @@ public void testConvertRejectRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.REJECT); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -182,7 +182,7 @@ public void testConvertNestedPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP_USER); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -197,7 +197,7 @@ public void testConvertNestedSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP_USER); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -215,7 +215,7 @@ public void testConvertNestedDefaultRule() { Rule rule = description.getRules().get(0); verifyRule(description.getRules().get(0), Policy.USER); assertEquals("Parent path", "root.abc", rule.getParentQueue()); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -245,7 +245,7 @@ public void testConvertMultiplePlacementRules() { verifyRule(description.getRules().get(0), Policy.USER); verifyRule(description.getRules().get(1), Policy.PRIMARY_GROUP); verifyRule(description.getRules().get(2), Policy.SECONDARY_GROUP); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -363,7 +363,7 @@ private void testConvertNestedRuleCreateFlagInWeightMode( any(Policy.class)); verifyNoMoreInteractions(ruleHandler); } else { - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } } diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 241e3bc237a0e..78f09a3971069 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -90,6 +90,11 @@ hadoop-yarn-applications-catalog-webapp war + + org.mockito + mockito-core + test + From 28c533a582f8b6070508ba2e57662d4576ac05d2 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 31 Aug 2023 14:54:53 +0100 Subject: [PATCH 45/48] Revert "HADOOP-18860. Upgrade mockito version to 4.11.0 (#5977)" This reverts commit 1046f9cf9888155c27923f3f56efa107d908ad5b. --- .../org/apache/hadoop/ipc/TestServer.java | 2 +- .../security/http/TestCrossOriginFilter.java | 9 ++++--- .../http/TestRestCsrfPreventionFilter.java | 10 +++---- .../federation/router/TestRouterAdmin.java | 16 +++++------- .../router/TestRouterRpcMultiDestination.java | 2 +- .../server/namenode/TestCacheDirectives.java | 2 +- .../namenode/TestSnapshotPathINodes.java | 2 +- hadoop-project/pom.xml | 18 +------------ .../containerlaunch/TestAbstractLauncher.java | 4 +-- .../resources/TestCGroupsHandlerImpl.java | 8 +++--- .../gpu/TestGpuResourceAllocator.java | 4 +-- .../runtime/TestDockerContainerRuntime.java | 4 +-- .../com/nec/TestNECVEPlugin.java | 6 ++--- ...TestFSConfigToCSConfigArgumentHandler.java | 6 ++--- .../TestFSConfigToCSConfigConverter.java | 4 +-- .../TestQueuePlacementConverter.java | 26 +++++++++---------- hadoop-yarn-project/pom.xml | 5 ---- 17 files changed, 53 insertions(+), 75 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java index b6e696fddeefa..748d99e2a0d34 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java @@ -151,7 +151,7 @@ public Writable call( // Nothing should be logged for a suppressed exception. server.logException(logger, new TestException1(), dummyCall); - verifyNoInteractions(logger); + verifyZeroInteractions(logger); // No stack trace should be logged for a terse exception. server.logException(logger, new TestException2(), dummyCall); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java index 9f7a6114b0ac3..0b396be48f983 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java @@ -58,7 +58,8 @@ public void testSameOrigin() throws ServletException, IOException { CrossOriginFilter filter = new CrossOriginFilter(); filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -223,7 +224,7 @@ public void testDisallowedOrigin() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -251,7 +252,7 @@ public void testDisallowedMethod() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -282,7 +283,7 @@ public void testDisallowedHeader() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java index f125ffac1e191..6052ef059a732 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java @@ -75,7 +75,7 @@ public void testNoHeaderDefaultConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -110,7 +110,7 @@ public void testNoHeaderCustomAgentConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -228,7 +228,7 @@ public void testMissingHeaderWithCustomHeaderConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -260,7 +260,7 @@ public void testMissingHeaderNoMethodsToIgnoreConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -356,6 +356,6 @@ public void testMissingHeaderMultipleIgnoreMethodsConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index dac5be209ab11..c2eaddc17a2a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; import java.util.Collections; import java.util.HashMap; @@ -69,6 +68,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; +import org.mockito.internal.util.reflection.FieldSetter; /** * The administrator interface of the {@link Router} implemented by @@ -118,20 +118,18 @@ public static void globalSetUp() throws Exception { * @throws IOException * @throws NoSuchFieldException */ - private static void setUpMocks() throws IOException, NoSuchFieldException, - IllegalAccessException { + private static void setUpMocks() throws IOException, NoSuchFieldException { RouterRpcServer spyRpcServer = Mockito.spy(routerContext.getRouter().createRpcServer()); - Field rpcServerField = Router.class.getDeclaredField("rpcServer"); - rpcServerField.setAccessible(true); - rpcServerField.set(routerContext.getRouter(), spyRpcServer); + FieldSetter.setField(routerContext.getRouter(), + Router.class.getDeclaredField("rpcServer"), spyRpcServer); Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString()); // mock rpc client for destination check when editing mount tables. mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient()); - Field rpcClientField = RouterRpcServer.class.getDeclaredField("rpcClient"); - rpcClientField.setAccessible(true); - rpcClientField.set(spyRpcServer, mockRpcClient); + FieldSetter.setField(spyRpcServer, + RouterRpcServer.class.getDeclaredField("rpcClient"), + mockRpcClient); RemoteLocation remoteLocation0 = new RemoteLocation("ns0", "/testdir", null); RemoteLocation remoteLocation1 = diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java index ab51a8224271a..336ea3913859e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.apache.hadoop.test.Whitebox.getInternalState; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index 48c1527de1031..1331c50e80b3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -1575,7 +1575,7 @@ public void testNoLookupsWhenNotUsed() throws Exception { CacheManager cm = cluster.getNamesystem().getCacheManager(); LocatedBlocks locations = Mockito.mock(LocatedBlocks.class); cm.setCachedLocations(locations); - Mockito.verifyNoInteractions(locations); + Mockito.verifyZeroInteractions(locations); } @Test(timeout=120000) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java index cda5b39f1d03e..b62a4180d43ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java @@ -447,6 +447,6 @@ public void testShortCircuitSnapshotSearch() throws SnapshotException { INodesInPath iip = Mockito.mock(INodesInPath.class); List snapDirs = new ArrayList<>(); FSDirSnapshotOp.checkSnapshot(fsn.getFSDirectory(), iip, snapDirs); - Mockito.verifyNoInteractions(iip); + Mockito.verifyZeroInteractions(iip); } } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b2176ccaa90d7..9fca0fa159b87 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -84,9 +84,6 @@ 1.1 - - 4.11.0 - 2.5.0 @@ -1306,20 +1303,7 @@ org.mockito mockito-core - ${mockito.version} - test - - - org.mockito - mockito-inline - ${mockito.version} - test - - - org.mockito - mockito-core - - + 2.28.2 org.mockito diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java index 6b3da713964cc..31ca38297c856 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java @@ -43,7 +43,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; /** @@ -90,7 +90,7 @@ public void testContainerRetries() throws Exception { providerService.buildContainerRetry(mockLauncher, getConfig(), componentLaunchContext, componentInstance); - verifyNoInteractions(mockLauncher); + verifyZeroInteractions(mockLauncher); //OnFailure restart policy diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java index ff5b9be6ddc08..b1e8989213ba1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java @@ -51,8 +51,8 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; /** * Tests for the CGroups handler implementation. @@ -192,7 +192,7 @@ public void testMountController() throws IOException { assertTrue("cgroup dir should be cerated", cgroup.mkdirs()); //Since we enabled (deferred) cgroup controller mounting, no interactions //should have occurred, with this mock - verifyNoInteractions(privilegedOperationExecutorMock); + verifyZeroInteractions(privilegedOperationExecutorMock); File emptyMtab = createEmptyCgroups(); try { @@ -238,7 +238,7 @@ public void testMountController() throws IOException { public void testCGroupPaths() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - verifyNoInteractions(privilegedOperationExecutorMock); + verifyZeroInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); @@ -281,7 +281,7 @@ public void testCGroupPaths() throws IOException { public void testCGroupOperations() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - verifyNoInteractions(privilegedOperationExecutorMock); + verifyZeroInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java index 07ce7b256d4c6..ba8a9309d02e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java @@ -27,7 +27,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.io.IOException; @@ -210,7 +210,7 @@ private void assertAllocatedGpus(int gpus, int deniedGpus, private void assertNoAllocation(GpuAllocation allocation) { assertEquals(1, allocation.getDeniedGPUs().size()); assertEquals(0, allocation.getAllowedGPUs().size()); - verifyNoInteractions(nmStateStore); + verifyZeroInteractions(nmStateStore); } private void assertAssignmentInStateStore(GpuDevice expectedGpu, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index f13d8388f9a32..ea7c213809330 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -1272,7 +1272,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added here - Mockito.verifyNoMoreInteractions(command); + Mockito.verifyZeroInteractions(command); String resourceOptionsCpu = "/sys/fs/cgroup/cpu/" + hierarchy + containerIdStr; @@ -1296,7 +1296,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added in either case - Mockito.verifyNoMoreInteractions(command); + Mockito.verifyZeroInteractions(command); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java index a9b01bca6fac7..86ef9058f26fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java @@ -25,7 +25,7 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.anyString; @@ -394,7 +394,7 @@ public void testFindDevicesWithUdev() assertEquals("No. of devices", 1, devices.size()); Device device = devices.iterator().next(); assertSame("Device", device, testDevice); - verifyNoInteractions(mockCommandExecutor); + verifyZeroInteractions(mockCommandExecutor); verify(mockEnvProvider).apply(eq("NEC_USE_UDEV")); verifyNoMoreInteractions(mockEnvProvider); } @@ -442,6 +442,6 @@ private Device getTestDevice(int id) { private void verifyBinaryPathSet(Path expectedPath) { assertEquals("Binary path", expectedPath.toString(), plugin.getBinaryPath()); - verifyNoInteractions(udevUtil); + verifyZeroInteractions(udevUtil); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java index 33cc3f0dba266..cb8cc587f68f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -23,7 +23,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import java.io.File; import java.io.IOException; @@ -666,7 +666,7 @@ public void testValidationSkippedWhenCmdLineSwitchIsDefined() FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s"); argumentHandler.parseAndConvert(args); - verifyNoInteractions(mockValidator); + verifyZeroInteractions(mockValidator); } @Test @@ -681,7 +681,7 @@ public void testValidationSkippedWhenOutputIsConsole() throws Exception { FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s", "-p"); argumentHandler.parseAndConvert(args); - verifyNoInteractions(mockValidator); + verifyZeroInteractions(mockValidator); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 530c6ddc55301..55c43666cdb5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -36,7 +36,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import java.io.ByteArrayInputStream; import java.io.File; @@ -673,7 +673,7 @@ public void testPlacementRulesConversionDisabled() throws Exception { converter.setPlacementConverter(placementConverter); converter.convert(params); - verifyNoInteractions(placementConverter); + verifyZeroInteractions(placementConverter); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java index 6871444baa03c..6599080aab59c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java @@ -21,7 +21,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.util.List; @@ -85,7 +85,7 @@ public void testConvertUserRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.USER); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -96,7 +96,7 @@ public void testConvertSpecifiedRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SPECIFIED); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -108,7 +108,7 @@ public void testConvertPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -120,7 +120,7 @@ public void testConvertSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -134,7 +134,7 @@ public void testConvertDefaultRuleWithQueueName() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.CUSTOM); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -147,7 +147,7 @@ public void testConvertDefaultRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.DEFAULT_QUEUE); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -168,7 +168,7 @@ public void testConvertRejectRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.REJECT); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -182,7 +182,7 @@ public void testConvertNestedPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP_USER); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -197,7 +197,7 @@ public void testConvertNestedSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP_USER); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -215,7 +215,7 @@ public void testConvertNestedDefaultRule() { Rule rule = description.getRules().get(0); verifyRule(description.getRules().get(0), Policy.USER); assertEquals("Parent path", "root.abc", rule.getParentQueue()); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -245,7 +245,7 @@ public void testConvertMultiplePlacementRules() { verifyRule(description.getRules().get(0), Policy.USER); verifyRule(description.getRules().get(1), Policy.PRIMARY_GROUP); verifyRule(description.getRules().get(2), Policy.SECONDARY_GROUP); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -363,7 +363,7 @@ private void testConvertNestedRuleCreateFlagInWeightMode( any(Policy.class)); verifyNoMoreInteractions(ruleHandler); } else { - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } } diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 78f09a3971069..241e3bc237a0e 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -90,11 +90,6 @@ hadoop-yarn-applications-catalog-webapp war - - org.mockito - mockito-core - test - From 01cc6d0bc8236d87eefa646a9ad2161841984fd3 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 31 Aug 2023 19:40:04 +0530 Subject: [PATCH 46/48] HADOOP-18865. ABFS: Add "100-continue" in userAgent if enabled (#5987) Contributed by Anmol Asrani --- .../services/AppendRequestParameters.java | 10 ++++ .../fs/azurebfs/services/AbfsClient.java | 17 ++++++ .../fs/azurebfs/services/ITestAbfsClient.java | 58 +++++++++++++++---- 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java index 57e559a60ec84..9da6427d65c2c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java @@ -35,6 +35,7 @@ public enum Mode { private final boolean isAppendBlob; private final String leaseId; private boolean isExpectHeaderEnabled; + private boolean isRetryDueToExpect; public AppendRequestParameters(final long position, final int offset, @@ -50,6 +51,7 @@ public AppendRequestParameters(final long position, this.isAppendBlob = isAppendBlob; this.leaseId = leaseId; this.isExpectHeaderEnabled = isExpectHeaderEnabled; + this.isRetryDueToExpect = false; } public long getPosition() { @@ -80,6 +82,14 @@ public boolean isExpectHeaderEnabled() { return isExpectHeaderEnabled; } + public boolean isRetryDueToExpect() { + return isRetryDueToExpect; + } + + public void setRetryDueToExpect(boolean retryDueToExpect) { + isRetryDueToExpect = retryDueToExpect; + } + public void setExpectHeaderEnabled(boolean expectHeaderEnabled) { isExpectHeaderEnabled = expectHeaderEnabled; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 77b8dcb2b9891..45cb538d0b007 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -87,6 +87,7 @@ */ public class AbfsClient implements Closeable { public static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); + public static final String HUNDRED_CONTINUE_USER_AGENT = SINGLE_WHITE_SPACE + HUNDRED_CONTINUE + SEMICOLON; private final URL baseUrl; private final SharedKeyCredentials sharedKeyCredentials; @@ -751,6 +752,15 @@ public AbfsRestOperation append(final String path, final byte[] buffer, } } + // Check if the retry is with "Expect: 100-continue" header being present in the previous request. + if (reqParams.isRetryDueToExpect()) { + String userAgentRetry = userAgent; + // Remove the specific marker related to "Expect: 100-continue" from the User-Agent string. + userAgentRetry = userAgentRetry.replace(HUNDRED_CONTINUE_USER_AGENT, EMPTY_STRING); + requestHeaders.removeIf(header -> header.getName().equalsIgnoreCase(USER_AGENT)); + requestHeaders.add(new AbfsHttpHeader(USER_AGENT, userAgentRetry)); + } + // AbfsInputStream/AbfsOutputStream reuse SAS tokens for better performance String sasTokenForReuse = appendSASTokenToQuery(path, SASTokenProvider.WRITE_OPERATION, abfsUriQueryBuilder, cachedSasToken); @@ -780,6 +790,7 @@ public AbfsRestOperation append(final String path, final byte[] buffer, if (checkUserError(responseStatusCode) && reqParams.isExpectHeaderEnabled()) { LOG.debug("User error, retrying without 100 continue enabled for the given path {}", path); reqParams.setExpectHeaderEnabled(false); + reqParams.setRetryDueToExpect(true); return this.append(path, buffer, reqParams, cachedSasToken, tracingContext); } @@ -1371,6 +1382,12 @@ String initializeUserAgent(final AbfsConfiguration abfsConfiguration, appendIfNotEmpty(sb, ExtensionHelper.getUserAgentSuffix(tokenProvider, EMPTY_STRING), true); + if (abfsConfiguration.isExpectHeaderEnabled()) { + sb.append(SINGLE_WHITE_SPACE); + sb.append(HUNDRED_CONTINUE); + sb.append(SEMICOLON); + } + sb.append(SINGLE_WHITE_SPACE); sb.append(abfsConfiguration.getClusterName()); sb.append(FORWARD_SLASH); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java index c031e5daa6c44..18d1e3917f24e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java @@ -86,6 +86,7 @@ public final class ITestAbfsClient extends AbstractAbfsIntegrationTest { private static final String ACCOUNT_NAME = "bogusAccountName.dfs.core.windows.net"; private static final String FS_AZURE_USER_AGENT_PREFIX = "Partner Service"; + private static final String HUNDRED_CONTINUE_USER_AGENT = SINGLE_WHITE_SPACE + HUNDRED_CONTINUE + SEMICOLON; private static final String TEST_PATH = "/testfile"; public static final int REDUCED_RETRY_COUNT = 2; public static final int REDUCED_BACKOFF_INTERVAL = 100; @@ -143,15 +144,15 @@ private String getUserAgentString(AbfsConfiguration config, } @Test - public void verifybBasicInfo() throws Exception { + public void verifyBasicInfo() throws Exception { final Configuration configuration = new Configuration(); configuration.addResource(TEST_CONFIGURATION_FILE_NAME); AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, ACCOUNT_NAME); - verifybBasicInfo(getUserAgentString(abfsConfiguration, false)); + verifyBasicInfo(getUserAgentString(abfsConfiguration, false)); } - private void verifybBasicInfo(String userAgentStr) { + private void verifyBasicInfo(String userAgentStr) { Assertions.assertThat(userAgentStr) .describedAs("User-Agent string [" + userAgentStr + "] should be of the pattern: " + this.userAgentStringPattern.pattern()) @@ -180,7 +181,7 @@ public void verifyUserAgentPrefix() ACCOUNT_NAME); String userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should contain " + FS_AZURE_USER_AGENT_PREFIX) .contains(FS_AZURE_USER_AGENT_PREFIX); @@ -190,12 +191,47 @@ public void verifyUserAgentPrefix() ACCOUNT_NAME); userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should not contain " + FS_AZURE_USER_AGENT_PREFIX) .doesNotContain(FS_AZURE_USER_AGENT_PREFIX); } + /** + * This method represents a unit test for verifying the behavior of the User-Agent header + * with respect to the "Expect: 100-continue" header setting in the Azure Blob File System (ABFS) configuration. + * + * The test ensures that the User-Agent string includes or excludes specific information based on whether the + * "Expect: 100-continue" header is enabled or disabled in the configuration. + * + */ + @Test + public void verifyUserAgentExpectHeader() + throws IOException, IllegalAccessException { + final Configuration configuration = new Configuration(); + configuration.addResource(TEST_CONFIGURATION_FILE_NAME); + configuration.set(ConfigurationKeys.FS_AZURE_USER_AGENT_PREFIX_KEY, FS_AZURE_USER_AGENT_PREFIX); + configuration.setBoolean(ConfigurationKeys.FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED, true); + AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, + ACCOUNT_NAME); + String userAgentStr = getUserAgentString(abfsConfiguration, false); + + verifyBasicInfo(userAgentStr); + Assertions.assertThat(userAgentStr) + .describedAs("User-Agent string should contain " + HUNDRED_CONTINUE_USER_AGENT) + .contains(HUNDRED_CONTINUE_USER_AGENT); + + configuration.setBoolean(ConfigurationKeys.FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED, false); + abfsConfiguration = new AbfsConfiguration(configuration, + ACCOUNT_NAME); + userAgentStr = getUserAgentString(abfsConfiguration, false); + + verifyBasicInfo(userAgentStr); + Assertions.assertThat(userAgentStr) + .describedAs("User-Agent string should not contain " + HUNDRED_CONTINUE_USER_AGENT) + .doesNotContain(HUNDRED_CONTINUE_USER_AGENT); + } + @Test public void verifyUserAgentWithoutSSLProvider() throws Exception { final Configuration configuration = new Configuration(); @@ -206,14 +242,14 @@ public void verifyUserAgentWithoutSSLProvider() throws Exception { ACCOUNT_NAME); String userAgentStr = getUserAgentString(abfsConfiguration, true); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should contain sslProvider") .contains(DelegatingSSLSocketFactory.getDefaultFactory().getProviderName()); userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should not contain sslProvider") .doesNotContain(DelegatingSSLSocketFactory.getDefaultFactory().getProviderName()); @@ -229,7 +265,7 @@ public void verifyUserAgentClusterName() throws Exception { ACCOUNT_NAME); String userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should contain cluster name") .contains(clusterName); @@ -239,7 +275,7 @@ public void verifyUserAgentClusterName() throws Exception { ACCOUNT_NAME); userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should not contain cluster name") .doesNotContain(clusterName) @@ -257,7 +293,7 @@ public void verifyUserAgentClusterType() throws Exception { ACCOUNT_NAME); String userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should contain cluster type") .contains(clusterType); @@ -267,7 +303,7 @@ public void verifyUserAgentClusterType() throws Exception { ACCOUNT_NAME); userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should not contain cluster type") .doesNotContain(clusterType) From 2f739450be88df2929c25a59ab4a48f84df5cbc2 Mon Sep 17 00:00:00 2001 From: Yang Jiandan Date: Fri, 1 Sep 2023 00:14:46 +0800 Subject: [PATCH 47/48] YARN-11552. Timeline endpoint: /clusters/{clusterid}/apps/{appid}/entity-types Error when using hdfs store (#5978) Contributed by Jiandan Yang. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../storage/FileSystemTimelineReaderImpl.java | 25 +++++++++++++------ .../TestFileSystemTimelineReaderImpl.java | 24 ++++++++++-------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index 913b8360e2f26..dff21a31da952 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -164,7 +164,7 @@ private static void fillFields(TimelineEntity finalEntity, private String getFlowRunPath(String userId, String clusterId, String flowName, Long flowRunId, String appId) throws IOException { if (userId != null && flowName != null && flowRunId != null) { - return userId + File.separator + flowName + File.separator + flowRunId; + return userId + File.separator + flowName + File.separator + "*" + File.separator + flowRunId; } if (clusterId == null || appId == null) { throw new IOException("Unable to get flow info"); @@ -186,7 +186,7 @@ private String getFlowRunPath(String userId, String clusterId, continue; } return record.get(1).trim() + File.separator + record.get(2).trim() + - File.separator + record.get(3).trim(); + File.separator + "*" + File.separator + record.get(3).trim(); } parser.close(); } @@ -286,6 +286,7 @@ public int compare(Long l1, Long l2) { } } ); + dir = getNormalPath(dir); if (dir != null) { RemoteIterator fileStatuses = fs.listFiles(dir, false); @@ -394,9 +395,11 @@ public TimelineEntity getEntity(TimelineReaderContext context, Path flowRunPath = new Path(clusterIdPath, flowRunPathStr); Path appIdPath = new Path(flowRunPath, context.getAppId()); Path entityTypePath = new Path(appIdPath, context.getEntityType()); - Path entityFilePath = new Path(entityTypePath, - context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION); - + Path entityFilePath = getNormalPath(new Path(entityTypePath, + context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION)); + if (entityFilePath == null) { + return null; + } try (BufferedReader reader = new BufferedReader(new InputStreamReader( fs.open(entityFilePath), Charset.forName("UTF-8")))) { @@ -410,6 +413,14 @@ public TimelineEntity getEntity(TimelineReaderContext context, } } + private Path getNormalPath(Path globPath) throws IOException { + FileStatus[] status = fs.globStatus(globPath); + if (status == null || status.length < 1) { + LOG.info("{} do not exist.", globPath); + return null; + } + return status[0].getPath(); + } @Override public Set getEntities(TimelineReaderContext context, TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve) @@ -433,13 +444,13 @@ public Set getEntities(TimelineReaderContext context, context.getClusterId(), context.getFlowName(), context.getFlowRunId(), context.getAppId()); if (context.getUserId() == null) { - context.setUserId(new Path(flowRunPathStr).getParent().getParent(). + context.setUserId(new Path(flowRunPathStr).getParent().getParent().getParent(). getName()); } Path clusterIdPath = new Path(entitiesPath, context.getClusterId()); Path flowRunPath = new Path(clusterIdPath, flowRunPathStr); Path appIdPath = new Path(flowRunPath, context.getAppId()); - FileStatus[] fileStatuses = fs.listStatus(appIdPath); + FileStatus[] fileStatuses = fs.listStatus(getNormalPath(appIdPath)); for (FileStatus fileStatus : fileStatuses) { if (fileStatus.isDirectory()) { result.add(fileStatus.getPath().getName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java index cf94749e883e8..47e5a490514c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java @@ -66,6 +66,11 @@ public class TestFileSystemTimelineReaderImpl { private static final String ROOT_DIR = new File("target", TestFileSystemTimelineReaderImpl.class.getSimpleName()).getAbsolutePath(); + private static String cluster = "cluster1"; + private static String user = "user1"; + private static String flowVersion = "v1"; + private static String flowRunId = "1"; + private FileSystemTimelineReaderImpl reader; @BeforeAll @@ -125,7 +130,7 @@ private static void writeEntityFile(TimelineEntity entity, File dir) private static void loadEntityData(String rootDir) throws Exception { File appDir = - getAppDir(rootDir, "cluster1", "user1", "flow1", "1", "app1", "app"); + getAppDir(rootDir, "flow1", "app1", "app"); TimelineEntity entity11 = new TimelineEntity(); entity11.setId("id_1"); entity11.setType("app"); @@ -266,8 +271,9 @@ private static void loadEntityData(String rootDir) throws Exception { entity4.addEvent(event44); writeEntityFile(entity4, appDir); - File attemptDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", - "app1", TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString()); + + File attemptDir = getAppDir(rootDir, "flow1", "app1", + TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString()); ApplicationAttemptEntity attempt1 = new ApplicationAttemptEntity(); attempt1.setId("app-attempt-1"); attempt1.setCreatedTime(1425017502003L); @@ -277,8 +283,8 @@ private static void loadEntityData(String rootDir) throws Exception { attempt2.setCreatedTime(1425017502004L); writeEntityFile(attempt2, attemptDir); - File entityDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", - "app1", TimelineEntityType.YARN_CONTAINER.toString()); + File entityDir = getAppDir(rootDir, "flow1", "app1", + TimelineEntityType.YARN_CONTAINER.toString()); ContainerEntity containerEntity1 = new ContainerEntity(); containerEntity1.setId("container_1_1"); containerEntity1.setParent(attempt1.getIdentifier()); @@ -298,8 +304,7 @@ private static void loadEntityData(String rootDir) throws Exception { writeEntityFile(containerEntity3, entityDir); File appDir2 = - getAppDir(rootDir, "cluster1", "user1", "flow1,flow", "1", "app2", - "app"); + getAppDir(rootDir, "flow1,flow", "app2", "app"); TimelineEntity entity5 = new TimelineEntity(); entity5.setId("id_5"); entity5.setType("app"); @@ -307,10 +312,9 @@ private static void loadEntityData(String rootDir) throws Exception { writeEntityFile(entity5, appDir2); } - private static File getAppDir(String rootDir, String cluster, String user, - String flowName, String flowRunId, String appId, String entityName) { + private static File getAppDir(String rootDir, String flowName, String appId, String entityName) { return new File(rootDir + File.separator + "entities" + File.separator + - cluster + File.separator + user + File.separator + flowName + + cluster + File.separator + user + File.separator + flowName + File.separator + flowVersion + File.separator + flowRunId + File.separator + appId + File.separator + entityName + File.separator); } From b6d06c89282153a0b5607e6c1236e4138669a1d9 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Fri, 1 Sep 2023 01:46:33 +0800 Subject: [PATCH 48/48] YARN-11554. Fix TestRMFailover#testWebAppProxyInStandAloneMode Failed. (#5971) --- .../test/java/org/apache/hadoop/yarn/client/TestRMFailover.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index d9f6d4e470887..a744714846bce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -212,6 +212,8 @@ public void testAutomaticFailover() public void testWebAppProxyInStandAloneMode() throws YarnException, InterruptedException, IOException { conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); + conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); + WebAppProxyServer webAppProxyServer = new WebAppProxyServer(); try { conf.set(YarnConfiguration.PROXY_ADDRESS, "0.0.0.0:9099");