Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/trunk' into new_rbf_nonnavia
Browse files Browse the repository at this point in the history
  • Loading branch information
KeeProMise committed Sep 1, 2023
2 parents cb4e08c + b6d06c8 commit b8b1dea
Show file tree
Hide file tree
Showing 145 changed files with 6,481 additions and 681 deletions.
4 changes: 2 additions & 2 deletions LICENSE-binary
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ com.aliyun:aliyun-java-sdk-kms:2.11.0
com.aliyun:aliyun-java-sdk-ram:3.1.0
com.aliyun:aliyun-java-sdk-sts:3.0.0
com.aliyun.oss:aliyun-sdk-oss:3.13.2
com.amazonaws:aws-java-sdk-bundle:1.12.367
com.amazonaws:aws-java-sdk-bundle:1.12.499
com.cedarsoftware:java-util:1.9.0
com.cedarsoftware:json-io:2.5.1
com.fasterxml.jackson.core:jackson-annotations:2.12.7
Expand All @@ -242,7 +242,7 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
com.microsoft.azure:azure-storage:7.0.0
com.nimbusds:nimbus-jose-jwt:9.31
com.squareup.okhttp3:okhttp:4.10.0
com.squareup.okio:okio:3.2.0
com.squareup.okio:okio:3.4.0
com.zaxxer:HikariCP:4.0.3
commons-beanutils:commons-beanutils:1.9.4
commons-cli:commons-cli:1.5.0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,16 @@
A special value of "*" means all users are allowed.</description>
</property>

<property>
<name>security.datanode.lifeline.protocol.acl</name>
<value>*</value>
<description>ACL for DatanodeLifelineProtocol, which is used by a
DataNode to send lifeline messages to the NameNode.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>

<property>
<name>security.namenode.protocol.acl</name>
<value>*</value>
Expand All @@ -82,6 +92,16 @@
A special value of "*" means all users are allowed.</description>
</property>

<property>
<name>security.get.user.mappings.protocol.acl</name>
<value>*</value>
<description>ACL for GetUserMappingsProtocol, implemented by the NameNode
and Job Tracker which maps users to groups.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>

<property>
<name>security.refresh.user.mappings.protocol.acl</name>
<value>*</value>
Expand All @@ -92,6 +112,16 @@
users are allowed.</description>
</property>

<property>
<name>security.reconfiguration.protocol.acl</name>
<value>*</value>
<description>ACL for ReconfigurationProtocol, used by HDFS admin to
reload configuration for NameNode/DataNode without restarting them.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>

<property>
<name>security.refresh.policy.protocol.acl</name>
<value>*</value>
Expand All @@ -102,6 +132,26 @@
A special value of "*" means all users are allowed.</description>
</property>

<property>
<name>security.refresh.callqueue.protocol.acl</name>
<value>*</value>
<description>ACL for RefreshCallQueueProtocol, which is used to refresh
the call queue in use currently.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>

<property>
<name>security.refresh.generic.protocol.acl</name>
<value>*</value>
<description>ACL for GenericRefreshProtocol, which is used to refresh
arbitrary things at runtime.
The ACL is a comma-separated list of user and group names. The user and
group list is separated by a blank. For e.g. "alice,bob users,wheel".
A special value of "*" means all users are allowed.</description>
</property>

<property>
<name>security.ha.service.protocol.acl</name>
<value>*</value>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1214,10 +1214,10 @@ private void receiveRpcResponse() {
if (status == RpcStatusProto.SUCCESS) {
Writable value = packet.newInstance(valueClass, conf);
final Call call = calls.remove(callId);
call.setRpcResponse(value);
if (call.alignmentContext != null) {
call.alignmentContext.receiveResponseState(header);
}
call.setRpcResponse(value);
}
// verify that packet length was correct
if (packet.remaining() > 0) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -616,6 +616,9 @@ void updateMetrics(Call call, long processingStartTimeNanos, boolean connDropped
deltaNanos -= details.get(Timing.RESPONSE);
details.set(Timing.HANDLER, deltaNanos);

long enQueueTime = details.get(Timing.ENQUEUE, rpcMetrics.getMetricsTimeUnit());
rpcMetrics.addRpcEnQueueTime(enQueueTime);

long queueTime = details.get(Timing.QUEUE, rpcMetrics.getMetricsTimeUnit());
rpcMetrics.addRpcQueueTime(queueTime);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ public class RpcMetrics {
CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
metricsTimeUnit = getMetricsTimeUnit(conf);
if (rpcQuantileEnable) {
rpcEnQueueTimeQuantiles =
new MutableQuantiles[intervals.length];
rpcQueueTimeQuantiles =
new MutableQuantiles[intervals.length];
rpcLockWaitTimeQuantiles =
Expand All @@ -81,6 +83,9 @@ public class RpcMetrics {
new MutableQuantiles[intervals.length];
for (int i = 0; i < intervals.length; i++) {
int interval = intervals[i];
rpcEnQueueTimeQuantiles[i] = registry.newQuantiles("rpcEnQueueTime"
+ interval + "s", "rpc enqueue time in " + metricsTimeUnit, "ops",
"latency", interval);
rpcQueueTimeQuantiles[i] = registry.newQuantiles("rpcQueueTime"
+ interval + "s", "rpc queue time in " + metricsTimeUnit, "ops",
"latency", interval);
Expand Down Expand Up @@ -114,6 +119,8 @@ public static RpcMetrics create(Server server, Configuration conf) {

@Metric("Number of received bytes") MutableCounterLong receivedBytes;
@Metric("Number of sent bytes") MutableCounterLong sentBytes;
@Metric("EQueue time") MutableRate rpcEnQueueTime;
MutableQuantiles[] rpcEnQueueTimeQuantiles;
@Metric("Queue time") MutableRate rpcQueueTime;
MutableQuantiles[] rpcQueueTimeQuantiles;
@Metric("Lock wait time") MutableRate rpcLockWaitTime;
Expand Down Expand Up @@ -257,6 +264,23 @@ public void incrReceivedBytes(int count) {
receivedBytes.incr(count);
}

/**
* Sometimes, the request time observed by the client is much longer than
* the queue + process time on the RPC server.Perhaps the RPC request
* 'waiting enQueue' took too long on the RPC server, so we should add
* enQueue time to RpcMetrics. See HADOOP-18840 for details.
* Add an RPC enqueue time sample
* @param enQTime the queue time
*/
public void addRpcEnQueueTime(long enQTime) {
rpcEnQueueTime.add(enQTime);
if (rpcQuantileEnable) {
for (MutableQuantiles q : rpcEnQueueTimeQuantiles) {
q.add(enQTime);
}
}
}

/**
* Add an RPC queue time sample
* @param qTime the queue time
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,7 @@ private String formatTokenId(TokenIdent id) {
* Cache of currently valid tokens, mapping from DelegationTokenIdentifier
* to DelegationTokenInformation. Protected by this object lock.
*/
protected final Map<TokenIdent, DelegationTokenInformation> currentTokens
= new ConcurrentHashMap<>();
protected Map<TokenIdent, DelegationTokenInformation> currentTokens;

/**
* Map of token real owners to its token count. This is used to generate
Expand Down Expand Up @@ -155,6 +154,7 @@ public AbstractDelegationTokenSecretManager(long delegationKeyUpdateInterval,
this.tokenRenewInterval = delegationTokenRenewInterval;
this.tokenRemoverScanInterval = delegationTokenRemoverScanInterval;
this.storeTokenTrackingId = false;
this.currentTokens = new ConcurrentHashMap<>();
}

/**
Expand Down Expand Up @@ -190,6 +190,14 @@ public long getCurrentTokensSize() {
return currentTokens.size();
}

/**
* Interval for tokens to be renewed.
* @return Renew interval in milliseconds.
*/
protected long getTokenRenewInterval() {
return this.tokenRenewInterval;
}

/**
* Add a previously used master key to cache (when NN restarts),
* should be called before activate().
Expand Down Expand Up @@ -751,7 +759,7 @@ private void removeExpiredToken() throws IOException {
Set<TokenIdent> expiredTokens = new HashSet<>();
synchronized (this) {
Iterator<Map.Entry<TokenIdent, DelegationTokenInformation>> i =
currentTokens.entrySet().iterator();
getCandidateTokensForCleanup().entrySet().iterator();
while (i.hasNext()) {
Map.Entry<TokenIdent, DelegationTokenInformation> entry = i.next();
long renewDate = entry.getValue().getRenewDate();
Expand All @@ -766,15 +774,23 @@ private void removeExpiredToken() throws IOException {
logExpireTokens(expiredTokens);
}

protected Map<TokenIdent, DelegationTokenInformation> getCandidateTokensForCleanup() {
return this.currentTokens;
}

protected void logExpireTokens(
Collection<TokenIdent> expiredTokens) throws IOException {
for (TokenIdent ident : expiredTokens) {
logExpireToken(ident);
LOG.info("Removing expired token " + formatTokenId(ident));
removeStoredToken(ident);
removeExpiredStoredToken(ident);
}
}

protected void removeExpiredStoredToken(TokenIdent ident) throws IOException {
removeStoredToken(ident);
}

public void stopThreads() {
if (LOG.isDebugEnabled())
LOG.debug("Stopping expired delegation token remover thread");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.security.token.delegation;

import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader;
import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache;


/**
* Cache for delegation tokens that can handle high volume of tokens. A
* loading cache will prevent all active tokens from being in memory at the
* same time. It will also trigger more requests from the persistent token storage.
*/
public class DelegationTokenLoadingCache<K, V> implements Map<K, V> {
private LoadingCache<K, V> internalLoadingCache;

public DelegationTokenLoadingCache(long cacheExpirationMs, long maximumCacheSize,
Function<K, V> singleEntryFunction) {
this.internalLoadingCache = CacheBuilder.newBuilder()
.expireAfterWrite(cacheExpirationMs, TimeUnit.MILLISECONDS)
.maximumSize(maximumCacheSize)
.build(new CacheLoader<K, V>() {
@Override
public V load(K k) throws Exception {
return singleEntryFunction.apply(k);
}
});
}

@Override
public int size() {
return (int) this.internalLoadingCache.size();
}

@Override
public boolean isEmpty() {
return size() == 0;
}

@Override
public boolean containsKey(Object key) {
return this.internalLoadingCache.getIfPresent(key) != null;
}

@Override
public boolean containsValue(Object value) {
throw new UnsupportedOperationException();
}

@Override
public V get(Object key) {
try {
return this.internalLoadingCache.get((K) key);
} catch (Exception e) {
return null;
}
}

@Override
public V put(K key, V value) {
this.internalLoadingCache.put(key, value);
return this.internalLoadingCache.getIfPresent(key);
}

@Override
public V remove(Object key) {
V value = this.internalLoadingCache.getIfPresent(key);
this.internalLoadingCache.invalidate(key);
return value;
}

@Override
public void putAll(Map<? extends K, ? extends V> m) {
this.internalLoadingCache.putAll(m);
}

@Override
public void clear() {
this.internalLoadingCache.invalidateAll();
}

@Override
public Set<K> keySet() {
return this.internalLoadingCache.asMap().keySet();
}

@Override
public Collection<V> values() {
return this.internalLoadingCache.asMap().values();
}

@Override
public Set<Entry<K, V>> entrySet() {
return this.internalLoadingCache.asMap().entrySet();
}
}
Loading

0 comments on commit b8b1dea

Please sign in to comment.