From d02516c45f87d9fcf34b36f8fea653a0d5537ae7 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 16 Aug 2022 16:48:27 +0800 Subject: [PATCH 01/13] YARN-9708. Yarn Router Support DelegationToken. --- .../client/YARNDelegationTokenIdentifier.java | 13 + .../YARNDelegationTokenIdentifierPBImpl.java | 200 ++++++++++++++ .../security/client/impl/pb/package-info.java | 20 ++ .../yarn/api/BasePBImplRecordsTest.java | 6 +- .../hadoop-yarn-server-common/pom.xml | 3 + .../FederationDelegationTokenStateStore.java | 131 +++++++++ .../store/FederationStateStore.java | 3 +- .../impl/MemoryFederationStateStore.java | 148 +++++++++++ .../store/impl/SQLFederationStateStore.java | 47 ++++ .../impl/ZookeeperFederationStateStore.java | 46 ++++ .../store/records/RouterMasterKey.java | 128 +++++++++ .../store/records/RouterMasterKeyRequest.java | 43 +++ .../records/RouterMasterKeyResponse.java | 44 ++++ .../records/RouterRMDTSecretManagerState.java | 53 ++++ .../store/records/RouterRMTokenRequest.java | 43 +++ .../store/records/RouterRMTokenResponse.java | 43 +++ .../store/records/RouterStoreToken.java | 56 ++++ .../impl/pb/RouterMasterKeyPBImpl.java | 133 ++++++++++ .../impl/pb/RouterMasterKeyRequestPBImpl.java | 128 +++++++++ .../pb/RouterMasterKeyResponsePBImpl.java | 128 +++++++++ .../impl/pb/RouterRMTokenRequestPBImpl.java | 128 +++++++++ .../impl/pb/RouterRMTokenResponsePBImpl.java | 110 ++++++++ .../impl/pb/RouterStoreTokenPBImpl.java | 171 ++++++++++++ .../utils/FederationStateStoreFacade.java | 120 +++++++++ .../proto/yarn_server_federation_protos.proto | 30 ++- .../TestFederationProtocolRecords.java | 47 ++++ .../utils/TestFederationStateStoreFacade.java | 104 ++++++++ .../FederationStateStoreService.java | 48 ++++ .../AbstractClientRequestInterceptor.java | 12 + .../clientrm/ClientRequestInterceptor.java | 15 ++ .../clientrm/RouterClientRMService.java | 43 ++- .../RouterDelegationTokenSecretManager.java | 248 ++++++++++++++++++ .../server/router/security/package-info.java | 19 ++ .../secure/AbstractSecureRouterTest.java | 8 + ...estRouterDelegationTokenSecretManager.java | 192 ++++++++++++++ 35 files changed, 2707 insertions(+), 4 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java index 6d8bc4bc1da88..0020b3c407627 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java @@ -22,11 +22,14 @@ import java.io.DataOutputStream; import java.io.IOException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; +import org.apache.hadoop.yarn.util.Records; @Private public abstract class YARNDelegationTokenIdentifier extends @@ -112,4 +115,14 @@ public YARNDelegationTokenIdentifierProto getProto() { setBuilderFields(); return builder.build(); } + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static YARNDelegationTokenIdentifier newInstance(Text owner, Text renewer, Text realUser) { + YARNDelegationTokenIdentifier policy = Records.newRecord(YARNDelegationTokenIdentifier.class); + policy.setOwner(owner); + policy.setRenewer(renewer); + policy.setRenewer(realUser); + return policy; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java new file mode 100644 index 0000000000000..aea2d0eb7c29d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.security.client.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; +import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProtoOrBuilder; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class YARNDelegationTokenIdentifierPBImpl extends YARNDelegationTokenIdentifier { + + private YARNDelegationTokenIdentifierProto proto = + YARNDelegationTokenIdentifierProto.getDefaultInstance(); + private YARNDelegationTokenIdentifierProto.Builder builder = null; + private boolean viaProto = false; + + public YARNDelegationTokenIdentifierPBImpl() { + builder = YARNDelegationTokenIdentifierProto.newBuilder(); + } + + public YARNDelegationTokenIdentifierPBImpl(YARNDelegationTokenIdentifierProto identifierProto) { + this.proto = identifierProto; + viaProto = true; + } + + public YARNDelegationTokenIdentifierProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + // mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + if (proto == null) { + proto = YARNDelegationTokenIdentifierProto.getDefaultInstance(); + } + builder = YARNDelegationTokenIdentifierProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public Text getOwner() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return new Text(p.getOwner()); + } + + @Override + public void setOwner(Text owner) { + super.setOwner(owner); + maybeInitBuilder(); + if (owner == null) { + builder.clearOwner(); + return; + } + builder.setOwner(owner.toString()); + } + + @Override + public Text getRenewer() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return new Text(p.getRenewer()); + } + + @Override + public void setRenewer(Text renewer) { + super.setRenewer(renewer); + maybeInitBuilder(); + if (renewer == null) { + builder.clearRenewer(); + return; + } + builder.setOwner(renewer.toString()); + } + + @Override + public Text getRealUser() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return new Text(p.getRealUser()); + } + + @Override + public void setRealUser(Text realUser) { + super.setRealUser(realUser); + maybeInitBuilder(); + if (realUser == null) { + builder.clearRealUser(); + return; + } + builder.setRealUser(realUser.toString()); + } + + @Override + public void setIssueDate(long issueDate) { + super.setIssueDate(issueDate); + maybeInitBuilder(); + builder.setIssueDate(issueDate); + } + + @Override + public long getIssueDate() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return p.getIssueDate(); + } + + @Override + public void setMaxDate(long maxDate) { + super.setMaxDate(maxDate); + maybeInitBuilder(); + builder.setMaxDate(maxDate); + } + + @Override + public long getMaxDate() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return p.getMaxDate(); + } + + @Override + public void setSequenceNumber(int seqNum) { + super.setSequenceNumber(seqNum); + maybeInitBuilder(); + builder.setSequenceNumber(seqNum); + } + + @Override + public int getSequenceNumber() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return p.getSequenceNumber(); + } + + @Override + public void setMasterKeyId(int newId) { + super.setMasterKeyId(newId); + maybeInitBuilder(); + builder.setMasterKeyId(newId); + } + + @Override + public int getMasterKeyId() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return p.getMasterKeyId(); + } + + @Override + public Text getKind() { + return null; + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java new file mode 100644 index 0000000000000..5751632e316ad --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Public +package org.apache.hadoop.yarn.security.client.impl.pb; +import org.apache.hadoop.classification.InterfaceAudience; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java index 5697923c9974d..63a5b352eef02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java @@ -18,10 +18,12 @@ package org.apache.hadoop.yarn.api; import org.apache.commons.lang3.Range; +import org.apache.hadoop.io.Text; import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.Sets; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraints; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; @@ -80,6 +82,8 @@ private static Object genTypeValue(Type type) { 'a' + rand.nextInt(26)); } else if (type.equals(Float.class)) { return rand.nextFloat(); + } else if (type.equals(Text.class)) { + return new Text('a' + String.valueOf(rand.nextInt(1000000))); } else if (type instanceof Class) { Class clazz = (Class)type; if (clazz.isArray()) { @@ -167,7 +171,7 @@ protected static Object generateByNewInstance(Class clazz) throws Exception { " does not have newInstance method"); } Object [] args = new Object[paramTypes.length]; - for (int i=0;i ${basedir}/../../hadoop-yarn-api/src/main/proto + + ${basedir}/../../hadoop-yarn-common/src/main/proto + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java new file mode 100644 index 0000000000000..45b70aabec4fd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; + +import java.io.IOException; + +/** + * FederationDelegationTokenStateStore maintains the state of all + * DelegationToken that have been submitted to the federated cluster. + * + *

+ * It mainly includes the following operations: + *

+ * + * + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface FederationDelegationTokenStateStore { + + /** + * The Router Supports Store the new master key. + * + * @param request DelegationKey. + * @return StoreNewMasterKeyResponse. + * @throws YarnException exception occurred. + */ + RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException; + + /** + * The Router Supports Remove the master key. + * + * @param request DelegationKey. + * @return RemoveStoredMasterKeyResponse. + * @throws YarnException exception occurred. + */ + RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException; + + /** + * The Router supports obtaining MasterKey based on KeyId. + * + * @param request + * @return + * @throws YarnException + * @throws IOException + */ + RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException; + + /** + * The Router Supports Store new Token. + * + * @param request DelegationKey. + * @return RouterRMTokenResponse. + * @throws YarnException exception occurred. + */ + RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) + throws YarnException, IOException; + + /** + * The Router Supports Update Token. + * + * @param request DelegationKey. + * @return RouterRMTokenResponse. + * @throws YarnException exception occurred. + */ + RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException; + + /** + * The Router Supports Remove Token. + * + * @param request DelegationKey. + * @return RouterRMTokenResponse. + * @throws YarnException exception occurred. + */ + RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException; + + /** + * The Router Supports Remove Token. + * + * @param request DelegationKey. + * @return RouterRMTokenResponse. + * @throws YarnException exception occurred. + */ + RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java index 67461e6c30b86..3ca8ccc2bfb69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java @@ -31,7 +31,8 @@ */ public interface FederationStateStore extends FederationApplicationHomeSubClusterStore, FederationMembershipStateStore, - FederationPolicyStore, FederationReservationHomeSubClusterStore { + FederationPolicyStore, FederationReservationHomeSubClusterStore, + FederationDelegationTokenStateStore { /** * Initialize the FederationStore. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 24ed54d63668b..0e1363a66d714 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -17,15 +17,21 @@ package org.apache.hadoop.yarn.server.federation.store.impl; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.TimeZone; import java.util.concurrent.ConcurrentHashMap; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -66,7 +72,14 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMDTSecretManagerState; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationReservationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; @@ -86,6 +99,7 @@ public class MemoryFederationStateStore implements FederationStateStore { private Map applications; private Map reservations; private Map policies; + private RouterRMDTSecretManagerState routerRMSecretManagerState; private final MonotonicClock clock = new MonotonicClock(); @@ -98,6 +112,7 @@ public void init(Configuration conf) { applications = new ConcurrentHashMap(); reservations = new ConcurrentHashMap(); policies = new ConcurrentHashMap(); + routerRMSecretManagerState = new RouterRMDTSecretManagerState(); } @Override @@ -365,4 +380,137 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( return GetReservationsHomeSubClusterResponse.newInstance(result); } + + @Override + public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + + // Restore the DelegationKey from the request + RouterMasterKey masterKey = request.getRouterMasterKey(); + ByteBuffer keyByteBuf = masterKey.getKeyBytes(); + byte[] keyBytes = new byte[keyByteBuf.remaining()]; + keyByteBuf.get(keyBytes); + DelegationKey delegationKey = + new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + + Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); + if (rmDTMasterKeyState.contains(delegationKey)) { + LOG.info("Error storing info for RMDTMasterKey with keyID: {}.", delegationKey.getKeyId()); + throw new IOException("RMDTMasterKey with keyID: " + + delegationKey.getKeyId() + " is already stored"); + } + + routerRMSecretManagerState.getMasterKeyState().add(delegationKey); + LOG.info("Store Router-RMDT master key with key id: {}. Currently rmDTMasterKeyState size: {}", + delegationKey.getKeyId(), rmDTMasterKeyState.size()); + + return RouterMasterKeyResponse.newInstance(masterKey); + } + + @Override + public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + + // Restore the DelegationKey from the request + RouterMasterKey masterKey = request.getRouterMasterKey(); + ByteBuffer keyByteBuf = masterKey.getKeyBytes(); + byte[] keyBytes = new byte[keyByteBuf.remaining()]; + keyByteBuf.get(keyBytes); + DelegationKey delegationKey = + new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + + LOG.info("Remove Router-RMDT master key with key id: {}.", delegationKey.getKeyId()); + Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); + rmDTMasterKeyState.remove(delegationKey); + + return RouterMasterKeyResponse.newInstance(masterKey); + } + + @Override + public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + // Restore the DelegationKey from the request + RouterMasterKey masterKey = request.getRouterMasterKey(); + ByteBuffer keyByteBuf = masterKey.getKeyBytes(); + byte[] keyBytes = new byte[keyByteBuf.remaining()]; + keyByteBuf.get(keyBytes); + DelegationKey delegationKey = + new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + + Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); + if (!rmDTMasterKeyState.contains(delegationKey)) { + throw new IOException("GetMasterKey with keyID: " + masterKey.getKeyId() + + " does not exist."); + } + RouterMasterKey resultRouterMasterKey = RouterMasterKey.newInstance(delegationKey.getKeyId(), + ByteBuffer.wrap(delegationKey.getEncodedKey()), delegationKey.getExpiryDate()); + return RouterMasterKeyResponse.newInstance(resultRouterMasterKey); + } + + @Override + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws YarnException, IOException { + RouterStoreToken storeToken = request.getRouterStoreToken(); + RMDelegationTokenIdentifier tokenIdentifier = + (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); + Long renewDate = storeToken.getRenewDate(); + storeOrUpdateRouterRMDT(tokenIdentifier, renewDate, false); + return RouterRMTokenResponse.newInstance(storeToken); + } + + @Override + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws YarnException, IOException { + RouterStoreToken storeToken = request.getRouterStoreToken(); + RMDelegationTokenIdentifier tokenIdentifier = + (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); + Long renewDate = storeToken.getRenewDate(); + Map rmDTState = routerRMSecretManagerState.getTokenState(); + rmDTState.remove(tokenIdentifier); + storeOrUpdateRouterRMDT(tokenIdentifier, renewDate, true); + return RouterRMTokenResponse.newInstance(storeToken); + } + + @Override + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws YarnException, IOException { + RouterStoreToken storeToken = request.getRouterStoreToken(); + RMDelegationTokenIdentifier tokenIdentifier = + (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); + Map rmDTState = routerRMSecretManagerState.getTokenState(); + rmDTState.remove(tokenIdentifier); + return RouterRMTokenResponse.newInstance(storeToken); + } + + private void storeOrUpdateRouterRMDT(RMDelegationTokenIdentifier rmDTIdentifier, + Long renewDate, boolean isUpdate) throws IOException { + Map rmDTState = routerRMSecretManagerState.getTokenState(); + if (rmDTState.containsKey(rmDTIdentifier)) { + LOG.info("Error storing info for RMDelegationToken: {}.", rmDTIdentifier); + throw new IOException("RMDelegationToken: " + rmDTIdentifier + "is already stored."); + } + rmDTState.put(rmDTIdentifier, renewDate); + if(!isUpdate) { + routerRMSecretManagerState.setDtSequenceNumber(rmDTIdentifier.getSequenceNumber()); + } + LOG.info("Store RM-RMDT with sequence number {}.", rmDTIdentifier.getSequenceNumber()); + } + + @Override + public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException { + RouterStoreToken storeToken = request.getRouterStoreToken(); + RMDelegationTokenIdentifier tokenIdentifier = + (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); + Map rmDTState = routerRMSecretManagerState.getTokenState(); + if (!rmDTState.containsKey(tokenIdentifier)) { + LOG.info("RMDelegationToken: {} does not exist.", tokenIdentifier); + throw new IOException("RMDelegationToken: " + tokenIdentifier + " does not exist."); + } + RouterStoreToken resultToken = + RouterStoreToken.newInstance(tokenIdentifier, rmDTState.get(tokenIdentifier)); + return RouterRMTokenResponse.newInstance(resultToken); + } + + @VisibleForTesting + public RouterRMDTSecretManagerState getRouterRMSecretManagerState() { + return routerRMSecretManagerState; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index dfcfb06bb46b8..e0e7e610d643d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.federation.store.impl; +import java.io.IOException; import java.nio.ByteBuffer; import java.sql.CallableStatement; import java.sql.Connection; @@ -78,6 +79,10 @@ import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; @@ -1027,4 +1032,46 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( GetReservationsHomeSubClusterRequest request) throws YarnException { throw new NotImplementedException("Code is not implemented"); } + + @Override + public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index 686ba077c7da5..ea23251815c54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -75,6 +75,10 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterIdPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterInfoPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterPolicyConfigurationPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; @@ -662,4 +666,46 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( GetReservationsHomeSubClusterRequest request) throws YarnException { throw new NotImplementedException("Code is not implemented"); } + + @Override + public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException{ + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java new file mode 100644 index 0000000000000..f9cb89354a2ff --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterMasterKey { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterMasterKey newInstance(Integer keyId, ByteBuffer keyBytes, Long expiryDate) { + RouterMasterKey policy = Records.newRecord(RouterMasterKey.class); + policy.setKeyId(keyId); + policy.setKeyBytes(keyBytes); + policy.setExpiryDate(expiryDate); + return policy; + } + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterMasterKey newInstance(RouterMasterKey masterKey) { + RouterMasterKey routerMasterKey = Records.newRecord(RouterMasterKey.class); + routerMasterKey.setKeyId(masterKey.getKeyId()); + routerMasterKey.setKeyBytes(masterKey.getKeyBytes()); + routerMasterKey.setExpiryDate(masterKey.getExpiryDate()); + return routerMasterKey; + } + + /** + * Get the keyId of the MasterKey. + * + * @return MasterKeyId. + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract Integer getKeyId(); + + /** + * Set the keyId of the MasterKey. + * + * @param keyId MasterKeyId. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setKeyId(Integer keyId); + + /** + * Get the keyBytes of the DelegationKey. + * + * @return KeyBytes of the DelegationKey. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract ByteBuffer getKeyBytes(); + + /** + * Set the keyBytes of the DelegationKey. + * + * @param keyBytes KeyBytes of the DelegationKey. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setKeyBytes(ByteBuffer keyBytes); + + /** + * Get the ExpiryDate of the DelegationKey. + * + * @return ExpiryDate of the DelegationKey. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract Long getExpiryDate(); + + /** + * Set the expiryDate of the DelegationKey. + * + * @param expiryDate expiryDate of the DelegationKey. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setExpiryDate(Long expiryDate); + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (getExpiryDate() ^ (getExpiryDate() >>> 32)); + result = prime * result + Arrays.hashCode(getKeyBytes().array()); + result = prime * result + getKeyId(); + return result; + } + + @Override + public boolean equals(Object right) { + if (this == right) { + return true; + } else if (right == null || getClass() != right.getClass()) { + return false; + } else { + RouterMasterKey r = (RouterMasterKey) right; + return getKeyId().intValue() == r.getKeyId().intValue() && + getExpiryDate().longValue() == r.getExpiryDate().longValue() && + Arrays.equals(getKeyBytes().array(), r.getKeyBytes().array()); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java new file mode 100644 index 0000000000000..2bf30535162d5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterMasterKeyRequest { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterMasterKeyRequest newInstance(RouterMasterKey routerMasterKey) { + RouterMasterKeyRequest request = Records.newRecord(RouterMasterKeyRequest.class); + request.setRouterMasterKey(routerMasterKey); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterMasterKey getRouterMasterKey(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java new file mode 100644 index 0000000000000..6625e4c1911d8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterMasterKeyResponse { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterMasterKeyResponse newInstance(RouterMasterKey masterKey) { + RouterMasterKeyResponse request = + Records.newRecord(RouterMasterKeyResponse.class); + request.setRouterMasterKey(masterKey); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterMasterKey getRouterMasterKey(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterMasterKey(RouterMasterKey masterKey); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java new file mode 100644 index 0000000000000..e5010945286e0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class RouterRMDTSecretManagerState { + + // DTIdentifier -> renewDate + private Map delegationTokenState = + new HashMap(); + + private Set masterKeyState = new HashSet(); + + private int dtSequenceNumber = 0; + + public Map getTokenState() { + return delegationTokenState; + } + + public Set getMasterKeyState() { + return masterKeyState; + } + + public int getDTSequenceNumber() { + return dtSequenceNumber; + } + + public void setDtSequenceNumber(int dtSequenceNumber) { + this.dtSequenceNumber = dtSequenceNumber; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java new file mode 100644 index 0000000000000..0d4908de3539f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterRMTokenRequest { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterRMTokenRequest newInstance(RouterStoreToken routerStoreToken) { + RouterRMTokenRequest request = Records.newRecord(RouterRMTokenRequest.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java new file mode 100644 index 0000000000000..57671bda419f4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterRMTokenResponse { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterRMTokenResponse newInstance(RouterStoreToken routerStoreToken) { + RouterRMTokenResponse request = Records.newRecord(RouterRMTokenResponse.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java new file mode 100644 index 0000000000000..d6eade878020f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; +import org.apache.hadoop.yarn.util.Records; + +import java.io.IOException; + +@Private +@Unstable +public abstract class RouterStoreToken { + + @Private + @Unstable + public static RouterStoreToken newInstance(YARNDelegationTokenIdentifier identifier, + Long renewdate) { + RouterStoreToken storeToken = Records.newRecord(RouterStoreToken.class); + storeToken.setIdentifier(identifier); + storeToken.setRenewDate(renewdate); + return storeToken; + } + + @Private + @Unstable + public abstract YARNDelegationTokenIdentifier getTokenIdentifier() throws IOException; + + @Private + @Unstable + public abstract void setIdentifier(YARNDelegationTokenIdentifier identifier); + + @Private + @Unstable + public abstract Long getRenewDate(); + + @Private + @Unstable + public abstract void setRenewDate(Long renewDate); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java new file mode 100644 index 0000000000000..e2adccc955205 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; + +import java.nio.ByteBuffer; + +public class RouterMasterKeyPBImpl extends RouterMasterKey { + + private RouterMasterKeyProto proto = RouterMasterKeyProto.getDefaultInstance(); + private RouterMasterKeyProto.Builder builder = null; + private boolean viaProto = false; + + public RouterMasterKeyPBImpl() { + builder = RouterMasterKeyProto.newBuilder(); + } + + public RouterMasterKeyPBImpl(RouterMasterKeyProto masterKeyProto) { + this.proto = masterKeyProto; + viaProto = true; + } + + public RouterMasterKeyProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterMasterKeyProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public Integer getKeyId() { + RouterMasterKeyProtoOrBuilder p = viaProto ? proto : builder; + return p.getKeyId(); + } + + @Override + public void setKeyId(Integer keyId) { + maybeInitBuilder(); + if (keyId == null) { + builder.clearKeyId(); + return; + } + builder.setKeyId(keyId); + } + + @Override + public ByteBuffer getKeyBytes() { + RouterMasterKeyProtoOrBuilder p = viaProto ? proto : builder; + return convertFromProtoFormat(p.getKeyBytes()); + } + + @Override + public void setKeyBytes(ByteBuffer keyBytes) { + maybeInitBuilder(); + if (keyBytes == null) { + builder.clearKeyBytes(); + return; + } + builder.setKeyBytes(convertToProtoFormat(keyBytes)); + } + + @Override + public Long getExpiryDate() { + RouterMasterKeyProtoOrBuilder p = viaProto ? proto : builder; + return p.getExpiryDate(); + } + + @Override + public void setExpiryDate(Long expiryDate) { + maybeInitBuilder(); + if (expiryDate == null) { + builder.clearExpiryDate(); + return; + } + builder.setExpiryDate(expiryDate); + } + + protected final ByteBuffer convertFromProtoFormat(ByteString byteString) { + return ProtoUtils.convertFromProtoFormat(byteString); + } + + protected final ByteString convertToProtoFormat(ByteBuffer byteBuffer) { + return ProtoUtils.convertToProtoFormat(byteBuffer); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java new file mode 100644 index 0000000000000..93fcdd8e6166d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyRequestProtoOrBuilder; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +@Private +@Unstable +public class RouterMasterKeyRequestPBImpl extends RouterMasterKeyRequest { + + private RouterMasterKeyRequestProto proto = RouterMasterKeyRequestProto.getDefaultInstance(); + private RouterMasterKeyRequestProto.Builder builder = null; + private boolean viaProto = false; + private RouterMasterKey routerMasterKey = null; + + public RouterMasterKeyRequestPBImpl() { + builder = RouterMasterKeyRequestProto.newBuilder(); + } + + public RouterMasterKeyRequestPBImpl(RouterMasterKeyRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public RouterMasterKeyRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterMasterKeyRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + RouterMasterKeyPBImpl masterKeyRequest = (RouterMasterKeyPBImpl) this.routerMasterKey; + RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey(); + if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) { + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + } + } + + @Override + public RouterMasterKey getRouterMasterKey() { + RouterMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerMasterKey != null) { + return this.routerMasterKey; + } + if (!p.hasRouterMasterKey()) { + return null; + } + this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); + return this.routerMasterKey; + } + + @Override + public void setRouterMasterKey(RouterMasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) { + builder.clearRouterMasterKey(); + } + this.routerMasterKey = masterKey; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKeyProto) { + return new RouterMasterKeyPBImpl(masterKeyProto); + } + + private RouterMasterKeyProto convertToProtoFormat(RouterMasterKey masterKey) { + return ((RouterMasterKeyPBImpl) masterKey).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java new file mode 100644 index 0000000000000..3f0f3fbde7e22 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; + +@Private +@Unstable +public class RouterMasterKeyResponsePBImpl extends RouterMasterKeyResponse { + + private RouterMasterKeyResponseProto proto = RouterMasterKeyResponseProto.getDefaultInstance(); + private RouterMasterKeyResponseProto.Builder builder = null; + private boolean viaProto = false; + private RouterMasterKey routerMasterKey = null; + + public RouterMasterKeyResponsePBImpl() { + builder = RouterMasterKeyResponseProto.newBuilder(); + } + + public RouterMasterKeyResponsePBImpl(RouterMasterKeyResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public RouterMasterKeyResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterMasterKeyResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + RouterMasterKeyPBImpl masterKeyRequest = (RouterMasterKeyPBImpl) this.routerMasterKey; + RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey(); + if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) { + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + } + } + + @Override + public RouterMasterKey getRouterMasterKey() { + RouterMasterKeyResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerMasterKey != null) { + return this.routerMasterKey; + } + if (!p.hasRouterMasterKey()) { + return null; + } + this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); + return this.routerMasterKey; + } + + @Override + public void setRouterMasterKey(RouterMasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) { + builder.clearRouterMasterKey(); + } + this.routerMasterKey = masterKey; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKeyProto) { + return new RouterMasterKeyPBImpl(masterKeyProto); + } + + private RouterMasterKeyProto convertToProtoFormat(RouterMasterKey masterKey) { + return ((RouterMasterKeyPBImpl) masterKey).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java new file mode 100644 index 0000000000000..756d4745507e0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenRequestProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; + +public class RouterRMTokenRequestPBImpl extends RouterRMTokenRequest { + + private RouterRMTokenRequestProto proto = RouterRMTokenRequestProto.getDefaultInstance(); + private RouterRMTokenRequestProto.Builder builder = null; + private boolean viaProto = false; + private RouterStoreToken routerStoreToken = null; + + public RouterRMTokenRequestPBImpl() { + builder = RouterRMTokenRequestProto.newBuilder(); + } + + public RouterRMTokenRequestPBImpl(RouterRMTokenRequestProto requestProto) { + this.proto = requestProto; + viaProto = true; + } + + public RouterRMTokenRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterRMTokenRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerStoreToken != null) { + RouterStoreTokenPBImpl routerStoreTokenPBImpl = + (RouterStoreTokenPBImpl) this.routerStoreToken; + RouterStoreTokenProto storeTokenProto = routerStoreTokenPBImpl.getProto(); + if (!storeTokenProto.equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } + } + } + + @Override + public RouterStoreToken getRouterStoreToken() { + RouterRMTokenRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerStoreToken != null) { + return this.routerStoreToken; + } + if (!p.hasRouterStoreToken()) { + return null; + } + this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); + return this.routerStoreToken; + } + + @Override + public void setRouterStoreToken(RouterStoreToken storeToken) { + maybeInitBuilder(); + if (storeToken == null) { + builder.clearRouterStoreToken(); + } + this.routerStoreToken = storeToken; + this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { + return ((RouterStoreTokenPBImpl) storeToken).getProto(); + } + + private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { + return new RouterStoreTokenPBImpl(storeTokenProto); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java new file mode 100644 index 0000000000000..0c87a1a78c432 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java @@ -0,0 +1,110 @@ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenResponseProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; + +@Private +@Unstable +public class RouterRMTokenResponsePBImpl extends RouterRMTokenResponse { + + private RouterRMTokenResponseProto proto = RouterRMTokenResponseProto.getDefaultInstance(); + private RouterRMTokenResponseProto.Builder builder = null; + private boolean viaProto = false; + private RouterStoreToken routerStoreToken = null; + + public RouterRMTokenResponsePBImpl() { + builder = RouterRMTokenResponseProto.newBuilder(); + } + + public RouterRMTokenResponsePBImpl(RouterRMTokenResponseProto requestProto) { + this.proto = requestProto; + viaProto = true; + } + + public RouterRMTokenResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterRMTokenResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerStoreToken != null && + !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } + } + + @Override + public RouterStoreToken getRouterStoreToken() { + RouterRMTokenResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerStoreToken != null) { + return this.routerStoreToken; + } + if (!p.hasRouterStoreToken()) { + return null; + } + this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); + return this.routerStoreToken; + } + + @Override + public void setRouterStoreToken(RouterStoreToken storeToken) { + maybeInitBuilder(); + if (storeToken == null) { + builder.clearRouterStoreToken(); + } + this.routerStoreToken = storeToken; + } + + private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { + return ((RouterStoreTokenPBImpl) storeToken).getProto(); + } + + private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { + return new RouterStoreTokenPBImpl(storeTokenProto); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java new file mode 100644 index 0000000000000..4df7e40730531 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProtoOrBuilder; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; + +/** + * Protocol buffer based implementation of {@link RouterStoreToken}. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class RouterStoreTokenPBImpl extends RouterStoreToken { + + private RouterStoreTokenProto proto = RouterStoreTokenProto.getDefaultInstance(); + + private RouterStoreTokenProto.Builder builder = null; + + private boolean viaProto = false; + + private YARNDelegationTokenIdentifier rMDelegationTokenIdentifier = null; + private Long renewDate; + + public RouterStoreTokenPBImpl() { + builder = RouterStoreTokenProto.newBuilder(); + } + + public RouterStoreTokenPBImpl(RouterStoreTokenProto storeTokenProto) { + this.proto = storeTokenProto; + viaProto = true; + } + + public RouterStoreTokenProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.rMDelegationTokenIdentifier != null) { + YARNDelegationTokenIdentifierProto idProto = this.rMDelegationTokenIdentifier.getProto(); + if (!idProto.equals(builder.getTokenIdentifier())) { + builder.setTokenIdentifier(convertToProtoFormat(this.rMDelegationTokenIdentifier)); + } + } + + if (this.renewDate != null) { + builder.setRenewDate(this.renewDate); + } + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterStoreTokenProto.newBuilder(proto); + } + viaProto = false; + } + + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public YARNDelegationTokenIdentifier getTokenIdentifier() throws IOException { + RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder; + if (rMDelegationTokenIdentifier != null) { + return rMDelegationTokenIdentifier; + } + if(!p.hasTokenIdentifier()){ + return null; + } + YARNDelegationTokenIdentifierProto identifierProto = p.getTokenIdentifier(); + ByteArrayInputStream in = new ByteArrayInputStream(identifierProto.toByteArray()); + RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(); + identifier.readFields(new DataInputStream(in)); + this.rMDelegationTokenIdentifier = identifier; + return identifier; + } + + @Override + public Long getRenewDate() { + RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder; + if (this.renewDate != null) { + return this.renewDate; + } + if (!p.hasRenewDate()) { + return null; + } + this.renewDate = p.getRenewDate(); + return this.renewDate; + } + + @Override + public void setIdentifier(YARNDelegationTokenIdentifier identifier) { + maybeInitBuilder(); + if(identifier == null) { + builder.clearTokenIdentifier(); + return; + } + this.rMDelegationTokenIdentifier = identifier; + this.builder.setTokenIdentifier(identifier.getProto()); + } + + @Override + public void setRenewDate(Long renewDate) { + maybeInitBuilder(); + if(renewDate == null) { + builder.clearRenewDate(); + return; + } + this.renewDate = renewDate; + this.builder.setRenewDate(renewDate); + } + + private YARNDelegationTokenIdentifierProto convertToProtoFormat( + YARNDelegationTokenIdentifier delegationTokenIdentifier) { + return delegationTokenIdentifier.getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 2f65bc2fe15a3..b33e410f7ef86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.federation.utils; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,6 +43,7 @@ import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryProxy; +import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ReservationId; @@ -72,6 +75,13 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -446,6 +456,116 @@ public SubClusterId getReservationHomeSubCluster(ReservationId reservationId) return response.getReservationHomeSubCluster().getHomeSubCluster(); } + /** + * The Router supports saving the master key. + * + * @param newKey DelegationKey + * @throws Exception An error occurred + */ + public RouterMasterKeyResponse storeNewMasterKey(DelegationKey newKey) + throws YarnException, IOException { + LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); + ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); + RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), + keyBytes, newKey.getExpiryDate()); + RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey); + return stateStore.storeNewMasterKey(keyRequest); + } + + /** + * The Router supports remove the master key. + * + * @param newKey DelegationKey + * @throws Exception An error occurred + */ + public void removeStoredMasterKey(DelegationKey newKey) throws YarnException, IOException { + LOG.info("Removing master key with keyID {}.", newKey.getKeyId()); + ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); + RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), + keyBytes, newKey.getExpiryDate()); + RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey); + stateStore.removeStoredMasterKey(keyRequest); + } + + /** + * The Router supports obtaining MasterKey based on KeyId. + * + * @param newKey DelegationKey + * @throws Exception An error occurred + */ + public RouterMasterKeyResponse getMasterKeyByDelegationKey(DelegationKey newKey) + throws YarnException, IOException { + LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); + ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); + RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), + keyBytes, newKey.getExpiryDate()); + RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey); + return stateStore.getMasterKeyByDelegationKey(keyRequest); + } + + /** + * The Router Supports Store new Token. + * + * @param identifier Delegation Token + * @param renewDate renewDate + * @throws IOException IO exception occurred. + */ + public void storeNewToken(RMDelegationTokenIdentifier identifier, + long renewDate) throws YarnException, IOException { + LOG.info("storing RMDelegation token with sequence number: {}.", + identifier.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); + RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); + stateStore.storeNewToken(request); + } + + /** + * The Router Supports Update Token. + * + * @param identifier Delegation Token + * @param renewDate renewDate + * @throws IOException IO exception occurred. + */ + public void updateStoredToken(RMDelegationTokenIdentifier identifier, + long renewDate) throws YarnException, IOException { + LOG.info("updating RMDelegation token with sequence number: {}.", + identifier.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); + RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); + stateStore.updateStoredToken(request); + } + + /** + * The Router Supports Remove Token. + * + * @param identifier Delegation Token + * @throws IOException IO exception occurred. + */ + public void removeStoredToken(RMDelegationTokenIdentifier identifier) + throws YarnException, IOException{ + LOG.info("removing RMDelegation token with sequence number: {}.", + identifier.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L); + RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); + stateStore.removeStoredToken(request); + } + + /** + * The Router Supports Remove Token. + * + * @param identifier Delegation Token + * @return RouterRMTokenResponse. + * @throws YarnException exception occurred. + */ + public RouterRMTokenResponse getTokenByRouterStoreToken(RMDelegationTokenIdentifier identifier) + throws YarnException, IOException { + LOG.info("get RouterStoreToken token with sequence number: {}.", + identifier.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L); + RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); + return stateStore.getTokenByRouterStoreToken(request); + } + /** * Helper method to create instances of Object using the class name defined in * the configuration object. The instances creates {@link RetryProxy} using diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto index 33f5cb3fc14e0..ff2b97091bf43 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto @@ -25,6 +25,7 @@ package hadoop.yarn; import "yarn_protos.proto"; import "yarn_server_common_protos.proto"; +import "yarn_security_token.proto"; message SubClusterIdProto { optional string id = 1; @@ -213,4 +214,31 @@ message GetSubClusterPoliciesConfigurationsRequestProto { message GetSubClusterPoliciesConfigurationsResponseProto { repeated SubClusterPolicyConfigurationProto policies_configurations = 1; - } +} + +message RouterMasterKeyProto { + optional int32 key_id = 1; + optional bytes key_bytes = 2; + optional uint64 expiry_date = 3; +} + +message RouterMasterKeyRequestProto { + optional RouterMasterKeyProto router_master_key = 1; +} + +message RouterMasterKeyResponseProto { + optional RouterMasterKeyProto router_master_key = 1; +} + +message RouterStoreTokenProto { + optional YARNDelegationTokenIdentifierProto token_identifier = 1; + optional int64 renew_date = 2; +} + +message RouterRMTokenRequestProto { + optional RouterStoreTokenProto router_store_token = 1; +} + +message RouterRMTokenResponseProto { + optional RouterStoreTokenProto router_store_token = 1; +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java index cf8cf719d01d5..f87d370224309 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java @@ -19,6 +19,7 @@ import org.apache.hadoop.yarn.api.BasePBImplRecordsTest; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProto; @@ -47,6 +48,13 @@ import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyResponseProto; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterRequestPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterResponsePBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.DeleteApplicationHomeSubClusterRequestPBImpl; @@ -75,6 +83,12 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterResponsePBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterRequestPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterStoreTokenPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRMTokenRequestPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRMTokenResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyRequestPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyResponsePBImpl; import org.apache.hadoop.yarn.server.records.Version; import org.junit.BeforeClass; import org.junit.Test; @@ -92,6 +106,9 @@ public static void setup() throws Exception { generateByNewInstance(SubClusterInfo.class); generateByNewInstance(ApplicationHomeSubCluster.class); generateByNewInstance(SubClusterPolicyConfiguration.class); + generateByNewInstance(RouterMasterKey.class); + generateByNewInstance(YARNDelegationTokenIdentifier.class); + generateByNewInstance(RouterStoreToken.class); } @Test @@ -262,4 +279,34 @@ public void testGetSubClusterPoliciesConfigurationsResponse() GetSubClusterPoliciesConfigurationsResponsePBImpl.class, GetSubClusterPoliciesConfigurationsResponseProto.class); } + + @Test + public void testRouterMasterKey() throws Exception { + validatePBImplRecord(RouterMasterKeyPBImpl.class, RouterMasterKeyProto.class); + } + + @Test + public void testRouterStoreToken() throws Exception { + validatePBImplRecord(RouterStoreTokenPBImpl.class, RouterStoreTokenProto.class); + } + + @Test + public void testRouterRMTokenRequest() throws Exception { + validatePBImplRecord(RouterRMTokenRequestPBImpl.class, RouterRMTokenRequestProto.class); + } + + @Test + public void testRouterRMTokenResponse() throws Exception { + validatePBImplRecord(RouterRMTokenResponsePBImpl.class, RouterRMTokenResponseProto.class); + } + + @Test + public void testRouterMasterKeyRequest() throws Exception { + validatePBImplRecord(RouterMasterKeyRequestPBImpl.class, RouterMasterKeyRequestProto.class); + } + + @Test + public void testRouterMasterKeyResponse() throws Exception { + validatePBImplRecord(RouterMasterKeyResponsePBImpl.class, RouterMasterKeyResponseProto.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index 0606f5c454081..b321a66bbe130 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -22,17 +22,24 @@ import java.util.Arrays; import java.util.Collection; import java.util.Map; +import java.util.HashSet; +import java.util.HashMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMDTSecretManagerState; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -232,4 +239,101 @@ public void testGetApplicationHomeSubClusterCache() throws YarnException { } } + @Test + public void testStoreNewMasterKey() throws YarnException, IOException { + // store delegation key; + DelegationKey key = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + HashSet keySet = new HashSet(); + keySet.add(key); + facade.storeNewMasterKey(key); + + MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState secretManagerState = federationStateStore.getRouterRMSecretManagerState(); + Assert.assertEquals(keySet, secretManagerState.getMasterKeyState()); + } + + @Test + public void testRemoveStoredMasterKey() throws YarnException, IOException { + // store delegation key; + DelegationKey key = new DelegationKey(4567, 7654, "keyBytes".getBytes()); + HashSet keySet = new HashSet(); + keySet.add(key); + facade.storeNewMasterKey(key); + + // check to delete delegationKey + facade.removeStoredMasterKey(key); + keySet.clear(); + + MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState secretManagerState = federationStateStore.getRouterRMSecretManagerState(); + Assert.assertEquals(keySet, secretManagerState.getMasterKeyState()); + } + + @Test + public void testStoreNewToken() throws YarnException, IOException { + // store new rm-token + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + int sequenceNumber = 1; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + facade.storeNewToken(dtId1, renewDate1); + + Map token1 = new HashMap(); + token1.put(dtId1, renewDate1); + + MemoryFederationStateStore stateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState storeSecretManagerState = + stateStore.getRouterRMSecretManagerState(); + Assert.assertEquals(token1, storeSecretManagerState.getTokenState()); + } + + @Test + public void testUpdateNewToken() throws YarnException, IOException { + // store new rm-token + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner2"), new Text("renewer2"), new Text("realuser2")); + int sequenceNumber = 2; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + facade.storeNewToken(dtId1, renewDate1); + + Map token1 = new HashMap(); + token1.put(dtId1, renewDate1); + + renewDate1 = Time.now(); + facade.updateStoredToken(dtId1, renewDate1); + token1.put(dtId1, renewDate1); + + MemoryFederationStateStore stateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState updateSecretManagerState = + stateStore.getRouterRMSecretManagerState(); + Assert.assertEquals(token1, updateSecretManagerState.getTokenState()); + Assert.assertEquals(sequenceNumber, updateSecretManagerState.getDTSequenceNumber()); + } + + @Test + public void testRemoveStoredToken() throws YarnException, IOException { + // store new rm-token + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner3"), new Text("renewer3"), new Text("realuser3")); + int sequenceNumber = 3; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + facade.storeNewToken(dtId1, renewDate1); + + Map token1 = new HashMap(); + token1.put(dtId1, renewDate1); + + // remove rm-token + facade.removeStoredToken(dtId1); + token1.clear(); + + MemoryFederationStateStore stateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState deleteSecretManagerState = + stateStore.getRouterRMSecretManagerState(); + + Assert.assertEquals(token1, deleteSecretManagerState.getTokenState()); + Assert.assertEquals(sequenceNumber, deleteSecretManagerState.getDTSequenceNumber()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java index d2d69231f5ef6..f6ff41ab40dcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java @@ -18,10 +18,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.federation; +import java.io.IOException; import java.net.InetSocketAddress; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.net.NetUtils; @@ -66,6 +68,10 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -325,4 +331,46 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( GetReservationsHomeSubClusterRequest request) throws YarnException { return stateStoreClient.getReservationsHomeSubCluster(request); } + + @Override + public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws IOException, YarnException { + return stateStoreClient.storeNewMasterKey(request); + } + + @Override + public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws IOException, YarnException { + return stateStoreClient.removeStoredMasterKey(request); + } + + @Override + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) + throws IOException, YarnException { + return stateStoreClient.storeNewToken(request); + } + + @Override + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) + throws IOException, YarnException { + return stateStoreClient.updateStoredToken(request); + } + + @Override + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) + throws IOException, YarnException { + return stateStoreClient.removeStoredToken(request); + } + + @Override + public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java index 961026d014611..c71be3c46f755 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +45,8 @@ public abstract class AbstractClientRequestInterceptor @SuppressWarnings("checkstyle:visibilitymodifier") protected UserGroupInformation user = null; + private RouterDelegationTokenSecretManager tokenSecretManager = null; + /** * Sets the {@link ClientRequestInterceptor} in the chain. */ @@ -125,4 +128,13 @@ private void setupUser(String userName) { } } + @Override + public RouterDelegationTokenSecretManager getTokenSecretManager() { + return tokenSecretManager; + } + + @Override + public void setTokenSecretManager(RouterDelegationTokenSecretManager tokenSecretManager) { + this.tokenSecretManager = tokenSecretManager; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java index 3e3ffce5f4b3f..6e19cbadf9d38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; /** * Defines the contract to be implemented by the request interceptor classes, @@ -62,4 +63,18 @@ public interface ClientRequestInterceptor */ ClientRequestInterceptor getNextInterceptor(); + /** + * Set RouterDelegationTokenSecretManager for specific interceptor to support Token operations, + * including create Token, update Token, and delete Token. + * + * @param tokenSecretManager Router DelegationTokenSecretManager + */ + void setTokenSecretManager(RouterDelegationTokenSecretManager tokenSecretManager); + + /** + * Get RouterDelegationTokenSecretManager. + * + * @return Router DelegationTokenSecretManager. + */ + RouterDelegationTokenSecretManager getTokenSecretManager(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index b60a267746e4f..e986e05a0bbd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -105,6 +105,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.router.RouterServerUtil; +import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; import org.slf4j.Logger; @@ -136,6 +137,8 @@ public class RouterClientRMService extends AbstractService // and remove the oldest used ones. private Map userPipelineMap; + private RouterDelegationTokenSecretManager routerDTSecretManager; + public RouterClientRMService() { super(RouterClientRMService.class.getName()); } @@ -164,8 +167,12 @@ protected void serviceStart() throws Exception { serverConf.getInt(YarnConfiguration.RM_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_RM_CLIENT_THREAD_COUNT); + // Initialize RouterRMDelegationTokenSecretManager. + routerDTSecretManager = createRouterRMDelegationTokenSecretManager(conf); + routerDTSecretManager.startThreads(); + this.server = rpc.getServer(ApplicationClientProtocol.class, this, - listenerEndpoint, serverConf, null, numWorkerThreads); + listenerEndpoint, serverConf, routerDTSecretManager, numWorkerThreads); // Enable service authorization? if (conf.getBoolean( @@ -508,6 +515,12 @@ private RequestInterceptorChainWrapper initializePipeline(String user) { ClientRequestInterceptor interceptorChain = this.createRequestInterceptorChain(); interceptorChain.init(user); + + // + if (routerDTSecretManager != null) { + interceptorChain.setTokenSecretManager(routerDTSecretManager); + } + chainWrapper.init(interceptorChain); } catch (Exception e) { LOG.error("Init ClientRequestInterceptor error for user: {}.", user, e); @@ -558,4 +571,32 @@ protected void finalize() { public Map getUserPipelineMap() { return userPipelineMap; } + + /** + * Create RouterRMDelegationTokenSecretManager. + * In the YARN federation, the Router will replace the RM to + * manage the RMDelegationToken (generate, update, cancel), + * so the relevant configuration parameters still obtain the configuration parameters of the RM. + * + * @param conf Configuration + * @return RouterDelegationTokenSecretManager. + */ + protected RouterDelegationTokenSecretManager createRouterRMDelegationTokenSecretManager( + Configuration conf) { + + long secretKeyInterval = conf.getLong(YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY, + YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); + long tokenMaxLifetime = conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY, + YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); + long tokenRenewInterval = conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, + YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); + + return new RouterDelegationTokenSecretManager(secretKeyInterval, + tokenMaxLifetime, tokenRenewInterval, 3600000); + } + + @VisibleForTesting + public RouterDelegationTokenSecretManager getRouterDTSecretManager() { + return routerDTSecretManager; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java new file mode 100644 index 0000000000000..72ba2f5130fb1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.security; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * A Router specific delegation token secret manager. + * The secret manager is responsible for generating and accepting the password + * for each token. + */ +public class RouterDelegationTokenSecretManager + extends AbstractDelegationTokenSecretManager { + + private static final Logger LOG = LoggerFactory + .getLogger(RouterDelegationTokenSecretManager.class); + + private FederationStateStoreFacade federationFacade; + + /** + * Create a Router Secret manager. + * + * @param delegationKeyUpdateInterval the number of milliseconds for rolling + * new secret keys. + * @param delegationTokenMaxLifetime the maximum lifetime of the delegation + * tokens in milliseconds + * @param delegationTokenRenewInterval how often the tokens must be renewed + * in milliseconds + * @param delegationTokenRemoverScanInterval how often the tokens are scanned + */ + public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, + long delegationTokenMaxLifetime, + long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval) { + super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + this.federationFacade = FederationStateStoreFacade.getInstance(); + } + + @Override + public RMDelegationTokenIdentifier createIdentifier() { + return new RMDelegationTokenIdentifier(); + } + + private boolean shouldIgnoreException(Exception e) { + return !running && e.getCause() instanceof InterruptedException; + } + + /** + * The Router Supports Store the new master key. + * + * @param newKey DelegationKey + */ + @Override + public void storeNewMasterKey(DelegationKey newKey) { + try { + federationFacade.storeNewMasterKey(newKey); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in storing master key with KeyID: {}.", newKey.getKeyId()); + ExitUtil.terminate(1, e); + } + } + } + + /** + * The Router Supports Remove the master key. + * + * @param delegationKey DelegationKey + */ + @Override + public void removeStoredMasterKey(DelegationKey delegationKey) { + try { + federationFacade.removeStoredMasterKey(delegationKey); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in removing master key with KeyID: {}.", delegationKey.getKeyId()); + ExitUtil.terminate(1, e); + } + } + } + + /** + * The Router Supports Store new Token. + * + * @param identifier Delegation Token + * @param renewDate renewDate + * @throws IOException IO exception occurred. + */ + @Override + public void storeNewToken(RMDelegationTokenIdentifier identifier, + long renewDate) throws IOException { + try { + federationFacade.storeNewToken(identifier, renewDate); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in storing RMDelegationToken with sequence number: {}.", + identifier.getSequenceNumber()); + ExitUtil.terminate(1, e); + } + } + } + + /** + * The Router Supports Update Token. + * + * @param id Delegation Token + * @param renewDate renewDate + * @throws IOException IO exception occurred. + */ + @Override + public void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) throws IOException { + try { + federationFacade.updateStoredToken(id, renewDate); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in updating persisted RMDelegationToken with sequence number: {}.", + id.getSequenceNumber()); + ExitUtil.terminate(1, e); + } + } + } + + /** + * The Router Supports Remove Token. + * + * @param identifier Delegation Token + * @throws IOException IO exception occurred. + */ + @Override + public void removeStoredToken(RMDelegationTokenIdentifier identifier) throws IOException { + try { + federationFacade.removeStoredToken(identifier); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in removing RMDelegationToken with sequence number: {}", + identifier.getSequenceNumber()); + ExitUtil.terminate(1, e); + } + } + } + + /** + * The Router supports obtaining MasterKey based on KeyId. + * + * @param newKey DelegationKey + * @throws Exception An error occurred + * @return DelegationKey + */ + public DelegationKey getMasterKeyByDelegationKey(DelegationKey newKey) + throws YarnException, IOException { + try { + RouterMasterKeyResponse response = federationFacade.getMasterKeyByDelegationKey(newKey); + RouterMasterKey masterKey = response.getRouterMasterKey(); + ByteBuffer keyByteBuf = masterKey.getKeyBytes(); + byte[] keyBytes = new byte[keyByteBuf.remaining()]; + keyByteBuf.get(keyBytes); + DelegationKey delegationKey = + new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + return delegationKey; + } catch (Exception ex) { + throw new YarnException(ex); + } + } + + /** + * The Router supports obtaining MasterKey based on KeyId. + * + * @param identifier RMDelegationTokenIdentifier + * @return RMDelegationTokenIdentifier. + * @throws YarnException exception occurred. + */ + public RMDelegationTokenIdentifier getTokenByRouterStoreToken( + RMDelegationTokenIdentifier identifier) throws YarnException, IOException { + try { + RouterRMTokenResponse response = federationFacade.getTokenByRouterStoreToken(identifier); + YARNDelegationTokenIdentifier responseIdentifier = + response.getRouterStoreToken().getTokenIdentifier(); + return (RMDelegationTokenIdentifier) responseIdentifier; + } catch (Exception ex) { + throw new YarnException(ex); + } + } + + @InterfaceAudience.Private + @VisibleForTesting + public int getLatestDTSequenceNumber() { + return delegationTokenSequenceNumber; + } + + public void setFederationFacade(FederationStateStoreFacade federationFacade) { + this.federationFacade = federationFacade; + } + + @InterfaceAudience.Private + @VisibleForTesting + public synchronized Set getAllMasterKeys() { + HashSet keySet = new HashSet<>(); + keySet.addAll(allKeys.values()); + return keySet; + } + + @InterfaceAudience.Private + @VisibleForTesting + public synchronized Map getAllTokens() { + Map allTokens = new HashMap<>(); + for (Map.Entry entry : currentTokens.entrySet()) { + allTokens.put(entry.getKey(), entry.getValue().getRenewDate()); + } + return allTokens; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java new file mode 100644 index 0000000000000..b15c54d81e836 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.security; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java index 61a5e8923b42a..090cd05ce5bb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java @@ -23,7 +23,9 @@ import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart; import org.apache.hadoop.yarn.server.router.Router; @@ -175,6 +177,9 @@ public static File createKeytab(String principal, String filename) throws Except */ public synchronized void startSecureRouter() { assertNull("Router is already running", router); + MemoryFederationStateStore stateStore = new MemoryFederationStateStore(); + stateStore.init(getConf()); + FederationStateStoreFacade.getInstance().reinitialize(stateStore, getConf()); UserGroupInformation.setConfiguration(conf); router = new Router(); router.init(conf); @@ -228,4 +233,7 @@ public static ConcurrentHashMap getMockRMs() { return mockRMs; } + public static Configuration getConf() { + return conf; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java new file mode 100644 index 0000000000000..e4f3fbf0eaa47 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.secure; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService; +import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; + +public class TestRouterDelegationTokenSecretManager extends AbstractSecureRouterTest { + + private static final Logger LOG = LoggerFactory.getLogger(TestRouterDelegationTokenSecretManager.class); + + @Test + public void testRouterStoreNewMasterKey() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: StoreNewMasterKey."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store NewMasterKey + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + secretManager.storeNewMasterKey(storeKey); + + // Get DelegationKey + DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + DelegationKey responseKey = secretManager.getMasterKeyByDelegationKey(paramKey); + + Assert.assertNotNull(paramKey); + Assert.assertEquals(storeKey.getExpiryDate(), responseKey.getExpiryDate()); + Assert.assertEquals(storeKey.getKeyId(), responseKey.getKeyId()); + Assert.assertTrue(Arrays.equals(storeKey.getEncodedKey(), responseKey.getEncodedKey())); + Assert.assertEquals(storeKey, responseKey); + + stopSecureRouter(); + } + + @Test + public void testRouterRemoveStoredMasterKey() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: RemoveStoredMasterKey."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store NewMasterKey + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + secretManager.storeNewMasterKey(storeKey); + + // Remove DelegationKey + secretManager.removeStoredMasterKey(storeKey); + + // Get DelegationKey + DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + LambdaTestUtils.intercept(YarnException.class, + "GetMasterKey with keyID: " + storeKey.getKeyId() + " does not exist.", + () -> secretManager.getMasterKeyByDelegationKey(paramKey)); + + stopSecureRouter(); + } + + @Test + public void testRouterStoreNewToken() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: StoreNewToken."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store new rm-token + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + int sequenceNumber = 1; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + secretManager.storeNewToken(dtId1, renewDate1); + + // query rm-token + RMDelegationTokenIdentifier dtId2 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + dtId2.setSequenceNumber(sequenceNumber); + RMDelegationTokenIdentifier dtId3 = secretManager.getTokenByRouterStoreToken(dtId2); + Assert.assertEquals(dtId1, dtId3); + + // query rm-token2 not exists + sequenceNumber++; + dtId2.setSequenceNumber(2); + LambdaTestUtils.intercept(YarnException.class, + "RMDelegationToken: " + dtId2 + " does not exist.", + () -> secretManager.getTokenByRouterStoreToken(dtId2)); + + stopSecureRouter(); + } + + @Test + public void testRouterUpdateNewToken() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: UpdateNewToken."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store new rm-token + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + int sequenceNumber = 1; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + secretManager.storeNewToken(dtId1, renewDate1); + + sequenceNumber++; + dtId1.setSequenceNumber(sequenceNumber); + secretManager.updateStoredToken(dtId1, renewDate1); + + // query rm-token + RMDelegationTokenIdentifier dtId2 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + dtId2.setSequenceNumber(sequenceNumber); + RMDelegationTokenIdentifier dtId3 = secretManager.getTokenByRouterStoreToken(dtId2); + Assert.assertNotNull(dtId3); + Assert.assertEquals(dtId1.getKind(), dtId3.getKind()); + Assert.assertEquals(dtId1.getOwner(), dtId3.getOwner()); + Assert.assertEquals(dtId1.getRealUser(), dtId3.getRealUser()); + Assert.assertEquals(dtId1.getRenewer(), dtId3.getRenewer()); + Assert.assertEquals(dtId1.getIssueDate(), dtId3.getIssueDate()); + Assert.assertEquals(dtId1.getMasterKeyId(), dtId3.getMasterKeyId()); + Assert.assertEquals(dtId1.getSequenceNumber(), dtId3.getSequenceNumber()); + Assert.assertEquals(sequenceNumber, dtId3.getSequenceNumber()); + Assert.assertEquals(dtId1, dtId3); + + stopSecureRouter(); + } + + @Test + public void testRouterRemoveToken() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: RouterRemoveToken."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store new rm-token + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + int sequenceNumber = 1; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + secretManager.storeNewToken(dtId1, renewDate1); + + // Remove rm-token + secretManager.removeStoredToken(dtId1); + + // query rm-token + LambdaTestUtils.intercept(YarnException.class, + "RMDelegationToken: " + dtId1 + " does not exist.", + () -> secretManager.getTokenByRouterStoreToken(dtId1)); + + stopSecureRouter(); + } +} From 690c035d4960cfaa2f0bc97c087bb25ffd1ee6f5 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 16 Aug 2022 08:55:57 -0700 Subject: [PATCH 02/13] YARN-9708. Fix CheckStyle. --- .../yarn/api/BasePBImplRecordsTest.java | 1 - .../impl/MemoryFederationStateStore.java | 9 ++++-- .../store/impl/SQLFederationStateStore.java | 8 ++--- .../impl/pb/RouterMasterKeyRequestPBImpl.java | 6 ++-- .../pb/RouterMasterKeyResponsePBImpl.java | 6 ++-- .../impl/pb/RouterRMTokenResponsePBImpl.java | 10 ++++-- .../TestFederationProtocolRecords.java | 1 - .../utils/TestFederationStateStoreFacade.java | 32 ++++++++++++------- ...estRouterDelegationTokenSecretManager.java | 21 +++++++----- 9 files changed, 57 insertions(+), 37 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java index 63a5b352eef02..6a908d57b1d31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java @@ -23,7 +23,6 @@ import org.apache.hadoop.util.Sets; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraints; -import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 0e1363a66d714..62d195e782461 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -448,7 +448,8 @@ public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyReques } @Override - public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws YarnException, IOException { + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) + throws YarnException, IOException { RouterStoreToken storeToken = request.getRouterStoreToken(); RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); @@ -458,7 +459,8 @@ public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws } @Override - public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws YarnException, IOException { + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { RouterStoreToken storeToken = request.getRouterStoreToken(); RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); @@ -470,7 +472,8 @@ public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) thr } @Override - public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws YarnException, IOException { + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { RouterStoreToken storeToken = request.getRouterStoreToken(); RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index e0e7e610d643d..527a3476cbf24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -1053,25 +1053,25 @@ public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) @Override public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) - throws YarnException, IOException { + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) - throws YarnException, IOException { + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) - throws YarnException, IOException { + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) - throws YarnException, IOException { + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java index 93fcdd8e6166d..2d457e2f8516b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java @@ -71,7 +71,7 @@ private void mergeLocalToBuilder() { RouterMasterKeyPBImpl masterKeyRequest = (RouterMasterKeyPBImpl) this.routerMasterKey; RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey(); if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) { - builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); } } @@ -79,10 +79,10 @@ private void mergeLocalToBuilder() { public RouterMasterKey getRouterMasterKey() { RouterMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.routerMasterKey != null) { - return this.routerMasterKey; + return this.routerMasterKey; } if (!p.hasRouterMasterKey()) { - return null; + return null; } this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); return this.routerMasterKey; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java index 3f0f3fbde7e22..1fbe1cce2dd1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java @@ -71,7 +71,7 @@ private void mergeLocalToBuilder() { RouterMasterKeyPBImpl masterKeyRequest = (RouterMasterKeyPBImpl) this.routerMasterKey; RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey(); if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) { - builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); } } @@ -79,10 +79,10 @@ private void mergeLocalToBuilder() { public RouterMasterKey getRouterMasterKey() { RouterMasterKeyResponseProtoOrBuilder p = viaProto ? proto : builder; if (this.routerMasterKey != null) { - return this.routerMasterKey; + return this.routerMasterKey; } if (!p.hasRouterMasterKey()) { - return null; + return null; } this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); return this.routerMasterKey; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java index 0c87a1a78c432..59288db1d8400 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java @@ -51,9 +51,13 @@ private void maybeInitBuilder() { } private void mergeLocalToBuilder() { - if (this.routerStoreToken != null && - !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { - builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + if (this.routerStoreToken != null) { + RouterStoreTokenPBImpl routerStoreTokenPBImpl = + (RouterStoreTokenPBImpl) this.routerStoreToken; + RouterStoreTokenProto storeTokenProto = routerStoreTokenPBImpl.getProto(); + if (!storeTokenProto.equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java index f87d370224309..c84f4863f561b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java @@ -19,7 +19,6 @@ import org.apache.hadoop.yarn.api.BasePBImplRecordsTest; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProto; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index b321a66bbe130..f9c758de58732 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -247,8 +247,10 @@ public void testStoreNewMasterKey() throws YarnException, IOException { keySet.add(key); facade.storeNewMasterKey(key); - MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); - RouterRMDTSecretManagerState secretManagerState = federationStateStore.getRouterRMSecretManagerState(); + MemoryFederationStateStore federationStateStore = + (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState secretManagerState = + federationStateStore.getRouterRMSecretManagerState(); Assert.assertEquals(keySet, secretManagerState.getMasterKeyState()); } @@ -264,8 +266,10 @@ public void testRemoveStoredMasterKey() throws YarnException, IOException { facade.removeStoredMasterKey(key); keySet.clear(); - MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); - RouterRMDTSecretManagerState secretManagerState = federationStateStore.getRouterRMSecretManagerState(); + MemoryFederationStateStore federationStateStore = + (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState secretManagerState = + federationStateStore.getRouterRMSecretManagerState(); Assert.assertEquals(keySet, secretManagerState.getMasterKeyState()); } @@ -279,12 +283,14 @@ public void testStoreNewToken() throws YarnException, IOException { Long renewDate1 = Time.now(); facade.storeNewToken(dtId1, renewDate1); - Map token1 = new HashMap(); + Map token1 = + new HashMap(); token1.put(dtId1, renewDate1); - MemoryFederationStateStore stateStore = (MemoryFederationStateStore) facade.getStateStore(); + MemoryFederationStateStore federationStateStore = + (MemoryFederationStateStore) facade.getStateStore(); RouterRMDTSecretManagerState storeSecretManagerState = - stateStore.getRouterRMSecretManagerState(); + federationStateStore.getRouterRMSecretManagerState(); Assert.assertEquals(token1, storeSecretManagerState.getTokenState()); } @@ -298,14 +304,16 @@ public void testUpdateNewToken() throws YarnException, IOException { Long renewDate1 = Time.now(); facade.storeNewToken(dtId1, renewDate1); - Map token1 = new HashMap(); + Map token1 = + new HashMap(); token1.put(dtId1, renewDate1); renewDate1 = Time.now(); facade.updateStoredToken(dtId1, renewDate1); token1.put(dtId1, renewDate1); - MemoryFederationStateStore stateStore = (MemoryFederationStateStore) facade.getStateStore(); + MemoryFederationStateStore stateStore = + (MemoryFederationStateStore) facade.getStateStore(); RouterRMDTSecretManagerState updateSecretManagerState = stateStore.getRouterRMSecretManagerState(); Assert.assertEquals(token1, updateSecretManagerState.getTokenState()); @@ -322,14 +330,16 @@ public void testRemoveStoredToken() throws YarnException, IOException { Long renewDate1 = Time.now(); facade.storeNewToken(dtId1, renewDate1); - Map token1 = new HashMap(); + Map token1 = + new HashMap(); token1.put(dtId1, renewDate1); // remove rm-token facade.removeStoredToken(dtId1); token1.clear(); - MemoryFederationStateStore stateStore = (MemoryFederationStateStore) facade.getStateStore(); + MemoryFederationStateStore stateStore = + (MemoryFederationStateStore) facade.getStateStore(); RouterRMDTSecretManagerState deleteSecretManagerState = stateStore.getRouterRMSecretManagerState(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java index e4f3fbf0eaa47..1fd1c9655f076 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java @@ -30,12 +30,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Arrays; public class TestRouterDelegationTokenSecretManager extends AbstractSecureRouterTest { - private static final Logger LOG = LoggerFactory.getLogger(TestRouterDelegationTokenSecretManager.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestRouterDelegationTokenSecretManager.class); @Test public void testRouterStoreNewMasterKey() throws Exception { @@ -46,7 +46,8 @@ public void testRouterStoreNewMasterKey() throws Exception { // Store NewMasterKey RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RouterDelegationTokenSecretManager secretManager = + routerClientRMService.getRouterDTSecretManager(); DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); secretManager.storeNewMasterKey(storeKey); @@ -72,7 +73,8 @@ public void testRouterRemoveStoredMasterKey() throws Exception { // Store NewMasterKey RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RouterDelegationTokenSecretManager secretManager = + routerClientRMService.getRouterDTSecretManager(); DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); secretManager.storeNewMasterKey(storeKey); @@ -83,7 +85,7 @@ public void testRouterRemoveStoredMasterKey() throws Exception { DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); LambdaTestUtils.intercept(YarnException.class, "GetMasterKey with keyID: " + storeKey.getKeyId() + " does not exist.", - () -> secretManager.getMasterKeyByDelegationKey(paramKey)); + () -> secretManager.getMasterKeyByDelegationKey(paramKey)); stopSecureRouter(); } @@ -97,7 +99,8 @@ public void testRouterStoreNewToken() throws Exception { // Store new rm-token RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RouterDelegationTokenSecretManager secretManager = + routerClientRMService.getRouterDTSecretManager(); RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( new Text("owner1"), new Text("renewer1"), new Text("realuser1")); int sequenceNumber = 1; @@ -131,7 +134,8 @@ public void testRouterUpdateNewToken() throws Exception { // Store new rm-token RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RouterDelegationTokenSecretManager secretManager = + routerClientRMService.getRouterDTSecretManager(); RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( new Text("owner1"), new Text("renewer1"), new Text("realuser1")); int sequenceNumber = 1; @@ -171,7 +175,8 @@ public void testRouterRemoveToken() throws Exception { // Store new rm-token RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RouterDelegationTokenSecretManager secretManager = + routerClientRMService.getRouterDTSecretManager(); RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( new Text("owner1"), new Text("renewer1"), new Text("realuser1")); int sequenceNumber = 1; From 8dfa838014c3f0494ff3457a922cc4944324c186 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 17 Aug 2022 08:56:28 +0800 Subject: [PATCH 03/13] YARN-9708. Fix CheckStyle. --- .../FederationDelegationTokenStateStore.java | 33 +++---------- .../impl/pb/RouterRMTokenResponsePBImpl.java | 17 +++++++ .../utils/FederationStateStoreFacade.java | 8 +-- .../RouterDelegationTokenSecretManager.java | 49 ++++++++++--------- 4 files changed, 55 insertions(+), 52 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java index 45b70aabec4fd..c51be9000eac6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -30,40 +30,19 @@ * FederationDelegationTokenStateStore maintains the state of all * DelegationToken that have been submitted to the federated cluster. * - *

- * It mainly includes the following operations: - *

- * - *
    - *
  • - * storeNewMasterKey
    - * Store the new MasterKey - *
  • - * - *
  • - * removeStoredMasterKey
    - * Remove MasterKey - *
  • - * - *
  • - * storeNewToken
    - * Store New delegationToken. - *
  • - * - * updateStoredToken - * removeStoredToken - *
*/ @InterfaceAudience.Private @InterfaceStability.Unstable public interface FederationDelegationTokenStateStore { /** - * The Router Supports Store the new master key. + * The Router Supports Store the New Master Key. + * During this Process, Facade will call the specific StateStore to store the MasterKey. * - * @param request DelegationKey. - * @return StoreNewMasterKeyResponse. - * @throws YarnException exception occurred. + * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey + * @return routerMasterKeyResponse + * @throws YarnException An internal conversion error occurred when store the Token + * @throws IOException An IO Error occurred */ RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) throws YarnException, IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java index 59288db1d8400..f50967d352be0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index b33e410f7ef86..06ad646beb17a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -457,10 +457,12 @@ public SubClusterId getReservationHomeSubCluster(ReservationId reservationId) } /** - * The Router supports saving the master key. + * The Router Supports Store New Token. + * Returns the RouterMasterKey{@link RouterMasterKey} for the delegation key. * - * @param newKey DelegationKey - * @throws Exception An error occurred + * @param newKey Key used for generating and verifying delegation tokens + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ public RouterMasterKeyResponse storeNewMasterKey(DelegationKey newKey) throws YarnException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java index 72ba2f5130fb1..69b52504c51a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -63,10 +63,8 @@ public class RouterDelegationTokenSecretManager * in milliseconds * @param delegationTokenRemoverScanInterval how often the tokens are scanned */ - public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, - long delegationTokenMaxLifetime, - long delegationTokenRenewInterval, - long delegationTokenRemoverScanInterval) { + public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, long delegationTokenMaxLifetime, + long delegationTokenRenewInterval, long delegationTokenRemoverScanInterval) { super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, delegationTokenRenewInterval, delegationTokenRemoverScanInterval); this.federationFacade = FederationStateStoreFacade.getInstance(); @@ -82,7 +80,8 @@ private boolean shouldIgnoreException(Exception e) { } /** - * The Router Supports Store the new master key. + * The Router Supports Store the New Master Key. + * During this Process, Facade will call the specific StateStore to store the MasterKey. * * @param newKey DelegationKey */ @@ -100,6 +99,7 @@ public void storeNewMasterKey(DelegationKey newKey) { /** * The Router Supports Remove the master key. + * During this Process, Facade will call the specific StateStore to remove the MasterKey. * * @param delegationKey DelegationKey */ @@ -118,7 +118,7 @@ public void removeStoredMasterKey(DelegationKey delegationKey) { /** * The Router Supports Store new Token. * - * @param identifier Delegation Token + * @param identifier RMDelegationToken * @param renewDate renewDate * @throws IOException IO exception occurred. */ @@ -139,9 +139,9 @@ public void storeNewToken(RMDelegationTokenIdentifier identifier, /** * The Router Supports Update Token. * - * @param id Delegation Token + * @param id RMDelegationToken * @param renewDate renewDate - * @throws IOException IO exception occurred. + * @throws IOException IO exception occurred */ @Override public void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) throws IOException { @@ -176,16 +176,18 @@ public void removeStoredToken(RMDelegationTokenIdentifier identifier) throws IOE } /** - * The Router supports obtaining MasterKey based on KeyId. + * The Router supports obtaining the DelegationKey stored in the Router StateStote + * according to the DelegationKey. * - * @param newKey DelegationKey - * @throws Exception An error occurred - * @return DelegationKey + * @param key Param DelegationKey + * @return Delegation Token + * @throws YarnException An internal conversion error occurred when getting the Token + * @throws IOException IO exception occurred */ - public DelegationKey getMasterKeyByDelegationKey(DelegationKey newKey) + public DelegationKey getMasterKeyByDelegationKey(DelegationKey key) throws YarnException, IOException { try { - RouterMasterKeyResponse response = federationFacade.getMasterKeyByDelegationKey(newKey); + RouterMasterKeyResponse response = federationFacade.getMasterKeyByDelegationKey(key); RouterMasterKey masterKey = response.getRouterMasterKey(); ByteBuffer keyByteBuf = masterKey.getKeyBytes(); byte[] keyBytes = new byte[keyByteBuf.remaining()]; @@ -193,17 +195,20 @@ public DelegationKey getMasterKeyByDelegationKey(DelegationKey newKey) DelegationKey delegationKey = new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); return delegationKey; - } catch (Exception ex) { + } catch (IOException ex) { + throw new IOException(ex); + } catch (YarnException ex) { throw new YarnException(ex); } } /** - * The Router supports obtaining MasterKey based on KeyId. + * Get RMDelegationTokenIdentifier according to RouterStoreToken. * * @param identifier RMDelegationTokenIdentifier - * @return RMDelegationTokenIdentifier. - * @throws YarnException exception occurred. + * @return RMDelegationTokenIdentifier + * @throws YarnException An internal conversion error occurred when getting the Token + * @throws IOException IO exception occurred */ public RMDelegationTokenIdentifier getTokenByRouterStoreToken( RMDelegationTokenIdentifier identifier) throws YarnException, IOException { @@ -217,16 +222,16 @@ public RMDelegationTokenIdentifier getTokenByRouterStoreToken( } } + public void setFederationFacade(FederationStateStoreFacade federationFacade) { + this.federationFacade = federationFacade; + } + @InterfaceAudience.Private @VisibleForTesting public int getLatestDTSequenceNumber() { return delegationTokenSequenceNumber; } - public void setFederationFacade(FederationStateStoreFacade federationFacade) { - this.federationFacade = federationFacade; - } - @InterfaceAudience.Private @VisibleForTesting public synchronized Set getAllMasterKeys() { From b00e0a13614532129a13011e4812d98b47738146 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 17 Aug 2022 09:47:06 +0800 Subject: [PATCH 04/13] YARN-9708. Fix CheckStyle. --- .../FederationDelegationTokenStateStore.java | 54 ++++++++++--------- .../utils/FederationStateStoreFacade.java | 47 ++++++++-------- 2 files changed, 56 insertions(+), 45 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java index c51be9000eac6..6407350536b40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -36,74 +36,80 @@ public interface FederationDelegationTokenStateStore { /** - * The Router Supports Store the New Master Key. + * The Router Supports Store NewMasterKey. * During this Process, Facade will call the specific StateStore to store the MasterKey. * * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey * @return routerMasterKeyResponse - * @throws YarnException An internal conversion error occurred when store the Token + * @throws YarnException if the call to the state store is unsuccessful * @throws IOException An IO Error occurred */ RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) throws YarnException, IOException; /** - * The Router Supports Remove the master key. + * The Router Supports Remove MasterKey. + * During this Process, Facade will call the specific StateStore to remove the MasterKey. * - * @param request DelegationKey. - * @return RemoveStoredMasterKeyResponse. - * @throws YarnException exception occurred. + * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey + * @return routerMasterKeyResponse + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) throws YarnException, IOException; /** - * The Router supports obtaining MasterKey based on KeyId. + * The Router Supports GetMasterKeyByDelegationKey. * - * @param request - * @return - * @throws YarnException - * @throws IOException + * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey + * @return routerMasterKeyResponse + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) throws YarnException, IOException; /** - * The Router Supports Store new Token. + * The Router Supports Store RMDelegationTokenIdentifier. * - * @param request DelegationKey. - * @return RouterRMTokenResponse. - * @throws YarnException exception occurred. + * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate) + * @return routerRMTokenResponse. + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws YarnException, IOException; /** - * The Router Supports Update Token. + * The Router Supports Update RMDelegationTokenIdentifier. * - * @param request DelegationKey. + * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate) * @return RouterRMTokenResponse. - * @throws YarnException exception occurred. + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws YarnException, IOException; /** - * The Router Supports Remove Token. + * The Router Supports Remove RMDelegationTokenIdentifier. * - * @param request DelegationKey. + * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate) * @return RouterRMTokenResponse. - * @throws YarnException exception occurred. + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws YarnException, IOException; /** - * The Router Supports Remove Token. + * The Router Supports GetTokenByRouterStoreToken. * - * @param request DelegationKey. + * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate) * @return RouterRMTokenResponse. - * @throws YarnException exception occurred. + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) throws YarnException, IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 06ad646beb17a..551852767fb54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -457,8 +457,7 @@ public SubClusterId getReservationHomeSubCluster(ReservationId reservationId) } /** - * The Router Supports Store New Token. - * Returns the RouterMasterKey{@link RouterMasterKey} for the delegation key. + * The Router Supports Store NewMasterKey (RouterMasterKey{@link RouterMasterKey}). * * @param newKey Key used for generating and verifying delegation tokens * @throws YarnException if the call to the state store is unsuccessful @@ -475,10 +474,11 @@ public RouterMasterKeyResponse storeNewMasterKey(DelegationKey newKey) } /** - * The Router supports remove the master key. + * The Router Supports Remove MasterKey (RouterMasterKey{@link RouterMasterKey}). * - * @param newKey DelegationKey - * @throws Exception An error occurred + * @param newKey Key used for generating and verifying delegation tokens + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ public void removeStoredMasterKey(DelegationKey newKey) throws YarnException, IOException { LOG.info("Removing master key with keyID {}.", newKey.getKeyId()); @@ -490,10 +490,11 @@ public void removeStoredMasterKey(DelegationKey newKey) throws YarnException, IO } /** - * The Router supports obtaining MasterKey based on KeyId. + * The Router Supports GetMasterKeyByDelegationKey. * - * @param newKey DelegationKey - * @throws Exception An error occurred + * @param newKey Key used for generating and verifying delegation tokens + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ public RouterMasterKeyResponse getMasterKeyByDelegationKey(DelegationKey newKey) throws YarnException, IOException { @@ -506,11 +507,12 @@ public RouterMasterKeyResponse getMasterKeyByDelegationKey(DelegationKey newKey) } /** - * The Router Supports Store new Token. + * The Router Supports Store RMDelegationTokenIdentifier{@link RMDelegationTokenIdentifier}. * - * @param identifier Delegation Token + * @param identifier delegation tokens from the RM * @param renewDate renewDate - * @throws IOException IO exception occurred. + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ public void storeNewToken(RMDelegationTokenIdentifier identifier, long renewDate) throws YarnException, IOException { @@ -522,11 +524,12 @@ public void storeNewToken(RMDelegationTokenIdentifier identifier, } /** - * The Router Supports Update Token. + * The Router Supports Update RMDelegationTokenIdentifier{@link RMDelegationTokenIdentifier}. * - * @param identifier Delegation Token + * @param identifier delegation tokens from the RM * @param renewDate renewDate - * @throws IOException IO exception occurred. + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ public void updateStoredToken(RMDelegationTokenIdentifier identifier, long renewDate) throws YarnException, IOException { @@ -538,10 +541,11 @@ public void updateStoredToken(RMDelegationTokenIdentifier identifier, } /** - * The Router Supports Remove Token. + * The Router Supports Remove RMDelegationTokenIdentifier{@link RMDelegationTokenIdentifier}. * - * @param identifier Delegation Token - * @throws IOException IO exception occurred. + * @param identifier delegation tokens from the RM + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ public void removeStoredToken(RMDelegationTokenIdentifier identifier) throws YarnException, IOException{ @@ -553,11 +557,12 @@ public void removeStoredToken(RMDelegationTokenIdentifier identifier) } /** - * The Router Supports Remove Token. + * The Router Supports GetTokenByRouterStoreToken{@link RMDelegationTokenIdentifier}. * - * @param identifier Delegation Token - * @return RouterRMTokenResponse. - * @throws YarnException exception occurred. + * @param identifier delegation tokens from the RM + * @return RouterStoreToken + * @throws YarnException if the call to the state store is unsuccessful + * @throws IOException An IO Error occurred */ public RouterRMTokenResponse getTokenByRouterStoreToken(RMDelegationTokenIdentifier identifier) throws YarnException, IOException { From a8ec3f59a09c3eb3c185d382510f05401ed064fc Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 17 Aug 2022 11:05:01 -0700 Subject: [PATCH 05/13] YARN-9708. Fix CheckStyle. --- .../store/records/impl/pb/RouterRMTokenRequestPBImpl.java | 1 + .../federation/utils/FederationStateStoreFacade.java | 2 ++ .../federation/utils/TestFederationStateStoreFacade.java | 8 ++++---- .../security/RouterDelegationTokenSecretManager.java | 5 +++-- .../hadoop/yarn/server/router/security/package-info.java | 2 +- .../secure/TestRouterDelegationTokenSecretManager.java | 2 +- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java index 756d4745507e0..1358a78326ff8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java @@ -92,6 +92,7 @@ public void setRouterStoreToken(RouterStoreToken storeToken) { maybeInitBuilder(); if (storeToken == null) { builder.clearRouterStoreToken(); + return; } this.routerStoreToken = storeToken; this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 551852767fb54..9bbb9a4dd9d18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -462,6 +462,7 @@ public SubClusterId getReservationHomeSubCluster(ReservationId reservationId) * @param newKey Key used for generating and verifying delegation tokens * @throws YarnException if the call to the state store is unsuccessful * @throws IOException An IO Error occurred + * @return RouterMasterKeyResponse */ public RouterMasterKeyResponse storeNewMasterKey(DelegationKey newKey) throws YarnException, IOException { @@ -495,6 +496,7 @@ public void removeStoredMasterKey(DelegationKey newKey) throws YarnException, IO * @param newKey Key used for generating and verifying delegation tokens * @throws YarnException if the call to the state store is unsuccessful * @throws IOException An IO Error occurred + * @return RouterMasterKeyResponse */ public RouterMasterKeyResponse getMasterKeyByDelegationKey(DelegationKey newKey) throws YarnException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index f9c758de58732..80dc6b80eacc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -312,10 +312,10 @@ public void testUpdateNewToken() throws YarnException, IOException { facade.updateStoredToken(dtId1, renewDate1); token1.put(dtId1, renewDate1); - MemoryFederationStateStore stateStore = + MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); RouterRMDTSecretManagerState updateSecretManagerState = - stateStore.getRouterRMSecretManagerState(); + federationStateStore.getRouterRMSecretManagerState(); Assert.assertEquals(token1, updateSecretManagerState.getTokenState()); Assert.assertEquals(sequenceNumber, updateSecretManagerState.getDTSequenceNumber()); } @@ -338,10 +338,10 @@ public void testRemoveStoredToken() throws YarnException, IOException { facade.removeStoredToken(dtId1); token1.clear(); - MemoryFederationStateStore stateStore = + MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); RouterRMDTSecretManagerState deleteSecretManagerState = - stateStore.getRouterRMSecretManagerState(); + federationStateStore.getRouterRMSecretManagerState(); Assert.assertEquals(token1, deleteSecretManagerState.getTokenState()); Assert.assertEquals(sequenceNumber, deleteSecretManagerState.getDTSequenceNumber()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java index 69b52504c51a2..b275f72460021 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -63,8 +63,9 @@ public class RouterDelegationTokenSecretManager * in milliseconds * @param delegationTokenRemoverScanInterval how often the tokens are scanned */ - public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, long delegationTokenMaxLifetime, - long delegationTokenRenewInterval, long delegationTokenRemoverScanInterval) { + public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, + long delegationTokenMaxLifetime, long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval) { super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, delegationTokenRenewInterval, delegationTokenRemoverScanInterval); this.federationFacade = FederationStateStoreFacade.getInstance(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java index b15c54d81e836..16a7488c07132 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java index 1fd1c9655f076..cf37c717f5e1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java @@ -85,7 +85,7 @@ public void testRouterRemoveStoredMasterKey() throws Exception { DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); LambdaTestUtils.intercept(YarnException.class, "GetMasterKey with keyID: " + storeKey.getKeyId() + " does not exist.", - () -> secretManager.getMasterKeyByDelegationKey(paramKey)); + () -> secretManager.getMasterKeyByDelegationKey(paramKey)); stopSecureRouter(); } From 9e6760b20adae91229949e11a04dd739d441429d Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 18 Aug 2022 20:17:28 +0800 Subject: [PATCH 06/13] YARN-9708. Fix Junit Test Error. --- .../router/secure/TestRouterDelegationTokenSecretManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java index cf37c717f5e1e..c98f9bae3742a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.Arrays; public class TestRouterDelegationTokenSecretManager extends AbstractSecureRouterTest { @@ -83,7 +84,7 @@ public void testRouterRemoveStoredMasterKey() throws Exception { // Get DelegationKey DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); - LambdaTestUtils.intercept(YarnException.class, + LambdaTestUtils.intercept(IOException.class, "GetMasterKey with keyID: " + storeKey.getKeyId() + " does not exist.", () -> secretManager.getMasterKeyByDelegationKey(paramKey)); From c9709504fefdc94c579acec06871ac94c362f71d Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 24 Aug 2022 12:36:35 +0800 Subject: [PATCH 07/13] YARN-9708. Fix CodeStyle. --- .../client/YARNDelegationTokenIdentifier.java | 7 +- .../impl/MemoryFederationStateStore.java | 32 +++--- .../store/records/RouterMasterKey.java | 63 ++++++----- .../store/records/RouterMasterKeyRequest.java | 22 ++-- .../records/RouterMasterKeyResponse.java | 24 ++-- .../records/RouterRMDTSecretManagerState.java | 5 +- .../store/records/RouterRMTokenRequest.java | 21 ++-- .../store/records/RouterRMTokenResponse.java | 21 ++-- .../utils/TestFederationStateStoreFacade.java | 104 +++++++++++------- .../clientrm/RouterClientRMService.java | 2 +- ...estRouterDelegationTokenSecretManager.java | 37 ++++--- 11 files changed, 187 insertions(+), 151 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java index 0020b3c407627..95fe4bbc64e3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java @@ -22,10 +22,9 @@ import java.io.DataOutputStream; import java.io.IOException; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; @@ -116,8 +115,8 @@ public YARNDelegationTokenIdentifierProto getProto() { return builder.build(); } - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public static YARNDelegationTokenIdentifier newInstance(Text owner, Text renewer, Text realUser) { YARNDelegationTokenIdentifier policy = Records.newRecord(YARNDelegationTokenIdentifier.class); policy.setOwner(owner); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 685909969a3bb..a390fe8475056 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -391,11 +391,7 @@ public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) // Restore the DelegationKey from the request RouterMasterKey masterKey = request.getRouterMasterKey(); - ByteBuffer keyByteBuf = masterKey.getKeyBytes(); - byte[] keyBytes = new byte[keyByteBuf.remaining()]; - keyByteBuf.get(keyBytes); - DelegationKey delegationKey = - new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + DelegationKey delegationKey = getDelegationKeyByMasterKey(masterKey); Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); if (rmDTMasterKeyState.contains(delegationKey)) { @@ -417,11 +413,7 @@ public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest requ // Restore the DelegationKey from the request RouterMasterKey masterKey = request.getRouterMasterKey(); - ByteBuffer keyByteBuf = masterKey.getKeyBytes(); - byte[] keyBytes = new byte[keyByteBuf.remaining()]; - keyByteBuf.get(keyBytes); - DelegationKey delegationKey = - new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + DelegationKey delegationKey = getDelegationKeyByMasterKey(masterKey); LOG.info("Remove Router-RMDT master key with key id: {}.", delegationKey.getKeyId()); Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); @@ -433,13 +425,10 @@ public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest requ @Override public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) throws YarnException, IOException { + // Restore the DelegationKey from the request RouterMasterKey masterKey = request.getRouterMasterKey(); - ByteBuffer keyByteBuf = masterKey.getKeyBytes(); - byte[] keyBytes = new byte[keyByteBuf.remaining()]; - keyByteBuf.get(keyBytes); - DelegationKey delegationKey = - new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + DelegationKey delegationKey = getDelegationKeyByMasterKey(masterKey); Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); if (!rmDTMasterKeyState.contains(delegationKey)) { @@ -547,4 +536,17 @@ public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( reservations.remove(reservationId); return DeleteReservationHomeSubClusterResponse.newInstance(); } + + /** + * Get DelegationKey By based on MasterKey. + * + * @param masterKey masterKey + * @return DelegationKey + */ + private DelegationKey getDelegationKeyByMasterKey(RouterMasterKey masterKey) { + ByteBuffer keyByteBuf = masterKey.getKeyBytes(); + byte[] keyBytes = new byte[keyByteBuf.remaining()]; + keyByteBuf.get(keyBytes); + return new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java index f9cb89354a2ff..ae0b30dec4fa7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java @@ -17,19 +17,21 @@ */ package org.apache.hadoop.yarn.server.federation.store.records; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; import java.nio.ByteBuffer; -import java.util.Arrays; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public abstract class RouterMasterKey { - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public static RouterMasterKey newInstance(Integer keyId, ByteBuffer keyBytes, Long expiryDate) { RouterMasterKey policy = Records.newRecord(RouterMasterKey.class); policy.setKeyId(keyId); @@ -38,8 +40,8 @@ public static RouterMasterKey newInstance(Integer keyId, ByteBuffer keyBytes, Lo return policy; } - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public static RouterMasterKey newInstance(RouterMasterKey masterKey) { RouterMasterKey routerMasterKey = Records.newRecord(RouterMasterKey.class); routerMasterKey.setKeyId(masterKey.getKeyId()); @@ -53,8 +55,8 @@ public static RouterMasterKey newInstance(RouterMasterKey masterKey) { * * @return MasterKeyId. */ - @InterfaceAudience.Public - @InterfaceStability.Unstable + @Public + @Unstable public abstract Integer getKeyId(); /** @@ -62,8 +64,8 @@ public static RouterMasterKey newInstance(RouterMasterKey masterKey) { * * @param keyId MasterKeyId. */ - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public abstract void setKeyId(Integer keyId); /** @@ -71,8 +73,8 @@ public static RouterMasterKey newInstance(RouterMasterKey masterKey) { * * @return KeyBytes of the DelegationKey. */ - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Public + @Unstable public abstract ByteBuffer getKeyBytes(); /** @@ -80,8 +82,8 @@ public static RouterMasterKey newInstance(RouterMasterKey masterKey) { * * @param keyBytes KeyBytes of the DelegationKey. */ - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public abstract void setKeyBytes(ByteBuffer keyBytes); /** @@ -89,8 +91,8 @@ public static RouterMasterKey newInstance(RouterMasterKey masterKey) { * * @return ExpiryDate of the DelegationKey. */ - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public abstract Long getExpiryDate(); /** @@ -98,18 +100,17 @@ public static RouterMasterKey newInstance(RouterMasterKey masterKey) { * * @param expiryDate expiryDate of the DelegationKey. */ - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public abstract void setExpiryDate(Long expiryDate); @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + (int) (getExpiryDate() ^ (getExpiryDate() >>> 32)); - result = prime * result + Arrays.hashCode(getKeyBytes().array()); - result = prime * result + getKeyId(); - return result; + return new HashCodeBuilder() + .append(this.getExpiryDate().longValue()) + .append(this.getKeyId().intValue()) + .append(getKeyBytes().array()) + .hashCode(); } @Override @@ -120,9 +121,11 @@ public boolean equals(Object right) { return false; } else { RouterMasterKey r = (RouterMasterKey) right; - return getKeyId().intValue() == r.getKeyId().intValue() && - getExpiryDate().longValue() == r.getExpiryDate().longValue() && - Arrays.equals(getKeyBytes().array(), r.getKeyBytes().array()); + return new EqualsBuilder() + .append(this.getKeyId().intValue(), r.getKeyId().intValue()) + .append(this.getExpiryDate().longValue(), this.getExpiryDate().longValue()) + .append(getKeyBytes().array(),r.getKeyBytes()) + .isEquals(); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java index 2bf30535162d5..9be6ed01ef2b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java @@ -16,28 +16,28 @@ */ package org.apache.hadoop.yarn.server.federation.store.records; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public abstract class RouterMasterKeyRequest { - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public static RouterMasterKeyRequest newInstance(RouterMasterKey routerMasterKey) { RouterMasterKeyRequest request = Records.newRecord(RouterMasterKeyRequest.class); request.setRouterMasterKey(routerMasterKey); return request; } - @InterfaceAudience.Public - @InterfaceStability.Unstable + @Public + @Unstable public abstract RouterMasterKey getRouterMasterKey(); - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java index 6625e4c1911d8..27c0a93eef335 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java @@ -17,28 +17,28 @@ */ package org.apache.hadoop.yarn.server.federation.store.records; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public abstract class RouterMasterKeyResponse { - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public static RouterMasterKeyResponse newInstance(RouterMasterKey masterKey) { - RouterMasterKeyResponse request = - Records.newRecord(RouterMasterKeyResponse.class); + RouterMasterKeyResponse request = Records.newRecord(RouterMasterKeyResponse.class); request.setRouterMasterKey(masterKey); return request; } - @InterfaceAudience.Public - @InterfaceStability.Unstable + @Public + @Unstable public abstract RouterMasterKey getRouterMasterKey(); - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public abstract void setRouterMasterKey(RouterMasterKey masterKey); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java index e5010945286e0..85a8002c91c9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java @@ -28,10 +28,9 @@ public class RouterRMDTSecretManagerState { // DTIdentifier -> renewDate - private Map delegationTokenState = - new HashMap(); + private Map delegationTokenState = new HashMap<>(); - private Set masterKeyState = new HashSet(); + private Set masterKeyState = new HashSet<>(); private int dtSequenceNumber = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java index 0d4908de3539f..790ee513bbd32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java @@ -17,27 +17,28 @@ */ package org.apache.hadoop.yarn.server.federation.store.records; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public abstract class RouterRMTokenRequest { - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public static RouterRMTokenRequest newInstance(RouterStoreToken routerStoreToken) { RouterRMTokenRequest request = Records.newRecord(RouterRMTokenRequest.class); request.setRouterStoreToken(routerStoreToken); return request; } - @InterfaceAudience.Public - @InterfaceStability.Unstable + @Public + @Unstable public abstract RouterStoreToken getRouterStoreToken(); - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java index 57671bda419f4..c629e46a048c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java @@ -17,27 +17,28 @@ */ package org.apache.hadoop.yarn.server.federation.store.records; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public abstract class RouterRMTokenResponse { - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public static RouterRMTokenResponse newInstance(RouterStoreToken routerStoreToken) { RouterRMTokenResponse request = Records.newRecord(RouterRMTokenResponse.class); request.setRouterStoreToken(routerStoreToken); return request; } - @InterfaceAudience.Public - @InterfaceStability.Unstable + @Public + @Unstable public abstract RouterStoreToken getRouterStoreToken(); - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index 80dc6b80eacc8..6e7ff20f8697c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -22,17 +22,19 @@ import java.util.Arrays; import java.util.Collection; import java.util.Map; +import java.util.Set; import java.util.HashSet; -import java.util.HashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; @@ -40,6 +42,9 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; import org.apache.hadoop.yarn.server.federation.store.records.RouterRMDTSecretManagerState; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -243,7 +248,7 @@ public void testGetApplicationHomeSubClusterCache() throws YarnException { public void testStoreNewMasterKey() throws YarnException, IOException { // store delegation key; DelegationKey key = new DelegationKey(1234, 4321, "keyBytes".getBytes()); - HashSet keySet = new HashSet(); + Set keySet = new HashSet<>(); keySet.add(key); facade.storeNewMasterKey(key); @@ -258,7 +263,7 @@ public void testStoreNewMasterKey() throws YarnException, IOException { public void testRemoveStoredMasterKey() throws YarnException, IOException { // store delegation key; DelegationKey key = new DelegationKey(4567, 7654, "keyBytes".getBytes()); - HashSet keySet = new HashSet(); + Set keySet = new HashSet<>(); keySet.add(key); facade.storeNewMasterKey(key); @@ -283,15 +288,22 @@ public void testStoreNewToken() throws YarnException, IOException { Long renewDate1 = Time.now(); facade.storeNewToken(dtId1, renewDate1); - Map token1 = - new HashMap(); - token1.put(dtId1, renewDate1); - - MemoryFederationStateStore federationStateStore = - (MemoryFederationStateStore) facade.getStateStore(); - RouterRMDTSecretManagerState storeSecretManagerState = - federationStateStore.getRouterRMSecretManagerState(); - Assert.assertEquals(token1, storeSecretManagerState.getTokenState()); + // get RouterStoreToken from StateStore + FederationStateStore stateStore = facade.getStateStore(); + RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1); + RouterRMTokenRequest routerRMTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); + RouterRMTokenResponse routerRMTokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); + Assert.assertNotNull(routerRMTokenResponse); + + RouterStoreToken resultStoreToken = routerRMTokenResponse.getRouterStoreToken(); + YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier(); + Assert.assertNotNull(resultStoreToken); + Assert.assertNotNull(resultTokenIdentifier); + Assert.assertNotNull(resultStoreToken.getRenewDate()); + + Assert.assertEquals(dtId1, resultTokenIdentifier); + Assert.assertEquals(renewDate1, resultStoreToken.getRenewDate()); + Assert.assertEquals(sequenceNumber, resultTokenIdentifier.getSequenceNumber()); } @Test @@ -304,24 +316,32 @@ public void testUpdateNewToken() throws YarnException, IOException { Long renewDate1 = Time.now(); facade.storeNewToken(dtId1, renewDate1); - Map token1 = - new HashMap(); - token1.put(dtId1, renewDate1); - - renewDate1 = Time.now(); - facade.updateStoredToken(dtId1, renewDate1); - token1.put(dtId1, renewDate1); - - MemoryFederationStateStore federationStateStore = - (MemoryFederationStateStore) facade.getStateStore(); - RouterRMDTSecretManagerState updateSecretManagerState = - federationStateStore.getRouterRMSecretManagerState(); - Assert.assertEquals(token1, updateSecretManagerState.getTokenState()); - Assert.assertEquals(sequenceNumber, updateSecretManagerState.getDTSequenceNumber()); + Long renewDate2 = Time.now(); + int sequenceNumber2 = 3; + dtId1.setSequenceNumber(sequenceNumber2); + facade.updateStoredToken(dtId1, renewDate2); + + // get RouterStoreToken from StateStore + FederationStateStore stateStore = facade.getStateStore(); + RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1); + RouterRMTokenRequest routerRMTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); + RouterRMTokenResponse routerRMTokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); + Assert.assertNotNull(routerRMTokenResponse); + + RouterStoreToken resultStoreToken = routerRMTokenResponse.getRouterStoreToken(); + YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier(); + Assert.assertNotNull(resultStoreToken); + Assert.assertNotNull(resultTokenIdentifier); + Assert.assertNotNull(resultStoreToken.getRenewDate()); + + Assert.assertEquals(dtId1, resultTokenIdentifier); + Assert.assertEquals(renewDate2, resultStoreToken.getRenewDate()); + Assert.assertEquals(sequenceNumber2, resultTokenIdentifier.getSequenceNumber()); } @Test - public void testRemoveStoredToken() throws YarnException, IOException { + public void testRemoveStoredToken() throws Exception { + // store new rm-token RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( new Text("owner3"), new Text("renewer3"), new Text("realuser3")); @@ -330,20 +350,28 @@ public void testRemoveStoredToken() throws YarnException, IOException { Long renewDate1 = Time.now(); facade.storeNewToken(dtId1, renewDate1); - Map token1 = - new HashMap(); - token1.put(dtId1, renewDate1); + // get RouterStoreToken from StateStore + FederationStateStore stateStore = facade.getStateStore(); + RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1); + RouterRMTokenRequest routerRMTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); + RouterRMTokenResponse routerRMTokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); + Assert.assertNotNull(routerRMTokenResponse); + + RouterStoreToken resultStoreToken = routerRMTokenResponse.getRouterStoreToken(); + YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier(); + Assert.assertNotNull(resultStoreToken); + Assert.assertNotNull(resultTokenIdentifier); + Assert.assertNotNull(resultStoreToken.getRenewDate()); + + Assert.assertEquals(dtId1, resultTokenIdentifier); + Assert.assertEquals(renewDate1, resultStoreToken.getRenewDate()); + Assert.assertEquals(sequenceNumber, resultTokenIdentifier.getSequenceNumber()); // remove rm-token facade.removeStoredToken(dtId1); - token1.clear(); - - MemoryFederationStateStore federationStateStore = - (MemoryFederationStateStore) facade.getStateStore(); - RouterRMDTSecretManagerState deleteSecretManagerState = - federationStateStore.getRouterRMSecretManagerState(); - Assert.assertEquals(token1, deleteSecretManagerState.getTokenState()); - Assert.assertEquals(sequenceNumber, deleteSecretManagerState.getDTSequenceNumber()); + // Call again(getTokenByRouterStoreToken) after remove will throw IOException(not exist) + LambdaTestUtils.intercept(IOException.class, "RMDelegationToken: " + dtId1 + " does not exist.", + () -> stateStore.getTokenByRouterStoreToken(routerRMTokenRequest)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index e986e05a0bbd9..6921f7e1839bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -592,7 +592,7 @@ protected RouterDelegationTokenSecretManager createRouterRMDelegationTokenSecret YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); return new RouterDelegationTokenSecretManager(secretKeyInterval, - tokenMaxLifetime, tokenRenewInterval, 3600000); + tokenMaxLifetime, tokenRenewInterval, 60 * 60 * 1000); } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java index c98f9bae3742a..eac2c5a03ba61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java @@ -31,12 +31,15 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Arrays; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertArrayEquals; public class TestRouterDelegationTokenSecretManager extends AbstractSecureRouterTest { private static final Logger LOG = - LoggerFactory.getLogger(TestRouterDelegationTokenSecretManager.class); + LoggerFactory.getLogger(TestRouterDelegationTokenSecretManager.class); @Test public void testRouterStoreNewMasterKey() throws Exception { @@ -56,11 +59,11 @@ public void testRouterStoreNewMasterKey() throws Exception { DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); DelegationKey responseKey = secretManager.getMasterKeyByDelegationKey(paramKey); - Assert.assertNotNull(paramKey); - Assert.assertEquals(storeKey.getExpiryDate(), responseKey.getExpiryDate()); - Assert.assertEquals(storeKey.getKeyId(), responseKey.getKeyId()); - Assert.assertTrue(Arrays.equals(storeKey.getEncodedKey(), responseKey.getEncodedKey())); - Assert.assertEquals(storeKey, responseKey); + assertNotNull(paramKey); + assertEquals(storeKey.getExpiryDate(), responseKey.getExpiryDate()); + assertEquals(storeKey.getKeyId(), responseKey.getKeyId()); + assertArrayEquals(storeKey.getEncodedKey(), responseKey.getEncodedKey()); + assertEquals(storeKey, responseKey); stopSecureRouter(); } @@ -153,16 +156,16 @@ public void testRouterUpdateNewToken() throws Exception { new Text("owner1"), new Text("renewer1"), new Text("realuser1")); dtId2.setSequenceNumber(sequenceNumber); RMDelegationTokenIdentifier dtId3 = secretManager.getTokenByRouterStoreToken(dtId2); - Assert.assertNotNull(dtId3); - Assert.assertEquals(dtId1.getKind(), dtId3.getKind()); - Assert.assertEquals(dtId1.getOwner(), dtId3.getOwner()); - Assert.assertEquals(dtId1.getRealUser(), dtId3.getRealUser()); - Assert.assertEquals(dtId1.getRenewer(), dtId3.getRenewer()); - Assert.assertEquals(dtId1.getIssueDate(), dtId3.getIssueDate()); - Assert.assertEquals(dtId1.getMasterKeyId(), dtId3.getMasterKeyId()); - Assert.assertEquals(dtId1.getSequenceNumber(), dtId3.getSequenceNumber()); - Assert.assertEquals(sequenceNumber, dtId3.getSequenceNumber()); - Assert.assertEquals(dtId1, dtId3); + assertNotNull(dtId3); + assertEquals(dtId1.getKind(), dtId3.getKind()); + assertEquals(dtId1.getOwner(), dtId3.getOwner()); + assertEquals(dtId1.getRealUser(), dtId3.getRealUser()); + assertEquals(dtId1.getRenewer(), dtId3.getRenewer()); + assertEquals(dtId1.getIssueDate(), dtId3.getIssueDate()); + assertEquals(dtId1.getMasterKeyId(), dtId3.getMasterKeyId()); + assertEquals(dtId1.getSequenceNumber(), dtId3.getSequenceNumber()); + assertEquals(sequenceNumber, dtId3.getSequenceNumber()); + assertEquals(dtId1, dtId3); stopSecureRouter(); } From 1b8690e10a189a8ec712a2d5f10baef958c76367 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 24 Aug 2022 07:28:17 -0700 Subject: [PATCH 08/13] YARN-9708. Fix CheckStyle. --- .../impl/MemoryFederationStateStore.java | 2 +- .../store/impl/SQLFederationStateStore.java | 8 +++---- .../impl/ZookeeperFederationStateStore.java | 6 +++--- .../store/records/RouterMasterKey.java | 2 +- .../utils/FederationStateStoreFacade.java | 4 ++-- .../utils/TestFederationStateStoreFacade.java | 21 ++++++++----------- .../FederationStateStoreService.java | 2 +- 7 files changed, 21 insertions(+), 24 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 32c7e96ad7aa1..529336f88110e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -508,7 +508,7 @@ public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest req public RouterRMDTSecretManagerState getRouterRMSecretManagerState() { return routerRMSecretManagerState; } - + @Override public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( UpdateReservationHomeSubClusterRequest request) throws YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index 07c226e6ca01b..d8b7e41b34212 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -1039,9 +1039,9 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( @Override public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); - } - + throw new NotImplementedException("Code is not implemented"); + } + @Override public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( DeleteReservationHomeSubClusterRequest request) throws YarnException { @@ -1083,7 +1083,7 @@ public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest req throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } - + @Override public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( UpdateReservationHomeSubClusterRequest request) throws YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index 77ac562183ae7..eae9418b8d531 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -716,9 +716,9 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( @Override public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) throws YarnException, IOException{ - throw new NotImplementedException("Code is not implemented"); + throw new NotImplementedException("Code is not implemented"); } - + @Override public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( DeleteReservationHomeSubClusterRequest request) throws YarnException { @@ -760,7 +760,7 @@ public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest req throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } - + @Override public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( UpdateReservationHomeSubClusterRequest request) throws YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java index ae0b30dec4fa7..e6f3dd36b2018 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java @@ -124,7 +124,7 @@ public boolean equals(Object right) { return new EqualsBuilder() .append(this.getKeyId().intValue(), r.getKeyId().intValue()) .append(this.getExpiryDate().longValue(), this.getExpiryDate().longValue()) - .append(getKeyBytes().array(),r.getKeyBytes()) + .append(getKeyBytes().array(), r.getKeyBytes()) .isEquals(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index c29b28c8d1c22..547f2bf7c3b65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -575,8 +575,8 @@ public RouterRMTokenResponse getTokenByRouterStoreToken(RMDelegationTokenIdentif RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L); RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); return stateStore.getTokenByRouterStoreToken(request); - } - + } + /** * Updates the home {@link SubClusterId} for the specified * {@link ReservationId}. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index 6e7ff20f8697c..09d11d6b47065 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -289,13 +289,12 @@ public void testStoreNewToken() throws YarnException, IOException { facade.storeNewToken(dtId1, renewDate1); // get RouterStoreToken from StateStore - FederationStateStore stateStore = facade.getStateStore(); RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1); RouterRMTokenRequest routerRMTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); - RouterRMTokenResponse routerRMTokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); - Assert.assertNotNull(routerRMTokenResponse); + RouterRMTokenResponse tokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); + Assert.assertNotNull(tokenResponse); - RouterStoreToken resultStoreToken = routerRMTokenResponse.getRouterStoreToken(); + RouterStoreToken resultStoreToken = tokenResponse.getRouterStoreToken(); YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier(); Assert.assertNotNull(resultStoreToken); Assert.assertNotNull(resultTokenIdentifier); @@ -322,13 +321,12 @@ public void testUpdateNewToken() throws YarnException, IOException { facade.updateStoredToken(dtId1, renewDate2); // get RouterStoreToken from StateStore - FederationStateStore stateStore = facade.getStateStore(); RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1); RouterRMTokenRequest routerRMTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); - RouterRMTokenResponse routerRMTokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); - Assert.assertNotNull(routerRMTokenResponse); + RouterRMTokenResponse tokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); + Assert.assertNotNull(tokenResponse); - RouterStoreToken resultStoreToken = routerRMTokenResponse.getRouterStoreToken(); + RouterStoreToken resultStoreToken = tokenResponse.getRouterStoreToken(); YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier(); Assert.assertNotNull(resultStoreToken); Assert.assertNotNull(resultTokenIdentifier); @@ -351,13 +349,12 @@ public void testRemoveStoredToken() throws Exception { facade.storeNewToken(dtId1, renewDate1); // get RouterStoreToken from StateStore - FederationStateStore stateStore = facade.getStateStore(); RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1); RouterRMTokenRequest routerRMTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); - RouterRMTokenResponse routerRMTokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); - Assert.assertNotNull(routerRMTokenResponse); + RouterRMTokenResponse tokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); + Assert.assertNotNull(tokenResponse); - RouterStoreToken resultStoreToken = routerRMTokenResponse.getRouterStoreToken(); + RouterStoreToken resultStoreToken = tokenResponse.getRouterStoreToken(); YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier(); Assert.assertNotNull(resultStoreToken); Assert.assertNotNull(resultTokenIdentifier); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java index 84bca07c65aea..f309ef404efcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java @@ -392,7 +392,7 @@ public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest req throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } - + @Override public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( UpdateReservationHomeSubClusterRequest request) throws YarnException { From 1b918a4d848c5453e5acab055a5acea50ab92701 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 25 Aug 2022 14:40:36 +0800 Subject: [PATCH 09/13] YARN-11253. Fix CheckStyle. --- .../security/client/impl/pb/package-info.java | 4 +-- .../impl/MemoryFederationStateStore.java | 2 +- .../utils/TestFederationStateStoreFacade.java | 26 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java index 5751632e316ad..f4f427b86db48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java @@ -15,6 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@InterfaceAudience.Public +@Public package org.apache.hadoop.yarn.security.client.impl.pb; -import org.apache.hadoop.classification.InterfaceAudience; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Public; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 529336f88110e..bdc565994c068 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -542,7 +542,7 @@ public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( * @param masterKey masterKey * @return DelegationKey */ - private DelegationKey getDelegationKeyByMasterKey(RouterMasterKey masterKey) { + private static DelegationKey getDelegationKeyByMasterKey(RouterMasterKey masterKey) { ByteBuffer keyByteBuf = masterKey.getKeyBytes(); byte[] keyBytes = new byte[keyByteBuf.remaining()]; keyByteBuf.get(keyBytes); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index 09d11d6b47065..1d8890d47fe77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -290,11 +290,11 @@ public void testStoreNewToken() throws YarnException, IOException { // get RouterStoreToken from StateStore RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1); - RouterRMTokenRequest routerRMTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); - RouterRMTokenResponse tokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); - Assert.assertNotNull(tokenResponse); + RouterRMTokenRequest rmTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); + RouterRMTokenResponse rmTokenResponse = stateStore.getTokenByRouterStoreToken(rmTokenRequest); + Assert.assertNotNull(rmTokenResponse); - RouterStoreToken resultStoreToken = tokenResponse.getRouterStoreToken(); + RouterStoreToken resultStoreToken = rmTokenResponse.getRouterStoreToken(); YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier(); Assert.assertNotNull(resultStoreToken); Assert.assertNotNull(resultTokenIdentifier); @@ -322,11 +322,11 @@ public void testUpdateNewToken() throws YarnException, IOException { // get RouterStoreToken from StateStore RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1); - RouterRMTokenRequest routerRMTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); - RouterRMTokenResponse tokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); - Assert.assertNotNull(tokenResponse); + RouterRMTokenRequest rmTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); + RouterRMTokenResponse rmTokenResponse = stateStore.getTokenByRouterStoreToken(rmTokenRequest); + Assert.assertNotNull(rmTokenResponse); - RouterStoreToken resultStoreToken = tokenResponse.getRouterStoreToken(); + RouterStoreToken resultStoreToken = rmTokenResponse.getRouterStoreToken(); YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier(); Assert.assertNotNull(resultStoreToken); Assert.assertNotNull(resultTokenIdentifier); @@ -350,11 +350,11 @@ public void testRemoveStoredToken() throws Exception { // get RouterStoreToken from StateStore RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1); - RouterRMTokenRequest routerRMTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); - RouterRMTokenResponse tokenResponse = stateStore.getTokenByRouterStoreToken(routerRMTokenRequest); - Assert.assertNotNull(tokenResponse); + RouterRMTokenRequest rmTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken); + RouterRMTokenResponse rmTokenResponse = stateStore.getTokenByRouterStoreToken(rmTokenRequest); + Assert.assertNotNull(rmTokenResponse); - RouterStoreToken resultStoreToken = tokenResponse.getRouterStoreToken(); + RouterStoreToken resultStoreToken = rmTokenResponse.getRouterStoreToken(); YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier(); Assert.assertNotNull(resultStoreToken); Assert.assertNotNull(resultTokenIdentifier); @@ -369,6 +369,6 @@ public void testRemoveStoredToken() throws Exception { // Call again(getTokenByRouterStoreToken) after remove will throw IOException(not exist) LambdaTestUtils.intercept(IOException.class, "RMDelegationToken: " + dtId1 + " does not exist.", - () -> stateStore.getTokenByRouterStoreToken(routerRMTokenRequest)); + () -> stateStore.getTokenByRouterStoreToken(rmTokenRequest)); } } From 745c5b8a24c985c739fb0787cee234d40daf1e17 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 25 Aug 2022 18:40:23 +0800 Subject: [PATCH 10/13] YARN-9708. Fix CheckStyle. --- .../store/FederationDelegationTokenStateStore.java | 8 ++++---- .../store/records/impl/pb/RouterStoreTokenPBImpl.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java index 6407350536b40..180e00c95cf60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -16,8 +16,8 @@ */ package org.apache.hadoop.yarn.server.federation.store; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; @@ -31,8 +31,8 @@ * DelegationToken that have been submitted to the federated cluster. * */ -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public interface FederationDelegationTokenStateStore { /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java index 4df7e40730531..32e148cb5b7ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.thirdparty.protobuf.TextFormat; import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; @@ -34,8 +34,8 @@ /** * Protocol buffer based implementation of {@link RouterStoreToken}. */ -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public class RouterStoreTokenPBImpl extends RouterStoreToken { private RouterStoreTokenProto proto = RouterStoreTokenProto.getDefaultInstance(); From 414140c265be948ff07ee444b0195e71d7f9dbfa Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 27 Aug 2022 19:58:02 -0700 Subject: [PATCH 11/13] YARN-9708. Fix CheckStyle. --- .../server/router/clientrm/RouterClientRMService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index 6921f7e1839bb..0f2ceb063c611 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -22,6 +22,7 @@ import java.net.InetSocketAddress; import java.util.Collections; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; @@ -590,9 +591,12 @@ protected RouterDelegationTokenSecretManager createRouterRMDelegationTokenSecret YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); long tokenRenewInterval = conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); - + long removeScanInterval = + conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY, + YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); return new RouterDelegationTokenSecretManager(secretKeyInterval, - tokenMaxLifetime, tokenRenewInterval, 60 * 60 * 1000); + tokenMaxLifetime, tokenRenewInterval, removeScanInterval); } @VisibleForTesting From ca65a329eb5ed64d9938765c231ac1c62c9319dd Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 30 Aug 2022 13:15:32 +0800 Subject: [PATCH 12/13] YARN-9708. Fix CheckStyle. --- .../impl/pb/YARNDelegationTokenIdentifierPBImpl.java | 8 ++++---- .../security/RouterDelegationTokenSecretManager.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java index aea2d0eb7c29d..f977a3391fe2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java @@ -17,16 +17,16 @@ */ package org.apache.hadoop.yarn.security.client.impl.pb; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.io.Text; import org.apache.hadoop.thirdparty.protobuf.TextFormat; import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProtoOrBuilder; import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public class YARNDelegationTokenIdentifierPBImpl extends YARNDelegationTokenIdentifier { private YARNDelegationTokenIdentifierProto proto = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java index b275f72460021..67318d1c29ea9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.yarn.server.router.security; -import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.DelegationKey; @@ -227,13 +227,13 @@ public void setFederationFacade(FederationStateStoreFacade federationFacade) { this.federationFacade = federationFacade; } - @InterfaceAudience.Private + @Public @VisibleForTesting public int getLatestDTSequenceNumber() { return delegationTokenSequenceNumber; } - @InterfaceAudience.Private + @Public @VisibleForTesting public synchronized Set getAllMasterKeys() { HashSet keySet = new HashSet<>(); @@ -241,7 +241,7 @@ public synchronized Set getAllMasterKeys() { return keySet; } - @InterfaceAudience.Private + @Public @VisibleForTesting public synchronized Map getAllTokens() { Map allTokens = new HashMap<>(); From bba520a82c45c5ceaa02786f74e11a3f19e07446 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 2 Sep 2022 13:57:43 +0800 Subject: [PATCH 13/13] YARN-9708. Fix CheckStyle. --- .../federation/store/impl/ZookeeperFederationStateStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index c9e6e7c896ddf..def8cb9adfadf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.TimeZone; +import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.curator.ZKCuratorManager;