From c9e5a2b9b4e2853b75d63581d93f306e4b1caf77 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 4 Nov 2023 08:05:09 +0800 Subject: [PATCH 1/7] YARN-11483. [Federation] Router AdminCLI Supports Clean Finish Apps. --- .../hadoop/yarn/client/cli/RouterCLI.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 2da584f9d61bb..e52dbaafd9163 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -227,12 +227,34 @@ public class RouterCLI extends Configured implements Tool { .addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_1) .addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_2); + // Command3: application + private static final String CMD_APPLICATION = "-application"; + + // Application Delete + protected final static UsageInfo APPLICATION_DELETE_USAGE = new UsageInfo( + "--delete ", + "This command is used to delete the specified application."); + + protected final static String APPLICATION_DELETE_USAGE_EXAMPLE_DESC = + "If we want to delete application_1440536969523_0001."; + + protected final static String APPLICATION_DELETE_USAGE_EXAMPLE_1 = + "yarn routeradmin -application --delete application_1440536969523_0001"; + + protected final static RouterCmdUsageInfos APPLICATION_USAGEINFOS = new RouterCmdUsageInfos() + // application delete + .addUsageInfo(APPLICATION_DELETE_USAGE) + .addExampleDescs(APPLICATION_DELETE_USAGE.args, APPLICATION_DELETE_USAGE_EXAMPLE_DESC) + .addExample(APPLICATION_DELETE_USAGE.args, APPLICATION_DELETE_USAGE_EXAMPLE_1); + protected final static Map ADMIN_USAGE = ImmutableMap.builder() // Command1: deregisterSubCluster .put(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_USAGEINFOS) // Command2: policy .put(CMD_POLICY, POLICY_USAGEINFOS) + // Command3: application + .put(CMD_APPLICATION, APPLICATION_USAGEINFOS) .build(); public RouterCLI() { From ea909373470ced9b225bf60b835bbd5a3fd0711e Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 4 Nov 2023 11:51:47 +0800 Subject: [PATCH 2/7] YARN-11483. Fix CheckStyle. --- .../DeleteFederationApplicationRequest.java | 50 ++++++++++ .../DeleteFederationApplicationResponse.java | 47 ++++++++++ ...erver_resourcemanager_service_protos.proto | 8 ++ .../hadoop/yarn/client/cli/RouterCLI.java | 56 ++++++++++-- ...eteFederationApplicationRequestPBImpl.java | 91 +++++++++++++++++++ 5 files changed, 246 insertions(+), 6 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationRequest.java new file mode 100644 index 0000000000000..8444d6ac9cb6a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationRequest.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * This class is used for cleaning up an application that exists in the FederationStateStore. + * This is a user-specified operation; we typically use this command to clean up an expired application. + * However, it can also be used to clean up non-expired application, although it is not recommended. + */ +@Private +@Unstable +public abstract class DeleteFederationApplicationRequest { + + @Private + @Unstable + public static DeleteFederationApplicationRequest newInstance(String application) { + DeleteFederationApplicationRequest request = + Records.newRecord(DeleteFederationApplicationRequest.class); + request.setApplication(application); + return request; + } + + @Public + @Unstable + public abstract String getApplication(); + + @Public + @Unstable + public abstract void setApplication(String application); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationResponse.java new file mode 100644 index 0000000000000..d75515ff633ec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationResponse.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +@Private +@Unstable +public abstract class DeleteFederationApplicationResponse { + + public static DeleteFederationApplicationResponse newInstance() { + return Records.newRecord(DeleteFederationApplicationResponse.class); + } + + public static DeleteFederationApplicationResponse newInstance(String msg) { + DeleteFederationApplicationResponse response = + Records.newRecord(DeleteFederationApplicationResponse.class); + response.setMessage(msg); + return response; + } + + @Public + @Unstable + public abstract String getMessage(); + + @Public + @Unstable + public abstract void setMessage(String msg); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index a2945f1eb1ee1..ba18e514a7dfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -203,6 +203,14 @@ message QueryFederationQueuePoliciesResponseProto { repeated FederationQueueWeightProto federationQueueWeights = 5; } +message DeleteFederationApplicationRequestProto { + optional string application = 1; +} + +message DeleteFederationApplicationResponseProto { + optional string message = 1; +} + ////////////////////////////////////////////////////////////////// ///////////// RM Failover related records //////////////////////// ////////////////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index e52dbaafd9163..e977dc8746c4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -17,12 +17,7 @@ */ package org.apache.hadoop.yarn.client.cli; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.MissingArgumentException; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.*; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -247,6 +242,9 @@ public class RouterCLI extends Configured implements Tool { .addExampleDescs(APPLICATION_DELETE_USAGE.args, APPLICATION_DELETE_USAGE_EXAMPLE_DESC) .addExample(APPLICATION_DELETE_USAGE.args, APPLICATION_DELETE_USAGE_EXAMPLE_1); + // delete application + private static final String OPTION_DELETE_APP = "delete"; + protected final static Map ADMIN_USAGE = ImmutableMap.builder() // Command1: deregisterSubCluster @@ -836,6 +834,50 @@ protected int handListPolicies(int pageSize, int currentPage, String queue, List } } + + private int handleDeleteApplication(String application) { + LOG.info("Delete Application = {}.", application); + try { + /*SaveFederationQueuePolicyRequest request = parsePolicy(policy); + ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); + SaveFederationQueuePolicyResponse response = adminProtocol.saveFederationQueuePolicy(request); + System.out.println(response.getMessage());*/ + return EXIT_SUCCESS; + } catch (Exception e) { + LOG.error("handleSavePolicy error.", e); + return EXIT_ERROR; + } + } + + private int handleApplication(String[] args) + throws IOException, YarnException, ParseException { + // Prepare Options. + Options opts = new Options(); + opts.addOption("application", false, + "We provide a set of commands to query and clean applications."); + Option deleteOpt = new Option(null, OPTION_DELETE_APP, true, + "We will clean up the provided application."); + opts.addOption(deleteOpt); + + // Parse command line arguments. + CommandLine cliParser; + try { + cliParser = new DefaultParser().parse(opts, args); + } catch (MissingArgumentException ex) { + System.out.println("Missing argument for options"); + printUsage(args[0]); + return EXIT_ERROR; + } + + if (cliParser.hasOption(OPTION_DELETE_APP)) { + String application = cliParser.getOptionValue(OPTION_DELETE_APP); + // return handleSavePolicy(policy); + } else { + + } + return 0; + } + @Override public int run(String[] args) throws Exception { YarnConfiguration yarnConf = getConf() == null ? @@ -861,6 +903,8 @@ public int run(String[] args) throws Exception { return handleDeregisterSubCluster(args); } else if (CMD_POLICY.equals(cmd)) { return handlePolicy(args); + } else if (CMD_APPLICATION.equals(cmd)) { + return handleApplication(args); } else { System.out.println("No related commands found."); printHelp(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java new file mode 100644 index 0000000000000..ee11f48cfaeee --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; + +@Private +@Unstable +public class DeleteFederationApplicationRequestPBImpl extends DeleteFederationApplicationRequest { + + private DeleteFederationApplicationRequestProto proto = + DeleteFederationApplicationRequestProto.getDefaultInstance(); + private DeleteFederationApplicationRequestProto.Builder builder = null; + private boolean viaProto = false; + + public DeleteFederationApplicationRequestPBImpl() { + builder = DeleteFederationApplicationRequestProto.newBuilder(); + } + + public DeleteFederationApplicationRequestPBImpl(DeleteFederationApplicationRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = DeleteFederationApplicationRequestProto.newBuilder(proto); + } + viaProto = false; + } + + public DeleteFederationApplicationRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof DeleteFederationApplicationRequest)) { + return false; + } + DeleteFederationApplicationRequestPBImpl otherImpl = this.getClass().cast(other); + return new EqualsBuilder().append(this.getProto(), otherImpl.getProto()).isEquals(); + } + + @Override + public String getApplication() { + DeleteFederationApplicationRequestProtoOrBuilder p = viaProto ? proto : builder; + boolean hasApplication = p.hasApplication(); + if (hasApplication) { + return p.getApplication(); + } + return null; + } + + @Override + public void setApplication(String application) { + maybeInitBuilder(); + if (application == null) { + builder.clearApplication(); + return; + } + builder.setApplication(application); + } +} From f4d3468141a757bb0440d82c6de857b794b1500b Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sun, 5 Nov 2023 08:19:16 +0800 Subject: [PATCH 3/7] YARN-11483. Fix CheckStyle. --- ...ResourceManagerAdministrationProtocol.java | 15 +++ ...ourcemanager_administration_protocol.proto | 1 + ...gerAdministrationProtocolPBClientImpl.java | 19 ++++ ...erAdministrationProtocolPBServiceImpl.java | 21 ++++ ...eteFederationApplicationRequestPBImpl.java | 6 ++ ...teFederationApplicationResponsePBImpl.java | 98 +++++++++++++++++++ .../server/MockResourceManagerFacade.java | 43 ++------ .../server/resourcemanager/AdminService.java | 10 ++ .../DefaultRMAdminRequestInterceptor.java | 9 ++ .../rmadmin/FederationRMAdminInterceptor.java | 8 ++ .../router/rmadmin/RouterRMAdminService.java | 9 ++ .../PassThroughRMAdminRequestInterceptor.java | 8 ++ 12 files changed, 211 insertions(+), 36 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java index 916dcd50aa0d9..0b4f1ad6429fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java @@ -64,6 +64,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; @Private public interface ResourceManagerAdministrationProtocol extends GetUserMappingsProtocol { @@ -218,4 +220,17 @@ BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( @Idempotent QueryFederationQueuePoliciesResponse listFederationQueuePolicies( QueryFederationQueuePoliciesRequest request) throws YarnException, IOException; + + /** + * In YARN-Federation mode, this method provides a way to delete federation application. + * + * @param request DeleteFederationApplicationRequest Request. + * @return Response from deleteFederationApplication. + * @throws YarnException exceptions from yarn servers. + * @throws IOException if an IO error occurred. + */ + @Private + @Idempotent + DeleteFederationApplicationResponse deleteFederationApplication( + DeleteFederationApplicationRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto index fcae14128d80f..f0f3fa563f59d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto @@ -51,4 +51,5 @@ service ResourceManagerAdministrationProtocolService { rpc saveFederationQueuePolicy(SaveFederationQueuePolicyRequestProto) returns (SaveFederationQueuePolicyResponseProto); rpc batchSaveFederationQueuePolicies(BatchSaveFederationQueuePoliciesRequestProto) returns (BatchSaveFederationQueuePoliciesResponseProto); rpc listFederationQueuePolicies(QueryFederationQueuePoliciesRequestProto) returns (QueryFederationQueuePoliciesResponseProto); + rpc deleteFederationApplication(DeleteFederationApplicationRequestProto) returns (DeleteFederationApplicationResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java index c7c6dc510424c..9d9c018a46962 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.QueryFederationQueuePoliciesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationRequestProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; @@ -86,6 +87,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl; @@ -122,6 +125,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeleteFederationApplicationRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeleteFederationApplicationResponsePBImpl; import org.apache.hadoop.thirdparty.protobuf.ServiceException; @@ -420,4 +425,18 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( } return null; } + + @Override + public DeleteFederationApplicationResponse deleteFederationApplication( + DeleteFederationApplicationRequest request) throws YarnException, IOException { + DeleteFederationApplicationRequestProto requestProto = + ((DeleteFederationApplicationRequestPBImpl) request).getProto(); + try { + return new DeleteFederationApplicationResponsePBImpl( + proxy.deleteFederationApplication(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + } + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java index 0eab80b9ad770..e331ef8c0e147 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java @@ -60,6 +60,8 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeregisterSubClusterResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationRequestProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; @@ -87,6 +89,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl; @@ -123,6 +127,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeleteFederationApplicationRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeleteFederationApplicationResponsePBImpl; import org.apache.hadoop.thirdparty.protobuf.RpcController; import org.apache.hadoop.thirdparty.protobuf.ServiceException; @@ -445,4 +451,19 @@ public QueryFederationQueuePoliciesResponseProto listFederationQueuePolicies( throw new ServiceException(e); } } + + @Override + public DeleteFederationApplicationResponseProto deleteFederationApplication(RpcController controller, + DeleteFederationApplicationRequestProto proto) throws ServiceException { + DeleteFederationApplicationRequest request = + new DeleteFederationApplicationRequestPBImpl(proto); + try { + DeleteFederationApplicationResponse response = real.deleteFederationApplication(request); + return ((DeleteFederationApplicationResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java index ee11f48cfaeee..96f3fecdd68ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationRequestProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; @@ -88,4 +89,9 @@ public void setApplication(String application) { } builder.setApplication(application); } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java new file mode 100644 index 0000000000000..759e2ceecf31b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; + +@Private +@Unstable +public class DeleteFederationApplicationResponsePBImpl extends DeleteFederationApplicationResponse { + + private DeleteFederationApplicationResponseProto proto = + DeleteFederationApplicationResponseProto.getDefaultInstance(); + private DeleteFederationApplicationResponseProto.Builder builder = null; + private boolean viaProto = false; + + public DeleteFederationApplicationResponsePBImpl() { + builder = DeleteFederationApplicationResponseProto.newBuilder(); + } + + public DeleteFederationApplicationResponsePBImpl(DeleteFederationApplicationResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public DeleteFederationApplicationResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof DeleteFederationApplicationRequest)) { + return false; + } + DeleteFederationApplicationResponsePBImpl otherImpl = this.getClass().cast(other); + return new EqualsBuilder().append(this.getProto(), otherImpl.getProto()).isEquals(); + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public String getMessage() { + DeleteFederationApplicationResponseProtoOrBuilder p = viaProto ? proto : builder; + boolean hasMessage = p.hasMessage(); + if (hasMessage) { + return p.getMessage(); + } + return null; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = DeleteFederationApplicationResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public void setMessage(String msg) { + maybeInitBuilder(); + if (msg == null) { + builder.clearMessage(); + return; + } + builder.setMessage(msg); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index ed66dadefb497..b9f4af15d080e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -141,46 +141,11 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; -import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.*; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; /** @@ -988,6 +953,12 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( return null; } + @Override + public DeleteFederationApplicationResponse deleteFederationApplication( + DeleteFederationApplicationRequest request) throws YarnException, IOException { + return null; + } + @VisibleForTesting public HashMap> getApplicationContainerIdMap() { return applicationContainerIdMap; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 8f9e8caa53670..c45c34dc85c80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -103,6 +103,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; @@ -1105,6 +1107,14 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( " Please call Router's listFederationQueuePolicies to list Policies."); } + @Override + public DeleteFederationApplicationResponse deleteFederationApplication( + DeleteFederationApplicationRequest request) throws YarnException, IOException { + throw new YarnException("It is not allowed to call the RM's " + + " deleteFederationApplication. " + + " Please call Router's deleteFederationApplication to delete Application."); + } + private void validateAttributesExists( List nodesToAttributes) throws IOException { NodeAttributesManager nodeAttributesManager = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java index 23517b97b4e19..2633a90d62cc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java @@ -64,6 +64,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -233,4 +235,11 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( QueryFederationQueuePoliciesRequest request) throws YarnException, IOException { return rmAdminProxy.listFederationQueuePolicies(request); } + + @Override + public DeleteFederationApplicationResponse deleteFederationApplication( + DeleteFederationApplicationRequest request) + throws YarnException, IOException { + return rmAdminProxy.deleteFederationApplication(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java index d269cfe0971cf..4bea6793940c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java @@ -71,6 +71,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; @@ -1088,6 +1090,12 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( throw new YarnException("Unable to listFederationQueuePolicies."); } + @Override + public DeleteFederationApplicationResponse deleteFederationApplication( + DeleteFederationApplicationRequest request) throws YarnException, IOException { + return null; + } + /** * According to the configuration information of the queue filtering queue, * this part should only return 1 result. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index fc2278d3bb318..718abd2894493 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -71,6 +71,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; @@ -419,4 +421,11 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( RequestInterceptorChainWrapper pipeline = getInterceptorChain(); return pipeline.getRootInterceptor().listFederationQueuePolicies(request); } + + @Override + public DeleteFederationApplicationResponse deleteFederationApplication( + DeleteFederationApplicationRequest request) throws YarnException, IOException { + RequestInterceptorChainWrapper pipeline = getInterceptorChain(); + return pipeline.getRootInterceptor().deleteFederationApplication(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java index 6fe218c7b2a00..0dab931a7ab97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java @@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; /** * Mock interceptor that does not do anything other than forwarding it to the @@ -185,4 +187,10 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( QueryFederationQueuePoliciesRequest request) throws YarnException, IOException { return getNextInterceptor().listFederationQueuePolicies(request); } + + @Override + public DeleteFederationApplicationResponse deleteFederationApplication( + DeleteFederationApplicationRequest request) throws YarnException, IOException { + return getNextInterceptor().deleteFederationApplication(request); + } } From 5f8abdf28ebefca7aa5b2f61089e1ef103b7826c Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Mon, 6 Nov 2023 23:12:28 +0800 Subject: [PATCH 4/7] YARN-11483. Fix CheckStyle. --- .../hadoop/yarn/client/cli/RouterCLI.java | 24 +++++++---- .../rmadmin/FederationRMAdminInterceptor.java | 20 ++++++++- .../TestFederationRMAdminInterceptor.java | 43 +++++-------------- 3 files changed, 45 insertions(+), 42 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index e977dc8746c4e..7031b8658a7e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -17,7 +17,13 @@ */ package org.apache.hadoop.yarn.client.cli; -import org.apache.commons.cli.*; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -37,6 +43,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusters; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; @@ -834,14 +842,15 @@ protected int handListPolicies(int pageSize, int currentPage, String queue, List } } - private int handleDeleteApplication(String application) { LOG.info("Delete Application = {}.", application); try { - /*SaveFederationQueuePolicyRequest request = parsePolicy(policy); + DeleteFederationApplicationRequest request = + DeleteFederationApplicationRequest.newInstance(application); ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); - SaveFederationQueuePolicyResponse response = adminProtocol.saveFederationQueuePolicy(request); - System.out.println(response.getMessage());*/ + DeleteFederationApplicationResponse response = + adminProtocol.deleteFederationApplication(request); + System.out.println(response.getMessage()); return EXIT_SUCCESS; } catch (Exception e) { LOG.error("handleSavePolicy error.", e); @@ -871,10 +880,9 @@ private int handleApplication(String[] args) if (cliParser.hasOption(OPTION_DELETE_APP)) { String application = cliParser.getOptionValue(OPTION_DELETE_APP); - // return handleSavePolicy(policy); - } else { - + return handleDeleteApplication(application); } + return 0; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java index 4bea6793940c0..16ad3f31f1293 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java @@ -28,6 +28,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -1093,7 +1094,24 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( @Override public DeleteFederationApplicationResponse deleteFederationApplication( DeleteFederationApplicationRequest request) throws YarnException, IOException { - return null; + // Parameter validation. + if (request == null) { + RouterServerUtil.logAndThrowException( + "Missing deleteFederationApplication Request.", null); + } + + + try { + ApplicationId applicationId = ApplicationId.fromString(request.getApplication()); + federationFacade.deleteApplicationHomeSubCluster(applicationId); + return DeleteFederationApplicationResponse.newInstance( + "application = " + applicationId + " delete success."); + } catch (Exception e) { + RouterServerUtil.logAndThrowException(e, + "Unable to deleteFederationApplication due to exception. " + e.getMessage()); + } + + throw new YarnException("Unable to listFederationQueuePolicies."); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java index 520c25d22cb09..a33249291c810 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java @@ -31,39 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeAttributeType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType; -import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; -import org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight; -import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.*; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; @@ -989,4 +957,13 @@ public void testFilterPoliciesConfigurationsByQueues() throws Exception { "The index of the records to be retrieved has exceeded the maximum index.", () -> interceptor.listFederationQueuePolicies(request8)); } + + + @Test + public void testDeleteFederationApplication() throws IOException, YarnException { + DeleteFederationApplicationRequest request = + DeleteFederationApplicationRequest.newInstance("application_1"); + DeleteFederationApplicationResponse response = interceptor.deleteFederationApplication(request); + assertNotNull(response); + } } From d789bf6bd38d3da496f87a6b323cbd72bcb50468 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 8 Nov 2023 10:15:28 +0800 Subject: [PATCH 5/7] YARN-11483. Fix CheckStyle. --- .../protocolrecords/DeleteFederationApplicationRequest.java | 3 ++- .../java/org/apache/hadoop/yarn/client/cli/RouterCLI.java | 4 ++-- .../ResourceManagerAdministrationProtocolPBServiceImpl.java | 5 +++-- .../impl/pb/DeleteFederationApplicationRequestPBImpl.java | 2 +- .../impl/pb/DeleteFederationApplicationResponsePBImpl.java | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationRequest.java index 8444d6ac9cb6a..a035aa8275c12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DeleteFederationApplicationRequest.java @@ -24,7 +24,8 @@ /** * This class is used for cleaning up an application that exists in the FederationStateStore. - * This is a user-specified operation; we typically use this command to clean up an expired application. + * This is a user-specified operation; + * we typically use this command to clean up an expired application. * However, it can also be used to clean up non-expired application, although it is not recommended. */ @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 7031b8658a7e1..9a68794078e2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -235,8 +235,8 @@ public class RouterCLI extends Configured implements Tool { // Application Delete protected final static UsageInfo APPLICATION_DELETE_USAGE = new UsageInfo( - "--delete ", - "This command is used to delete the specified application."); + "--delete ", + "This command is used to delete the specified application."); protected final static String APPLICATION_DELETE_USAGE_EXAMPLE_DESC = "If we want to delete application_1440536969523_0001."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java index e331ef8c0e147..1e0b09a619568 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java @@ -453,8 +453,9 @@ public QueryFederationQueuePoliciesResponseProto listFederationQueuePolicies( } @Override - public DeleteFederationApplicationResponseProto deleteFederationApplication(RpcController controller, - DeleteFederationApplicationRequestProto proto) throws ServiceException { + public DeleteFederationApplicationResponseProto deleteFederationApplication( + RpcController controller, DeleteFederationApplicationRequestProto proto) + throws ServiceException { DeleteFederationApplicationRequest request = new DeleteFederationApplicationRequestPBImpl(proto); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java index 96f3fecdd68ff..0309632c40ac2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationRequestPBImpl.java @@ -64,7 +64,7 @@ public int hashCode() { @Override public boolean equals(Object other) { if (!(other instanceof DeleteFederationApplicationRequest)) { - return false; + return false; } DeleteFederationApplicationRequestPBImpl otherImpl = this.getClass().cast(other); return new EqualsBuilder().append(this.getProto(), otherImpl.getProto()).isEquals(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java index 759e2ceecf31b..2b5bac072cc97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java @@ -81,7 +81,7 @@ public String getMessage() { private synchronized void maybeInitBuilder() { if (viaProto || builder == null) { - builder = DeleteFederationApplicationResponseProto.newBuilder(proto); + builder = DeleteFederationApplicationResponseProto.newBuilder(proto); } viaProto = false; } From 4721e3feae980f0109cacb2b64145a808fe28607 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 8 Nov 2023 20:21:24 +0800 Subject: [PATCH 6/7] YARN-11483. Fix CheckStyle. --- .../hadoop/yarn/client/cli/TestRouterCLI.java | 9 ++- ...teFederationApplicationResponsePBImpl.java | 3 +- .../yarn/server/router/RouterMetrics.java | 32 ++++++++++ .../rmadmin/FederationRMAdminInterceptor.java | 18 +++++- .../TestFederationRMAdminInterceptor.java | 63 +++++++++++++++++-- 5 files changed, 113 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java index a86878dac3f81..8e697dd6c01db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java @@ -94,6 +94,7 @@ public void setup() throws Exception { return QueryFederationQueuePoliciesResponse.newInstance(1, 1, 1, 10, weights); }); + Configuration config = new Configuration(); config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); @@ -271,7 +272,7 @@ public void testListPolicies() throws Exception { @Test public void testBuildHelpMsg() throws Exception { Map adminUsage = rmAdminCLI.getAdminUsage(); - assertEquals(2, adminUsage.size()); + assertEquals(3, adminUsage.size()); RouterCLI.RouterCmdUsageInfos deregisterSubClusterUsageInfos = adminUsage.get("-deregisterSubCluster"); @@ -291,5 +292,11 @@ public void testBuildHelpMsg() throws Exception { policyExamplesMap.forEach((cmd, cmdExamples) -> { assertEquals(2, cmdExamples.size()); }); + + RouterCLI.RouterCmdUsageInfos applicationUsageInfos = adminUsage.get("-application"); + assertNotNull(applicationUsageInfos); + Map> applicationExamplesMap = applicationUsageInfos.getExamples(); + assertNotNull(applicationExamplesMap); + assertEquals(1, applicationExamplesMap.size()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java index 2b5bac072cc97..f21f6fbde18f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/DeleteFederationApplicationResponsePBImpl.java @@ -23,7 +23,6 @@ import org.apache.hadoop.thirdparty.protobuf.TextFormat; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeleteFederationApplicationResponseProtoOrBuilder; -import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; @Private @@ -57,7 +56,7 @@ public int hashCode() { @Override public boolean equals(Object other) { - if (!(other instanceof DeleteFederationApplicationRequest)) { + if (!(other instanceof DeleteFederationApplicationResponse)) { return false; } DeleteFederationApplicationResponsePBImpl otherImpl = this.getClass().cast(other); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java index d0e4825fed39b..66157797a936e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java @@ -155,6 +155,8 @@ public final class RouterMetrics { private MutableGaugeInt numBatchSaveFederationQueuePoliciesFailedRetrieved; @Metric("# of listFederationQueuePolicies failed to be retrieved") private MutableGaugeInt numListFederationQueuePoliciesFailedRetrieved; + @Metric("# of deleteFederationApplication failed to be retrieved") + private MutableGaugeInt numDeleteFederationApplicationFailedRetrieved; @Metric("# of refreshAdminAcls failed to be retrieved") private MutableGaugeInt numRefreshAdminAclsFailedRetrieved; @Metric("# of refreshServiceAcls failed to be retrieved") @@ -307,6 +309,8 @@ public final class RouterMetrics { private MutableRate totalSucceededBatchSaveFederationQueuePoliciesRetrieved; @Metric("Total number of successful Retrieved ListFederationQueuePolicies and latency(ms)") private MutableRate totalSucceededListFederationQueuePoliciesFailedRetrieved; + @Metric("Total number of successful Retrieved DeleteFederationApplication and latency(ms)") + private MutableRate totalSucceededDeleteFederationApplicationFailedRetrieved; @Metric("Total number of successful Retrieved RefreshAdminAcls and latency(ms)") private MutableRate totalSucceededRefreshAdminAclsRetrieved; @Metric("Total number of successful Retrieved RefreshServiceAcls and latency(ms)") @@ -396,6 +400,7 @@ public final class RouterMetrics { private MutableQuantiles saveFederationQueuePolicyLatency; private MutableQuantiles batchSaveFederationQueuePoliciesLatency; private MutableQuantiles listFederationQueuePoliciesLatency; + private MutableQuantiles deleteFederationApplicationLatency; private MutableQuantiles refreshAdminAclsLatency; private MutableQuantiles refreshServiceAclsLatency; private MutableQuantiles replaceLabelsOnNodesLatency; @@ -618,6 +623,10 @@ private RouterMetrics() { "listFederationQueuePoliciesLatency", "latency of list federationqueuepolicies timeouts", "ops", "latency", 10); + deleteFederationApplicationLatency = registry.newQuantiles( + "deleteFederationApplicationLatency", + "latency of delete FederationApplication timeouts", "ops", "latency", 10); + refreshAdminAclsLatency = registry.newQuantiles("refreshAdminAclsLatency", "latency of refresh admin acls timeouts", "ops", "latency", 10); @@ -962,6 +971,11 @@ public long getNumSucceededListFederationQueuePoliciesFailedRetrieved() { return totalSucceededListFederationQueuePoliciesFailedRetrieved.lastStat().numSamples(); } + @VisibleForTesting + public long getNumSucceededDeleteFederationApplicationFailedRetrieved() { + return totalSucceededDeleteFederationApplicationFailedRetrieved.lastStat().numSamples(); + } + @VisibleForTesting public long getNumSucceededRefreshAdminAclsRetrieved() { return totalSucceededRefreshAdminAclsRetrieved.lastStat().numSamples(); @@ -1322,6 +1336,11 @@ public double getLatencySucceededListFederationQueuePoliciesRetrieved() { return totalSucceededListFederationQueuePoliciesFailedRetrieved.lastStat().mean(); } + @VisibleForTesting + public double getLatencySucceededDeleteFederationApplicationFailedRetrieved() { + return totalSucceededDeleteFederationApplicationFailedRetrieved.lastStat().mean(); + } + @VisibleForTesting public double getLatencySucceededRefreshAdminAclsRetrieved() { return totalSucceededRefreshAdminAclsRetrieved.lastStat().mean(); @@ -1629,6 +1648,10 @@ public int getListFederationQueuePoliciesFailedRetrieved() { return numListFederationQueuePoliciesFailedRetrieved.value(); } + public int getDeleteFederationApplicationFailedRetrieved() { + return numDeleteFederationApplicationFailedRetrieved.value(); + } + public int getNumRefreshAdminAclsFailedRetrieved() { return numRefreshAdminAclsFailedRetrieved.value(); } @@ -1996,6 +2019,11 @@ public void succeededListFederationQueuePoliciesRetrieved(long duration) { listFederationQueuePoliciesLatency.add(duration); } + public void succeededDeleteFederationApplicationFailedRetrieved(long duration) { + totalSucceededDeleteFederationApplicationFailedRetrieved.add(duration); + deleteFederationApplicationLatency.add(duration); + } + public void succeededRefreshAdminAclsRetrieved(long duration) { totalSucceededRefreshAdminAclsRetrieved.add(duration); refreshAdminAclsLatency.add(duration); @@ -2286,6 +2314,10 @@ public void incrListFederationQueuePoliciesFailedRetrieved() { numListFederationQueuePoliciesFailedRetrieved.incr(); } + public void incrDeleteFederationApplicationFailedRetrieved() { + numDeleteFederationApplicationFailedRetrieved.incr(); + } + public void incrRefreshAdminAclsFailedRetrieved() { numRefreshAdminAclsFailedRetrieved.incr(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java index 16ad3f31f1293..c98f5dcb56c60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java @@ -1094,24 +1094,36 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( @Override public DeleteFederationApplicationResponse deleteFederationApplication( DeleteFederationApplicationRequest request) throws YarnException, IOException { + // Parameter validation. if (request == null) { + routerMetrics.incrDeleteFederationApplicationFailedRetrieved(); RouterServerUtil.logAndThrowException( "Missing deleteFederationApplication Request.", null); } + String application = request.getApplication(); + if (StringUtils.isBlank(application)) { + routerMetrics.incrDeleteFederationApplicationFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "ApplicationId cannot be null.", null); + } + // Try calling deleteApplicationHomeSubCluster to delete the application. try { - ApplicationId applicationId = ApplicationId.fromString(request.getApplication()); + long startTime = clock.getTime(); + ApplicationId applicationId = ApplicationId.fromString(application); federationFacade.deleteApplicationHomeSubCluster(applicationId); + long stopTime = clock.getTime(); + routerMetrics.succeededDeleteFederationApplicationFailedRetrieved(stopTime - startTime); return DeleteFederationApplicationResponse.newInstance( - "application = " + applicationId + " delete success."); + "applicationId = " + applicationId + " delete success."); } catch (Exception e) { RouterServerUtil.logAndThrowException(e, "Unable to deleteFederationApplication due to exception. " + e.getMessage()); } - throw new YarnException("Unable to listFederationQueuePolicies."); + throw new YarnException("Unable to deleteFederationApplication."); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java index a33249291c810..62f1eee845b56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.DecommissionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; @@ -31,13 +32,48 @@ import org.apache.hadoop.yarn.api.records.NodeAttributeType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.api.protocolrecords.*; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; +import org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight; +import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil; import org.junit.Assert; @@ -960,10 +996,25 @@ public void testFilterPoliciesConfigurationsByQueues() throws Exception { @Test - public void testDeleteFederationApplication() throws IOException, YarnException { - DeleteFederationApplicationRequest request = - DeleteFederationApplicationRequest.newInstance("application_1"); - DeleteFederationApplicationResponse response = interceptor.deleteFederationApplication(request); - assertNotNull(response); + public void testDeleteFederationApplication() throws Exception { + ApplicationId applicationId = ApplicationId.newInstance(10, 1); + DeleteFederationApplicationRequest request1 = + DeleteFederationApplicationRequest.newInstance(applicationId.toString()); + LambdaTestUtils.intercept(YarnException.class, + "Application application_10_0001 does not exist.", + () -> interceptor.deleteFederationApplication(request1)); + + ApplicationId applicationId2 = ApplicationId.newInstance(10, 2); + SubClusterId homeSubCluster = SubClusterId.newInstance("SC-1"); + ApplicationHomeSubCluster appHomeSubCluster = + ApplicationHomeSubCluster.newInstance(applicationId2, homeSubCluster); + facade.addApplicationHomeSubCluster(appHomeSubCluster); + DeleteFederationApplicationRequest request2 = + DeleteFederationApplicationRequest.newInstance(applicationId2.toString()); + DeleteFederationApplicationResponse deleteFederationApplicationResponse = + interceptor.deleteFederationApplication(request2); + assertNotNull(deleteFederationApplicationResponse); + assertEquals("applicationId = " + applicationId2 + " delete success.", + deleteFederationApplicationResponse.getMessage()); } } From 6e9b8e1c6964c100e2178616bfd6eb30f82d2ba2 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 16 Nov 2023 22:49:48 +0800 Subject: [PATCH 7/7] YARN-11483. Fix CheckStyle. --- .../server/MockResourceManagerFacade.java | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index b9f4af15d080e..0823660bc0404 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -141,11 +141,48 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; -import org.apache.hadoop.yarn.server.api.protocolrecords.*; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DeleteFederationApplicationResponse; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; /**