Skip to content

Commit

Permalink
YARN-11536. Fix CheckStyle.
Browse files Browse the repository at this point in the history
  • Loading branch information
@slfan1989
slfan1989 committed Jul 27, 2023
1 parent 04e7aa8 commit 6de2cf6
Show file tree
Hide file tree
Showing 12 changed files with 231 additions and 39 deletions.
16 changes: 15 additions & 1 deletion ...rc/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;

import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;

@Private
public interface ResourceManagerAdministrationProtocol extends GetUserMappingsProtocol {
Expand Down Expand Up @@ -189,4 +190,17 @@ DeregisterSubClusterResponse deregisterSubCluster(DeregisterSubClusterRequest re
@Idempotent
SaveFederationQueuePolicyResponse saveFederationQueuePolicy(
SaveFederationQueuePolicyRequest request) throws YarnException, IOException;

/**
* In YARN-Federation mode, this method provides a way to save queue policies in batches.
*
* @param request BatchSaveFederationQueuePolicies Request
* @return Response from batchSaveFederationQueuePolicies.
* @throws YarnException exceptions from yarn servers.
* @throws IOException if an IO error occurred.
*/
@Private
@Idempotent
BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException;
}
20 changes: 20 additions & 0 deletions ...oop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto;
Expand All @@ -47,6 +48,7 @@
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeregisterSubClusterRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
Expand Down Expand Up @@ -81,6 +83,8 @@
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl;
Expand Down Expand Up @@ -113,6 +117,8 @@
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeregisterSubClusterResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl;

import org.apache.hadoop.thirdparty.protobuf.ServiceException;

Expand Down Expand Up @@ -381,4 +387,18 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy(
return null;
}
}

@Override
public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException {
BatchSaveFederationQueuePoliciesRequestProto requestProto =
((BatchSaveFederationQueuePoliciesRequestPBImpl) request).getProto();
try {
return new BatchSaveFederationQueuePoliciesResponsePBImpl(
proxy.batchSaveFederationQueuePolicies(null, requestProto));
} catch (ServiceException e) {
RPCUtil.unwrapAndThrowException(e);
return null;
}
}
}
21 changes: 18 additions & 3 deletions ...p/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@

import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto;
Expand Down Expand Up @@ -80,6 +81,8 @@
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl;
Expand Down Expand Up @@ -112,6 +115,8 @@
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DeregisterSubClusterResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl;

import org.apache.hadoop.thirdparty.protobuf.RpcController;
import org.apache.hadoop.thirdparty.protobuf.ServiceException;
Expand Down Expand Up @@ -402,7 +407,17 @@ public SaveFederationQueuePolicyResponseProto saveFederationQueuePolicy(RpcContr
}

@Override
public YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProto batchSaveFederationQueuePolicies(RpcController controller, YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto request) throws ServiceException {
return null;
public BatchSaveFederationQueuePoliciesResponseProto batchSaveFederationQueuePolicies(
RpcController controller, BatchSaveFederationQueuePoliciesRequestProto proto)
throws ServiceException {
BatchSaveFederationQueuePoliciesRequest request = new BatchSaveFederationQueuePoliciesRequestPBImpl(proto);
try {
BatchSaveFederationQueuePoliciesResponse response = real.batchSaveFederationQueuePolicies(request);
return ((BatchSaveFederationQueuePoliciesResponsePBImpl) response).getProto();
} catch (YarnException e) {
throw new ServiceException(e);
} catch (IOException e) {
throw new ServiceException(e);
}
}
}
3 changes: 3 additions & 0 deletions ...arn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesRequestPBImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
import java.util.ArrayList;
import java.util.List;

/**
* The class is responsible for batch-saving queue policies requests.
*/
@Private
@Unstable
public class BatchSaveFederationQueuePoliciesRequestPBImpl
Expand Down
4 changes: 3 additions & 1 deletion ...rn/server/api/protocolrecords/impl/pb/BatchSaveFederationQueuePoliciesResponsePBImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@
package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;

import org.apache.hadoop.thirdparty.protobuf.TextFormat;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesResponseProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;

/**
* The class is responsible for batch-saving queue policies responses.
*/
public class BatchSaveFederationQueuePoliciesResponsePBImpl
extends BatchSaveFederationQueuePoliciesResponse {

Expand Down
8 changes: 8 additions & 0 deletions ...-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,8 @@
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;

/**
Expand Down Expand Up @@ -972,6 +974,12 @@ public SaveFederationQueuePolicyResponse saveFederationQueuePolicy(
return null;
}

@Override
public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException {
return null;
}

@VisibleForTesting
public HashMap<ApplicationId, List<ContainerId>> getApplicationContainerIdMap() {
return applicationContainerIdMap;
Expand Down
12 changes: 11 additions & 1 deletion ...urcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@
import org.apache.hadoop.yarn.server.api.protocolrecords.DeregisterSubClusterResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration;
Expand Down Expand Up @@ -1060,7 +1062,15 @@ public DeregisterSubClusterResponse deregisterSubCluster(
public SaveFederationQueuePolicyResponse saveFederationQueuePolicy(
SaveFederationQueuePolicyRequest request) throws YarnException, IOException {
throw new YarnException("It is not allowed to call the RM's saveFederationQueuePolicy. " +
" Please call Router's deregisterSubCluster to set Policy.");
" Please call Router's saveFederationQueuePolicy to set Policy.");
}

@Override
public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException {
throw new YarnException("It is not allowed to call the RM's " +
" batchSaveFederationQueuePolicies. " +
" Please call Router's batchSaveFederationQueuePolicies to set Policies.");
}

private void validateAttributesExists(
Expand Down
34 changes: 33 additions & 1 deletion ...-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,8 @@ public final class RouterMetrics {
private MutableGaugeInt numDeregisterSubClusterFailedRetrieved;
@Metric("# of saveFederationQueuePolicy failed to be retrieved")
private MutableGaugeInt numSaveFederationQueuePolicyFailedRetrieved;
@Metric("# of batchSaveFederationQueuePolicies failed to be retrieved")
private MutableGaugeInt numBatchSaveFederationQueuePoliciesFailedRetrieved;
@Metric("# of refreshAdminAcls failed to be retrieved")
private MutableGaugeInt numRefreshAdminAclsFailedRetrieved;
@Metric("# of refreshServiceAcls failed to be retrieved")
Expand Down Expand Up @@ -299,6 +301,8 @@ public final class RouterMetrics {
private MutableRate totalSucceededDeregisterSubClusterRetrieved;
@Metric("Total number of successful Retrieved SaveFederationQueuePolicy and latency(ms)")
private MutableRate totalSucceededSaveFederationQueuePolicyRetrieved;
@Metric("Total number of successful Retrieved BatchSaveFederationQueuePolicies and latency(ms)")
private MutableRate totalSucceededBatchSaveFederationQueuePoliciesRetrieved;
@Metric("Total number of successful Retrieved RefreshAdminAcls and latency(ms)")
private MutableRate totalSucceededRefreshAdminAclsRetrieved;
@Metric("Total number of successful Retrieved RefreshServiceAcls and latency(ms)")
Expand Down Expand Up @@ -386,6 +390,7 @@ public final class RouterMetrics {
private MutableQuantiles refreshUserToGroupsMappingsLatency;
private MutableQuantiles refreshDeregisterSubClusterLatency;
private MutableQuantiles saveFederationQueuePolicyLatency;
private MutableQuantiles batchSaveFederationQueuePoliciesLatency;
private MutableQuantiles refreshAdminAclsLatency;
private MutableQuantiles refreshServiceAclsLatency;
private MutableQuantiles replaceLabelsOnNodesLatency;
Expand Down Expand Up @@ -598,7 +603,11 @@ private RouterMetrics() {
"latency of deregister subcluster timeouts", "ops", "latency", 10);

saveFederationQueuePolicyLatency = registry.newQuantiles("saveFederationQueuePolicyLatency",
"latency of refresh subcluster timeouts", "ops", "latency", 10);
"latency of save federation queue policy timeouts", "ops", "latency", 10);

batchSaveFederationQueuePoliciesLatency = registry.newQuantiles(
"batchSaveFederationQueuePoliciesLatency",
"latency of batch save federationqueuepolicies timeouts", "ops", "latency", 10);

refreshAdminAclsLatency = registry.newQuantiles("refreshAdminAclsLatency",
"latency of refresh admin acls timeouts", "ops", "latency", 10);
Expand Down Expand Up @@ -934,6 +943,11 @@ public long getNumSucceededSaveFederationQueuePolicyRetrieved() {
return totalSucceededSaveFederationQueuePolicyRetrieved.lastStat().numSamples();
}

@VisibleForTesting
public long getNumSucceededBatchSaveFederationQueuePoliciesRetrieved() {
return totalSucceededBatchSaveFederationQueuePoliciesRetrieved.lastStat().numSamples();
}

@VisibleForTesting
public long getNumSucceededRefreshAdminAclsRetrieved() {
return totalSucceededRefreshAdminAclsRetrieved.lastStat().numSamples();
Expand Down Expand Up @@ -1284,6 +1298,11 @@ public double getLatencySucceededSaveFederationQueuePolicyRetrieved() {
return totalSucceededSaveFederationQueuePolicyRetrieved.lastStat().mean();
}

@VisibleForTesting
public double getLatencySucceededBatchSaveFederationQueuePoliciesRetrieved() {
return totalSucceededBatchSaveFederationQueuePoliciesRetrieved.lastStat().mean();
}

@VisibleForTesting
public double getLatencySucceededRefreshAdminAclsRetrieved() {
return totalSucceededRefreshAdminAclsRetrieved.lastStat().mean();
Expand Down Expand Up @@ -1583,6 +1602,10 @@ public int getSaveFederationQueuePolicyFailedRetrieved() {
return numSaveFederationQueuePolicyFailedRetrieved.value();
}

public int getBatchSaveFederationQueuePoliciesFailedRetrieved() {
return numBatchSaveFederationQueuePoliciesFailedRetrieved.value();
}

public int getNumRefreshAdminAclsFailedRetrieved() {
return numRefreshAdminAclsFailedRetrieved.value();
}
Expand Down Expand Up @@ -1940,6 +1963,11 @@ public void succeededSaveFederationQueuePolicyRetrieved(long duration) {
saveFederationQueuePolicyLatency.add(duration);
}

public void succeededBatchSaveFederationQueuePoliciesRetrieved(long duration) {
totalSucceededBatchSaveFederationQueuePoliciesRetrieved.add(duration);
batchSaveFederationQueuePoliciesLatency.add(duration);
}

public void succeededRefreshAdminAclsRetrieved(long duration) {
totalSucceededRefreshAdminAclsRetrieved.add(duration);
refreshAdminAclsLatency.add(duration);
Expand Down Expand Up @@ -2222,6 +2250,10 @@ public void incrSaveFederationQueuePolicyFailedRetrieved() {
numSaveFederationQueuePolicyFailedRetrieved.incr();
}

public void incrBatchSaveFederationQueuePoliciesFailedRetrieved() {
numBatchSaveFederationQueuePoliciesFailedRetrieved.incr();
}

public void incrRefreshAdminAclsFailedRetrieved() {
numRefreshAdminAclsFailedRetrieved.incr();
}
Expand Down
Loading

0 comments on commit 6de2cf6

Please sign in to comment.