metrics: measure rate of RPC requests that serve API

.changelog/15876.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+metrics: Added metrics for rate of RPC requests
+```

nomad/acl_endpoint.go

@@ -79,6 +79,7 @@ func (a *ACL) UpsertPolicies(args *structs.ACLPolicyUpsertRequest, reply *struct
 	if done, err := a.srv.forward("ACL.UpsertPolicies", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -127,6 +128,7 @@ func (a *ACL) DeletePolicies(args *structs.ACLPolicyDeleteRequest, reply *struct
 	if done, err := a.srv.forward("ACL.DeletePolicies", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -164,6 +166,7 @@ func (a *ACL) ListPolicies(args *structs.ACLPolicyListRequest, reply *structs.AC
 	if done, err := a.srv.forward("ACL.ListPolicies", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -257,6 +260,7 @@ func (a *ACL) GetPolicy(args *structs.ACLPolicySpecificRequest, reply *structs.S
 	if done, err := a.srv.forward("ACL.GetPolicy", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -560,6 +564,7 @@ func (a *ACL) UpsertTokens(args *structs.ACLTokenUpsertRequest, reply *structs.A
 	if done, err := a.srv.forward(structs.ACLUpsertTokensRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -701,6 +706,7 @@ func (a *ACL) DeleteTokens(args *structs.ACLTokenDeleteRequest, reply *structs.G
 	if done, err := a.srv.forward("ACL.DeleteTokens", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -778,6 +784,7 @@ func (a *ACL) ListTokens(args *structs.ACLTokenListRequest, reply *structs.ACLTo
 	if done, err := a.srv.forward("ACL.ListTokens", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -866,6 +873,7 @@ func (a *ACL) GetToken(args *structs.ACLTokenSpecificRequest, reply *structs.Sin
 	if done, err := a.srv.forward("ACL.GetToken", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
 	if authErr != nil {
 		return authErr
 	}
@@ -931,6 +939,7 @@ func (a *ACL) GetTokens(args *structs.ACLTokenSetRequest, reply *structs.ACLToke
 	if done, err := a.srv.forward("ACL.GetTokens", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
 	if authErr != nil {
 		return authErr
 	}
@@ -1175,6 +1184,7 @@ func (a *ACL) UpsertRoles(
 	if done, err := a.srv.forward(structs.ACLUpsertRolesRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -1323,6 +1333,7 @@ func (a *ACL) DeleteRolesByID(
 	if done, err := a.srv.forward(structs.ACLDeleteRolesByIDRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -1375,6 +1386,7 @@ func (a *ACL) ListRoles(
 	if done, err := a.srv.forward(structs.ACLListRolesRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -1530,6 +1542,7 @@ func (a *ACL) GetRoleByID(
 	if done, err := a.srv.forward(structs.ACLGetRoleByIDRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -1618,6 +1631,7 @@ func (a *ACL) GetRoleByName(
 	if done, err := a.srv.forward(structs.ACLGetRoleByNameRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -1764,6 +1778,7 @@ func (a *ACL) UpsertAuthMethods(
 	if done, err := a.srv.forward(structs.ACLUpsertAuthMethodsRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -1871,6 +1886,7 @@ func (a *ACL) DeleteAuthMethods(
 		structs.ACLDeleteAuthMethodsRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -1971,6 +1987,7 @@ func (a *ACL) GetAuthMethod(
 		structs.ACLGetAuthMethodRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -2105,6 +2122,7 @@ func (a *ACL) UpsertBindingRules(
 	if done, err := a.srv.forward(structs.ACLUpsertBindingRulesRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -2236,6 +2254,7 @@ func (a *ACL) DeleteBindingRules(
 	if done, err := a.srv.forward(structs.ACLDeleteBindingRulesRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -2289,6 +2308,7 @@ func (a *ACL) ListBindingRules(
 	if done, err := a.srv.forward(structs.ACLListBindingRulesRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -2347,6 +2367,7 @@ func (a *ACL) GetBindingRules(
 	if done, err := a.srv.forward(structs.ACLGetBindingRulesRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -2401,6 +2422,7 @@ func (a *ACL) GetBindingRule(
 	if done, err := a.srv.forward(structs.ACLGetBindingRuleRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}

nomad/alloc_endpoint.go

@@ -34,6 +34,7 @@ func (a *Alloc) List(args *structs.AllocListRequest, reply *structs.AllocListRes
 	if done, err := a.srv.forward("Alloc.List", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricList, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -143,6 +144,7 @@ func (a *Alloc) GetAlloc(args *structs.AllocSpecificRequest,
 	if done, err := a.srv.forward("Alloc.GetAlloc", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -270,6 +272,7 @@ func (a *Alloc) Stop(args *structs.AllocStopRequest, reply *structs.AllocStopRes
 	if done, err := a.srv.forward("Alloc.Stop", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -334,6 +337,7 @@ func (a *Alloc) UpdateDesiredTransition(args *structs.AllocUpdateDesiredTransiti
 	if done, err := a.srv.forward("Alloc.UpdateDesiredTransition", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -374,6 +378,7 @@ func (a *Alloc) GetServiceRegistrations(
 	if done, err := a.srv.forward(structs.AllocServiceRegistrationsRPCMethod, args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("alloc", structs.RateMetricList, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}

nomad/client_agent_endpoint.go

@@ -36,6 +36,7 @@ func (a *Agent) register() {
 
 func (a *Agent) Profile(args *structs.AgentPprofRequest, reply *structs.AgentPprofResponse) error {
 	authErr := a.srv.Authenticate(nil, args)
+	a.srv.MeasureRPCRate("agent", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -134,6 +135,7 @@ func (a *Agent) monitor(conn io.ReadWriteCloser) {
 		return
 	}
 	authErr := a.srv.Authenticate(nil, &args)
+	a.srv.MeasureRPCRate("agent", structs.RateMetricRead, &args)
 	if authErr != nil {
 		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
 		return
@@ -409,6 +411,7 @@ func (a *Agent) forwardProfileClient(args *structs.AgentPprofRequest, reply *str
 // Host returns data about the agent's host system for the `debug` command.
 func (a *Agent) Host(args *structs.HostDataRequest, reply *structs.HostDataResponse) error {
 	authErr := a.srv.Authenticate(nil, args)
+	a.srv.MeasureRPCRate("agent", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}

nomad/client_alloc_endpoint.go

@@ -45,6 +45,7 @@ func (a *ClientAllocations) GarbageCollectAll(args *structs.NodeSpecificRequest,
 	if done, err := a.srv.forward("ClientAllocations.GarbageCollectAll", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -96,6 +97,7 @@ func (a *ClientAllocations) Signal(args *structs.AllocSignalRequest, reply *stru
 	if done, err := a.srv.forward("ClientAllocations.Signal", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -153,6 +155,7 @@ func (a *ClientAllocations) GarbageCollect(args *structs.AllocSpecificRequest, r
 	if done, err := a.srv.forward("ClientAllocations.GarbageCollect", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -210,6 +213,7 @@ func (a *ClientAllocations) Restart(args *structs.AllocRestartRequest, reply *st
 	if done, err := a.srv.forward("ClientAllocations.Restart", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -262,6 +266,7 @@ func (a *ClientAllocations) Stats(args *cstructs.AllocStatsRequest, reply *cstru
 	if done, err := a.srv.forward("ClientAllocations.Stats", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -318,6 +323,7 @@ func (a *ClientAllocations) Checks(args *cstructs.AllocChecksRequest, reply *cst
 	if done, err := a.srv.forward("ClientAllocations.Checks", args, args, reply); done {
 		return err
 	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -382,6 +388,7 @@ func (a *ClientAllocations) exec(conn io.ReadWriteCloser) {
 			args.AllocID, &args.QueryOptions)
 		return
 	}
+	a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, &args)
 	if authErr != nil {
 		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
 		return

nomad/client_fs_endpoint.go

@@ -112,6 +112,7 @@ func (f *FileSystem) List(args *cstructs.FsListRequest, reply *cstructs.FsListRe
 	if done, err := f.srv.forward("FileSystem.List", args, args, reply); done {
 		return err
 	}
+	f.srv.MeasureRPCRate("file_system", structs.RateMetricList, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -171,6 +172,7 @@ func (f *FileSystem) Stat(args *cstructs.FsStatRequest, reply *cstructs.FsStatRe
 	if done, err := f.srv.forward("FileSystem.Stat", args, args, reply); done {
 		return err
 	}
+	f.srv.MeasureRPCRate("file_system", structs.RateMetricRead, args)
 	if authErr != nil {
 		return structs.ErrPermissionDenied
 	}
@@ -239,6 +241,7 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) {
 			args.AllocID, &args.QueryOptions)
 		return
 	}
+	f.srv.MeasureRPCRate("file_system", structs.RateMetricRead, &args)
 	if authErr != nil {
 		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
 		return
@@ -363,6 +366,7 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) {
 			args.AllocID, &args.QueryOptions)
 		return
 	}
+	f.srv.MeasureRPCRate("file_system", structs.RateMetricRead, &args)
 	if authErr != nil {
 		handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
 		return

nomad/client_stats_endpoint.go

@@ -34,6 +34,7 @@ func (s *ClientStats) Stats(args *nstructs.NodeSpecificRequest, reply *structs.C
 	if done, err := s.srv.forward("ClientStats.Stats", args, args, reply); done {
 		return err
 	}
+	s.srv.MeasureRPCRate("client_stats", nstructs.RateMetricRead, args)
 	if authErr != nil {
 		return nstructs.ErrPermissionDenied
 	}