Skip to content

Commit

Permalink
metrics: measure rate of RPC requests that serve API (#15876)
Browse files Browse the repository at this point in the history 
This changeset configures the RPC rate metrics that were added in #15515 to all
the RPCs that support authenticated HTTP API requests. These endpoints already
configured with pre-forwarding authentication in #15870, and a handful of others
were done already as part of the proof-of-concept work. So this changeset is
entirely copy-and-pasting one method call into a whole mess of handlers.

Upcoming PRs will wire up pre-forwarding auth and rate metrics for the remaining
set of RPCs that have no API consumers or aren't authenticated, in smaller
chunks that can be more thoughtfully reviewed.
  • Loading branch information
@tgross
tgross authored Jan 25, 2023
1 parent 6b01bbb commit 187f0b8
Show file tree
Hide file tree
Showing 23 changed files with 141 additions and 0 deletions.
3 changes: 3 additions & 0 deletions .changelog/15876.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:improvement
metrics: Added metrics for rate of RPC requests
```
22 changes: 22 additions & 0 deletions nomad/acl_endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ func (a *ACL) UpsertPolicies(args *structs.ACLPolicyUpsertRequest, reply *struct
if done, err := a.srv.forward("ACL.UpsertPolicies", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -127,6 +128,7 @@ func (a *ACL) DeletePolicies(args *structs.ACLPolicyDeleteRequest, reply *struct
if done, err := a.srv.forward("ACL.DeletePolicies", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -164,6 +166,7 @@ func (a *ACL) ListPolicies(args *structs.ACLPolicyListRequest, reply *structs.AC
if done, err := a.srv.forward("ACL.ListPolicies", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -257,6 +260,7 @@ func (a *ACL) GetPolicy(args *structs.ACLPolicySpecificRequest, reply *structs.S
if done, err := a.srv.forward("ACL.GetPolicy", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -560,6 +564,7 @@ func (a *ACL) UpsertTokens(args *structs.ACLTokenUpsertRequest, reply *structs.A
if done, err := a.srv.forward(structs.ACLUpsertTokensRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -701,6 +706,7 @@ func (a *ACL) DeleteTokens(args *structs.ACLTokenDeleteRequest, reply *structs.G
if done, err := a.srv.forward("ACL.DeleteTokens", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -778,6 +784,7 @@ func (a *ACL) ListTokens(args *structs.ACLTokenListRequest, reply *structs.ACLTo
if done, err := a.srv.forward("ACL.ListTokens", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -866,6 +873,7 @@ func (a *ACL) GetToken(args *structs.ACLTokenSpecificRequest, reply *structs.Sin
if done, err := a.srv.forward("ACL.GetToken", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
if authErr != nil {
return authErr
}
Expand Down Expand Up @@ -931,6 +939,7 @@ func (a *ACL) GetTokens(args *structs.ACLTokenSetRequest, reply *structs.ACLToke
if done, err := a.srv.forward("ACL.GetTokens", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
if authErr != nil {
return authErr
}
Expand Down Expand Up @@ -1175,6 +1184,7 @@ func (a *ACL) UpsertRoles(
if done, err := a.srv.forward(structs.ACLUpsertRolesRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -1323,6 +1333,7 @@ func (a *ACL) DeleteRolesByID(
if done, err := a.srv.forward(structs.ACLDeleteRolesByIDRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -1375,6 +1386,7 @@ func (a *ACL) ListRoles(
if done, err := a.srv.forward(structs.ACLListRolesRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -1530,6 +1542,7 @@ func (a *ACL) GetRoleByID(
if done, err := a.srv.forward(structs.ACLGetRoleByIDRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -1618,6 +1631,7 @@ func (a *ACL) GetRoleByName(
if done, err := a.srv.forward(structs.ACLGetRoleByNameRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -1764,6 +1778,7 @@ func (a *ACL) UpsertAuthMethods(
if done, err := a.srv.forward(structs.ACLUpsertAuthMethodsRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -1871,6 +1886,7 @@ func (a *ACL) DeleteAuthMethods(
structs.ACLDeleteAuthMethodsRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -1971,6 +1987,7 @@ func (a *ACL) GetAuthMethod(
structs.ACLGetAuthMethodRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -2105,6 +2122,7 @@ func (a *ACL) UpsertBindingRules(
if done, err := a.srv.forward(structs.ACLUpsertBindingRulesRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -2236,6 +2254,7 @@ func (a *ACL) DeleteBindingRules(
if done, err := a.srv.forward(structs.ACLDeleteBindingRulesRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -2289,6 +2308,7 @@ func (a *ACL) ListBindingRules(
if done, err := a.srv.forward(structs.ACLListBindingRulesRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricList, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -2347,6 +2367,7 @@ func (a *ACL) GetBindingRules(
if done, err := a.srv.forward(structs.ACLGetBindingRulesRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -2401,6 +2422,7 @@ func (a *ACL) GetBindingRule(
if done, err := a.srv.forward(structs.ACLGetBindingRuleRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("acl", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down
5 changes: 5 additions & 0 deletions nomad/alloc_endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func (a *Alloc) List(args *structs.AllocListRequest, reply *structs.AllocListRes
if done, err := a.srv.forward("Alloc.List", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("alloc", structs.RateMetricList, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -143,6 +144,7 @@ func (a *Alloc) GetAlloc(args *structs.AllocSpecificRequest,
if done, err := a.srv.forward("Alloc.GetAlloc", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("alloc", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -270,6 +272,7 @@ func (a *Alloc) Stop(args *structs.AllocStopRequest, reply *structs.AllocStopRes
if done, err := a.srv.forward("Alloc.Stop", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("alloc", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -334,6 +337,7 @@ func (a *Alloc) UpdateDesiredTransition(args *structs.AllocUpdateDesiredTransiti
if done, err := a.srv.forward("Alloc.UpdateDesiredTransition", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("alloc", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -374,6 +378,7 @@ func (a *Alloc) GetServiceRegistrations(
if done, err := a.srv.forward(structs.AllocServiceRegistrationsRPCMethod, args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("alloc", structs.RateMetricList, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down
3 changes: 3 additions & 0 deletions nomad/client_agent_endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ func (a *Agent) register() {

func (a *Agent) Profile(args *structs.AgentPprofRequest, reply *structs.AgentPprofResponse) error {
authErr := a.srv.Authenticate(nil, args)
a.srv.MeasureRPCRate("agent", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -134,6 +135,7 @@ func (a *Agent) monitor(conn io.ReadWriteCloser) {
return
}
authErr := a.srv.Authenticate(nil, &args)
a.srv.MeasureRPCRate("agent", structs.RateMetricRead, &args)
if authErr != nil {
handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
return
Expand Down Expand Up @@ -409,6 +411,7 @@ func (a *Agent) forwardProfileClient(args *structs.AgentPprofRequest, reply *str
// Host returns data about the agent's host system for the `debug` command.
func (a *Agent) Host(args *structs.HostDataRequest, reply *structs.HostDataResponse) error {
authErr := a.srv.Authenticate(nil, args)
a.srv.MeasureRPCRate("agent", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down
7 changes: 7 additions & 0 deletions nomad/client_alloc_endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ func (a *ClientAllocations) GarbageCollectAll(args *structs.NodeSpecificRequest,
if done, err := a.srv.forward("ClientAllocations.GarbageCollectAll", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -96,6 +97,7 @@ func (a *ClientAllocations) Signal(args *structs.AllocSignalRequest, reply *stru
if done, err := a.srv.forward("ClientAllocations.Signal", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -153,6 +155,7 @@ func (a *ClientAllocations) GarbageCollect(args *structs.AllocSpecificRequest, r
if done, err := a.srv.forward("ClientAllocations.GarbageCollect", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -210,6 +213,7 @@ func (a *ClientAllocations) Restart(args *structs.AllocRestartRequest, reply *st
if done, err := a.srv.forward("ClientAllocations.Restart", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -262,6 +266,7 @@ func (a *ClientAllocations) Stats(args *cstructs.AllocStatsRequest, reply *cstru
if done, err := a.srv.forward("ClientAllocations.Stats", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("client_allocations", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -318,6 +323,7 @@ func (a *ClientAllocations) Checks(args *cstructs.AllocChecksRequest, reply *cst
if done, err := a.srv.forward("ClientAllocations.Checks", args, args, reply); done {
return err
}
a.srv.MeasureRPCRate("client_allocations", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -382,6 +388,7 @@ func (a *ClientAllocations) exec(conn io.ReadWriteCloser) {
args.AllocID, &args.QueryOptions)
return
}
a.srv.MeasureRPCRate("client_allocations", structs.RateMetricWrite, &args)
if authErr != nil {
handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
return
Expand Down
4 changes: 4 additions & 0 deletions nomad/client_fs_endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ func (f *FileSystem) List(args *cstructs.FsListRequest, reply *cstructs.FsListRe
if done, err := f.srv.forward("FileSystem.List", args, args, reply); done {
return err
}
f.srv.MeasureRPCRate("file_system", structs.RateMetricList, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -171,6 +172,7 @@ func (f *FileSystem) Stat(args *cstructs.FsStatRequest, reply *cstructs.FsStatRe
if done, err := f.srv.forward("FileSystem.Stat", args, args, reply); done {
return err
}
f.srv.MeasureRPCRate("file_system", structs.RateMetricRead, args)
if authErr != nil {
return structs.ErrPermissionDenied
}
Expand Down Expand Up @@ -239,6 +241,7 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) {
args.AllocID, &args.QueryOptions)
return
}
f.srv.MeasureRPCRate("file_system", structs.RateMetricRead, &args)
if authErr != nil {
handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
return
Expand Down Expand Up @@ -363,6 +366,7 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) {
args.AllocID, &args.QueryOptions)
return
}
f.srv.MeasureRPCRate("file_system", structs.RateMetricRead, &args)
if authErr != nil {
handleStreamResultError(structs.ErrPermissionDenied, nil, encoder)
return
Expand Down
1 change: 1 addition & 0 deletions nomad/client_stats_endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func (s *ClientStats) Stats(args *nstructs.NodeSpecificRequest, reply *structs.C
if done, err := s.srv.forward("ClientStats.Stats", args, args, reply); done {
return err
}
s.srv.MeasureRPCRate("client_stats", nstructs.RateMetricRead, args)
if authErr != nil {
return nstructs.ErrPermissionDenied
}
Expand Down
Loading

0 comments on commit 187f0b8

Please sign in to comment.