expose configuration for envoy's RateLimitedAsResourceExhausted

The Rate Limit filter in Envoy translates a 429 HTTP response code to UNAVAILABLE as specified in the gRPC mapping document, but Google recommends translating it to RESOURCE_EXHAUSTED (see https://github.com/grpc/grpc/blob/master/doc/http-grpc-status-mapping.md) This commit introduces a new setting to allow contour to forward the same parameter introduced in envoyproxy/envoy#4879 The default value is disabled to retain the original behaviour of returning UNAVAILABLE, as changing it would be a breaking change. Signed-off-by: Víctor Roldán Betancort <[email protected]>
vroldanbet · Jan 12, 2023 · 5ecbdb7 · 5ecbdb7
1 parent 63dd999
commit 5ecbdb7
Show file tree

Hide file tree

Showing 19 changed files with 212 additions and 68 deletions.
diff --git a/apis/projectcontour/v1alpha1/contourconfig.go b/apis/projectcontour/v1alpha1/contourconfig.go
@@ -619,6 +619,12 @@ type RateLimitServiceConfig struct {
 	// ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
 	// +optional
 	EnableXRateLimitHeaders *bool `json:"enableXRateLimitHeaders,omitempty"`
+
+	// EnableResourceExhaustedCode enables translating error code 429 to
+	// grpc code RESOURCE_EXHAUSTED. When disabled it's translated to UNAVAILABLE
+	//
+	// +optional
+	EnableResourceExhaustedCode *bool `json:"enableResourceExhaustedCode,omitempty"`
 }
 
 // PolicyConfig holds default policy used if not explicitly set by the user

diff --git a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go
diff --git a/changelogs/unreleased/4971-vroldanbet-small.md b/changelogs/unreleased/4971-vroldanbet-small.md
@@ -0,0 +1 @@
+expose configuration for envoy's RateLimitedAsResourceExhausted
diff --git a/cmd/contour/serve.go b/cmd/contour/serve.go
@@ -610,12 +610,13 @@ func (s *Server) setupRateLimitService(contourConfiguration contour_api_v1alpha1
 	}
 
 	return &xdscache_v3.RateLimitConfig{
-		ExtensionService:        key,
-		SNI:                     sni,
-		Domain:                  contourConfiguration.RateLimitService.Domain,
-		Timeout:                 responseTimeout,
-		FailOpen:                ref.Val(contourConfiguration.RateLimitService.FailOpen, false),
-		EnableXRateLimitHeaders: ref.Val(contourConfiguration.RateLimitService.EnableXRateLimitHeaders, false),
+		ExtensionService:            key,
+		SNI:                         sni,
+		Domain:                      contourConfiguration.RateLimitService.Domain,
+		Timeout:                     responseTimeout,
+		FailOpen:                    ref.Val(contourConfiguration.RateLimitService.FailOpen, false),
+		EnableXRateLimitHeaders:     ref.Val(contourConfiguration.RateLimitService.EnableXRateLimitHeaders, false),
+		EnableResourceExhaustedCode: ref.Val(contourConfiguration.RateLimitService.EnableResourceExhaustedCode, false),
 	}, nil
 }
 

diff --git a/cmd/contour/servecontext.go b/cmd/contour/servecontext.go
@@ -371,9 +371,10 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_api_v1alpha
 				Name:      nsedName.Name,
 				Namespace: nsedName.Namespace,
 			},
-			Domain:                  ctx.Config.RateLimitService.Domain,
-			FailOpen:                ref.To(ctx.Config.RateLimitService.FailOpen),
-			EnableXRateLimitHeaders: ref.To(ctx.Config.RateLimitService.EnableXRateLimitHeaders),
+			Domain:                      ctx.Config.RateLimitService.Domain,
+			FailOpen:                    ref.To(ctx.Config.RateLimitService.FailOpen),
+			EnableXRateLimitHeaders:     ref.To(ctx.Config.RateLimitService.EnableXRateLimitHeaders),
+			EnableResourceExhaustedCode: ref.To(ctx.Config.RateLimitService.EnableResourceExhaustedCode),
 		}
 	}
 

diff --git a/cmd/contour/servecontext_test.go b/cmd/contour/servecontext_test.go
@@ -623,10 +623,11 @@ func TestConvertServeContext(t *testing.T) {
 		"ratelimit": {
 			getServeContext: func(ctx *serveContext) *serveContext {
 				ctx.Config.RateLimitService = config.RateLimitService{
-					ExtensionService:        "ratens/ratelimitext",
-					Domain:                  "contour",
-					FailOpen:                true,
-					EnableXRateLimitHeaders: true,
+					ExtensionService:            "ratens/ratelimitext",
+					Domain:                      "contour",
+					FailOpen:                    true,
+					EnableXRateLimitHeaders:     true,
+					EnableResourceExhaustedCode: true,
 				}
 				return ctx
 			},
@@ -636,9 +637,10 @@ func TestConvertServeContext(t *testing.T) {
 						Name:      "ratelimitext",
 						Namespace: "ratens",
 					},
-					Domain:                  "contour",
-					FailOpen:                ref.To(true),
-					EnableXRateLimitHeaders: ref.To(true),
+					Domain:                      "contour",
+					FailOpen:                    ref.To(true),
+					EnableXRateLimitHeaders:     ref.To(true),
+					EnableResourceExhaustedCode: ref.To(true),
 				}
 				return cfg
 			},

diff --git a/examples/contour/01-contour-config.yaml b/examples/contour/01-contour-config.yaml
@@ -145,6 +145,9 @@ data:
     #   Limit Service is consulted for a request.
     #   ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
     #   enableXRateLimitHeaders: false
+    #   Defines whether to translate status code 429 to grpc code RESOURCE_EXHAUSTED
+    #   instead of the default UNAVAILABLE
+    #   enableResourceExhaustedCode: false
     #
     # Global Policy settings.
     # policy:

diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml
@@ -521,6 +521,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3521,6 +3526,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml
@@ -178,6 +178,9 @@ data:
     #   Limit Service is consulted for a request.
     #   ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
     #   enableXRateLimitHeaders: false
+    #   Defines whether to translate status code 429 to grpc code RESOURCE_EXHAUSTED
+    #   instead of the default UNAVAILABLE
+    #   enableResourceExhaustedCode: false
     #
     # Global Policy settings.
     # policy:
@@ -731,6 +734,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3731,6 +3739,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml
@@ -535,6 +535,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3535,6 +3540,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml
@@ -184,6 +184,9 @@ data:
     #   Limit Service is consulted for a request.
     #   ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
     #   enableXRateLimitHeaders: false
+    #   Defines whether to translate status code 429 to grpc code RESOURCE_EXHAUSTED
+    #   instead of the default UNAVAILABLE
+    #   enableResourceExhaustedCode: false
     #
     # Global Policy settings.
     # policy:
@@ -737,6 +740,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3737,6 +3745,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml
@@ -178,6 +178,9 @@ data:
     #   Limit Service is consulted for a request.
     #   ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html
     #   enableXRateLimitHeaders: false
+    #   Defines whether to translate status code 429 to grpc code RESOURCE_EXHAUSTED
+    #   instead of the default UNAVAILABLE
+    #   enableResourceExhaustedCode: false
     #
     # Global Policy settings.
     # policy:
@@ -731,6 +734,11 @@ spec:
                   domain:
                     description: Domain is passed to the Rate Limit Service.
                     type: string
+                  enableResourceExhaustedCode:
+                    description: EnableResourceExhaustedCode enables translating error
+                      code 429 to grpc code RESOURCE_EXHAUSTED. When disabled it's
+                      translated to UNAVAILABLE
+                    type: boolean
                   enableXRateLimitHeaders:
                     description: "EnableXRateLimitHeaders defines whether to include
                       the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,
@@ -3731,6 +3739,11 @@ spec:
                       domain:
                         description: Domain is passed to the Rate Limit Service.
                         type: string
+                      enableResourceExhaustedCode:
+                        description: EnableResourceExhaustedCode enables translating
+                          error code 429 to grpc code RESOURCE_EXHAUSTED. When disabled
+                          it's translated to UNAVAILABLE
+                        type: boolean
                       enableXRateLimitHeaders:
                         description: "EnableXRateLimitHeaders defines whether to include
                           the X-RateLimit headers X-RateLimit-Limit, X-RateLimit-Remaining,

diff --git a/internal/envoy/v3/ratelimit.go b/internal/envoy/v3/ratelimit.go
@@ -123,12 +123,13 @@ func GlobalRateLimits(descriptors []*dag.RateLimitDescriptor) []*envoy_route_v3.
 // GlobalRateLimitConfig stores configuration for
 // an HTTP global rate limiting filter.
 type GlobalRateLimitConfig struct {
-	ExtensionService        types.NamespacedName
-	SNI                     string
-	FailOpen                bool
-	Timeout                 timeout.Setting
-	Domain                  string
-	EnableXRateLimitHeaders bool
+	ExtensionService            types.NamespacedName
+	SNI                         string
+	FailOpen                    bool
+	Timeout                     timeout.Setting
+	Domain                      string
+	EnableXRateLimitHeaders     bool
+	EnableResourceExhaustedCode bool
 }
 
 // GlobalRateLimitFilter returns a configured HTTP global rate limit filter,
@@ -149,7 +150,8 @@ func GlobalRateLimitFilter(config *GlobalRateLimitConfig) *http.HttpFilter {
 					GrpcService:         GrpcService(dag.ExtensionClusterName(config.ExtensionService), config.SNI, timeout.DefaultSetting()),
 					TransportApiVersion: envoy_core_v3.ApiVersion_V3,
 				},
-				EnableXRatelimitHeaders: enableXRateLimitHeaders(config.EnableXRateLimitHeaders),
+				EnableXRatelimitHeaders:        enableXRateLimitHeaders(config.EnableXRateLimitHeaders),
+				RateLimitedAsResourceExhausted: config.EnableResourceExhaustedCode,
 			}),
 		},
 	}

diff --git a/internal/envoy/v3/ratelimit_test.go b/internal/envoy/v3/ratelimit_test.go
@@ -372,6 +372,37 @@ func TestGlobalRateLimitFilter(t *testing.T) {
 				},
 			},
 		},
+		"EnableResourceExhaustedCode=true is configured correctly": {
+			cfg: &GlobalRateLimitConfig{
+				ExtensionService:            k8s.NamespacedNameFrom("projectcontour/ratelimit"),
+				Timeout:                     timeout.DurationSetting(7 * time.Second),
+				Domain:                      "domain",
+				FailOpen:                    true,
+				EnableResourceExhaustedCode: true,
+			},
+			want: &http.HttpFilter{
+				Name: wellknown.HTTPRateLimit,
+				ConfigType: &http.HttpFilter_TypedConfig{
+					TypedConfig: protobuf.MustMarshalAny(&ratelimit_filter_v3.RateLimit{
+						Domain:          "domain",
+						Timeout:         durationpb.New(7 * time.Second),
+						FailureModeDeny: false,
+						RateLimitService: &ratelimit_config_v3.RateLimitServiceConfig{
+							GrpcService: &envoy_core_v3.GrpcService{
+								TargetSpecifier: &envoy_core_v3.GrpcService_EnvoyGrpc_{
+									EnvoyGrpc: &envoy_core_v3.GrpcService_EnvoyGrpc{
+										ClusterName: "extension/projectcontour/ratelimit",
+										Authority:   "extension.projectcontour.ratelimit",
+									},
+								},
+							},
+							TransportApiVersion: envoy_core_v3.ApiVersion_V3,
+						},
+						RateLimitedAsResourceExhausted: true,
+					}),
+				},
+			},
+		},
 	}
 
 	for name, tc := range tests {

diff --git a/internal/xdscache/v3/listener.go b/internal/xdscache/v3/listener.go
@@ -140,12 +140,13 @@ type ListenerConfig struct {
 }
 
 type RateLimitConfig struct {
-	ExtensionService        types.NamespacedName
-	SNI                     string
-	Domain                  string
-	Timeout                 timeout.Setting
-	FailOpen                bool
-	EnableXRateLimitHeaders bool
+	ExtensionService            types.NamespacedName
+	SNI                         string
+	Domain                      string
+	Timeout                     timeout.Setting
+	FailOpen                    bool
+	EnableXRateLimitHeaders     bool
+	EnableResourceExhaustedCode bool
 }
 
 // DefaultListeners returns the configured Listeners or a single
@@ -558,12 +559,13 @@ func envoyGlobalRateLimitConfig(config *RateLimitConfig) *envoy_v3.GlobalRateLim
 	}
 
 	return &envoy_v3.GlobalRateLimitConfig{
-		ExtensionService:        config.ExtensionService,
-		SNI:                     config.SNI,
-		FailOpen:                config.FailOpen,
-		Timeout:                 config.Timeout,
-		Domain:                  config.Domain,
-		EnableXRateLimitHeaders: config.EnableXRateLimitHeaders,
+		ExtensionService:            config.ExtensionService,
+		SNI:                         config.SNI,
+		FailOpen:                    config.FailOpen,
+		Timeout:                     config.Timeout,
+		Domain:                      config.Domain,
+		EnableXRateLimitHeaders:     config.EnableXRateLimitHeaders,
+		EnableResourceExhaustedCode: config.EnableResourceExhaustedCode,
 	}
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		expose configuration for envoy's RateLimitedAsResourceExhausted