diff --git a/deployments/deployment/nginx-plus-ingress.yaml b/deployments/deployment/nginx-plus-ingress.yaml index 09553e3845..ed0760b16f 100644 --- a/deployments/deployment/nginx-plus-ingress.yaml +++ b/deployments/deployment/nginx-plus-ingress.yaml @@ -12,54 +12,55 @@ spec: metadata: labels: app: nginx-ingress - #annotations: - #prometheus.io/scrape: "true" - #prometheus.io/port: "9113" - #prometheus.io/scheme: http + #annotations: + #prometheus.io/scrape: "true" + #prometheus.io/port: "9113" + #prometheus.io/scheme: http + spec: serviceAccountName: nginx-ingress containers: - - image: nginx-plus-ingress:2.0.3 - imagePullPolicy: IfNotPresent - name: nginx-plus-ingress - ports: - - name: http - containerPort: 80 - - name: https - containerPort: 443 - - name: readiness-port - containerPort: 8081 - - name: prometheus - containerPort: 9113 - readinessProbe: - httpGet: - path: /nginx-ready - port: readiness-port - periodSeconds: 1 - securityContext: - allowPrivilegeEscalation: true - runAsUser: 101 #nginx - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - args: - - -nginx-plus - - -nginx-configmaps=$(POD_NAMESPACE)/nginx-config - - -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret - #- -enable-app-protect - #- -v=3 # Enables extensive logging. Useful for troubleshooting. - #- -report-ingress-status - #- -external-service=nginx-ingress - #- -enable-prometheus-metrics - #- -global-configuration=$(POD_NAMESPACE)/nginx-configuration + - image: nginx/nginx-ingress:2.0.3-SNAPSHOT-68238f2 + imagePullPolicy: IfNotPresent + name: nginx-plus-ingress + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + - name: readiness-port + containerPort: 8081 + - name: prometheus + containerPort: 9113 + readinessProbe: + httpGet: + path: /nginx-ready + port: readiness-port + periodSeconds: 1 + securityContext: + allowPrivilegeEscalation: true + runAsUser: 101 #nginx + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + args: + - -nginx-plus + - -nginx-configmaps=$(POD_NAMESPACE)/nginx-config + - -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret +#- -enable-app-protect +#- -v=3 # Enables extensive logging. Useful for troubleshooting. +#- -report-ingress-status +#- -external-service=nginx-ingress +#- -enable-prometheus-metrics +#- -global-configuration=$(POD_NAMESPACE)/nginx-configuration diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl index 83a5e286ec..b5c24ebdf6 100644 --- a/internal/configs/version2/nginx-plus.virtualserver.tmpl +++ b/internal/configs/version2/nginx-plus.virtualserver.tmpl @@ -340,6 +340,28 @@ server { {{ end }} {{ end }} + {{ if $l.GRPCPass }} + error_page 400 = @grpc_internal; + error_page 401 = @grpc_unauthenticated; + error_page 403 = @grpc_permission_denied; + error_page 404 = @grpc_unimplemented; + error_page 429 = @grpc_unavailable; + error_page 502 = @grpc_unavailable; + error_page 503 = @grpc_unavailable; + error_page 504 = @grpc_unavailable; + error_page 405 = @grpc_internal; + error_page 408 = @grpc_deadline_exceeded; + error_page 413 = @grpc_resource_exhausted; + error_page 414 = @grpc_resource_exhausted; + error_page 415 = @grpc_internal; + error_page 426 = @grpc_internal; + error_page 495 = @grpc_unauthenticated; + error_page 496 = @grpc_unauthenticated; + error_page 497 = @grpc_internal; + error_page 500 = @grpc_internal; + error_page 501 = @grpc_internal; + {{ end }} + {{ range $e := $l.ErrorPages }} error_page {{ $e.Codes }} {{ if ne 0 $e.ResponseCode }}={{ $e.ResponseCode }}{{ end }} "{{ $e.Name }}"; {{ end }} @@ -372,28 +394,6 @@ server { {{ if $l.ProxyBufferSize }} {{ $proxyOrGRPC }}_buffer_size {{ $l.ProxyBufferSize }}; {{ end }} - - {{ if $l.GRPCPass }} - error_page 400 = @grpc_internal; - error_page 401 = @grpc_unauthenticated; - error_page 403 = @grpc_permission_denied; - error_page 404 = @grpc_unimplemented; - error_page 429 = @grpc_unavailable; - error_page 502 = @grpc_unavailable; - error_page 503 = @grpc_unavailable; - error_page 504 = @grpc_unavailable; - error_page 405 = @grpc_internal; - error_page 408 = @grpc_deadline_exceeded; - error_page 413 = @grpc_resource_exhausted; - error_page 414 = @grpc_resource_exhausted; - error_page 415 = @grpc_internal; - error_page 426 = @grpc_internal; - error_page 495 = @grpc_unauthenticated; - error_page 496 = @grpc_unauthenticated; - error_page 497 = @grpc_internal; - error_page 500 = @grpc_internal; - error_page 501 = @grpc_internal; - {{ end }} {{ if not $l.GRPCPass }} proxy_http_version 1.1; diff --git a/internal/configs/version2/nginx.virtualserver.tmpl b/internal/configs/version2/nginx.virtualserver.tmpl index a004d37228..a64abce37d 100644 --- a/internal/configs/version2/nginx.virtualserver.tmpl +++ b/internal/configs/version2/nginx.virtualserver.tmpl @@ -244,6 +244,28 @@ server { {{ $proxyOrGRPC }}_ssl_name {{ .SSLName }}; {{ end }} + {{ if $l.GRPCPass }} + error_page 400 = @grpc_internal; + error_page 401 = @grpc_unauthenticated; + error_page 403 = @grpc_permission_denied; + error_page 404 = @grpc_unimplemented; + error_page 429 = @grpc_unavailable; + error_page 502 = @grpc_unavailable; + error_page 503 = @grpc_unavailable; + error_page 504 = @grpc_unavailable; + error_page 405 = @grpc_internal; + error_page 408 = @grpc_deadline_exceeded; + error_page 413 = @grpc_resource_exhausted; + error_page 414 = @grpc_resource_exhausted; + error_page 415 = @grpc_internal; + error_page 426 = @grpc_internal; + error_page 495 = @grpc_unauthenticated; + error_page 496 = @grpc_unauthenticated; + error_page 497 = @grpc_internal; + error_page 500 = @grpc_internal; + error_page 501 = @grpc_internal; + {{ end }} + {{ range $e := $l.ErrorPages }} error_page {{ $e.Codes }} {{ if ne 0 $e.ResponseCode }}={{ $e.ResponseCode }}{{ end }} "{{ $e.Name }}"; {{ end }} @@ -276,28 +298,6 @@ server { {{ if $l.ProxyBufferSize }} {{ $proxyOrGRPC }}_buffer_size {{ $l.ProxyBufferSize }}; {{ end }} - - {{ if $l.GRPCPass }} - error_page 400 = @grpc_internal; - error_page 401 = @grpc_unauthenticated; - error_page 403 = @grpc_permission_denied; - error_page 404 = @grpc_unimplemented; - error_page 429 = @grpc_unavailable; - error_page 502 = @grpc_unavailable; - error_page 503 = @grpc_unavailable; - error_page 504 = @grpc_unavailable; - error_page 405 = @grpc_internal; - error_page 408 = @grpc_deadline_exceeded; - error_page 413 = @grpc_resource_exhausted; - error_page 414 = @grpc_resource_exhausted; - error_page 415 = @grpc_internal; - error_page 426 = @grpc_internal; - error_page 495 = @grpc_unauthenticated; - error_page 496 = @grpc_unauthenticated; - error_page 497 = @grpc_internal; - error_page 500 = @grpc_internal; - error_page 501 = @grpc_internal; - {{ end }} {{ if not $l.GRPCPass }} proxy_http_version 1.1; diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go index b688901af8..e015e95034 100644 --- a/internal/configs/virtualserver.go +++ b/internal/configs/virtualserver.go @@ -27,6 +27,28 @@ const ( subRouteContext = "subroute" ) +var grpcConflictingErrors = map[int]bool{ + 400: true, + 401: true, + 403: true, + 404: true, + 405: true, + 408: true, + 413: true, + 414: true, + 415: true, + 426: true, + 429: true, + 495: true, + 496: true, + 497: true, + 500: true, + 501: true, + 502: true, + 503: true, + 504: true, +} + var incompatibleLBMethodsForSlowStart = map[string]bool{ "random": true, "ip_hash": true, @@ -441,24 +463,34 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig(vsEx *VirtualS internalRedirectLocations = append(internalRedirectLocations, cfg.InternalRedirectLocation) returnLocations = append(returnLocations, cfg.ReturnLocations...) splitClients = append(splitClients, cfg.SplitClients...) + for _, w := range cfg.Warnings { + vsc.addWarningf(vsEx.VirtualServer, w) + } matchesRoutes++ } else if len(r.Splits) > 0 { cfg := generateDefaultSplitsConfig(r, virtualServerUpstreamNamer, crUpstreams, variableNamer, len(splitClients), - vsc.cfgParams, r.ErrorPages, errorPageIndex, r.Path, vsLocSnippets, vsc.enableSnippets, len(returnLocations), isVSR, "", "") + vsc.cfgParams, r.ErrorPages, errorPageIndex, r.Path, vsLocSnippets, vsc.enableSnippets, len(returnLocations), + isVSR, "", "") addPoliciesCfgToLocations(routePoliciesCfg, cfg.Locations) splitClients = append(splitClients, cfg.SplitClients...) locations = append(locations, cfg.Locations...) internalRedirectLocations = append(internalRedirectLocations, cfg.InternalRedirectLocation) returnLocations = append(returnLocations, cfg.ReturnLocations...) + for _, w := range cfg.Warnings { + vsc.addWarningf(vsEx.VirtualServer, w) + } } else { upstreamName := virtualServerUpstreamNamer.GetNameForUpstreamFromAction(r.Action) upstream := crUpstreams[upstreamName] proxySSLName := generateProxySSLName(upstream.Service, vsEx.VirtualServer.Namespace) - loc, returnLoc := generateLocation(r.Path, upstreamName, upstream, r.Action, vsc.cfgParams, r.ErrorPages, false, + loc, returnLoc, warnings := generateLocation(r.Path, upstreamName, upstream, r.Action, vsc.cfgParams, r.ErrorPages, false, errorPageIndex, proxySSLName, r.Path, vsLocSnippets, vsc.enableSnippets, len(returnLocations), isVSR, "", "") + for _, w := range warnings { + vsc.addWarningf(vsEx.VirtualServer, w) + } addPoliciesCfgToLocation(routePoliciesCfg, &loc) locations = append(locations, loc) @@ -544,6 +576,9 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig(vsEx *VirtualS internalRedirectLocations = append(internalRedirectLocations, cfg.InternalRedirectLocation) returnLocations = append(returnLocations, cfg.ReturnLocations...) splitClients = append(splitClients, cfg.SplitClients...) + for _, w := range cfg.Warnings { + vsc.addWarningf(vsEx.VirtualServer, w) + } matchesRoutes++ } else if len(r.Splits) > 0 { cfg := generateDefaultSplitsConfig(r, upstreamNamer, crUpstreams, variableNamer, len(splitClients), vsc.cfgParams, @@ -554,15 +589,23 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig(vsEx *VirtualS locations = append(locations, cfg.Locations...) internalRedirectLocations = append(internalRedirectLocations, cfg.InternalRedirectLocation) returnLocations = append(returnLocations, cfg.ReturnLocations...) + for _, w := range cfg.Warnings { + vsc.addWarningf(vsEx.VirtualServer, w) + } } else { upstreamName := upstreamNamer.GetNameForUpstreamFromAction(r.Action) upstream := crUpstreams[upstreamName] proxySSLName := generateProxySSLName(upstream.Service, vsr.Namespace) - loc, returnLoc := generateLocation(r.Path, upstreamName, upstream, r.Action, vsc.cfgParams, errorPages, false, - errorPageIndex, proxySSLName, r.Path, locSnippets, vsc.enableSnippets, len(returnLocations), isVSR, vsr.Name, vsr.Namespace) + loc, returnLoc, warnings := generateLocation(r.Path, upstreamName, upstream, r.Action, vsc.cfgParams, errorPages, false, + errorPageIndex, proxySSLName, r.Path, locSnippets, vsc.enableSnippets, len(returnLocations), isVSR, vsr.Name, + vsr.Namespace) addPoliciesCfgToLocation(routePoliciesCfg, &loc) + for _, w := range warnings { + vsc.addWarningf(vsEx.VirtualServer, w) + } + locations = append(locations, loc) if returnLoc != nil { returnLocations = append(returnLocations, *returnLoc) @@ -1497,19 +1540,22 @@ func generateReturnBlock(text string, code int, defaultCode int) *version2.Retur func generateLocation(path string, upstreamName string, upstream conf_v1.Upstream, action *conf_v1.Action, cfgParams *ConfigParams, errorPages []conf_v1.ErrorPage, internal bool, errPageIndex int, proxySSLName string, - originalPath string, locSnippets string, enableSnippets bool, retLocIndex int, isVSR bool, vsrName string, vsrNamespace string) (version2.Location, *version2.ReturnLocation) { + originalPath string, locSnippets string, enableSnippets bool, retLocIndex int, isVSR bool, vsrName string, + vsrNamespace string) (version2.Location, *version2.ReturnLocation, []string) { locationSnippets := generateSnippets(enableSnippets, locSnippets, cfgParams.LocationSnippets) if action.Redirect != nil { - return generateLocationForRedirect(path, locationSnippets, action.Redirect), nil + return generateLocationForRedirect(path, locationSnippets, action.Redirect), nil, nil } if action.Return != nil { return generateLocationForReturn(path, cfgParams.LocationSnippets, action.Return, retLocIndex) } + warnings := checkGrpcErrorPageCodes(errorPages, isGRPC(upstream.Type), upstream.Name) + return generateLocationForProxying(path, upstreamName, upstream, cfgParams, errorPages, internal, - errPageIndex, proxySSLName, action.Proxy, originalPath, locationSnippets, isVSR, vsrName, vsrNamespace), nil + errPageIndex, proxySSLName, action.Proxy, originalPath, locationSnippets, isVSR, vsrName, vsrNamespace), nil, warnings } func generateProxySetHeaders(proxy *conf_v1.ActionProxy) []version2.Header { @@ -1661,7 +1707,7 @@ func generateLocationForRedirect( } func generateLocationForReturn(path string, locationSnippets []string, actionReturn *conf_v1.ActionReturn, - retLocIndex int) (version2.Location, *version2.ReturnLocation) { + retLocIndex int) (version2.Location, *version2.ReturnLocation, []string) { defaultType := actionReturn.Type if defaultType == "" { defaultType = "text/plain" @@ -1692,7 +1738,7 @@ func generateLocationForReturn(path string, locationSnippets []string, actionRet Return: version2.Return{ Text: actionReturn.Body, }, - } + }, nil } type routingCfg struct { @@ -1701,6 +1747,7 @@ type routingCfg struct { Locations []version2.Location InternalRedirectLocation version2.InternalRedirectLocation ReturnLocations []version2.ReturnLocation + Warnings []string } func generateSplits( @@ -1719,7 +1766,7 @@ func generateSplits( isVSR bool, vsrName string, vsrNamespace string, -) (version2.SplitClient, []version2.Location, []version2.ReturnLocation) { +) (version2.SplitClient, []version2.Location, []version2.ReturnLocation, []string) { var distributions []version2.Distribution for i, s := range splits { @@ -1738,6 +1785,7 @@ func generateSplits( var locations []version2.Location var returnLocations []version2.ReturnLocation + var warnings []string for i, s := range splits { path := fmt.Sprintf("/%vsplits_%d_split_%d", internalLocationPrefix, scIndex, i) @@ -1745,15 +1793,18 @@ func generateSplits( upstream := crUpstreams[upstreamName] proxySSLName := generateProxySSLName(upstream.Service, upstreamNamer.namespace) newRetLocIndex := retLocIndex + len(returnLocations) - loc, returnLoc := generateLocation(path, upstreamName, upstream, s.Action, cfgParams, errorPages, true, + loc, returnLoc, w := generateLocation(path, upstreamName, upstream, s.Action, cfgParams, errorPages, true, errPageIndex, proxySSLName, originalPath, locSnippets, enableSnippets, newRetLocIndex, isVSR, vsrName, vsrNamespace) locations = append(locations, loc) if returnLoc != nil { returnLocations = append(returnLocations, *returnLoc) } + if w != nil { + warnings = append(warnings, w...) + } } - return splitClient, locations, returnLocations + return splitClient, locations, returnLocations, warnings } func generateDefaultSplitsConfig( @@ -1773,7 +1824,7 @@ func generateDefaultSplitsConfig( vsrName string, vsrNamespace string, ) routingCfg { - sc, locs, returnLocs := generateSplits(route.Splits, upstreamNamer, crUpstreams, variableNamer, scIndex, cfgParams, + sc, locs, returnLocs, warnings := generateSplits(route.Splits, upstreamNamer, crUpstreams, variableNamer, scIndex, cfgParams, errorPages, errPageIndex, originalPath, locSnippets, enableSnippets, retLocIndex, isVSR, vsrName, vsrNamespace) splitClientVarName := variableNamer.GetNameForSplitClientVariable(scIndex) @@ -1788,6 +1839,7 @@ func generateDefaultSplitsConfig( Locations: locs, InternalRedirectLocation: irl, ReturnLocations: returnLocs, + Warnings: warnings, } } @@ -1863,12 +1915,13 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr var locations []version2.Location var returnLocations []version2.ReturnLocation var splitClients []version2.SplitClient + var warnings []string scLocalIndex = 0 for i, m := range route.Matches { if len(m.Splits) > 0 { newRetLocIndex := retLocIndex + len(returnLocations) - sc, locs, returnLocs := generateSplits( + sc, locs, returnLocs, w := generateSplits( m.Splits, upstreamNamer, crUpstreams, @@ -1889,25 +1942,32 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr splitClients = append(splitClients, sc) locations = append(locations, locs...) returnLocations = append(returnLocations, returnLocs...) + if w != nil { + warnings = append(warnings, w...) + } } else { path := fmt.Sprintf("/%vmatches_%d_match_%d", internalLocationPrefix, index, i) upstreamName := upstreamNamer.GetNameForUpstreamFromAction(m.Action) upstream := crUpstreams[upstreamName] proxySSLName := generateProxySSLName(upstream.Service, upstreamNamer.namespace) newRetLocIndex := retLocIndex + len(returnLocations) - loc, returnLoc := generateLocation(path, upstreamName, upstream, m.Action, cfgParams, errorPages, true, - errPageIndex, proxySSLName, route.Path, locSnippets, enableSnippets, newRetLocIndex, isVSR, vsrName, vsrNamespace) + loc, returnLoc, w := generateLocation(path, upstreamName, upstream, m.Action, cfgParams, errorPages, true, + errPageIndex, proxySSLName, route.Path, locSnippets, enableSnippets, newRetLocIndex, isVSR, vsrName, + vsrNamespace) locations = append(locations, loc) if returnLoc != nil { returnLocations = append(returnLocations, *returnLoc) } + if w != nil { + warnings = append(warnings, w...) + } } } // Generate default splits or default action if len(route.Splits) > 0 { newRetLocIndex := retLocIndex + len(returnLocations) - sc, locs, returnLocs := generateSplits( + sc, locs, returnLocs, w := generateSplits( route.Splits, upstreamNamer, crUpstreams, @@ -1927,18 +1987,25 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr splitClients = append(splitClients, sc) locations = append(locations, locs...) returnLocations = append(returnLocations, returnLocs...) + if w != nil { + warnings = append(warnings, w...) + } } else { path := fmt.Sprintf("/%vmatches_%d_default", internalLocationPrefix, index) upstreamName := upstreamNamer.GetNameForUpstreamFromAction(route.Action) upstream := crUpstreams[upstreamName] proxySSLName := generateProxySSLName(upstream.Service, upstreamNamer.namespace) newRetLocIndex := retLocIndex + len(returnLocations) - loc, returnLoc := generateLocation(path, upstreamName, upstream, route.Action, cfgParams, errorPages, true, - errPageIndex, proxySSLName, route.Path, locSnippets, enableSnippets, newRetLocIndex, isVSR, vsrName, vsrNamespace) + loc, returnLoc, w := generateLocation(path, upstreamName, upstream, route.Action, cfgParams, errorPages, true, + errPageIndex, proxySSLName, route.Path, locSnippets, enableSnippets, newRetLocIndex, isVSR, vsrName, + vsrNamespace) locations = append(locations, loc) if returnLoc != nil { returnLocations = append(returnLocations, *returnLoc) } + if w != nil { + warnings = append(warnings, w...) + } } // Generate an InternalRedirectLocation to the location defined by the main map variable @@ -1953,6 +2020,7 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr InternalRedirectLocation: irl, SplitClients: splitClients, ReturnLocations: returnLocations, + Warnings: warnings, } } @@ -2179,6 +2247,26 @@ func generateErrorPageName(errPageIndex int, index int) string { return fmt.Sprintf("@error_page_%v_%v", errPageIndex, index) } +func checkGrpcErrorPageCodes(errorPages []conf_v1.ErrorPage, isGRPC bool, uName string) []string { + if errorPages == nil { + return nil + } + + var c []int + var warnings []string + for _, e := range errorPages { + for _, code := range e.Codes { + if isGRPC && grpcConflictingErrors[code] { + c = append(c, code) + } + } + } + if len(c) > 0 { + warnings = append(warnings, fmt.Sprintf("The error page configuration for the upstream %s is ignored for status code(s) %v, which cannot be used for GRPC upstreams.", uName, c)) + } + return warnings +} + func generateErrorPageCodes(codes []int) string { var c []string for _, code := range codes { diff --git a/internal/configs/virtualserver_test.go b/internal/configs/virtualserver_test.go index 6c40b631b9..f3a14dca4e 100644 --- a/internal/configs/virtualserver_test.go +++ b/internal/configs/virtualserver_test.go @@ -657,6 +657,371 @@ func TestGenerateVirtualServerConfig(t *testing.T) { } } +func TestGenerateVirtualServerConfigGrpcErrorPageWarning(t *testing.T) { + virtualServerEx := VirtualServerEx{ + VirtualServer: &conf_v1.VirtualServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "cafe", + Namespace: "default", + }, + Spec: conf_v1.VirtualServerSpec{ + Host: "cafe.example.com", + TLS: &conf_v1.TLS{ + Secret: "", + }, + Upstreams: []conf_v1.Upstream{ + { + Name: "grpc-app-1", + Service: "grpc-svc", + Port: 50051, + Type: "grpc", + TLS: conf_v1.UpstreamTLS{ + Enable: true, + }, + }, + { + Name: "grpc-app-2", + Service: "grpc-svc2", + Port: 50052, + Type: "grpc", + TLS: conf_v1.UpstreamTLS{ + Enable: true, + }, + }, + { + Name: "tea", + Service: "tea-svc", + Port: 80, + }, + }, + Routes: []conf_v1.Route{ + { + Path: "/grpc-errorpage", + Action: &conf_v1.Action{ + Pass: "grpc-app-1", + }, + ErrorPages: []conf_v1.ErrorPage{ + { + Codes: []int{404, 405}, + Return: &conf_v1.ErrorPageReturn{ + ActionReturn: conf_v1.ActionReturn{ + Code: 200, + Type: "text/plain", + Body: "All Good", + }, + }, + }, + }, + }, + { + Path: "/grpc-matches", + Matches: []conf_v1.Match{ + { + Conditions: []conf_v1.Condition{ + { + Variable: "$request_method", + Value: "POST", + }, + }, + Action: &conf_v1.Action{ + Pass: "grpc-app-2", + }, + }, + }, + Action: &conf_v1.Action{ + Pass: "tea", + }, + ErrorPages: []conf_v1.ErrorPage{ + { + Codes: []int{404}, + Return: &conf_v1.ErrorPageReturn{ + ActionReturn: conf_v1.ActionReturn{ + Code: 200, + Type: "text/plain", + Body: "Original resource not found, but success!", + }, + }, + }, + }, + }, + { + Path: "/grpc-splits", + Splits: []conf_v1.Split{ + { + Weight: 90, + Action: &conf_v1.Action{ + Pass: "grpc-app-1", + }, + }, + { + Weight: 10, + Action: &conf_v1.Action{ + Pass: "grpc-app-2", + }, + }, + }, + ErrorPages: []conf_v1.ErrorPage{ + { + Codes: []int{404, 405}, + Return: &conf_v1.ErrorPageReturn{ + ActionReturn: conf_v1.ActionReturn{ + Code: 200, + Type: "text/plain", + Body: "All Good", + }, + }, + }, + }, + }, + }, + }, + }, + Endpoints: map[string][]string{ + "default/grpc-svc:50051": { + "10.0.0.20:80", + }, + }, + } + + baseCfgParams := ConfigParams{ + HTTP2: true, + } + + expected := version2.VirtualServerConfig{ + Upstreams: []version2.Upstream{ + { + UpstreamLabels: version2.UpstreamLabels{ + Service: "grpc-svc", + ResourceType: "virtualserver", + ResourceName: "cafe", + ResourceNamespace: "default", + }, + Name: "vs_default_cafe_grpc-app-1", + Servers: []version2.UpstreamServer{ + { + Address: "10.0.0.20:80", + }, + }, + }, + { + Name: "vs_default_cafe_grpc-app-2", + UpstreamLabels: version2.UpstreamLabels{ + Service: "grpc-svc2", + ResourceType: "virtualserver", + ResourceName: "cafe", + ResourceNamespace: "default", + }, + Servers: []version2.UpstreamServer{ + { + Address: "unix:/var/lib/nginx/nginx-502-server.sock", + }, + }, + }, + { + Name: "vs_default_cafe_tea", + UpstreamLabels: version2.UpstreamLabels{ + Service: "tea-svc", + ResourceType: "virtualserver", + ResourceName: "cafe", + ResourceNamespace: "default", + }, + Servers: []version2.UpstreamServer{ + { + Address: "unix:/var/lib/nginx/nginx-502-server.sock", + }, + }, + }, + }, + HTTPSnippets: []string{}, + LimitReqZones: []version2.LimitReqZone{}, + Maps: []version2.Map{ + { + Source: "$request_method", + Variable: "$vs_default_cafe_matches_0_match_0_cond_0", + Parameters: []version2.Parameter{ + { + Value: `"POST"`, + Result: "1", + }, + { + Value: "default", + Result: "0", + }, + }, + }, + { + Source: "$vs_default_cafe_matches_0_match_0_cond_0", + Variable: "$vs_default_cafe_matches_0", + Parameters: []version2.Parameter{ + { + Value: "~^1", + Result: "/internal_location_matches_0_match_0", + }, + { + Value: "default", + Result: "/internal_location_matches_0_default", + }, + }, + }, + }, + Server: version2.Server{ + ServerName: "cafe.example.com", + StatusZone: "cafe.example.com", + VSNamespace: "default", + VSName: "cafe", + SSL: &version2.SSL{ + HTTP2: true, + Certificate: "/etc/nginx/secrets/wildcard", + CertificateKey: "/etc/nginx/secrets/wildcard", + }, + InternalRedirectLocations: []version2.InternalRedirectLocation{ + { + Path: "/grpc-matches", + Destination: "$vs_default_cafe_matches_0", + }, + { + Path: "/grpc-splits", + Destination: "$vs_default_cafe_splits_0", + }, + }, + Locations: []version2.Location{ + { + Path: "/grpc-errorpage", + ProxyPass: "https://vs_default_cafe_grpc-app-1", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "0s", + ProxyNextUpstreamTries: 0, + ErrorPages: []version2.ErrorPage{{Name: "@error_page_0_0", Codes: "404 405", ResponseCode: 200}}, + ProxyInterceptErrors: true, + ProxySSLName: "grpc-svc.default.svc", + ProxyPassRequestHeaders: true, + ProxySetHeaders: []version2.Header{{Name: "Host", Value: "$host"}}, + ServiceName: "grpc-svc", + GRPCPass: "grpcs://vs_default_cafe_grpc-app-1", + }, + { + Path: "/internal_location_matches_0_match_0", + Internal: true, + ProxyPass: "https://vs_default_cafe_grpc-app-2$request_uri", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "0s", + ProxyNextUpstreamTries: 0, + Rewrites: []string{"^ $request_uri break"}, + ErrorPages: []version2.ErrorPage{{Name: "@error_page_1_0", Codes: "404", ResponseCode: 200}}, + ProxyInterceptErrors: true, + ProxySSLName: "grpc-svc2.default.svc", + ProxyPassRequestHeaders: true, + ProxySetHeaders: []version2.Header{{Name: "Host", Value: "$host"}}, + ServiceName: "grpc-svc2", + GRPCPass: "grpcs://vs_default_cafe_grpc-app-2", + }, + { + Path: "/internal_location_matches_0_default", + Internal: true, + ProxyPass: "http://vs_default_cafe_tea$request_uri", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "0s", + ProxyNextUpstreamTries: 0, + ErrorPages: []version2.ErrorPage{{Name: "@error_page_1_0", Codes: "404", ResponseCode: 200}}, + ProxyInterceptErrors: true, + ProxySSLName: "tea-svc.default.svc", + ProxyPassRequestHeaders: true, + ProxySetHeaders: []version2.Header{{Name: "Host", Value: "$host"}}, + ServiceName: "tea-svc", + }, + { + Path: "/internal_location_splits_0_split_0", + Internal: true, + ProxyPass: "https://vs_default_cafe_grpc-app-1$request_uri", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "0s", + ProxyNextUpstreamTries: 0, + HasKeepalive: false, + ErrorPages: []version2.ErrorPage{{Name: "@error_page_2_0", Codes: "404 405", ResponseCode: 200}}, + ProxyInterceptErrors: true, + Rewrites: []string{"^ $request_uri break"}, + ProxySSLName: "grpc-svc.default.svc", + ProxyPassRequestHeaders: true, + ProxySetHeaders: []version2.Header{{Name: "Host", Value: "$host"}}, + ServiceName: "grpc-svc", + GRPCPass: "grpcs://vs_default_cafe_grpc-app-1", + }, + { + Path: "/internal_location_splits_0_split_1", + Internal: true, + ProxyPass: "https://vs_default_cafe_grpc-app-2$request_uri", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "0s", + ProxyNextUpstreamTries: 0, + HasKeepalive: false, + ErrorPages: []version2.ErrorPage{{Name: "@error_page_2_0", Codes: "404 405", ResponseCode: 200}}, + ProxyInterceptErrors: true, + Rewrites: []string{"^ $request_uri break"}, + ProxySSLName: "grpc-svc2.default.svc", + ProxyPassRequestHeaders: true, + ProxySetHeaders: []version2.Header{{Name: "Host", Value: "$host"}}, + ServiceName: "grpc-svc2", + GRPCPass: "grpcs://vs_default_cafe_grpc-app-2", + }, + }, + ErrorPageLocations: []version2.ErrorPageLocation{ + { + Name: "@error_page_0_0", + DefaultType: "text/plain", + Return: &version2.Return{Text: "All Good"}, + }, + { + Name: "@error_page_1_0", + DefaultType: "text/plain", + Return: &version2.Return{Text: "Original resource not found, but success!"}, + }, + { + Name: "@error_page_2_0", + DefaultType: "text/plain", + Return: &version2.Return{Text: "All Good"}, + }, + }, + }, + SplitClients: []version2.SplitClient{ + { + Source: "$request_id", + Variable: "$vs_default_cafe_splits_0", + Distributions: []version2.Distribution{ + { + Weight: "90%", + Value: "/internal_location_splits_0_split_0", + }, + { + Weight: "10%", + Value: "/internal_location_splits_0_split_1", + }, + }, + }, + }, + } + expectedWarnings := Warnings{ + virtualServerEx.VirtualServer: { + `The error page configuration for the upstream grpc-app-1 is ignored for status code(s) [404 405], which cannot be used for GRPC upstreams.`, + `The error page configuration for the upstream grpc-app-2 is ignored for status code(s) [404], which cannot be used for GRPC upstreams.`, + `The error page configuration for the upstream grpc-app-1 is ignored for status code(s) [404 405], which cannot be used for GRPC upstreams.`, + `The error page configuration for the upstream grpc-app-2 is ignored for status code(s) [404 405], which cannot be used for GRPC upstreams.`, + }, + } + isPlus := false + isResolverConfigured := false + isWildcardEnabled := true + vsc := newVirtualServerConfigurator(&baseCfgParams, isPlus, isResolverConfigured, &StaticConfigParams{}, isWildcardEnabled) + + result, warnings := vsc.GenerateVirtualServerConfig(&virtualServerEx, nil) + if diff := cmp.Diff(expected, result); diff != "" { + t.Errorf("TestGenerateVirtualServerConfigGrpcErrorPageWarning() mismatch (-want +got):\n%s", diff) + } + + if !reflect.DeepEqual(vsc.warnings, expectedWarnings) { + t.Errorf("GenerateVirtualServerConfig() returned warnings of \n%v but expected \n%v", warnings, expectedWarnings) + } +} + func TestGenerateVirtualServerConfigWithSpiffeCerts(t *testing.T) { virtualServerEx := VirtualServerEx{ VirtualServer: &conf_v1.VirtualServer{ @@ -4062,7 +4427,7 @@ func TestGenerateLocationForReturn(t *testing.T) { returnLocationIndex := 1 for _, test := range tests { - location, returnLocation := generateLocationForReturn(path, snippets, test.actionReturn, returnLocationIndex) + location, returnLocation, _ := generateLocationForReturn(path, snippets, test.actionReturn, returnLocationIndex) if !reflect.DeepEqual(location, test.expectedLocation) { t.Errorf("generateLocationForReturn() returned \n%+v but expected \n%+v for the case of %s", location, test.expectedLocation, test.msg) @@ -4652,7 +5017,6 @@ func TestGenerateSplits(t *testing.T) { }, }, } - expectedSplitClient := version2.SplitClient{ Source: "$request_id", Variable: "$vs_default_cafe_splits_1", @@ -4759,7 +5123,7 @@ func TestGenerateSplits(t *testing.T) { } returnLocationIndex := 1 - resultSplitClient, resultLocations, resultReturnLocations := generateSplits( + resultSplitClient, resultLocations, resultReturnLocations, _ := generateSplits( splits, upstreamNamer, crUpstreams, @@ -5675,6 +6039,7 @@ func TestGenerateMatchesConfigWithMultipleSplits(t *testing.T) { "vs_default_cafe_coffee-v1": {Service: "coffee-v1"}, "vs_default_cafe_coffee-v2": {Service: "coffee-v2"}, } + result := generateMatchesConfig( route, upstreamNamer, diff --git a/tests/data/virtual-server-grpc/virtual-server-error-page.yaml b/tests/data/virtual-server-grpc/virtual-server-error-page.yaml new file mode 100644 index 0000000000..6da25e300e --- /dev/null +++ b/tests/data/virtual-server-grpc/virtual-server-error-page.yaml @@ -0,0 +1,29 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server +spec: + host: virtual-server.example.com + tls: + secret: virtual-server-tls-grpc-secret + upstreams: + - name: grpc1 + service: grpc1-svc + port: 50051 + type: grpc + - name: grpc2 + service: grpc2-svc + port: 50051 + type: grpc + routes: + - path: "/helloworld.Greeter" + action: + pass: grpc1 + errorPages: + - codes: [504] + return: + code: 200 + body: "Original resource not found, but success!" + - path: "/notimplemented" + action: + pass: grpc2 diff --git a/tests/suite/test_virtual_server_grpc.py b/tests/suite/test_virtual_server_grpc.py index 29a9aeb41d..ff25f3f9a7 100644 --- a/tests/suite/test_virtual_server_grpc.py +++ b/tests/suite/test_virtual_server_grpc.py @@ -1,16 +1,17 @@ import grpc import pytest -from kubernetes.client.rest import ApiException +import time from settings import TEST_DATA, DEPLOYMENTS from suite.custom_assertions import assert_event_starts_with_text_and_contains_errors, \ - assert_grpc_entries_exist, assert_proxy_entries_do_not_exist, assert_vs_conf_not_exists + assert_grpc_entries_exist, assert_proxy_entries_do_not_exist, \ + assert_vs_conf_not_exists, assert_event_and_count from suite.custom_resources_utils import read_custom_resource from suite.grpc.helloworld_pb2 import HelloRequest from suite.grpc.helloworld_pb2_grpc import GreeterStub from suite.resources_utils import create_example_app, wait_until_all_pods_are_ready, \ delete_common_app, create_secret_from_yaml, replace_configmap_from_yaml, \ - delete_items_from_yaml, get_first_pod_name, get_events + delete_items_from_yaml, get_first_pod_name, get_events, scale_deployment from suite.ssl_utils import get_certificate from suite.vs_vsr_resources_utils import get_vs_nginx_template_conf, \ patch_virtual_server_from_yaml @@ -63,6 +64,7 @@ def fin(): request.addfinalizer(fin) @pytest.mark.vs +@pytest.mark.ciara @pytest.mark.smoke @pytest.mark.parametrize('crd_ingress_controller, virtual_server_setup', [({"type": "complete", "extra_args": [f"-enable-custom-resources"]}, @@ -151,6 +153,51 @@ def test_config_after_enable_tls(self, kube_apis, ingress_controller_prerequisit ingress_controller_prerequisites.namespace) assert 'grpc_pass grpcs://' in config + @pytest.mark.parametrize("backend_setup", [{"app_type": "grpc-vs"}], indirect=True) + def test_config_error_page_warning(self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller, + backend_setup, virtual_server_setup): + text = f"{virtual_server_setup.namespace}/{virtual_server_setup.vs_name}" + vs_event_warning_text = f"Configuration for {text} was added or updated ; with warning(s):" + patch_virtual_server_from_yaml(kube_apis.custom_objects, + virtual_server_setup.vs_name, + f"{TEST_DATA}/virtual-server-grpc/virtual-server-error-page.yaml", + virtual_server_setup.namespace) + wait_before_test() + + events = get_events(kube_apis.v1, virtual_server_setup.namespace) + assert_event_and_count(vs_event_warning_text, 1, events) + + cert = get_certificate(virtual_server_setup.public_endpoint.public_ip, + virtual_server_setup.vs_host, + virtual_server_setup.public_endpoint.port_ssl) + target = f'{virtual_server_setup.public_endpoint.public_ip}:{virtual_server_setup.public_endpoint.port_ssl}' + credentials = grpc.ssl_channel_credentials(root_certificates=cert.encode()) + options = (('grpc.ssl_target_name_override', virtual_server_setup.vs_host),) + + with grpc.secure_channel(target, credentials, options) as channel: + stub = GreeterStub(channel) + response = "" + try: + response = stub.SayHello(HelloRequest(name=virtual_server_setup.public_endpoint.public_ip)) + valid_message = "Hello {}".format(virtual_server_setup.public_endpoint.public_ip) + assert valid_message in response.message + except grpc.RpcError as e: + print(e.details()) + pytest.fail("RPC error was not expected during call, exiting...") + + scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "grpc1", virtual_server_setup.namespace, 0) + scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "grpc2", virtual_server_setup.namespace, 0) + time.sleep(1) + + with grpc.secure_channel(target, credentials, options) as channel: + stub = GreeterStub(channel) + try: + response = stub.SayHello(HelloRequest(name=virtual_server_setup.public_endpoint.public_ip)) + assert response.status == 14 + pytest.fail("RPC error was expected during call, exiting...") + except grpc.RpcError as e: + print(e) + @pytest.mark.vs @pytest.mark.smoke @pytest.mark.skip_for_nginx_oss diff --git a/tests/suite/test_virtual_server_mixed_grpc.py b/tests/suite/test_virtual_server_mixed_grpc.py index 3f8e795b5f..533ad5cd7d 100644 --- a/tests/suite/test_virtual_server_mixed_grpc.py +++ b/tests/suite/test_virtual_server_mixed_grpc.py @@ -7,10 +7,12 @@ from suite.grpc.helloworld_pb2_grpc import GreeterStub from suite.resources_utils import create_example_app, wait_until_all_pods_are_ready, \ delete_common_app, create_secret_from_yaml, replace_configmap_from_yaml, \ - delete_items_from_yaml, get_first_pod_name + delete_items_from_yaml, get_first_pod_name, wait_before_test, get_events from suite.ssl_utils import get_certificate -from suite.vs_vsr_resources_utils import get_vs_nginx_template_conf -from suite.custom_assertions import assert_grpc_entries_exist, assert_proxy_entries_exist +from suite.vs_vsr_resources_utils import get_vs_nginx_template_conf, \ + patch_virtual_server_from_yaml +from suite.custom_assertions import assert_grpc_entries_exist, assert_proxy_entries_exist, \ + assert_event_and_count @pytest.fixture(scope="function") @@ -102,3 +104,17 @@ def test_config_after_setup(self, kube_apis, ingress_controller_prerequisites, c except grpc.RpcError as e: print(e.details()) pytest.fail("RPC error was not expected during call, exiting...") + + @pytest.mark.parametrize("backend_setup", [{"app_type": "grpc-vs-mixed"}], indirect=True) + def test_config_error_page_warning(self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller, + backend_setup, virtual_server_setup): + text = f"{virtual_server_setup.namespace}/{virtual_server_setup.vs_name}" + vs_event_warning_text = f"Configuration for {text} was added or updated ; with warning(s):" + patch_virtual_server_from_yaml(kube_apis.custom_objects, + virtual_server_setup.vs_name, + f"{TEST_DATA}/virtual-server-grpc-mixed/virtual-server-updated.yaml", + virtual_server_setup.namespace) + wait_before_test() + + events = get_events(kube_apis.v1, virtual_server_setup.namespace) + assert_event_and_count(vs_event_warning_text, 1, events)