Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add all-features flag to add ability to enable all supported feature conformance tests. #1642

Merged
24 changes: 13 additions & 11 deletions conformance/conformance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"sigs.k8s.io/gateway-api/conformance/utils/flags"
"sigs.k8s.io/gateway-api/conformance/utils/suite"

"k8s.io/apimachinery/pkg/util/sets"
_ "k8s.io/client-go/plugin/pkg/client/auth"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
Expand All @@ -47,29 +48,30 @@ func TestConformance(t *testing.T) {
supportedFeatures := parseSupportedFeatures(*flags.SupportedFeatures)
exemptFeatures := parseSupportedFeatures(*flags.ExemptFeatures)
for feature := range exemptFeatures {
supportedFeatures[feature] = false
supportedFeatures.Delete(feature)
}

t.Logf("Running conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n supported features: [%v]\n exempt features: [%v]",
*flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.SupportedFeatures, *flags.ExemptFeatures)
t.Logf("Running conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]",
*flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures)

cSuite := suite.New(suite.Options{
Client: client,
GatewayClassName: *flags.GatewayClassName,
Debug: *flags.ShowDebug,
CleanupBaseResources: *flags.CleanupBaseResources,
SupportedFeatures: supportedFeatures,
Client: client,
GatewayClassName: *flags.GatewayClassName,
Debug: *flags.ShowDebug,
CleanupBaseResources: *flags.CleanupBaseResources,
SupportedFeatures: supportedFeatures,
EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
})
cSuite.Setup(t)
cSuite.Run(t, tests.ConformanceTests)
}

// parseSupportedFeatures parses flag arguments and converts the string to
// map[suite.SupportedFeature]bool
func parseSupportedFeatures(f string) map[suite.SupportedFeature]bool {
res := map[suite.SupportedFeature]bool{}
func parseSupportedFeatures(f string) sets.Set[suite.SupportedFeature] {
res := sets.Set[suite.SupportedFeature]{}
for _, value := range strings.Split(f, ",") {
res[suite.SupportedFeature(value)] = true
res.Insert(suite.SupportedFeature(value))
}
return res
}
11 changes: 6 additions & 5 deletions conformance/utils/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ import (
)

var (
GatewayClassName = flag.String("gateway-class", "gateway-conformance", "Name of GatewayClass to use for tests")
ShowDebug = flag.Bool("debug", false, "Whether to print debug logs")
CleanupBaseResources = flag.Bool("cleanup-base-resources", true, "Whether to cleanup base test resources after the run")
SupportedFeatures = flag.String("supported-features", "", "Supported features included in conformance tests suites")
ExemptFeatures = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites")
GatewayClassName = flag.String("gateway-class", "gateway-conformance", "Name of GatewayClass to use for tests")
ShowDebug = flag.Bool("debug", false, "Whether to print debug logs")
CleanupBaseResources = flag.Bool("cleanup-base-resources", true, "Whether to cleanup base test resources after the run")
SupportedFeatures = flag.String("supported-features", "", "Supported features included in conformance tests suites")
ExemptFeatures = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites")
EnableAllSupportedFeatures = flag.Bool("all-features", false, "Whether to enable all supported features for conformance tests")
)
47 changes: 34 additions & 13 deletions conformance/utils/suite/suite.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,29 @@ const (

// StandardCoreFeatures are the features that are required to be conformant with
// the Core API features that are part of the Standard release channel.
var StandardCoreFeatures = map[SupportedFeature]bool{
SupportReferenceGrant: true,
}
var StandardCoreFeatures = sets.New(
SupportReferenceGrant,
)

// AllFeatures contains all the supported features and can be used to run all
// conformance tests with `all-features` flag.
//
// Note that the AllFeatures must in sync with defined features when the
// feature constants change.
var AllFeatures = sets.New(
SupportReferenceGrant,
SupportTLSRoute,
SupportHTTPRouteQueryParamMatching,
SupportHTTPRouteMethodMatching,
SupportHTTPResponseHeaderModification,
SupportRouteDestinationPortMatching,
SupportGatewayClassObservedGenerationBump,
SupportHTTPRoutePortRedirect,
SupportHTTPRouteSchemeRedirect,
SupportHTTPRoutePathRedirect,
SupportHTTPRouteHostRewrite,
SupportHTTPRoutePathRewrite,
)

// ConformanceTestSuite defines the test suite used to run Gateway API
// conformance tests.
Expand All @@ -89,7 +109,7 @@ type ConformanceTestSuite struct {
Cleanup bool
BaseManifests string
Applier kubernetes.Applier
SupportedFeatures map[SupportedFeature]bool
SupportedFeatures sets.Set[SupportedFeature]
TimeoutConfig config.TimeoutConfig
SkipTests sets.Set[string]
}
Expand All @@ -112,9 +132,10 @@ type Options struct {

// CleanupBaseResources indicates whether or not the base test
// resources such as Gateways should be cleaned up after the run.
CleanupBaseResources bool
SupportedFeatures map[SupportedFeature]bool
TimeoutConfig config.TimeoutConfig
CleanupBaseResources bool
SupportedFeatures sets.Set[SupportedFeature]
EnableAllSupportedFeatures bool
TimeoutConfig config.TimeoutConfig
// SkipTests contains all the tests not to be run and can be used to opt out
// of specific tests
SkipTests []string
Expand All @@ -129,13 +150,13 @@ func New(s Options) *ConformanceTestSuite {
roundTripper = &roundtripper.DefaultRoundTripper{Debug: s.Debug, TimeoutConfig: s.TimeoutConfig}
}

if s.SupportedFeatures == nil {
if s.EnableAllSupportedFeatures == true {
s.SupportedFeatures = AllFeatures
} else if s.SupportedFeatures == nil {
s.SupportedFeatures = StandardCoreFeatures
} else {
for feature, val := range StandardCoreFeatures {
if _, ok := s.SupportedFeatures[feature]; !ok {
s.SupportedFeatures[feature] = val
}
for feature := range StandardCoreFeatures {
s.SupportedFeatures.Insert(feature)
}
}

Expand Down Expand Up @@ -222,7 +243,7 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) {
// Check that all features exercised by the test have been opted into by
// the suite.
for _, feature := range test.Features {
if supported, ok := suite.SupportedFeatures[feature]; !ok || !supported {
if !suite.SupportedFeatures.Has(feature) {
t.Skipf("Skipping %s: suite does not support %s", test.ShortName, feature)
}
}
Expand Down