Skip to content

Commit

Permalink
support skipping tests by labels and feature name
Browse files Browse the repository at this point in the history
  • Loading branch information
ShwethaKumbla committed Oct 13, 2021
1 parent 0610338 commit 3771367
Show file tree
Hide file tree
Showing 5 changed files with 164 additions and 27 deletions.
4 changes: 2 additions & 2 deletions docs/design/test-harness-framework.md
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,7 @@ go test -tags=e2e ./test/... --feature="special"
The framework shall provide predefined flagsets that will be automatically applied during test execution. Possible filters that could be supported by the framework implementation:

* `--feature` - a regular expression that target features to run
* `--assert` - a regular expression that targets the name of an assertive steps
* `--assess` - a regular expression that targets the name of an assesment steps
* `--labels` - a comma-separated list of key/value pairs used to filter features by their assigned labels

The framework should automatically inject these filter values into the environment component when it is created.
Expand All @@ -470,7 +470,7 @@ The framework should automatically inject these filter values into the environme
The test framework should provide the ability to explicitly exclude features during a test run. This could be done with the following flags:

* `--skip-feature` - a regular expression that skips features with matching names
* `--skip-assert` - a regular expression that skips assertions with matching name
* `--skip-assessment` - a regular expression that skips assessment with matching name
* `--skip-lables` - a comma-separated list of key/value pairs used to skip features with matching lables

## Test support
Expand Down
19 changes: 19 additions & 0 deletions pkg/env/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -290,10 +290,23 @@ func (e *testEnv) execFeature(ctx context.Context, t *testing.T, f types.Feature
// feature-level subtest
t.Run(featName, func(t *testing.T) {
// skip if feature name does not match
// skip feature which matches with --skip-features
if e.cfg.SkipFeatureRegex() != nil && e.cfg.SkipFeatureRegex().MatchString(featName) {
t.Skipf(`Skipping feature "%s": name matched`, featName)
}

// skip feature which does not match with --feature
if e.cfg.FeatureRegex() != nil && !e.cfg.FeatureRegex().MatchString(featName) {
t.Skipf(`Skipping feature "%s": name not matched`, featName)
}

// skip running a feature if labels matches with --skip-labels
for k, v := range e.cfg.SkipLabels() {
if f.Labels()[k] == v {
t.Skipf(`Skipping feature "%s": matched label provided in --skip-lables "%s=%s"`, featName, k, f.Labels()[k])
}
}

// skip if labels does not match
// run tests if --labels values matches the feature labels
for k, v := range e.cfg.Labels() {
Expand All @@ -313,6 +326,12 @@ func (e *testEnv) execFeature(ctx context.Context, t *testing.T, f types.Feature

for _, assess := range assessments {
t.Run(assess.Name(), func(t *testing.T) {
// skip assessments which matches with --skip-assessments
if e.cfg.SkipAssessmentRegex() != nil && e.cfg.SkipAssessmentRegex().MatchString(assess.Name()) {
t.Skipf(`Skipping assessment "%s": name matched`, assess.Name())
}

// skip assessments which does not matches with --assess
if e.cfg.AssessmentRegex() != nil && !e.cfg.AssessmentRegex().MatchString(assess.Name()) {
t.Skipf(`Skipping assessment "%s": name not matched`, assess.Name())
}
Expand Down
50 changes: 44 additions & 6 deletions pkg/envconf/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,15 @@ import (

// Config represents and environment configuration
type Config struct {
kubeconfig string
client klient.Client
namespace string
assessmentRegex *regexp.Regexp
featureRegex *regexp.Regexp
labels map[string]string
client klient.Client
kubeconfig string
namespace string
assessmentRegex *regexp.Regexp
featureRegex *regexp.Regexp
labels map[string]string
skipFeatureRegex *regexp.Regexp
skipLabels map[string]string
skipAssessmentRegex *regexp.Regexp
}

// New creates and initializes an empty environment configuration
Expand All @@ -56,6 +59,8 @@ func NewFromFlags() (*Config, error) {
e.labels = envFlags.Labels()
e.namespace = envFlags.Namespace()
e.kubeconfig = envFlags.Kubeconfig()
e.skipFeatureRegex = regexp.MustCompile(envFlags.SkipFeatures())
e.skipLabels = envFlags.SkipLabels()
return e, nil
}

Expand Down Expand Up @@ -124,6 +129,17 @@ func (c *Config) AssessmentRegex() *regexp.Regexp {
return c.assessmentRegex
}

// WithSkipAssessmentRegex sets the environment assessment regex filter
func (c *Config) WithSkipAssessmentRegex(regex string) *Config {
c.skipAssessmentRegex = regexp.MustCompile(regex)
return c
}

// SkipAssessmentRegex returns the environment assessment filter
func (c *Config) SkipAssessmentRegex() *regexp.Regexp {
return c.skipAssessmentRegex
}

// WithFeatureRegex sets the environment's feature regex filter
func (c *Config) WithFeatureRegex(regex string) *Config {
c.featureRegex = regexp.MustCompile(regex)
Expand All @@ -135,6 +151,17 @@ func (c *Config) FeatureRegex() *regexp.Regexp {
return c.featureRegex
}

// WithSkipFeatureRegex sets the environment's skip feature regex filter
func (c *Config) WithSkipFeatureRegex(regex string) *Config {
c.skipFeatureRegex = regexp.MustCompile(regex)
return c
}

// SkipFeatureRegex returns the environment's skipfeature regex filter
func (c *Config) SkipFeatureRegex() *regexp.Regexp {
return c.skipFeatureRegex
}

// WithLabels sets the environment label filters
func (c *Config) WithLabels(lbls map[string]string) *Config {
c.labels = lbls
Expand All @@ -146,6 +173,17 @@ func (c *Config) Labels() map[string]string {
return c.labels
}

// WithSkipLabels sets the environment label filters
func (c *Config) WithSkipLabels(lbls map[string]string) *Config {
c.skipLabels = lbls
return c
}

// SkipLabels returns the environment's label filters
func (c *Config) SkipLabels() map[string]string {
return c.skipLabels
}

func randNS() string {
return RandomName("testns-", 32)
}
Expand Down
89 changes: 72 additions & 17 deletions pkg/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,14 @@ import (
)

const (
flagNamespaceName = "namespace"
flagKubecofigName = "kubeconfig"
flagFeatureName = "feature"
flagAssessName = "assess"
flagLabelsName = "labels"
flagNamespaceName = "namespace"
flagKubecofigName = "kubeconfig"
flagFeatureName = "feature"
flagAssessName = "assess"
flagLabelsName = "labels"
flagSkipLabelName = "skip-labels"
flagSkipFeatureName = "skip-features"
flagSkipAssessmentName = "skip-assessment"
)

// Supported flag definitions
Expand All @@ -53,17 +56,30 @@ var (
Name: flagNamespaceName,
Usage: "A namespace value to use for testing (optional)",
}
skipLabelsFlag = flag.Flag{
Name: flagSkipLabelName,
Usage: "Regular expression to skip label(s) to run",
}
skipFeatureFlag = flag.Flag{
Name: flagSkipFeatureName,
Usage: "Regular expression to skip feature(s) to run",
}
skipAssessmentFlag = flag.Flag{
Name: flagSkipAssessmentName,
Usage: "Regular expression to skip assessment(s) to run",
}
)

// EnvFlags surfaces all resolved flag values for the testing framework
type EnvFlags struct {
feature string
assess string
labels LabelsMap

// optional kube flags
kubeconfig string
namespace string
feature string
assess string
labels LabelsMap
kubeconfig string
namespace string
skiplabels LabelsMap
skipFeatures string
skipAssessments string
}

// Feature returns value for `-feature` flag
Expand All @@ -86,6 +102,18 @@ func (f *EnvFlags) Namespace() string {
return f.namespace
}

func (f *EnvFlags) SkipFeatures() string {
return f.skipFeatures
}

func (f *EnvFlags) SkipAssessment() string {
return f.skipAssessments
}

func (f *EnvFlags) SkipLabels() LabelsMap {
return f.skiplabels
}

// Kubeconfig returns an optional path for kubeconfig file
func (f *EnvFlags) Kubeconfig() string {
return f.kubeconfig
Expand All @@ -99,11 +127,17 @@ func Parse() (*EnvFlags, error) {
// ParseArgs parses the specified args from global flag.CommandLine
// and returns a set of environment flag values.
func ParseArgs(args []string) (*EnvFlags, error) {
var feature string
var assess string
var (
feature string
assess string
namespace string
kubeconfig string
skipFeature string
skipAssessment string
)

labels := make(LabelsMap)
var namespace string
var kubeconfig string
skipLabels := make(LabelsMap)

if flag.Lookup(featureFlag.Name) == nil {
flag.StringVar(&feature, featureFlag.Name, featureFlag.DefValue, featureFlag.Usage)
Expand All @@ -125,11 +159,32 @@ func ParseArgs(args []string) (*EnvFlags, error) {
flag.Var(&labels, labelsFlag.Name, labelsFlag.Usage)
}

if flag.Lookup(skipLabelsFlag.Name) == nil {
flag.Var(&skipLabels, skipLabelsFlag.Name, skipLabelsFlag.Usage)
}

if flag.Lookup(skipAssessmentFlag.Name) == nil {
flag.StringVar(&skipAssessment, skipAssessmentFlag.Name, skipAssessmentFlag.DefValue, skipAssessmentFlag.Usage)
}

if flag.Lookup(skipFeatureFlag.Name) == nil {
flag.StringVar(&skipFeature, skipFeatureFlag.Name, skipFeatureFlag.DefValue, skipFeatureFlag.Usage)
}

if err := flag.CommandLine.Parse(args); err != nil {
return nil, fmt.Errorf("flags parsing: %w", err)
}

return &EnvFlags{feature: feature, assess: assess, labels: labels, namespace: namespace, kubeconfig: kubeconfig}, nil
return &EnvFlags{
feature: feature,
assess: assess,
labels: labels,
namespace: namespace,
kubeconfig: kubeconfig,
skiplabels: skipLabels,
skipFeatures: skipFeature,
skipAssessments: skipAssessment,
}, nil
}

type LabelsMap map[string]string
Expand Down
29 changes: 27 additions & 2 deletions pkg/flags/flags_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,18 @@ func TestParseFlags(t *testing.T) {
}{
{
name: "with all",
args: []string{"-assess", "volume test", "--feature", "beta", "--labels", "k0=v0, k1=v1, k2=v2"},
flags: &EnvFlags{assess: "volume test", feature: "beta", labels: LabelsMap{"k0": "v0", "k1": "v1", "k2": "v2"}},
args: []string{"-assess", "volume test", "--feature", "beta", "--labels", "k0=v0, k1=v1, k2=v2", "--skip-labels", "k0=v0, k1=v1"},
flags: &EnvFlags{assess: "volume test", feature: "beta", labels: LabelsMap{"k0": "v0", "k1": "v1", "k2": "v2"}, skiplabels: LabelsMap{"k0": "v0", "k1": "v1"}},
},
{
name: "with skip feature only",
args: []string{"-skip-features", "networking"},
flags: &EnvFlags{skipFeatures: "networking"},
},
{
name: "with skip assesment only",
args: []string{"-skip-assessment", "volume test"},
flags: &EnvFlags{skipAssessments: "volume test"},
},
}

Expand All @@ -39,6 +49,7 @@ func TestParseFlags(t *testing.T) {
if err != nil {
t.Fatal(err)
}

if testFlags.Feature() != test.flags.Feature() {
t.Errorf("unmatched feature: %s; %s", testFlags.Feature(), test.flags.Feature())
}
Expand All @@ -51,6 +62,20 @@ func TestParseFlags(t *testing.T) {
t.Errorf("unmatched label %s=%s", k, test.flags.Labels()[k])
}
}

for k, v := range testFlags.SkipLabels() {
if test.flags.SkipLabels()[k] != v {
t.Errorf("unmatched skip label %s=%s", k, test.flags.SkipLabels()[k])
}
}

if testFlags.SkipFeatures() != test.flags.SkipFeatures() {
t.Errorf("unmatched feature for skip: %s", testFlags.SkipFeatures())
}

if testFlags.SkipAssessment() != test.flags.SkipAssessment() {
t.Errorf("unmatched assesment name for skip: %s", testFlags.SkipFeatures())
}
})
}
}

0 comments on commit 3771367

Please sign in to comment.