Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add certified conformance mode and non-disruptive-conformance mode #885

Merged
merged 1 commit into from
Sep 17, 2019
Merged

Add certified conformance mode and non-disruptive-conformance mode #885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/sonobuoy/app/args.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func AddNamespaceFlag(str *string, flags *pflag.FlagSet) {
// Mode can be partially or fully overridden by specifying config, e2e-focus, and e2e-skip.
// The variables specified by those flags will overlay the defaults provided by the given mode.
func AddModeFlag(mode *ops.Mode, flags *pflag.FlagSet) {
*mode = ops.Conformance // default
*mode = ops.NonDisruptiveConformance // default
flags.VarP(
mode, "mode", "m",
fmt.Sprintf("What mode to run sonobuoy in. Valid modes are %s.", strings.Join(ops.GetModes(), ", ")),
Expand Down Expand Up @@ -132,7 +132,7 @@ const (
// users. Using e2e-parallel incorrectly has the potential to destroy clusters!
func AddE2EConfigFlags(flags *pflag.FlagSet) *pflag.FlagSet {
e2eFlags := pflag.NewFlagSet("e2e", pflag.ExitOnError)
modeName := ops.Conformance
modeName := ops.NonDisruptiveConformance
defaultMode := modeName.Get()
e2eFlags.String(
e2eFocusFlag, defaultMode.E2EConfig.Focus,
Expand Down
9 changes: 4 additions & 5 deletions cmd/sonobuoy/app/args_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,15 @@ func TestGetE2EConfig(t *testing.T) {
}{
{
desc: "Default",
mode: "Conformance",
mode: "certified-conformance",
flagArgs: []string{},
expect: &ops.E2EConfig{
Focus: `\[Conformance\]`,
Skip: `Alpha|\[(Disruptive|Feature:[^\]]+|Flaky)\]`,
Parallel: "1",
},
}, {
desc: "Flags settable",
mode: "Conformance",
mode: "certified-conformance",
flagArgs: []string{"--e2e-focus=foo", "--e2e-skip=bar", "--e2e-parallel=2"},
expect: &ops.E2EConfig{
Focus: `foo`,
Expand All @@ -52,12 +51,12 @@ func TestGetE2EConfig(t *testing.T) {
},
}, {
desc: "Focus regexp validated settable",
mode: "Conformance",
mode: "certified-conformance",
flagArgs: []string{"--e2e-focus=*"},
expectErr: true,
}, {
desc: "Skip regexp validated settable",
mode: "Conformance",
mode: "certified-conformance",
flagArgs: []string{"--e2e-skip=*"},
expectErr: true,
},
Expand Down
8 changes: 4 additions & 4 deletions cmd/sonobuoy/app/gen_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,9 @@ func TestResolveConfig(t *testing.T) {
cliInput string
}{
{
name: "Conformance mode when supplied config is nil (nothing interesting happens)",
name: "NonDisruptiveConformance mode when supplied config is nil (nothing interesting happens)",
input: &genFlags{
mode: client.Conformance,
mode: client.NonDisruptiveConformance,
sonobuoyConfig: SonobuoyConfig{},
},
expected: &config.Config{
Expand Down Expand Up @@ -139,9 +139,9 @@ func TestResolveConfig(t *testing.T) {
Resources: config.DefaultResources,
},
}, {
name: "Conformance mode with plugin selection specified",
name: "NonDisruptiveConformance mode with plugin selection specified",
input: &genFlags{
mode: client.Conformance,
mode: client.NonDisruptiveConformance,
sonobuoyConfig: SonobuoyConfig{
Config: config.Config{
PluginSelections: []plugin.Selection{
Expand Down
4 changes: 2 additions & 2 deletions pkg/client/defaults.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ import (
"github.com/heptio/sonobuoy/pkg/config"
)

// NewGenConfig is a GenConfig using the default config and Conformance mode
// NewGenConfig is a GenConfig using the default config and NonDisruptiveConformance mode
func NewGenConfig() *GenConfig {
modeName := Conformance
modeName := NonDisruptiveConformance
defaultE2E := modeName.Get().E2EConfig

return &GenConfig{
Expand Down
43 changes: 31 additions & 12 deletions pkg/client/mode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,26 @@ type Mode string

const (
// Quick runs a single E2E test and the systemd log tests.
Quick Mode = "Quick"
Quick Mode = "quick"

// Conformance runs all of the E2E tests and the systemd log tests.
Conformance Mode = "Conformance"
// NonDisruptiveConformance runs all of the `Conformance` E2E tests which are not marked as disuprtive and the systemd log tests.
NonDisruptiveConformance Mode = "non-disruptive-conformance"

// CertifiedConformance runs all of the `Conformance` E2E tests and the systemd log tests.
CertifiedConformance Mode = "certified-conformance"
)

const defaultSkipList = `Alpha|\[(Disruptive|Feature:[^\]]+|Flaky)\]`
// nonDisruptiveSkipList should generally just need to skip disruptive tests since upstream
// will disallow the other types of tests from being tagged as Conformance. However, in v1.16
// two disruptive tests were not marked as such, meaning we needed to specify them here to ensure
// user workload safety. See https://github.com/kubernetes/kubernetes/issues/82663
// and https://github.com/kubernetes/kubernetes/issues/82787
const nonDisruptiveSkipList = `\[Disruptive\]|NoExecuteTaintManager`

var modeMap = map[string]Mode{
string(Conformance): Conformance,
string(Quick): Quick,
string(NonDisruptiveConformance): NonDisruptiveConformance,
string(Quick): Quick,
string(CertifiedConformance): CertifiedConformance,
}

// ModeConfig represents the sonobuoy configuration for a given mode.
Expand All @@ -60,9 +69,9 @@ func (m *Mode) Type() string { return "Mode" }

// Set the name with a given string. Returns error on unknown mode.
func (m *Mode) Set(str string) error {
// Allow lowercase "conformance", "quick" etc in command line
upcase := strings.Title(str)
mode, ok := modeMap[upcase]
// Allow other casing from user in command line (e.g. Quick & quick)
lcase := strings.ToLower(str)
mode, ok := modeMap[lcase]
if !ok {
return fmt.Errorf("unknown mode %s", str)
}
Expand All @@ -74,11 +83,22 @@ func (m *Mode) Set(str string) error {
// if there's no associated mode
func (m *Mode) Get() *ModeConfig {
switch *m {
case Conformance:
case CertifiedConformance:
return &ModeConfig{
E2EConfig: E2EConfig{
Focus: `\[Conformance\]`,
Parallel: "1",
},
Selectors: []plugin.Selection{
{Name: "e2e"},
{Name: "systemd-logs"},
},
}
case NonDisruptiveConformance:
return &ModeConfig{
E2EConfig: E2EConfig{
Focus: `\[Conformance\]`,
Skip: defaultSkipList,
Skip: nonDisruptiveSkipList,
Parallel: "1",
},
Selectors: []plugin.Selection{
Expand All @@ -90,7 +110,6 @@ func (m *Mode) Get() *ModeConfig {
return &ModeConfig{
E2EConfig: E2EConfig{
Focus: "Pods should be submitted and removed",
Skip: defaultSkipList,
Parallel: "1",
},
Selectors: []plugin.Selection{
Expand Down