More cmd/ refactoring to simplify test loading and support integration tests

cmd/archive.go

@@ -23,13 +23,9 @@ package cmd
 import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
-
-	"go.k6.io/k6/errext"
-	"go.k6.io/k6/errext/exitcodes"
-	"go.k6.io/k6/lib/metrics"
 )
 
-func getArchiveCmd(globalState *globalState) *cobra.Command { // nolint: funlen
+func getArchiveCmd(gs *globalState) *cobra.Command {
 	archiveOut := "archive.tar"
 	// archiveCmd represents the archive command
 	archiveCmd := &cobra.Command{
@@ -46,60 +42,24 @@ An archive is a fully self-contained test run, and can be executed identically e
   k6 run myarchive.tar`[1:],
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			src, filesystems, err := readSource(globalState, args[0])
-			if err != nil {
-				return err
-			}
-
-			runtimeOptions, err := getRuntimeOptions(cmd.Flags(), globalState.envVars)
-			if err != nil {
-				return err
-			}
-
-			registry := metrics.NewRegistry()
-			builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-			r, err := newRunner(
-				globalState.logger, src, globalState.flags.runType,
-				filesystems, runtimeOptions, builtinMetrics, registry,
-			)
-			if err != nil {
-				return err
-			}
-
-			cliOpts, err := getOptions(cmd.Flags())
-			if err != nil {
-				return err
-			}
-			conf, err := getConsolidatedConfig(globalState, Config{Options: cliOpts}, r.GetOptions())
-			if err != nil {
-				return err
-			}
-
-			// Parse the thresholds, only if the --no-threshold flag is not set.
-			// If parsing the threshold expressions failed, consider it as an
-			// invalid configuration error.
-			if !runtimeOptions.NoThresholds.Bool {
-				for _, thresholds := range conf.Options.Thresholds {
-					err = thresholds.Parse()
-					if err != nil {
-						return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig)
-					}
-				}
-			}
-
-			_, err = deriveAndValidateConfig(conf, r.IsExecutable, globalState.logger)
+			test, err := loadTest(gs, cmd, args, getPartialConfig)
 			if err != nil {
 				return err
 			}
 
-			err = r.SetOptions(conf.Options)
+			// It's important to NOT set the derived options back to the runner
+			// here, only the consolidated ones. Otherwise, if the script used
+			// an execution shortcut option (e.g. `iterations` or `duration`),
+			// we will have multiple conflicting execution options since the
+			// derivation will set `scenarios` as well.
+			err = test.initRunner.SetOptions(test.consolidatedConfig.Options)
 			if err != nil {
 				return err
 			}
 
 			// Archive.
-			arc := r.MakeArchive()
-			f, err := globalState.fs.Create(archiveOut)
+			arc := test.initRunner.MakeArchive()
+			f, err := gs.fs.Create(archiveOut)
 			if err != nil {
 				return err
 			}

cmd/cloud.go

@@ -42,7 +42,6 @@ import (
 	"go.k6.io/k6/errext/exitcodes"
 	"go.k6.io/k6/lib"
 	"go.k6.io/k6/lib/consts"
-	"go.k6.io/k6/lib/metrics"
 	"go.k6.io/k6/ui/pb"
 )
 
@@ -91,56 +90,23 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 		RunE: func(cmd *cobra.Command, args []string) error {
 			printBanner(globalState)
 
-			logger := globalState.logger
 			progressBar := pb.New(
 				pb.WithConstLeft("Init"),
-				pb.WithConstProgress(0, "Parsing script"),
+				pb.WithConstProgress(0, "Loading test script..."),
 			)
 			printBar(globalState, progressBar)
 
-			// Runner
-			filename := args[0]
-			src, filesystems, err := readSource(globalState, filename)
+			test, err := loadTest(globalState, cmd, args, getPartialConfig)
 			if err != nil {
 				return err
 			}
 
-			runtimeOptions, err := getRuntimeOptions(cmd.Flags(), globalState.envVars)
-			if err != nil {
-				return err
-			}
-
-			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Getting script options"))
-			registry := metrics.NewRegistry()
-			builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-			r, err := newRunner(logger, src, globalState.flags.runType, filesystems, runtimeOptions, builtinMetrics, registry)
-			if err != nil {
-				return err
-			}
-
-			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Consolidating options"))
-			cliOpts, err := getOptions(cmd.Flags())
-			if err != nil {
-				return err
-			}
-			conf, err := getConsolidatedConfig(globalState, Config{Options: cliOpts}, r.GetOptions())
-			if err != nil {
-				return err
-			}
-
-			// Parse the thresholds, only if the --no-threshold flag is not set.
-			// If parsing the threshold expressions failed, consider it as an
-			// invalid configuration error.
-			if !runtimeOptions.NoThresholds.Bool {
-				for _, thresholds := range conf.Options.Thresholds {
-					err = thresholds.Parse()
-					if err != nil {
-						return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig)
-					}
-				}
-			}
-
-			derivedConf, err := deriveAndValidateConfig(conf, r.IsExecutable, logger)
+			// It's important to NOT set the derived options back to the runner
+			// here, only the consolidated ones. Otherwise, if the script used
+			// an execution shortcut option (e.g. `iterations` or `duration`),
+			// we will have multiple conflicting execution options since the
+			// derivation will set `scenarios` as well.
+			err = test.initRunner.SetOptions(test.consolidatedConfig.Options)
 			if err != nil {
 				return err
 			}
@@ -149,13 +115,9 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 			// TODO: validate for externally controlled executor (i.e. executors that aren't distributable)
 			// TODO: move those validations to a separate function and reuse validateConfig()?
 
-			err = r.SetOptions(conf.Options)
-			if err != nil {
-				return err
-			}
+			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Building the archive..."))
+			arc := test.initRunner.MakeArchive()
 
-			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Building the archive"))
-			arc := r.MakeArchive()
 			// TODO: Fix this
 			// We reuse cloud.Config for parsing options.ext.loadimpact, but this probably shouldn't be
 			// done, as the idea of options.ext is that they are extensible without touching k6. But in
@@ -174,7 +136,7 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 
 			// Cloud config
 			cloudConfig, err := cloudapi.GetConsolidatedConfig(
-				derivedConf.Collectors["cloud"], globalState.envVars, "", arc.Options.External)
+				test.derivedConfig.Collectors["cloud"], globalState.envVars, "", arc.Options.External)
 			if err != nil {
 				return err
 			}
@@ -205,12 +167,14 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 
 			name := cloudConfig.Name.String
 			if !cloudConfig.Name.Valid || cloudConfig.Name.String == "" {
-				name = filepath.Base(filename)
+				name = filepath.Base(test.testPath)
 			}
 
 			globalCtx, globalCancel := context.WithCancel(globalState.ctx)
 			defer globalCancel()
 
+			logger := globalState.logger
+
 			// Start cloud test run
 			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Validating script options"))
 			client := cloudapi.NewClient(
@@ -248,14 +212,14 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 				os.Exit(int(exitcodes.ExternalAbort))
 			}()
 
-			et, err := lib.NewExecutionTuple(derivedConf.ExecutionSegment, derivedConf.ExecutionSegmentSequence)
+			et, err := lib.NewExecutionTuple(test.derivedConfig.ExecutionSegment, test.derivedConfig.ExecutionSegmentSequence)
 			if err != nil {
 				return err
 			}
 			testURL := cloudapi.URLForResults(refID, cloudConfig)
-			executionPlan := derivedConf.Scenarios.GetFullExecutionRequirements(et)
+			executionPlan := test.derivedConfig.Scenarios.GetFullExecutionRequirements(et)
 			printExecutionDescription(
-				globalState, "cloud", filename, testURL, derivedConf, et, executionPlan, nil,
+				globalState, "cloud", test.testPath, testURL, test.derivedConfig, et, executionPlan, nil,
 			)
 
 			modifyAndPrintBar(

cmd/common.go

@@ -21,17 +21,13 @@
 package cmd
 
 import (
-	"archive/tar"
-	"bytes"
 	"fmt"
 
-	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"gopkg.in/guregu/null.v3"
 
 	"go.k6.io/k6/lib/types"
-	"go.k6.io/k6/loader"
 )
 
 // Panic if the given error is not nil.
@@ -86,26 +82,6 @@ func exactArgsWithMsg(n int, msg string) cobra.PositionalArgs {
 	}
 }
 
-// readSource is a small wrapper around loader.ReadSource returning
-// result of the load and filesystems map
-func readSource(globalState *globalState, filename string) (*loader.SourceData, map[string]afero.Fs, error) {
-	pwd, err := globalState.getwd()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	filesystems := loader.CreateFilesystems(globalState.fs)
-	src, err := loader.ReadSource(globalState.logger, filename, pwd, filesystems, globalState.stdIn)
-	return src, filesystems, err
-}
-
-func detectType(data []byte) string {
-	if _, err := tar.NewReader(bytes.NewReader(data)).Next(); err == nil {
-		return typeArchive
-	}
-	return typeJS
-}
-
 func printToStdout(gs *globalState, s string) {
 	if _, err := fmt.Fprint(gs.stdOut, s); err != nil {
 		gs.logger.Errorf("could not print '%s' to stdout: %s", s, err.Error())

cmd/config.go

@@ -91,6 +91,16 @@ func (c Config) Apply(cfg Config) Config {
 	return c
 }
 
+// Returns a Config but only parses the Options inside.
+func getPartialConfig(flags *pflag.FlagSet) (Config, error) {
+	opts, err := getOptions(flags)
+	if err != nil {
+		return Config{}, err
+	}
+
+	return Config{Options: opts}, nil
+}
+
 // Gets configuration from CLI flags.
 func getConfig(flags *pflag.FlagSet) (Config, error) {
 	opts, err := getOptions(flags)