diff --git a/.buildkite/build/ssh.conf b/.buildkite/build/ssh.conf index 5ecdd5298c..4a1dc4a77b 100644 --- a/.buildkite/build/ssh.conf +++ b/.buildkite/build/ssh.conf @@ -1,4 +1,4 @@ -# The following config is used by bootstrap/git_test.go +# The following config is used by job/git_test.go Host github.aaakk.us.kg-alias1 Hostname github.com diff --git a/.buildkite/steps/tests.sh b/.buildkite/steps/tests.sh index d3a0f507c4..248cb6e46d 100755 --- a/.buildkite/steps/tests.sh +++ b/.buildkite/steps/tests.sh @@ -10,4 +10,4 @@ echo '+++ Running tests' gotestsum --junitfile "junit-${BUILDKITE_JOB_ID}.xml" -- -count=1 -failfast "$@" ./... echo '+++ Running integration tests for git-mirrors experiment' -TEST_EXPERIMENT=git-mirrors gotestsum --junitfile "junit-${BUILDKITE_JOB_ID}-git-mirrors.xml" -- -count=1 -failfast "$@" ./bootstrap/integration +TEST_EXPERIMENT=git-mirrors gotestsum --junitfile "junit-${BUILDKITE_JOB_ID}-git-mirrors.xml" -- -count=1 -failfast "$@" ./job/integration diff --git a/README.md b/README.md index 931e72a04e..f9e7868c7a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ Available commands are: meta-data Get/set data from Buildkite jobs pipeline Make changes to the pipeline of the currently running build step Make changes to a step (this includes any jobs that were created from the step) - bootstrap Run a Buildkite job locally + bootstrap [DEPRECATED] Run a Buildkite job locally + job Interact with Buildkite jobs help Shows a list of commands or help for one command Use "buildkite-agent --help" for more information about a command. diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go index a116e7cf30..78e912e224 100644 --- a/agent/agent_configuration.go +++ b/agent/agent_configuration.go @@ -4,7 +4,7 @@ package agent // has been loaded from the config file and command-line params type AgentConfiguration struct { ConfigPath string - BootstrapScript string + JobRunScript string BuildPath string HooksPath string SocketsPath string diff --git a/agent/integration/job_runner_integration_test.go b/agent/integration/job_runner_integration_test.go index 951eb43e9c..0768a40c80 100644 --- a/agent/integration/job_runner_integration_test.go +++ b/agent/integration/job_runner_integration_test.go @@ -41,7 +41,7 @@ func TestJobRunner_WhenJobHasToken_ItOverridesAccessToken(t *testing.T) { }) } -func TestJobRunnerPassesAccessTokenToBootstrap(t *testing.T) { +func TestJobRunnerPassesAccessTokenToJobExecute(t *testing.T) { ag := &api.AgentRegisterResponse{ AccessToken: "llamasrock", } @@ -91,20 +91,20 @@ func TestJobRunnerIgnoresPipelineChangesToProtectedVars(t *testing.T) { } -func runJob(t *testing.T, ag *api.AgentRegisterResponse, j *api.Job, cfg agent.AgentConfiguration, bootstrap func(c *bintest.Call)) { +func runJob(t *testing.T, ag *api.AgentRegisterResponse, j *api.Job, cfg agent.AgentConfiguration, executor func(c *bintest.Call)) { // create a mock agent API server := createTestAgentEndpoint(t, "my-job-id") defer server.Close() - // set up a mock bootstrap that the runner will call - bs, err := bintest.NewMock("buildkite-agent-bootstrap") + // set up a mock executor that the runner will call + bs, err := bintest.NewMock("buildkite-agent-job-execute") if err != nil { t.Fatalf("bintest.NewMock() error = %v", err) } defer bs.CheckAndClose(t) - // execute the callback we have inside the bootstrap mock - bs.Expect().Once().AndExitWith(0).AndCallFunc(bootstrap) + // execute the callback we have inside the executor mock + bs.Expect().Once().AndExitWith(0).AndCallFunc(executor) l := logger.Discard @@ -112,8 +112,8 @@ func runJob(t *testing.T, ag *api.AgentRegisterResponse, j *api.Job, cfg agent.A m := metrics.NewCollector(l, metrics.CollectorConfig{}) scope := m.Scope(metrics.Tags{}) - // set the bootstrap into the config - cfg.BootstrapScript = bs.Path + // set the executor into the config + cfg.JobRunScript = bs.Path client := api.NewClient(l, api.Config{ Endpoint: server.URL, diff --git a/agent/job_runner.go b/agent/job_runner.go index fc10b696aa..098a6856d4 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -12,9 +12,9 @@ import ( "time" "github.com/buildkite/agent/v3/api" - "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/experiments" "github.com/buildkite/agent/v3/hook" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/agent/v3/kubernetes" "github.com/buildkite/agent/v3/logger" "github.com/buildkite/agent/v3/metrics" @@ -26,7 +26,7 @@ import ( const ( // BuildkiteMessageMax is the maximum length of "BUILDKITE_MESSAGE=...\0" - // environment entry passed to bootstrap, beyond which it will be truncated + // environment entry passed to job executor, beyond which it will be truncated // to avoid exceeding the system limit. Note that it includes the variable // name, equals sign, and null terminator. // @@ -199,11 +199,11 @@ func NewJobRunner(l logger.Logger, scope *metrics.Scope, ag *api.AgentRegisterRe return nil, err } - // The bootstrap-script gets parsed based on the operating system - cmd, err := shellwords.Split(conf.AgentConfiguration.BootstrapScript) + // The job executor script gets parsed based on the operating system + cmd, err := shellwords.Split(conf.AgentConfiguration.JobRunScript) if err != nil { - return nil, fmt.Errorf("Failed to split bootstrap-script (%q) into tokens: %v", - conf.AgentConfiguration.BootstrapScript, err) + return nil, fmt.Errorf("Failed to split job executor script (%q) into tokens: %v", + conf.AgentConfiguration.JobRunScript, err) } // Our log streamer works off a buffer of output @@ -311,6 +311,7 @@ func NewJobRunner(l logger.Logger, scope *metrics.Scope, ag *api.AgentRegisterRe ClientCount: containerCount, }) } else { + // The process that will run the job executor script runner.process = process.New(l, process.Config{ Path: cmd[0], Args: cmd[1:], @@ -339,6 +340,37 @@ func NewJobRunner(l logger.Logger, scope *metrics.Scope, ag *api.AgentRegisterRe return runner, nil } +type hookExit struct { + ExitStatus int + SignalReason string + Ok bool +} + +func (r *JobRunner) preExecHook(ctx context.Context, hookName string) hookExit { + exit := hookExit{Ok: true} + hookPath, _ := hook.Find(r.conf.AgentConfiguration.HooksPath, hookName) + if hookPath == "" { + return exit + } + + // Once we have a hook any failure to run it MUST be fatal to the job to guarantee a true + // positive result from the hook + okay, err := r.executePreExecHook(ctx, hookName, hookPath) + if !okay { + exit.Ok = false + + // Ensure the Job UI knows why this job resulted in failure + r.logStreamer.Process([]byte(fmt.Sprintf("%s hook rejected this job, see the buildkite-agent logs for more details", hookName))) + // But disclose more information in the agent logs + r.logger.Error("%s hook rejected this job: %s", hookName, err) + + exit.ExitStatus = -1 + exit.SignalReason = "agent_refused" + } + + return exit +} + // Runs the job func (r *JobRunner) Run(ctx context.Context) error { r.logger.Info("Starting job %s", r.job.ID) @@ -374,139 +406,87 @@ func (r *JobRunner) Run(ctx context.Context) error { return err } + hookExit := r.preExecHooks(ctx) + // Before executing the job process with the received Job env, + // execute the pre-bootstrap/pre-exec hook (if present) for it to tell us + // whether it is happy to proceed. // Default exit status is no exit status - exitStatus := "" signal := "" - signalReason := "" - - // Before executing the bootstrap process with the received Job env, - // execute the pre-bootstrap hook (if present) for it to tell us - // whether it is happy to proceed. - environmentCommandOkay := true - - if hook, _ := hook.Find(r.conf.AgentConfiguration.HooksPath, "pre-bootstrap"); hook != "" { - // Once we have a hook any failure to run it MUST be fatal to the job to guarantee a true - // positive result from the hook - okay, err := r.executePreBootstrapHook(ctx, hook) - if !okay { - environmentCommandOkay = false - - // Ensure the Job UI knows why this job resulted in failure - r.logStreamer.Process([]byte("pre-bootstrap hook rejected this job, see the buildkite-agent logs for more details")) - // But disclose more information in the agent logs - r.logger.Error("pre-bootstrap hook rejected this job: %s", err) - - exitStatus = "-1" - signalReason = "agent_refused" - } - } + exitStatus := strconv.Itoa(hookExit.ExitStatus) + signalReason := hookExit.SignalReason - // Used to wait on various routines that we spin up var wg sync.WaitGroup - // Set up a child context for helper goroutines related to running the job. cctx, cancel := context.WithCancel(ctx) - defer cancel() - - if environmentCommandOkay { - // Kick off log streaming and job status checking when the process - // starts. - wg.Add(2) - go r.jobLogStreamer(cctx, &wg) - go r.jobCancellationChecker(cctx, &wg) - - // Run the process. This will block until it finishes. - if err := r.process.Run(cctx); err != nil { - // Send the error as output - r.logStreamer.Process([]byte(err.Error())) - - // The process did not run at all, so make sure it fails - exitStatus = "-1" - signalReason = "process_run_error" - } else { - // Intended to capture situations where the job-exec (aka bootstrap) container did not - // start. Normally such errors are hidden in the Kubernetes events. Let's feed them up - // to the user as they may be the caused by errors in the pipeline definition. - if r.cancelled && !r.stopped { - k8sProcess, ok := r.process.(*kubernetes.Runner) - if ok && k8sProcess.ClientStateUnknown() { - r.logStreamer.Process([]byte( - "Some containers had unknown exit statuses. Perhaps they were in ImagePullBackOff.", - )) - } - } + defer func() { + // Finish the build in the Buildkite Agent API + // + // Once we tell the API we're finished it might assign us new work, so make + // sure everything else is done first. + r.finishJob(ctx, startedAt, cancel, &wg, exitStatus, signal, signalReason) - // Add the final output to the streamer - r.logStreamer.Process(r.output.ReadAndTruncate()) + r.logger.Info("Finished job %s", r.job.ID) + }() - // Collect the finished process' exit status - exitStatus = fmt.Sprintf("%d", r.process.WaitStatus().ExitStatus()) - if ws := r.process.WaitStatus(); ws.Signaled() { - signal = process.SignalString(ws.Signal()) - } - if r.stopped { - // The agent is being gracefully stopped, and we signaled the job to end. Often due - // to pending host shutdown or EC2 spot instance termination - signalReason = "agent_stop" - } else if r.cancelled { - // The job was signaled because it was cancelled via the buildkite web UI - signalReason = "cancel" - } - } + if !hookExit.Ok { + return nil } - // Store the finished at time - finishedAt := time.Now() - - // Stop the header time streamer. This will block until all the chunks - // have been uploaded - r.headerTimesStreamer.Stop() + // Kick off log streaming and job status checking when the process starts. + wg.Add(2) + go r.jobLogStreamer(cctx, &wg) + go r.jobCancellationChecker(cctx, &wg) - // Stop the log streamer. This will block until all the chunks have - // been uploaded - r.logStreamer.Stop() + // Run the process. This will block until it finishes. + if err := r.process.Run(cctx); err != nil { + // Send the error as output + r.logStreamer.Process([]byte(err.Error())) - // Warn about failed chunks - if count := r.logStreamer.FailedChunks(); count > 0 { - r.logger.Warn("%d chunks failed to upload for this job", count) - } - - // Ensure the additional goroutines are stopped. - cancel() + // The process did not run at all, so make sure it fails + exitStatus = "-1" + signalReason = "process_run_error" + } else { + // Intended to capture situations where the job-exec (aka bootstrap) container did not + // start. Normally such errors are hidden in the Kubernetes events. Let's feed them up + // to the user as they may be the caused by errors in the pipeline definition. + if r.cancelled && !r.stopped { + k8sProcess, ok := r.process.(*kubernetes.Runner) + if ok && k8sProcess.ClientStateUnknown() { + r.logStreamer.Process([]byte( + "Some containers had unknown exit statuses. Perhaps they were in ImagePullBackOff.", + )) + } + } - // Wait for the routines that we spun up to finish - r.logger.Debug("[JobRunner] Waiting for all other routines to finish") - wg.Wait() + // Add the final output to the streamer + r.logStreamer.Process(r.output.ReadAndTruncate()) - // Remove the env file, if any - if r.envFile != nil { - if err := os.Remove(r.envFile.Name()); err != nil { - r.logger.Warn("[JobRunner] Error cleaning up env file: %s", err) + // Collect the finished process' exit status + exitStatus = fmt.Sprintf("%d", r.process.WaitStatus().ExitStatus()) + if ws := r.process.WaitStatus(); ws.Signaled() { + signal = process.SignalString(ws.Signal()) + } + if r.stopped { + // The agent is being gracefully stopped, and we signaled the job to end. Often due + // to pending host shutdown or EC2 spot instance termination + signalReason = "agent_stop" + } else if r.cancelled { + // The job was signaled because it was cancelled via the buildkite web UI + signalReason = "cancel" } - r.logger.Debug("[JobRunner] Deleted env file: %s", r.envFile.Name()) } - // Write some metrics about the job run - jobMetrics := r.metrics.With(metrics.Tags{ - "exit_code": exitStatus, - }) - if exitStatus == "0" { - jobMetrics.Timing("jobs.duration.success", finishedAt.Sub(startedAt)) - jobMetrics.Count("jobs.success", 1) - } else { - jobMetrics.Timing("jobs.duration.error", finishedAt.Sub(startedAt)) - jobMetrics.Count("jobs.failed", 1) - } + return nil +} - // Finish the build in the Buildkite Agent API - // - // Once we tell the API we're finished it might assign us new work, so make - // sure everything else is done first. - r.finishJob(ctx, finishedAt, exitStatus, signal, signalReason, r.logStreamer.FailedChunks()) +func (r *JobRunner) preExecHooks(ctx context.Context) hookExit { + hookExit := r.preExecHook(ctx, "pre-bootstrap") + if !hookExit.Ok { + return hookExit + } - r.logger.Info("Finished job %s", r.job.ID) + return r.preExecHook(ctx, "pre-exec") - return nil } func (r *JobRunner) CancelAndStop() error { @@ -595,7 +575,7 @@ func (r *JobRunner) createEnvironment() ([]string, error) { } } - // Set BUILDKITE_IGNORED_ENV so the bootstrap can show warnings + // Set BUILDKITE_IGNORED_ENV so the job executor can show warnings if len(ignoredEnv) > 0 { env["BUILDKITE_IGNORED_ENV"] = strings.Join(ignoredEnv, ",") } @@ -645,19 +625,19 @@ func (r *JobRunner) createEnvironment() ([]string, error) { env["BUILDKITE_AGENT_EXPERIMENT"] = strings.Join(experiments.Enabled(), ",") env["BUILDKITE_REDACTED_VARS"] = strings.Join(r.conf.AgentConfiguration.RedactedVars, ",") - // propagate CancelSignal to bootstrap, unless it's the default SIGTERM + // propagate CancelSignal to job executor, unless it's the default SIGTERM if r.conf.CancelSignal != process.SIGTERM { env["BUILDKITE_CANCEL_SIGNAL"] = r.conf.CancelSignal.String() } - // Whether to enable profiling in the bootstrap + // Whether to enable profiling in the job executor if r.conf.AgentConfiguration.Profile != "" { env["BUILDKITE_AGENT_PROFILE"] = r.conf.AgentConfiguration.Profile } - // PTY-mode is enabled by default in `start` and `bootstrap`, so we only need + // PTY-mode is enabled by default in `start` and `job-exec`, so we only need // to propagate it if it's explicitly disabled. - if r.conf.AgentConfiguration.RunInPty == false { + if !r.conf.AgentConfiguration.RunInPty { env["BUILDKITE_PTY"] = "false" } @@ -721,8 +701,8 @@ func (w LogWriter) Write(bytes []byte) (int, error) { return len(bytes), nil } -func (r *JobRunner) executePreBootstrapHook(ctx context.Context, hook string) (bool, error) { - r.logger.Info("Running pre-bootstrap hook %q", hook) +func (r *JobRunner) executePreExecHook(ctx context.Context, hookName, hookPath string) (bool, error) { + r.logger.Info("Running %s hook %q", hookName, hookPath) sh, err := shell.New() if err != nil { @@ -730,19 +710,19 @@ func (r *JobRunner) executePreBootstrapHook(ctx context.Context, hook string) (b } // This (plus inherited) is the only ENV that should be exposed - // to the pre-bootstrap hook. + // to the pre-exec hook. sh.Env.Set("BUILDKITE_ENV_FILE", r.envFile.Name()) sh.Writer = LogWriter{ l: r.logger, } - if err := sh.RunWithoutPrompt(ctx, hook); err != nil { - r.logger.Error("Finished pre-bootstrap hook %q: job rejected", hook) + if err := sh.RunWithoutPrompt(ctx, hookPath); err != nil { + r.logger.Error("Finished %s hook %q: job rejected", hookName, hookPath) return false, err } - r.logger.Info("Finished pre-bootstrap hook %q: job accepted", hook) + r.logger.Info("Finished %s hook %q: job accepted", hookName, hookPath) return true, nil } @@ -776,18 +756,61 @@ func (r *JobRunner) startJob(ctx context.Context, startedAt time.Time) error { // finishJob finishes the job in the Buildkite Agent API. If the FinishJob call // cannot return successfully, this will retry for a long time. -func (r *JobRunner) finishJob(ctx context.Context, finishedAt time.Time, exitStatus, signal, signalReason string, failedChunkCount int) error { +func (r *JobRunner) finishJob(ctx context.Context, startedAt time.Time, cancel func(), wg *sync.WaitGroup, exitStatus, signal, signalReason string) error { + finishedAt := time.Now() + + // Stop the header time streamer. This will block until all the chunks + // have been uploaded + r.headerTimesStreamer.Stop() + + // Stop the log streamer. This will block until all the chunks have + // been uploaded + r.logStreamer.Stop() + + // Warn about failed chunks + if count := r.logStreamer.FailedChunks(); count > 0 { + r.logger.Warn("%d chunks failed to upload for this job", count) + } + + // Ensure the additional goroutines are stopped. + cancel() + + // Wait for the routines that we spun up to finish + r.logger.Debug("[JobRunner] Waiting for all other routines to finish") + wg.Wait() + + // Remove the env file, if any + if r.envFile != nil { + if err := os.Remove(r.envFile.Name()); err != nil { + r.logger.Warn("[JobRunner] Error cleaning up env file: %s", err) + } + r.logger.Debug("[JobRunner] Deleted env file: %s", r.envFile.Name()) + } + + // Write some metrics about the job run + jobMetrics := r.metrics.With(metrics.Tags{ + "exit_code": exitStatus, + }) + + if exitStatus == "0" { + jobMetrics.Timing("jobs.duration.success", finishedAt.Sub(startedAt)) + jobMetrics.Count("jobs.success", 1) + } else { + jobMetrics.Timing("jobs.duration.error", finishedAt.Sub(startedAt)) + jobMetrics.Count("jobs.failed", 1) + } + r.job.FinishedAt = finishedAt.UTC().Format(time.RFC3339Nano) r.job.ExitStatus = exitStatus r.job.Signal = signal r.job.SignalReason = signalReason - r.job.ChunksFailedCount = failedChunkCount + r.job.ChunksFailedCount = r.logStreamer.FailedChunks() r.logger.Debug("[JobRunner] Finishing job with exit_status=%s, signal=%s and signal_reason=%s", r.job.ExitStatus, r.job.Signal, r.job.SignalReason) - ctx, cancel := context.WithTimeout(ctx, 48*time.Hour) - defer cancel() + ctx, ccancel := context.WithTimeout(ctx, 48*time.Hour) + defer ccancel() return roko.NewRetrier( roko.TryForever(), diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index eac2c559a9..db4e803127 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -18,11 +18,11 @@ import ( "github.com/buildkite/agent/v3/agent" "github.com/buildkite/agent/v3/api" - "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/cliconfig" "github.com/buildkite/agent/v3/experiments" "github.com/buildkite/agent/v3/hook" "github.com/buildkite/agent/v3/internal/agentapi" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/agent/v3/logger" "github.com/buildkite/agent/v3/metrics" "github.com/buildkite/agent/v3/process" @@ -42,7 +42,7 @@ const startDescription = `Usage: Description: - When a job is ready to run it will call the "bootstrap-script" + When a job is ready to run it will call the "job-run-script" and pass it all the environment variables required for the job to run. This script is responsible for checking out the code, and running the actual build script defined in the pipeline. @@ -56,8 +56,8 @@ Example: // Adding config requires changes in a few different spots // - The AgentStartConfig struct with a cli parameter // - As a flag in the AgentStartCommand (with matching env) -// - Into an env to be passed to the bootstrap in agent/job_runner.go, createEnvironment() -// - Into clicommand/bootstrap.go to read it from the env into the bootstrap config +// - Into an env to be passed to the job executor in agent/job_runner.go, createEnvironment() +// - Into clicommand/run_job.go to read it from the env into the job executor config type AgentStartConfig struct { Config string `cli:"config"` @@ -66,7 +66,7 @@ type AgentStartConfig struct { AcquireJob string `cli:"acquire-job"` DisconnectAfterJob bool `cli:"disconnect-after-job"` DisconnectAfterIdleTimeout int `cli:"disconnect-after-idle-timeout"` - BootstrapScript string `cli:"bootstrap-script" normalize:"commandpath"` + JobRunScript string `cli:"job-run-script" normalize:"commandpath"` CancelGracePeriod int `cli:"cancel-grace-period"` EnableJobLogTmpfile bool `cli:"enable-job-log-tmpfile"` WriteJobLogsToStdout bool `cli:"write-job-logs-to-stdout"` @@ -138,6 +138,7 @@ type AgentStartConfig struct { MetaDataGCP bool `cli:"meta-data-gcp" deprecated-and-renamed-to:"TagsFromGCP"` TagsFromEC2 bool `cli:"tags-from-ec2" deprecated-and-renamed-to:"TagsFromEC2MetaData"` TagsFromGCP bool `cli:"tags-from-gcp" deprecated-and-renamed-to:"TagsFromGCPMetaData"` + BootstrapScript string `cli:"bootstrap-script" deprecated-and-renamed-to:"JobRunScript" normalize:"commandpath"` DisconnectAfterJobTimeout int `cli:"disconnect-after-job-timeout" deprecated:"Use disconnect-after-idle-timeout instead"` } @@ -426,9 +427,15 @@ var AgentStartCommand = cli.Command{ cli.StringFlag{ Name: "bootstrap-script", Value: "", - Usage: "The command that is executed for bootstrapping a job, defaults to the bootstrap sub-command of this binary", + Usage: "[DEPRECATED] The command that is executed for running a job, defaults to the `buildkite-agent job run`", EnvVar: "BUILDKITE_BOOTSTRAP_SCRIPT_PATH", }, + cli.StringFlag{ + Name: "job-run-script", + Value: "", + Usage: "The command that is executed for running a job, defaults to the `buildkite-agent job run`", + EnvVar: "BUILDKITE_JOB_EXECUTOR_SCRIPT_PATH", + }, cli.StringFlag{ Name: "build-path", Value: "", @@ -649,7 +656,7 @@ var AgentStartCommand = cli.Command{ done := HandleGlobalFlags(l, cfg) defer done() - // Remove any config env from the environment to prevent them propagating to bootstrap + // Remove any config env from the environment to prevent them propagating to job execution err = UnsetConfigFromEnvironment(c) if err != nil { fmt.Printf("%s", err) @@ -668,13 +675,13 @@ var AgentStartCommand = cli.Command{ cfg.NoPTY = true } - // Set a useful default for the bootstrap script - if cfg.BootstrapScript == "" { + // Set a useful default for the job exec script + if cfg.JobRunScript == "" { exePath, err := os.Executable() if err != nil { - l.Fatal("Unable to find executable path for bootstrap") + l.Fatal("Unable to find our executable path to construct the job executor script: %v", err) } - cfg.BootstrapScript = fmt.Sprintf("%s bootstrap", shellwords.Quote(exePath)) + cfg.JobRunScript = fmt.Sprintf("%s job run", shellwords.Quote(exePath)) } isSetNoPlugins := c.IsSet("no-plugins") @@ -757,7 +764,7 @@ var AgentStartCommand = cli.Command{ DatadogDistributions: cfg.MetricsDatadogDistributions, }) - // Sense check supported tracing backends, we don't want bootstrapped jobs to silently have no tracing + // Sense check supported tracing backends, we don't want jobs to silently have no tracing if _, has := tracetools.ValidTracingBackends[cfg.TracingBackend]; !has { l.Fatal("The given tracing backend %q is not supported. Valid backends are: %q", cfg.TracingBackend, maps.Keys(tracetools.ValidTracingBackends)) } @@ -769,7 +776,7 @@ var AgentStartCommand = cli.Command{ // AgentConfiguration is the runtime configuration for an agent agentConf := agent.AgentConfiguration{ - BootstrapScript: cfg.BootstrapScript, + JobRunScript: cfg.JobRunScript, BuildPath: cfg.BuildPath, SocketsPath: cfg.SocketsPath, GitMirrorsPath: cfg.GitMirrorsPath, @@ -837,7 +844,7 @@ var AgentStartCommand = cli.Command{ l.WithFields(logger.StringField(`path`, agentConf.ConfigPath)).Info("Configuration loaded") } - l.Debug("Bootstrap command: %s", agentConf.BootstrapScript) + l.Debug("Job Exec command: %s", agentConf.JobRunScript) l.Debug("Build path: %s", agentConf.BuildPath) l.Debug("Hooks directory: %s", agentConf.HooksPath) l.Debug("Plugins directory: %s", agentConf.PluginsPath) @@ -871,7 +878,7 @@ var AgentStartCommand = cli.Command{ l.Fatal("Failed to parse cancel-signal: %v", err) } - // confirm the BuildPath is exists. The bootstrap is going to write to it when a job executes, + // confirm the BuildPath is exists. The job executor is going to write to it when a job executes, // so we may as well check that'll work now and fail early if it's a problem if !utils.FileExists(agentConf.BuildPath) { l.Info("Build Path doesn't exist, creating it (%s)", agentConf.BuildPath) diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go deleted file mode 100644 index 28bf29b118..0000000000 --- a/clicommand/bootstrap.go +++ /dev/null @@ -1,523 +0,0 @@ -package clicommand - -import ( - "context" - "fmt" - "os" - "os/signal" - "runtime" - "sync" - "syscall" - - "github.com/buildkite/agent/v3/bootstrap" - "github.com/buildkite/agent/v3/cliconfig" - "github.com/buildkite/agent/v3/experiments" - "github.com/buildkite/agent/v3/process" - "github.com/urfave/cli" -) - -const bootstrapHelpDescription = `Usage: - - buildkite-agent bootstrap [options...] - -Description: - - The bootstrap command executes a Buildkite job locally. - - Generally the bootstrap command is run as a sub-process of the buildkite-agent to execute - a given job sent from buildkite.com, but you can also invoke the bootstrap manually. - - Execution is broken down into phases. By default, the bootstrap runs a plugin phase which - sets up any plugins specified, then a checkout phase which pulls down your code and then a - command phase that executes the specified command in the created environment. - - You can run only specific phases with the --phases flag. - - The bootstrap is also responsible for executing hooks around the phases. - See https://buildkite.com/docs/agent/v3/hooks for more details. - -Example: - - $ eval $(curl -s -H "Authorization: Bearer xxx" \ - "https://api.buildkite.com/v2/organizations/[org]/pipelines/[proj]/builds/[build]/jobs/[job]/env.txt" | sed 's/^/export /') - $ buildkite-agent bootstrap --build-path builds` - -type BootstrapConfig struct { - Command string `cli:"command"` - JobID string `cli:"job" validate:"required"` - Repository string `cli:"repository" validate:"required"` - Commit string `cli:"commit" validate:"required"` - Branch string `cli:"branch" validate:"required"` - Tag string `cli:"tag"` - RefSpec string `cli:"refspec"` - Plugins string `cli:"plugins"` - PullRequest string `cli:"pullrequest"` - GitSubmodules bool `cli:"git-submodules"` - SSHKeyscan bool `cli:"ssh-keyscan"` - AgentName string `cli:"agent" validate:"required"` - Queue string `cli:"queue"` - OrganizationSlug string `cli:"organization" validate:"required"` - PipelineSlug string `cli:"pipeline" validate:"required"` - PipelineProvider string `cli:"pipeline-provider" validate:"required"` - AutomaticArtifactUploadPaths string `cli:"artifact-upload-paths"` - ArtifactUploadDestination string `cli:"artifact-upload-destination"` - CleanCheckout bool `cli:"clean-checkout"` - GitCheckoutFlags string `cli:"git-checkout-flags"` - GitCloneFlags string `cli:"git-clone-flags"` - GitFetchFlags string `cli:"git-fetch-flags"` - GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"` - GitCleanFlags string `cli:"git-clean-flags"` - GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"` - GitMirrorsLockTimeout int `cli:"git-mirrors-lock-timeout"` - GitMirrorsSkipUpdate bool `cli:"git-mirrors-skip-update"` - GitSubmoduleCloneConfig []string `cli:"git-submodule-clone-config"` - BinPath string `cli:"bin-path" normalize:"filepath"` - BuildPath string `cli:"build-path" normalize:"filepath"` - HooksPath string `cli:"hooks-path" normalize:"filepath"` - SocketsPath string `cli:"sockets-path" normalize:"filepath"` - PluginsPath string `cli:"plugins-path" normalize:"filepath"` - CommandEval bool `cli:"command-eval"` - PluginsEnabled bool `cli:"plugins-enabled"` - PluginValidation bool `cli:"plugin-validation"` - PluginsAlwaysCloneFresh bool `cli:"plugins-always-clone-fresh"` - LocalHooksEnabled bool `cli:"local-hooks-enabled"` - PTY bool `cli:"pty"` - LogLevel string `cli:"log-level"` - Debug bool `cli:"debug"` - Shell string `cli:"shell"` - Experiments []string `cli:"experiment" normalize:"list"` - Phases []string `cli:"phases" normalize:"list"` - Profile string `cli:"profile"` - CancelSignal string `cli:"cancel-signal"` - RedactedVars []string `cli:"redacted-vars" normalize:"list"` - TracingBackend string `cli:"tracing-backend"` - TracingServiceName string `cli:"tracing-service-name"` -} - -var BootstrapCommand = cli.Command{ - Name: "bootstrap", - Usage: "Run a Buildkite job locally", - Description: bootstrapHelpDescription, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "command", - Value: "", - Usage: "The command to run", - EnvVar: "BUILDKITE_COMMAND", - }, - cli.StringFlag{ - Name: "job", - Value: "", - Usage: "The ID of the job being run", - EnvVar: "BUILDKITE_JOB_ID", - }, - cli.StringFlag{ - Name: "repository", - Value: "", - Usage: "The repository to clone and run the job from", - EnvVar: "BUILDKITE_REPO", - }, - cli.StringFlag{ - Name: "commit", - Value: "", - Usage: "The commit to checkout in the repository", - EnvVar: "BUILDKITE_COMMIT", - }, - cli.StringFlag{ - Name: "branch", - Value: "", - Usage: "The branch the commit is in", - EnvVar: "BUILDKITE_BRANCH", - }, - cli.StringFlag{ - Name: "tag", - Value: "", - Usage: "The tag the commit", - EnvVar: "BUILDKITE_TAG", - }, - cli.StringFlag{ - Name: "refspec", - Value: "", - Usage: "Optional refspec to override git fetch", - EnvVar: "BUILDKITE_REFSPEC", - }, - cli.StringFlag{ - Name: "plugins", - Value: "", - Usage: "The plugins for the job", - EnvVar: "BUILDKITE_PLUGINS", - }, - cli.StringFlag{ - Name: "pullrequest", - Value: "", - Usage: "The number/id of the pull request this commit belonged to", - EnvVar: "BUILDKITE_PULL_REQUEST", - }, - cli.StringFlag{ - Name: "agent", - Value: "", - Usage: "The name of the agent running the job", - EnvVar: "BUILDKITE_AGENT_NAME", - }, - cli.StringFlag{ - Name: "queue", - Value: "", - Usage: "The name of the queue the agent belongs to, if tagged", - EnvVar: "BUILDKITE_AGENT_META_DATA_QUEUE", - }, - cli.StringFlag{ - Name: "organization", - Value: "", - Usage: "The slug of the organization that the job is a part of", - EnvVar: "BUILDKITE_ORGANIZATION_SLUG", - }, - cli.StringFlag{ - Name: "pipeline", - Value: "", - Usage: "The slug of the pipeline that the job is a part of", - EnvVar: "BUILDKITE_PIPELINE_SLUG", - }, - cli.StringFlag{ - Name: "pipeline-provider", - Value: "", - Usage: "The id of the SCM provider that the repository is hosted on", - EnvVar: "BUILDKITE_PIPELINE_PROVIDER", - }, - cli.StringFlag{ - Name: "artifact-upload-paths", - Value: "", - Usage: "Paths to files to automatically upload at the end of a job", - EnvVar: "BUILDKITE_ARTIFACT_PATHS", - }, - cli.StringFlag{ - Name: "artifact-upload-destination", - Value: "", - Usage: "A custom location to upload artifact paths to (for example, s3://my-custom-bucket/and/prefix)", - EnvVar: "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION", - }, - cli.BoolFlag{ - Name: "clean-checkout", - Usage: "Whether or not the bootstrap should remove the existing repository before running the command", - EnvVar: "BUILDKITE_CLEAN_CHECKOUT", - }, - cli.StringFlag{ - Name: "git-checkout-flags", - Value: "-f", - Usage: "Flags to pass to \"git checkout\" command", - EnvVar: "BUILDKITE_GIT_CHECKOUT_FLAGS", - }, - cli.StringFlag{ - Name: "git-clone-flags", - Value: "-v", - Usage: "Flags to pass to \"git clone\" command", - EnvVar: "BUILDKITE_GIT_CLONE_FLAGS", - }, - cli.StringFlag{ - Name: "git-clone-mirror-flags", - Value: "-v", - Usage: "Flags to pass to \"git clone\" command when mirroring", - EnvVar: "BUILDKITE_GIT_CLONE_MIRROR_FLAGS", - }, - cli.StringFlag{ - Name: "git-clean-flags", - Value: "-ffxdq", - Usage: "Flags to pass to \"git clean\" command", - EnvVar: "BUILDKITE_GIT_CLEAN_FLAGS", - }, - cli.StringFlag{ - Name: "git-fetch-flags", - Value: "", - Usage: "Flags to pass to \"git fetch\" command", - EnvVar: "BUILDKITE_GIT_FETCH_FLAGS", - }, - cli.StringSliceFlag{ - Name: "git-submodule-clone-config", - Value: &cli.StringSlice{}, - Usage: "Comma separated key=value git config pairs applied before git submodule clone commands. For example, ′update --init′. If the config is needed to be applied to all git commands, supply it in a global git config file for the system that the agent runs in instead.", - EnvVar: "BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG", - }, - cli.StringFlag{ - Name: "git-mirrors-path", - Value: "", - Usage: "Path to where mirrors of git repositories are stored", - EnvVar: "BUILDKITE_GIT_MIRRORS_PATH", - }, - cli.IntFlag{ - Name: "git-mirrors-lock-timeout", - Value: 300, - Usage: "Seconds to lock a git mirror during clone, should exceed your longest checkout", - EnvVar: "BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT", - }, - cli.BoolFlag{ - Name: "git-mirrors-skip-update", - Usage: "Skip updating the Git mirror", - EnvVar: "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE", - }, - cli.StringFlag{ - Name: "bin-path", - Value: "", - Usage: "Directory where the buildkite-agent binary lives", - EnvVar: "BUILDKITE_BIN_PATH", - }, - cli.StringFlag{ - Name: "build-path", - Value: "", - Usage: "Directory where builds will be created", - EnvVar: "BUILDKITE_BUILD_PATH", - }, - cli.StringFlag{ - Name: "hooks-path", - Value: "", - Usage: "Directory where the hook scripts are found", - EnvVar: "BUILDKITE_HOOKS_PATH", - }, - cli.StringFlag{ - Name: "sockets-path", - Value: defaultSocketsPath(), - Usage: "Directory where the agent will place sockets", - EnvVar: "BUILDKITE_SOCKETS_PATH", - }, - cli.StringFlag{ - Name: "plugins-path", - Value: "", - Usage: "Directory where the plugins are saved to", - EnvVar: "BUILDKITE_PLUGINS_PATH", - }, - cli.BoolTFlag{ - Name: "command-eval", - Usage: "Allow running of arbitrary commands", - EnvVar: "BUILDKITE_COMMAND_EVAL", - }, - cli.BoolTFlag{ - Name: "plugins-enabled", - Usage: "Allow plugins to be run", - EnvVar: "BUILDKITE_PLUGINS_ENABLED", - }, - cli.BoolFlag{ - Name: "plugin-validation", - Usage: "Validate plugin configuration", - EnvVar: "BUILDKITE_PLUGIN_VALIDATION", - }, - cli.BoolFlag{ - Name: "plugins-always-clone-fresh", - Usage: "Always make a new clone of plugin source, even if already present", - EnvVar: "BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH", - }, - cli.BoolTFlag{ - Name: "local-hooks-enabled", - Usage: "Allow local hooks to be run", - EnvVar: "BUILDKITE_LOCAL_HOOKS_ENABLED", - }, - cli.BoolTFlag{ - Name: "ssh-keyscan", - Usage: "Automatically run ssh-keyscan before checkout", - EnvVar: "BUILDKITE_SSH_KEYSCAN", - }, - cli.BoolTFlag{ - Name: "git-submodules", - Usage: "Enable git submodules", - EnvVar: "BUILDKITE_GIT_SUBMODULES", - }, - cli.BoolTFlag{ - Name: "pty", - Usage: "Run jobs within a pseudo terminal", - EnvVar: "BUILDKITE_PTY", - }, - cli.StringFlag{ - Name: "shell", - Usage: "The shell to use to interpret build commands", - EnvVar: "BUILDKITE_SHELL", - Value: DefaultShell(), - }, - cli.StringSliceFlag{ - Name: "phases", - Usage: "The specific phases to execute. The order they're defined is irrelevant.", - EnvVar: "BUILDKITE_BOOTSTRAP_PHASES", - }, - cli.StringFlag{ - Name: "cancel-signal", - Usage: "The signal to use for cancellation", - EnvVar: "BUILDKITE_CANCEL_SIGNAL", - Value: "SIGTERM", - }, - cli.StringSliceFlag{ - Name: "redacted-vars", - Usage: "Pattern of environment variable names containing sensitive values", - EnvVar: "BUILDKITE_REDACTED_VARS", - }, - cli.StringFlag{ - Name: "tracing-backend", - Usage: "The name of the tracing backend to use.", - EnvVar: "BUILDKITE_TRACING_BACKEND", - Value: "", - }, - cli.StringFlag{ - Name: "tracing-service-name", - Usage: "Service name to use when reporting traces.", - EnvVar: "BUILDKITE_TRACING_SERVICE_NAME", - Value: "buildkite-agent", - }, - DebugFlag, - LogLevelFlag, - ExperimentsFlag, - ProfileFlag, - }, - Action: func(c *cli.Context) { - // The configuration will be loaded into this struct - cfg := BootstrapConfig{} - - loader := cliconfig.Loader{CLI: c, Config: &cfg} - warnings, err := loader.Load() - if err != nil { - fmt.Printf("%s", err) - os.Exit(1) - } - - l := CreateLogger(&cfg) - - // Now that we have a logger, log out the warnings that loading config generated - for _, warning := range warnings { - l.Warn("%s", warning) - } - - // Enable experiments - for _, name := range cfg.Experiments { - known := experiments.Enable(name) - if !known { - l.Warn("Unknown experiment enabled: %q", name) - } - } - - // Handle profiling flag - done := HandleProfileFlag(l, cfg) - defer done() - - // Turn of PTY support if we're on Windows - runInPty := cfg.PTY - if runtime.GOOS == "windows" { - runInPty = false - } - - // Validate phases - for _, phase := range cfg.Phases { - switch phase { - case "plugin", "checkout", "command": - // Valid phase - default: - l.Fatal("Invalid phase %q", phase) - } - } - - cancelSig, err := process.ParseSignal(cfg.CancelSignal) - if err != nil { - l.Fatal("Failed to parse cancel-signal: %v", err) - } - - // Configure the bootstraper - bootstrap := bootstrap.New(bootstrap.Config{ - AgentName: cfg.AgentName, - ArtifactUploadDestination: cfg.ArtifactUploadDestination, - AutomaticArtifactUploadPaths: cfg.AutomaticArtifactUploadPaths, - BinPath: cfg.BinPath, - Branch: cfg.Branch, - BuildPath: cfg.BuildPath, - SocketsPath: cfg.SocketsPath, - CancelSignal: cancelSig, - CleanCheckout: cfg.CleanCheckout, - Command: cfg.Command, - CommandEval: cfg.CommandEval, - Commit: cfg.Commit, - Debug: cfg.Debug, - GitCheckoutFlags: cfg.GitCheckoutFlags, - GitCleanFlags: cfg.GitCleanFlags, - GitCloneFlags: cfg.GitCloneFlags, - GitCloneMirrorFlags: cfg.GitCloneMirrorFlags, - GitFetchFlags: cfg.GitFetchFlags, - GitMirrorsLockTimeout: cfg.GitMirrorsLockTimeout, - GitMirrorsPath: cfg.GitMirrorsPath, - GitMirrorsSkipUpdate: cfg.GitMirrorsSkipUpdate, - GitSubmodules: cfg.GitSubmodules, - GitSubmoduleCloneConfig: cfg.GitSubmoduleCloneConfig, - HooksPath: cfg.HooksPath, - JobID: cfg.JobID, - LocalHooksEnabled: cfg.LocalHooksEnabled, - OrganizationSlug: cfg.OrganizationSlug, - Phases: cfg.Phases, - PipelineProvider: cfg.PipelineProvider, - PipelineSlug: cfg.PipelineSlug, - PluginValidation: cfg.PluginValidation, - Plugins: cfg.Plugins, - PluginsEnabled: cfg.PluginsEnabled, - PluginsAlwaysCloneFresh: cfg.PluginsAlwaysCloneFresh, - PluginsPath: cfg.PluginsPath, - PullRequest: cfg.PullRequest, - Queue: cfg.Queue, - RedactedVars: cfg.RedactedVars, - RefSpec: cfg.RefSpec, - Repository: cfg.Repository, - RunInPty: runInPty, - SSHKeyscan: cfg.SSHKeyscan, - Shell: cfg.Shell, - Tag: cfg.Tag, - TracingBackend: cfg.TracingBackend, - TracingServiceName: cfg.TracingServiceName, - }) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt, - syscall.SIGHUP, - syscall.SIGTERM, - syscall.SIGINT, - syscall.SIGQUIT) - defer signal.Stop(signals) - - var ( - cancelled bool - received os.Signal - signalMu sync.Mutex - ) - - // Listen for signals in the background and cancel the bootstrap - go func() { - sig := <-signals - signalMu.Lock() - defer signalMu.Unlock() - - // Cancel the bootstrap - bootstrap.Cancel() - - // Track the state and signal used - cancelled = true - received = sig - - // Remove our signal handler so subsequent signals kill - signal.Stop(signals) - }() - - // Run the bootstrap and get the exit code - exitCode := bootstrap.Run(ctx) - - signalMu.Lock() - defer signalMu.Unlock() - - // If cancelled and our child process returns a non-zero, we should terminate - // ourselves with the same signal so that our caller can detect and handle appropriately - if cancelled && runtime.GOOS != "windows" { - p, err := os.FindProcess(os.Getpid()) - if err != nil { - l.Error("Failed to find current process: %v", err) - } - - l.Debug("Terminating bootstrap after cancellation with %v", received) - err = p.Signal(received) - if err != nil { - l.Error("Failed to signal self: %v", err) - } - } - - os.Exit(exitCode) - }, -} diff --git a/clicommand/job_run.go b/clicommand/job_run.go new file mode 100644 index 0000000000..c353ae0a26 --- /dev/null +++ b/clicommand/job_run.go @@ -0,0 +1,570 @@ +package clicommand + +import ( + "context" + "fmt" + "os" + "os/signal" + "runtime" + "strings" + "sync" + "syscall" + "text/template" + + "github.com/buildkite/agent/v3/cliconfig" + "github.com/buildkite/agent/v3/experiments" + "github.com/buildkite/agent/v3/job" + "github.com/buildkite/agent/v3/process" + "github.com/urfave/cli" +) + +var jobRunHelpTpl = template.Must(template.New("jobRunHelp").Parse(`Usage: + + buildkite-agent {{.}} [options...] + +Description: + + The {{.}} command executes a Buildkite job locally. + + Generally the {{.}} command is run as a sub-process of the buildkite-agent to execute + a given job sent from buildkite.com, but you can also invoke the {{.}} manually. + + Execution is broken down into phases. By default, the {{.}} runs a plugin phase which + sets up any plugins specified, then a checkout phase which pulls down your code and then a + command phase that executes the specified command in the created environment. + + You can run only specific phases with the --phases flag. + + The {{.}} is also responsible for executing hooks around the phases. + See https://buildkite.com/docs/agent/v3/hooks for more details. + +Example: + + $ eval $(curl -s -H "Authorization: Bearer xxx" \ + "https://api.buildkite.com/v2/organizations/[org]/pipelines/[proj]/builds/[build]/jobs/[job]/env.txt" | sed 's/^/export /') + $ buildkite-agent {{.}} --build-path builds`)) + +type JobRunConfig struct { + Command string `cli:"command"` + JobID string `cli:"job" validate:"required"` + Repository string `cli:"repository" validate:"required"` + Commit string `cli:"commit" validate:"required"` + Branch string `cli:"branch" validate:"required"` + Tag string `cli:"tag"` + RefSpec string `cli:"refspec"` + Plugins string `cli:"plugins"` + PullRequest string `cli:"pullrequest"` + GitSubmodules bool `cli:"git-submodules"` + SSHKeyscan bool `cli:"ssh-keyscan"` + AgentName string `cli:"agent" validate:"required"` + Queue string `cli:"queue"` + OrganizationSlug string `cli:"organization" validate:"required"` + PipelineSlug string `cli:"pipeline" validate:"required"` + PipelineProvider string `cli:"pipeline-provider" validate:"required"` + AutomaticArtifactUploadPaths string `cli:"artifact-upload-paths"` + ArtifactUploadDestination string `cli:"artifact-upload-destination"` + CleanCheckout bool `cli:"clean-checkout"` + GitCheckoutFlags string `cli:"git-checkout-flags"` + GitCloneFlags string `cli:"git-clone-flags"` + GitFetchFlags string `cli:"git-fetch-flags"` + GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"` + GitCleanFlags string `cli:"git-clean-flags"` + GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"` + GitMirrorsLockTimeout int `cli:"git-mirrors-lock-timeout"` + GitMirrorsSkipUpdate bool `cli:"git-mirrors-skip-update"` + GitSubmoduleCloneConfig []string `cli:"git-submodule-clone-config"` + BinPath string `cli:"bin-path" normalize:"filepath"` + BuildPath string `cli:"build-path" normalize:"filepath"` + HooksPath string `cli:"hooks-path" normalize:"filepath"` + PluginsPath string `cli:"plugins-path" normalize:"filepath"` + CommandEval bool `cli:"command-eval"` + PluginsEnabled bool `cli:"plugins-enabled"` + PluginValidation bool `cli:"plugin-validation"` + PluginsAlwaysCloneFresh bool `cli:"plugins-always-clone-fresh"` + LocalHooksEnabled bool `cli:"local-hooks-enabled"` + PTY bool `cli:"pty"` + LogLevel string `cli:"log-level"` + Debug bool `cli:"debug"` + Shell string `cli:"shell"` + Experiments []string `cli:"experiment" normalize:"list"` + Phases []string `cli:"phases" normalize:"list"` + Profile string `cli:"profile"` + CancelSignal string `cli:"cancel-signal"` + RedactedVars []string `cli:"redacted-vars" normalize:"list"` + TracingBackend string `cli:"tracing-backend"` + TracingServiceName string `cli:"tracing-service-name"` +} + +var jobRunFlags = []cli.Flag{ + cli.StringFlag{ + Name: "command", + Value: "", + Usage: "The command to run", + EnvVar: "BUILDKITE_COMMAND", + }, + cli.StringFlag{ + Name: "job", + Value: "", + Usage: "The ID of the job being run", + EnvVar: "BUILDKITE_JOB_ID", + }, + cli.StringFlag{ + Name: "repository", + Value: "", + Usage: "The repository to clone and run the job from", + EnvVar: "BUILDKITE_REPO", + }, + cli.StringFlag{ + Name: "commit", + Value: "", + Usage: "The commit to checkout in the repository", + EnvVar: "BUILDKITE_COMMIT", + }, + cli.StringFlag{ + Name: "branch", + Value: "", + Usage: "The branch the commit is in", + EnvVar: "BUILDKITE_BRANCH", + }, + cli.StringFlag{ + Name: "tag", + Value: "", + Usage: "The tag the commit", + EnvVar: "BUILDKITE_TAG", + }, + cli.StringFlag{ + Name: "refspec", + Value: "", + Usage: "Optional refspec to override git fetch", + EnvVar: "BUILDKITE_REFSPEC", + }, + cli.StringFlag{ + Name: "plugins", + Value: "", + Usage: "The plugins for the job", + EnvVar: "BUILDKITE_PLUGINS", + }, + cli.StringFlag{ + Name: "pullrequest", + Value: "", + Usage: "The number/id of the pull request this commit belonged to", + EnvVar: "BUILDKITE_PULL_REQUEST", + }, + cli.StringFlag{ + Name: "agent", + Value: "", + Usage: "The name of the agent running the job", + EnvVar: "BUILDKITE_AGENT_NAME", + }, + cli.StringFlag{ + Name: "queue", + Value: "", + Usage: "The name of the queue the agent belongs to, if tagged", + EnvVar: "BUILDKITE_AGENT_META_DATA_QUEUE", + }, + cli.StringFlag{ + Name: "organization", + Value: "", + Usage: "The slug of the organization that the job is a part of", + EnvVar: "BUILDKITE_ORGANIZATION_SLUG", + }, + cli.StringFlag{ + Name: "pipeline", + Value: "", + Usage: "The slug of the pipeline that the job is a part of", + EnvVar: "BUILDKITE_PIPELINE_SLUG", + }, + cli.StringFlag{ + Name: "pipeline-provider", + Value: "", + Usage: "The id of the SCM provider that the repository is hosted on", + EnvVar: "BUILDKITE_PIPELINE_PROVIDER", + }, + cli.StringFlag{ + Name: "artifact-upload-paths", + Value: "", + Usage: "Paths to files to automatically upload at the end of a job", + EnvVar: "BUILDKITE_ARTIFACT_PATHS", + }, + cli.StringFlag{ + Name: "artifact-upload-destination", + Value: "", + Usage: "A custom location to upload artifact paths to (for example, s3://my-custom-bucket/and/prefix)", + EnvVar: "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION", + }, + cli.BoolFlag{ + Name: "clean-checkout", + Usage: "Whether or not the job executor should remove the existing repository before running the command", + EnvVar: "BUILDKITE_CLEAN_CHECKOUT", + }, + cli.StringFlag{ + Name: "git-checkout-flags", + Value: "-f", + Usage: "Flags to pass to \"git checkout\" command", + EnvVar: "BUILDKITE_GIT_CHECKOUT_FLAGS", + }, + cli.StringFlag{ + Name: "git-clone-flags", + Value: "-v", + Usage: "Flags to pass to \"git clone\" command", + EnvVar: "BUILDKITE_GIT_CLONE_FLAGS", + }, + cli.StringFlag{ + Name: "git-clone-mirror-flags", + Value: "-v", + Usage: "Flags to pass to \"git clone\" command when mirroring", + EnvVar: "BUILDKITE_GIT_CLONE_MIRROR_FLAGS", + }, + cli.StringFlag{ + Name: "git-clean-flags", + Value: "-ffxdq", + Usage: "Flags to pass to \"git clean\" command", + EnvVar: "BUILDKITE_GIT_CLEAN_FLAGS", + }, + cli.StringFlag{ + Name: "git-fetch-flags", + Value: "", + Usage: "Flags to pass to \"git fetch\" command", + EnvVar: "BUILDKITE_GIT_FETCH_FLAGS", + }, + cli.StringSliceFlag{ + Name: "git-submodule-clone-config", + Value: &cli.StringSlice{}, + Usage: "Comma separated key=value git config pairs applied before git submodule clone commands, e.g. `update --init`. If the config is needed to be applied to all git commands, supply it in a global git config file for the system that the agent runs in instead.", + EnvVar: "BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG", + }, + cli.StringFlag{ + Name: "git-mirrors-path", + Value: "", + Usage: "Path to where mirrors of git repositories are stored", + EnvVar: "BUILDKITE_GIT_MIRRORS_PATH", + }, + cli.IntFlag{ + Name: "git-mirrors-lock-timeout", + Value: 300, + Usage: "Seconds to lock a git mirror during clone, should exceed your longest checkout", + EnvVar: "BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT", + }, + cli.BoolFlag{ + Name: "git-mirrors-skip-update", + Usage: "Skip updating the Git mirror", + EnvVar: "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE", + }, + cli.StringFlag{ + Name: "bin-path", + Value: "", + Usage: "Directory where the buildkite-agent binary lives", + EnvVar: "BUILDKITE_BIN_PATH", + }, + cli.StringFlag{ + Name: "build-path", + Value: "", + Usage: "Directory where builds will be created", + EnvVar: "BUILDKITE_BUILD_PATH", + }, + cli.StringFlag{ + Name: "hooks-path", + Value: "", + Usage: "Directory where the hook scripts are found", + EnvVar: "BUILDKITE_HOOKS_PATH", + }, + cli.StringFlag{ + Name: "plugins-path", + Value: "", + Usage: "Directory where the plugins are saved to", + EnvVar: "BUILDKITE_PLUGINS_PATH", + }, + cli.BoolTFlag{ + Name: "command-eval", + Usage: "Allow running of arbitrary commands", + EnvVar: "BUILDKITE_COMMAND_EVAL", + }, + cli.BoolTFlag{ + Name: "plugins-enabled", + Usage: "Allow plugins to be run", + EnvVar: "BUILDKITE_PLUGINS_ENABLED", + }, + cli.BoolFlag{ + Name: "plugin-validation", + Usage: "Validate plugin configuration", + EnvVar: "BUILDKITE_PLUGIN_VALIDATION", + }, + cli.BoolFlag{ + Name: "plugins-always-clone-fresh", + Usage: "Always make a new clone of plugin source, even if already present", + EnvVar: "BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH", + }, + cli.BoolTFlag{ + Name: "local-hooks-enabled", + Usage: "Allow local hooks to be run", + EnvVar: "BUILDKITE_LOCAL_HOOKS_ENABLED", + }, + cli.BoolTFlag{ + Name: "ssh-keyscan", + Usage: "Automatically run ssh-keyscan before checkout", + EnvVar: "BUILDKITE_SSH_KEYSCAN", + }, + cli.BoolTFlag{ + Name: "git-submodules", + Usage: "Enable git submodules", + EnvVar: "BUILDKITE_GIT_SUBMODULES", + }, + cli.BoolTFlag{ + Name: "pty", + Usage: "Run jobs within a pseudo terminal", + EnvVar: "BUILDKITE_PTY", + }, + cli.StringFlag{ + Name: "shell", + Usage: "The shell to use to interpret build commands", + EnvVar: "BUILDKITE_SHELL", + Value: DefaultShell(), + }, + cli.StringSliceFlag{ + Name: "phases", + Usage: "The specific phases to execute. The order they're defined is irrelevant.", + EnvVar: "BUILDKITE_BOOTSTRAP_PHASES,BUILDKITE_JOB_RUN_PHASES", + }, + cli.StringFlag{ + Name: "cancel-signal", + Usage: "The signal to use for cancellation", + EnvVar: "BUILDKITE_CANCEL_SIGNAL", + Value: "SIGTERM", + }, + cli.StringSliceFlag{ + Name: "redacted-vars", + Usage: "Pattern of environment variable names containing sensitive values", + EnvVar: "BUILDKITE_REDACTED_VARS", + }, + cli.StringFlag{ + Name: "tracing-backend", + Usage: "The name of the tracing backend to use.", + EnvVar: "BUILDKITE_TRACING_BACKEND", + Value: "", + }, + cli.StringFlag{ + Name: "tracing-service-name", + Usage: "Service name to use when reporting traces.", + EnvVar: "BUILDKITE_TRACING_SERVICE_NAME", + Value: "buildkite-agent", + }, + DebugFlag, + LogLevelFlag, + ExperimentsFlag, + ProfileFlag, +} + +func jobRunAction(c *cli.Context) { + // The configuration will be loaded into this struct + cfg := JobRunConfig{} + + loader := cliconfig.Loader{CLI: c, Config: &cfg} + warnings, err := loader.Load() + if err != nil { + fmt.Printf("%s", err) + os.Exit(1) + } + + l := CreateLogger(&cfg) + + // Now that we have a logger, log out the warnings that loading config generated + for _, warning := range warnings { + l.Warn("%s", warning) + } + + // Enable experiments + for _, name := range cfg.Experiments { + experiments.Enable(name) + } + + // Handle profiling flag + done := HandleProfileFlag(l, cfg) + defer done() + + // Turn of PTY support if we're on Windows + runInPty := cfg.PTY + if runtime.GOOS == "windows" { + runInPty = false + } + + // Validate phases + for _, phase := range cfg.Phases { + switch phase { + case "plugin", "checkout", "command": + // Valid phase + default: + l.Fatal("Invalid phase %q", phase) + } + } + + cancelSig, err := process.ParseSignal(cfg.CancelSignal) + if err != nil { + l.Fatal("Failed to parse cancel-signal: %v", err) + } + + // Configure the job executor + executor := job.NewExecutor(job.Config{ + AgentName: cfg.AgentName, + ArtifactUploadDestination: cfg.ArtifactUploadDestination, + AutomaticArtifactUploadPaths: cfg.AutomaticArtifactUploadPaths, + BinPath: cfg.BinPath, + Branch: cfg.Branch, + BuildPath: cfg.BuildPath, + CancelSignal: cancelSig, + CleanCheckout: cfg.CleanCheckout, + Command: cfg.Command, + CommandEval: cfg.CommandEval, + Commit: cfg.Commit, + Debug: cfg.Debug, + GitCheckoutFlags: cfg.GitCheckoutFlags, + GitCleanFlags: cfg.GitCleanFlags, + GitCloneFlags: cfg.GitCloneFlags, + GitCloneMirrorFlags: cfg.GitCloneMirrorFlags, + GitFetchFlags: cfg.GitFetchFlags, + GitMirrorsLockTimeout: cfg.GitMirrorsLockTimeout, + GitMirrorsPath: cfg.GitMirrorsPath, + GitMirrorsSkipUpdate: cfg.GitMirrorsSkipUpdate, + GitSubmodules: cfg.GitSubmodules, + GitSubmoduleCloneConfig: cfg.GitSubmoduleCloneConfig, + HooksPath: cfg.HooksPath, + JobID: cfg.JobID, + LocalHooksEnabled: cfg.LocalHooksEnabled, + OrganizationSlug: cfg.OrganizationSlug, + Phases: cfg.Phases, + PipelineProvider: cfg.PipelineProvider, + PipelineSlug: cfg.PipelineSlug, + PluginValidation: cfg.PluginValidation, + Plugins: cfg.Plugins, + PluginsEnabled: cfg.PluginsEnabled, + PluginsAlwaysCloneFresh: cfg.PluginsAlwaysCloneFresh, + PluginsPath: cfg.PluginsPath, + PullRequest: cfg.PullRequest, + Queue: cfg.Queue, + RedactedVars: cfg.RedactedVars, + RefSpec: cfg.RefSpec, + Repository: cfg.Repository, + RunInPty: runInPty, + SSHKeyscan: cfg.SSHKeyscan, + Shell: cfg.Shell, + Tag: cfg.Tag, + TracingBackend: cfg.TracingBackend, + TracingServiceName: cfg.TracingServiceName, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, + syscall.SIGHUP, + syscall.SIGTERM, + syscall.SIGINT, + syscall.SIGQUIT) + defer signal.Stop(signals) + + var ( + cancelled bool + received os.Signal + signalMu sync.Mutex + ) + + // Listen for signals in the background and cancel the job execution + go func() { + sig := <-signals + signalMu.Lock() + defer signalMu.Unlock() + + // Cancel the job execution + executor.Cancel() + + // Track the state and signal used + cancelled = true + received = sig + + // Remove our signal handler so subsequent signals kill + signal.Stop(signals) + }() + + // Run the job and get the exit code + exitCode := executor.Run(ctx) + + signalMu.Lock() + defer signalMu.Unlock() + + // If cancelled and our child process returns a non-zero, we should terminate + // ourselves with the same signal so that our caller can detect and handle appropriately + if cancelled && runtime.GOOS != "windows" { + p, err := os.FindProcess(os.Getpid()) + if err != nil { + l.Error("Failed to find current process: %v", err) + } + + l.Debug("Terminating job execution after cancellation with %v", received) + err = p.Signal(received) + if err != nil { + l.Error("Failed to signal self: %v", err) + } + } + + os.Exit(exitCode) +} + +func genBootstrap() cli.Command { + var help strings.Builder + help.WriteString("⚠️ ⚠️ ⚠️\n") + help.WriteString("DEPRECATED: Use `buildkite-agent job run` instead\n") + help.WriteString("⚠️ ⚠️ ⚠️\n\n") + + err := jobRunHelpTpl.Execute(&help, "bootstrap") + if err != nil { + // This can only hapen if we've mangled the template or its parsing + // and will be caught by tests and local usage + // (famous last words) + panic(err) + } + + return cli.Command{ + Name: "bootstrap", + Usage: "[DEPRECATED] Run a Buildkite job locally", + Description: help.String(), + Flags: jobRunFlags, + Action: func(c *cli.Context) { + fmt.Println("⚠️ WARNING ⚠️") + fmt.Println("This command (`buildkite-agent bootstrap`) is deprecated and will be removed in the next major version of the Buildkite Agent") + fmt.Println() + fmt.Println("You're probably seeing this message because you're using the `--bootstrap-script` flag (or its associated environment variable) and manually calling `buildkite-agent bootstrap` from your custom bootstrap script, customising the behaviour of the agent when it runs a job") + fmt.Println() + fmt.Println("This workflow is still totally supported, but we've renamed the command to `buildkite-agent job-run` to make it more obvious what it does") + fmt.Println("You can update your bootstrap script to use `buildkite-agent job run` instead of `buildkite-agent bootstrap` and everything will work pretty much exactly the same") + fmt.Println("Also, the `--bootstrap-script` flag is now called `--job-run-script`, but the change is backwards compatible -- the old flag will still work for now") + fmt.Println() + fmt.Println("For more information, see https://github.com/buildkite/agent/pull/1958") + fmt.Println() + jobRunAction(c) + }, + } +} + +func genJobRun() cli.Command { + var help strings.Builder + err := jobRunHelpTpl.Execute(&help, "job run") + if err != nil { + // This can only hapen if we've mangled the template or its parsing + // and will be caught by tests and local usage + // (famous last words) + panic(err) + } + + return cli.Command{ + Name: "run", + Usage: "Run a Buildkite job locally", + Description: help.String(), + Flags: jobRunFlags, + Action: jobRunAction, + } +} + +var ( + BootstrapCommand = genBootstrap() + JobRunCommand = genJobRun() +) diff --git a/clicommand/pipeline_upload.go b/clicommand/pipeline_upload.go index 989f3cffd4..1de03a0e24 100644 --- a/clicommand/pipeline_upload.go +++ b/clicommand/pipeline_upload.go @@ -14,9 +14,9 @@ import ( "github.com/buildkite/agent/v3/agent" "github.com/buildkite/agent/v3/api" - "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/cliconfig" "github.com/buildkite/agent/v3/env" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/agent/v3/redaction" "github.com/buildkite/agent/v3/stdin" "github.com/urfave/cli" diff --git a/hook/hook.go b/hook/hook.go index d0c2283d57..29ce4bf04a 100644 --- a/hook/hook.go +++ b/hook/hook.go @@ -9,7 +9,7 @@ import ( "path/filepath" "runtime" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/agent/v3/utils" ) diff --git a/hook/scriptwrapper.go b/hook/scriptwrapper.go index 20e121e707..7c8b97c889 100644 --- a/hook/scriptwrapper.go +++ b/hook/scriptwrapper.go @@ -10,8 +10,8 @@ import ( "strings" "text/template" - "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/env" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/agent/v3/shellscript" "github.com/buildkite/agent/v3/utils" ) @@ -83,7 +83,7 @@ func (e *HookExitError) Error() string { type scriptWrapperOpt func(*ScriptWrapper) -// Hooks get "sourced" into the bootstrap in the sense that they get the +// Hooks get "sourced" into job execution in the sense that they get the // environment set for them and then we capture any extra environment variables // that are exported in the script. @@ -91,7 +91,7 @@ type scriptWrapperOpt func(*ScriptWrapper) // before it finishes, so we've got an awesome (ugly) hack to get around this. // We write the ENV to file, run the hook and then write the ENV back to another file. // Then we can use the diff of the two to figure out what changes to make to the -// bootstrap. Horrible, but effective. +// job executor. Horrible, but effective. // ScriptWrapper wraps a hook script with env collection and then provides // a way to get the difference between the environment before the hook is run and @@ -151,7 +151,7 @@ func NewScriptWrapper(opts ...scriptWrapperOpt) (*ScriptWrapper, error) { var isPOSIXHook, isPwshHook bool - scriptFileName := "buildkite-agent-bootstrap-hook-runner" + scriptFileName := "buildkite-agent-job-exec-hook-runner" isWindows := wrap.os == "windows" // we use bash hooks for scripts with no extension, otherwise on windows @@ -177,7 +177,7 @@ func NewScriptWrapper(opts ...scriptWrapperOpt) (*ScriptWrapper, error) { // We'll pump the ENV before the hook into this temp file wrap.beforeEnvFile, err = shell.TempFileWithExtension( - "buildkite-agent-bootstrap-hook-env-before", + "buildkite-agent-job-exec-hook-env-before", ) if err != nil { return nil, err @@ -186,7 +186,7 @@ func NewScriptWrapper(opts ...scriptWrapperOpt) (*ScriptWrapper, error) { // We'll then pump the ENV _after_ the hook into this temp file wrap.afterEnvFile, err = shell.TempFileWithExtension( - "buildkite-agent-bootstrap-hook-env-after", + "buildkite-agent-job-exec-hook-env-after", ) if err != nil { return nil, err diff --git a/hook/scriptwrapper_test.go b/hook/scriptwrapper_test.go index 0dd2dcc1e0..5cb22dc1c2 100644 --- a/hook/scriptwrapper_test.go +++ b/hook/scriptwrapper_test.go @@ -10,8 +10,8 @@ import ( "runtime" "testing" - "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/env" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/bintest/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/bootstrap/api.go b/job/api.go similarity index 94% rename from bootstrap/api.go rename to job/api.go index 47c34e8fb0..67d5e39ff2 100644 --- a/bootstrap/api.go +++ b/job/api.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "fmt" @@ -10,7 +10,7 @@ import ( // startJobAPI starts the job API server, iff the job API experiment is enabled, and the OS of the box supports it // otherwise it returns a noop cleanup function // It also sets the BUILDKITE_AGENT_JOB_API_SOCKET and BUILDKITE_AGENT_JOB_API_TOKEN environment variables -func (b *Bootstrap) startJobAPI() (cleanup func(), err error) { +func (b *Executor) startJobAPI() (cleanup func(), err error) { cleanup = func() {} if !experiments.IsEnabled(experiments.JobAPI) { diff --git a/bootstrap/config.go b/job/config.go similarity index 92% rename from bootstrap/config.go rename to job/config.go index 95352028fc..7918cebd90 100644 --- a/bootstrap/config.go +++ b/job/config.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "reflect" @@ -10,14 +10,14 @@ import ( "github.com/buildkite/agent/v3/process" ) -// Config provides the configuration for the Bootstrap. Some of the keys are +// Config provides the configuration for the Executor. Some of the keys are // read from the environment after hooks are run, so we use struct tags to provide // that mapping along with some reflection. It's a little bit magical but it's // less work to maintain in the long run. // // To add a new config option that is mapped from an environment variable, add a // struct tag, then don't forget to add a corresponding CLI flag over in the -// clicommand/bootstrap.go(BootstrapConfig) struct, otherwise it won't work. +// clicommand/executor.go(ExecutorConfig) struct, otherwise it won't work. type Config struct { // The command to run @@ -26,7 +26,7 @@ type Config struct { // The ID of the job being run JobID string - // If the bootstrap is in debug mode + // If the executor is in debug mode Debug bool // The repository that needs to be cloned @@ -62,13 +62,13 @@ type Config struct { // Slug of the current pipeline PipelineSlug string - // Name of the agent running the bootstrap + // Name of the agent running the executor AgentName string // Name of the queue the agent belongs to, if tagged Queue string - // Should the bootstrap remove an existing checkout before running the job + // Should the executor remove an existing checkout before running the job CleanCheckout bool `env:"BUILDKITE_CLEAN_CHECKOUT"` // Flags to pass to "git checkout" command @@ -196,7 +196,7 @@ func (c *Config) ReadFromEnvironment(environ *env.Environment) map[string]string v.SetBool(newBool) changed[tag] = newStr default: - log.Printf("warning: bootstrap.Config.ReadFromEnvironment does not support %v for %s", v.Kind(), tag) + log.Printf("warning: executor.Config.ReadFromEnvironment does not support %v for %s", v.Kind(), tag) } } } diff --git a/bootstrap/config_test.go b/job/config_test.go similarity index 99% rename from bootstrap/config_test.go rename to job/config_test.go index 9247a4aac1..31f13ad45d 100644 --- a/bootstrap/config_test.go +++ b/job/config_test.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "testing" diff --git a/bootstrap/docker.go b/job/docker.go similarity index 96% rename from bootstrap/docker.go rename to job/docker.go index ff50e1e166..e1344cc698 100644 --- a/bootstrap/docker.go +++ b/job/docker.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "context" @@ -6,7 +6,7 @@ import ( "fmt" "strings" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" ) var dockerEnv = []string{ @@ -83,7 +83,7 @@ func tearDownDeprecatedDockerIntegration(ctx context.Context, sh *shell.Shell) e } // runDockerCommand executes a command inside a docker container that is built as needed -// Ported from https://github.com/buildkite/agent/blob/2b8f1d569b659e07de346c0e3ae7090cb98e49ba/templates/bootstrap.sh#L439 +// Ported from https://github.com/buildkite/agent/blob/2b8f1d569b659e07de346c0e3ae7090cb98e49ba/templates/boostrap.sh#L439 func runDockerCommand(ctx context.Context, sh *shell.Shell, cmd []string) error { jobId, _ := sh.Env.Get("BUILDKITE_JOB_ID") dockerContainer := fmt.Sprintf("buildkite_%s_container", jobId) @@ -111,7 +111,7 @@ func runDockerCommand(ctx context.Context, sh *shell.Shell, cmd []string) error } // runDockerComposeCommand executes a command with docker-compose -// Ported from https://github.com/buildkite/agent/blob/2b8f1d569b659e07de346c0e3ae7090cb98e49ba/templates/bootstrap.sh#L462 +// Ported from https://github.com/buildkite/agent/blob/2b8f1d569b659e07de346c0e3ae7090cb98e49ba/templates/executor.sh#L462 func runDockerComposeCommand(ctx context.Context, sh *shell.Shell, cmd []string) error { composeContainer, _ := sh.Env.Get("BUILDKITE_DOCKER_COMPOSE_CONTAINER") jobId, _ := sh.Env.Get("BUILDKITE_JOB_ID") diff --git a/bootstrap/bootstrap.go b/job/executor.go similarity index 93% rename from bootstrap/bootstrap.go rename to job/executor.go index e412c26dc8..b90432804c 100644 --- a/bootstrap/bootstrap.go +++ b/job/executor.go @@ -1,8 +1,8 @@ -// Package bootstrap provides management of the phases of execution of a +// package job provides management of the phases of execution of a // Buildkite job. // // It is intended for internal use by buildkite-agent only. -package bootstrap +package job import ( "context" @@ -18,10 +18,10 @@ import ( "time" "github.com/buildkite/agent/v3/agent/plugin" - "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/env" "github.com/buildkite/agent/v3/experiments" "github.com/buildkite/agent/v3/hook" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/agent/v3/kubernetes" "github.com/buildkite/agent/v3/process" "github.com/buildkite/agent/v3/redaction" @@ -32,16 +32,17 @@ import ( "github.com/buildkite/shellwords" ) -// Bootstrap represents the phases of execution in a Buildkite Job. It's run as +// Executor represents the phases of execution in a Buildkite Job. It's run as // a sub-process of the buildkite-agent and finishes at the conclusion of a job. // -// Historically (prior to v3) the bootstrap was a shell script, but was ported +// Historically (prior to v3) the executor was a shell script, but was ported // to Go for portability and testability. -type Bootstrap struct { - // Config provides the bootstrap configuration +// It also used to be called a "bootstrap", so you might see that verbiage hanging around in some places. +type Executor struct { + // Config provides the executor configuration Config - // Shell is the shell environment for the bootstrap + // Shell is the shell environment for the executor shell *shell.Shell // Plugins to use @@ -50,23 +51,23 @@ type Bootstrap struct { // Plugin checkouts from the plugin phases pluginCheckouts []*pluginCheckout - // Directories to clean up at end of bootstrap + // Directories to clean up at end of executor cleanupDirs []string // A channel to track cancellation cancelCh chan struct{} } -// New returns a new Bootstrap instance -func New(conf Config) *Bootstrap { - return &Bootstrap{ +// NewExecutor returns a new Executor instance +func NewExecutor(conf Config) *Executor { + return &Executor{ Config: conf, cancelCh: make(chan struct{}), } } -// Run the bootstrap and return the exit code -func (b *Bootstrap) Run(ctx context.Context) (exitCode int) { +// Run the executor and return the exit code +func (b *Executor) Run(ctx context.Context) (exitCode int) { // Check if not nil to allow for tests to overwrite shell if b.shell == nil { var err error @@ -128,7 +129,7 @@ func (b *Bootstrap) Run(ctx context.Context) (exitCode int) { // Tear down the environment (and fire pre-exit hook) before we exit defer func() { if err = b.tearDown(ctx); err != nil { - b.shell.Errorf("Error tearing down bootstrap: %v", err) + b.shell.Errorf("Error tearing down executor: %v", err) // this gets passed back via the named return exitCode = shell.GetExitCode(err) @@ -137,7 +138,7 @@ func (b *Bootstrap) Run(ctx context.Context) (exitCode int) { // Initialize the environment, a failure here will still call the tearDown if err = b.setUp(ctx); err != nil { - b.shell.Errorf("Error setting up bootstrap: %v", err) + b.shell.Errorf("Error setting up executor: %v", err) return shell.GetExitCode(err) } @@ -153,7 +154,7 @@ func (b *Bootstrap) Run(ctx context.Context) (exitCode int) { return false } - // Execute the bootstrap phases in order + // Execute the executor phases in order var phaseErr error if includePhase("plugin") { @@ -235,8 +236,8 @@ func (b *Bootstrap) Run(ctx context.Context) (exitCode int) { return exitStatusCode } -// Cancel interrupts any running shell processes and causes the bootstrap to stop -func (b *Bootstrap) Cancel() error { +// Cancel interrupts any running shell processes and causes the executor to stop +func (b *Executor) Cancel() error { b.cancelCh <- struct{}{} return nil } @@ -250,7 +251,7 @@ type HookConfig struct { PluginName string } -func (b *Bootstrap) tracingImplementationSpecificHookScope(scope string) string { +func (b *Executor) tracingImplementationSpecificHookScope(scope string) string { if b.TracingBackend != tracetools.BackendOpenTelemetry { return scope } @@ -269,7 +270,7 @@ func (b *Bootstrap) tracingImplementationSpecificHookScope(scope string) string } // executeHook runs a hook script with the hookRunner -func (b *Bootstrap) executeHook(ctx context.Context, hookCfg HookConfig) error { +func (b *Executor) executeHook(ctx context.Context, hookCfg HookConfig) error { scopeName := b.tracingImplementationSpecificHookScope(hookCfg.Scope) spanName := b.implementationSpecificSpanName(fmt.Sprintf("%s %s hook", scopeName, hookCfg.Name), "hook.execute") span, ctx := tracetools.StartSpanFromContext(ctx, spanName, b.Config.TracingBackend) @@ -369,7 +370,7 @@ func (b *Bootstrap) executeHook(ctx context.Context, hookCfg HookConfig) error { return nil } -func (b *Bootstrap) applyEnvironmentChanges(changes hook.HookScriptChanges, redactors redaction.RedactorMux) { +func (b *Executor) applyEnvironmentChanges(changes hook.HookScriptChanges, redactors redaction.RedactorMux) { if afterWd, err := changes.GetAfterWd(); err == nil { if afterWd != b.shell.Getwd() { _ = b.shell.Chdir(afterWd) @@ -388,12 +389,12 @@ func (b *Bootstrap) applyEnvironmentChanges(changes hook.HookScriptChanges, reda redactors.Reset(redaction.GetValuesToRedact(b.shell, b.Config.RedactedVars, b.shell.Env.Dump())) // First, let see any of the environment variables are supposed - // to change the bootstrap configuration at run time. - bootstrapConfigEnvChanges := b.Config.ReadFromEnvironment(b.shell.Env) + // to change the executor configuration at run time. + executorConfigEnvChanges := b.Config.ReadFromEnvironment(b.shell.Env) // Print out the env vars that changed. As we go through each - // one, we'll determine if it was a special "bootstrap" - // environment variable that has changed the bootstrap + // one, we'll determine if it was a special "executor" + // environment variable that has changed the executor // configuration at runtime. // // If it's "special", we'll show the value it was changed to - @@ -402,21 +403,21 @@ func (b *Bootstrap) applyEnvironmentChanges(changes hook.HookScriptChanges, reda // THIRD_PARTY_API_KEY) we'll just not show any values for // anything not controlled by us. for k, v := range changes.Diff.Added { - if _, ok := bootstrapConfigEnvChanges[k]; ok { + if _, ok := executorConfigEnvChanges[k]; ok { b.shell.Commentf("%s is now %q", k, v) } else { b.shell.Commentf("%s added", k) } } for k, v := range changes.Diff.Changed { - if _, ok := bootstrapConfigEnvChanges[k]; ok { + if _, ok := executorConfigEnvChanges[k]; ok { b.shell.Commentf("%s is now %q", k, v) } else { b.shell.Commentf("%s changed", k) } } for k, v := range changes.Diff.Removed { - if _, ok := bootstrapConfigEnvChanges[k]; ok { + if _, ok := executorConfigEnvChanges[k]; ok { b.shell.Commentf("%s is now %q", k, v) } else { b.shell.Commentf("%s removed", k) @@ -424,18 +425,18 @@ func (b *Bootstrap) applyEnvironmentChanges(changes hook.HookScriptChanges, reda } } -func (b *Bootstrap) hasGlobalHook(name string) bool { +func (b *Executor) hasGlobalHook(name string) bool { _, err := b.globalHookPath(name) return err == nil } // Returns the absolute path to a global hook, or os.ErrNotExist if none is found -func (b *Bootstrap) globalHookPath(name string) (string, error) { +func (b *Executor) globalHookPath(name string) (string, error) { return hook.Find(b.HooksPath, name) } // Executes a global hook if one exists -func (b *Bootstrap) executeGlobalHook(ctx context.Context, name string) error { +func (b *Executor) executeGlobalHook(ctx context.Context, name string) error { if !b.hasGlobalHook(name) { return nil } @@ -451,18 +452,18 @@ func (b *Bootstrap) executeGlobalHook(ctx context.Context, name string) error { } // Returns the absolute path to a local hook, or os.ErrNotExist if none is found -func (b *Bootstrap) localHookPath(name string) (string, error) { +func (b *Executor) localHookPath(name string) (string, error) { dir := filepath.Join(b.shell.Getwd(), ".buildkite", "hooks") return hook.Find(dir, name) } -func (b *Bootstrap) hasLocalHook(name string) bool { +func (b *Executor) hasLocalHook(name string) bool { _, err := b.localHookPath(name) return err == nil } // Executes a local hook -func (b *Bootstrap) executeLocalHook(ctx context.Context, name string) error { +func (b *Executor) executeLocalHook(ctx context.Context, name string) error { if !b.hasLocalHook(name) { return nil } @@ -521,8 +522,8 @@ func addRepositoryHostToSSHKnownHosts(ctx context.Context, sh *shell.Shell, repo } // setUp is run before all the phases run. It's responsible for initializing the -// bootstrap environment -func (b *Bootstrap) setUp(ctx context.Context) error { +// executor environment +func (b *Executor) setUp(ctx context.Context) error { span, ctx := tracetools.StartSpanFromContext(ctx, "environment", b.Config.TracingBackend) var err error defer func() { span.FinishWithError(err) }() @@ -580,8 +581,8 @@ func (b *Bootstrap) setUp(ctx context.Context) error { return err } -// tearDown is called before the bootstrap exits, even on error -func (b *Bootstrap) tearDown(ctx context.Context) error { +// tearDown is called before the executor exits, even on error +func (b *Executor) tearDown(ctx context.Context) error { span, ctx := tracetools.StartSpanFromContext(ctx, "pre-exit", b.Config.TracingBackend) var err error defer func() { span.FinishWithError(err) }() @@ -612,11 +613,11 @@ func (b *Bootstrap) tearDown(ctx context.Context) error { return nil } -func (b *Bootstrap) hasPlugins() bool { +func (b *Executor) hasPlugins() bool { return b.Config.Plugins != "" } -func (b *Bootstrap) preparePlugins() error { +func (b *Executor) preparePlugins() error { if !b.hasPlugins() { return nil } @@ -651,7 +652,7 @@ func (b *Bootstrap) preparePlugins() error { return nil } -func (b *Bootstrap) validatePluginCheckout(checkout *pluginCheckout) error { +func (b *Executor) validatePluginCheckout(checkout *pluginCheckout) error { if !b.Config.PluginValidation { return nil } @@ -689,7 +690,7 @@ func (b *Bootstrap) validatePluginCheckout(checkout *pluginCheckout) error { // PluginPhase is where plugins that weren't filtered in the Environment phase are // checked out and made available to later phases -func (b *Bootstrap) PluginPhase(ctx context.Context) error { +func (b *Executor) PluginPhase(ctx context.Context) error { if len(b.plugins) == 0 { if b.Debug { b.shell.Commentf("Skipping plugin phase") @@ -730,7 +731,7 @@ func (b *Bootstrap) PluginPhase(ctx context.Context) error { // VendoredPluginPhase is where plugins that are included in the // checked out code are added -func (b *Bootstrap) VendoredPluginPhase(ctx context.Context) error { +func (b *Executor) VendoredPluginPhase(ctx context.Context) error { if !b.hasPlugins() { return nil } @@ -782,7 +783,7 @@ func (b *Bootstrap) VendoredPluginPhase(ctx context.Context) error { } // Executes a named hook on plugins that have it -func (b *Bootstrap) executePluginHook(ctx context.Context, name string, checkouts []*pluginCheckout) error { +func (b *Executor) executePluginHook(ctx context.Context, name string, checkouts []*pluginCheckout) error { for _, p := range checkouts { hookPath, err := hook.Find(p.HooksDir, name) if errors.Is(err, os.ErrNotExist) { @@ -814,7 +815,7 @@ func (b *Bootstrap) executePluginHook(ctx context.Context, name string, checkout } // If any plugin has a hook by this name -func (b *Bootstrap) hasPluginHook(name string) bool { +func (b *Executor) hasPluginHook(name string) bool { for _, p := range b.pluginCheckouts { if _, err := hook.Find(p.HooksDir, name); err == nil { return true @@ -824,7 +825,7 @@ func (b *Bootstrap) hasPluginHook(name string) bool { } // Checkout a given plugin to the plugins directory and return that directory -func (b *Bootstrap) checkoutPlugin(ctx context.Context, p *plugin.Plugin) (*pluginCheckout, error) { +func (b *Executor) checkoutPlugin(ctx context.Context, p *plugin.Plugin) (*pluginCheckout, error) { // Make sure we have a plugin path before trying to do anything if b.PluginsPath == "" { return nil, fmt.Errorf("Can't checkout plugin without a `plugins-path`") @@ -957,7 +958,7 @@ func (b *Bootstrap) checkoutPlugin(ctx context.Context, p *plugin.Plugin) (*plug return checkout, nil } -func (b *Bootstrap) removeCheckoutDir() error { +func (b *Executor) removeCheckoutDir() error { checkoutPath, _ := b.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH") // on windows, sometimes removing large dirs can fail for various reasons @@ -981,7 +982,7 @@ func (b *Bootstrap) removeCheckoutDir() error { return fmt.Errorf("Failed to remove %s", checkoutPath) } -func (b *Bootstrap) createCheckoutDir() error { +func (b *Executor) createCheckoutDir() error { checkoutPath, _ := b.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH") if !utils.FileExists(checkoutPath) { @@ -1003,7 +1004,7 @@ func (b *Bootstrap) createCheckoutDir() error { // CheckoutPhase creates the build directory and makes sure we're running the // build at the right commit. -func (b *Bootstrap) CheckoutPhase(ctx context.Context) error { +func (b *Executor) CheckoutPhase(ctx context.Context) error { span, ctx := tracetools.StartSpanFromContext(ctx, "checkout", b.Config.TracingBackend) var err error defer func() { span.FinishWithError(err) }() @@ -1035,7 +1036,7 @@ func (b *Bootstrap) CheckoutPhase(ctx context.Context) error { } b.shell.Env.Set("BUILDKITE_BUILD_CHECKOUT_PATH", buildDir) - // Track the directory so we can remove it at the end of the bootstrap + // Track the directory so we can remove it at the end of the executor b.cleanupDirs = append(b.cleanupDirs, buildDir) } @@ -1173,7 +1174,7 @@ func hasGitCommit(ctx context.Context, sh *shell.Shell, gitDir string, commit st return true } -func (b *Bootstrap) updateGitMirror(ctx context.Context, repository string) (string, error) { +func (b *Executor) updateGitMirror(ctx context.Context, repository string) (string, error) { // Create a unique directory for the repository mirror mirrorDir := filepath.Join(b.Config.GitMirrorsPath, dirForRepository(repository)) @@ -1266,7 +1267,7 @@ func (b *Bootstrap) updateGitMirror(ctx context.Context, repository string) (str return mirrorDir, nil } -func (b *Bootstrap) getOrUpdateMirrorDir(ctx context.Context, repository string) (string, error) { +func (b *Executor) getOrUpdateMirrorDir(ctx context.Context, repository string) (string, error) { var mirrorDir string var err error // Skip updating the Git mirror before using it? @@ -1291,7 +1292,7 @@ func (b *Bootstrap) getOrUpdateMirrorDir(ctx context.Context, repository string) // defaultCheckoutPhase is called by the CheckoutPhase if no global or plugin checkout // hook exists. It performs the default checkout on the Repository provided in the config -func (b *Bootstrap) defaultCheckoutPhase(ctx context.Context) error { +func (b *Executor) defaultCheckoutPhase(ctx context.Context) error { span, _ := tracetools.StartSpanFromContext(ctx, "repo-checkout", b.Config.TracingBackend) span.AddAttributes(map[string]string{ "checkout.repo_name": b.Repository, @@ -1427,7 +1428,7 @@ func (b *Bootstrap) defaultCheckoutPhase(ctx context.Context) error { // `submodule sync` will ensure the .git/config // matches the .gitmodules file. The command // is only available in git version 1.8.1, so - // if the call fails, continue the bootstrap + // if the call fails, continue the executor // script, and show an informative error. if err := b.shell.Run(ctx, "git", "submodule", "sync", "--recursive"); err != nil { gitVersionOutput, _ := b.shell.RunAndCapture(ctx, "git", "--version") @@ -1559,7 +1560,7 @@ func (b *Bootstrap) defaultCheckoutPhase(ctx context.Context) error { return nil } -func (b *Bootstrap) resolveCommit(ctx context.Context) { +func (b *Executor) resolveCommit(ctx context.Context) { commitRef, _ := b.shell.Env.Get("BUILDKITE_COMMIT") if commitRef == "" { b.shell.Warningf("BUILDKITE_COMMIT was empty") @@ -1578,7 +1579,7 @@ func (b *Bootstrap) resolveCommit(ctx context.Context) { } // runPreCommandHooks runs the pre-command hooks and adds tracing spans. -func (b *Bootstrap) runPreCommandHooks(ctx context.Context) error { +func (b *Executor) runPreCommandHooks(ctx context.Context) error { spanName := b.implementationSpecificSpanName("pre-command", "pre-command hooks") span, ctx := tracetools.StartSpanFromContext(ctx, spanName, b.Config.TracingBackend) var err error @@ -1597,7 +1598,7 @@ func (b *Bootstrap) runPreCommandHooks(ctx context.Context) error { } // runCommand runs the command and adds tracing spans. -func (b *Bootstrap) runCommand(ctx context.Context) error { +func (b *Executor) runCommand(ctx context.Context) error { var err error // There can only be one command hook, so we check them in order of plugin, local switch { @@ -1614,7 +1615,7 @@ func (b *Bootstrap) runCommand(ctx context.Context) error { } // runPostCommandHooks runs the post-command hooks and adds tracing spans. -func (b *Bootstrap) runPostCommandHooks(ctx context.Context) error { +func (b *Executor) runPostCommandHooks(ctx context.Context) error { spanName := b.implementationSpecificSpanName("post-command", "post-command hooks") span, ctx := tracetools.StartSpanFromContext(ctx, spanName, b.Config.TracingBackend) var err error @@ -1633,7 +1634,7 @@ func (b *Bootstrap) runPostCommandHooks(ctx context.Context) error { } // CommandPhase determines how to run the build, and then runs it -func (b *Bootstrap) CommandPhase(ctx context.Context) (error, error) { +func (b *Executor) CommandPhase(ctx context.Context) (error, error) { span, ctx := tracetools.StartSpanFromContext(ctx, "command", b.Config.TracingBackend) var err error defer func() { span.FinishWithError(err) }() @@ -1678,7 +1679,7 @@ func (b *Bootstrap) CommandPhase(ctx context.Context) (error, error) { } // defaultCommandPhase is executed if there is no global or plugin command hook -func (b *Bootstrap) defaultCommandPhase(ctx context.Context) error { +func (b *Executor) defaultCommandPhase(ctx context.Context) error { spanName := b.implementationSpecificSpanName("default command hook", "hook.execute") span, ctx := tracetools.StartSpanFromContext(ctx, spanName, b.Config.TracingBackend) var err error @@ -1722,7 +1723,7 @@ func (b *Bootstrap) defaultCommandPhase(ctx context.Context) error { } if len(shell) == 0 { - return fmt.Errorf("No shell set for bootstrap") + return fmt.Errorf("No shell set for executor") } // Windows CMD.EXE is horrible and can't handle newline delimited commands. We write @@ -1851,7 +1852,7 @@ func shouldCallBatchLine(line string) bool { return (strings.HasSuffix(first, ".bat") || strings.HasSuffix(first, ".cmd")) } -func (b *Bootstrap) writeBatchScript(cmd string) (string, error) { +func (b *Executor) writeBatchScript(cmd string) (string, error) { scriptFile, err := shell.TempFileWithExtension( "buildkite-script.bat", ) @@ -1882,7 +1883,7 @@ func (b *Bootstrap) writeBatchScript(cmd string) (string, error) { } -func (b *Bootstrap) artifactPhase(ctx context.Context) error { +func (b *Executor) artifactPhase(ctx context.Context) error { if b.AutomaticArtifactUploadPaths == "" { return nil } @@ -1911,7 +1912,7 @@ func (b *Bootstrap) artifactPhase(ctx context.Context) error { } // Run the pre-artifact hooks -func (b *Bootstrap) preArtifactHooks(ctx context.Context) error { +func (b *Executor) preArtifactHooks(ctx context.Context) error { span, ctx := tracetools.StartSpanFromContext(ctx, "pre-artifact", b.Config.TracingBackend) var err error defer func() { span.FinishWithError(err) }() @@ -1932,7 +1933,7 @@ func (b *Bootstrap) preArtifactHooks(ctx context.Context) error { } // Run the artifact upload command -func (b *Bootstrap) uploadArtifacts(ctx context.Context) error { +func (b *Executor) uploadArtifacts(ctx context.Context) error { span, _ := tracetools.StartSpanFromContext(ctx, "artifact-upload", b.Config.TracingBackend) var err error defer func() { span.FinishWithError(err) }() @@ -1953,7 +1954,7 @@ func (b *Bootstrap) uploadArtifacts(ctx context.Context) error { } // Run the post-artifact hooks -func (b *Bootstrap) postArtifactHooks(ctx context.Context) error { +func (b *Executor) postArtifactHooks(ctx context.Context) error { span, _ := tracetools.StartSpanFromContext(ctx, "post-artifact", b.Config.TracingBackend) var err error defer func() { span.FinishWithError(err) }() @@ -1977,7 +1978,7 @@ func (b *Bootstrap) postArtifactHooks(ctx context.Context) error { // env (for example, BUILDKITE_BUILD_PATH) can only be set from config or by hooks. // If these env are set at a pipeline level, we rewrite them to BUILDKITE_X_BUILD_PATH // and warn on them here so that users know what is going on -func (b *Bootstrap) ignoredEnv() []string { +func (b *Executor) ignoredEnv() []string { var ignored []string for _, env := range os.Environ() { if strings.HasPrefix(env, "BUILDKITE_X_") { @@ -1992,7 +1993,7 @@ func (b *Bootstrap) ignoredEnv() []string { // is necessary based on RedactedVars configuration and the existence of // matching environment vars. // redaction.RedactorMux (possibly empty) is returned so the caller can `defer redactor.Flush()` -func (b *Bootstrap) setupRedactors() redaction.RedactorMux { +func (b *Executor) setupRedactors() redaction.RedactorMux { valuesToRedact := redaction.GetValuesToRedact(b.shell, b.Config.RedactedVars, b.shell.Env.Dump()) if len(valuesToRedact) == 0 { return nil @@ -2048,7 +2049,7 @@ type pluginCheckout struct { HooksDir string } -func (b *Bootstrap) startKubernetesClient(ctx context.Context, kubernetesClient *kubernetes.Client) error { +func (b *Executor) startKubernetesClient(ctx context.Context, kubernetesClient *kubernetes.Client) error { b.shell.Commentf("Using experimental Kubernetes support") err := roko.NewRetrier( roko.WithMaxAttempts(7), diff --git a/bootstrap/bootstrap_test.go b/job/executor_test.go similarity index 96% rename from bootstrap/bootstrap_test.go rename to job/executor_test.go index b7fb0a9260..cb72c0b39f 100644 --- a/bootstrap/bootstrap_test.go +++ b/job/executor_test.go @@ -1,10 +1,10 @@ -package bootstrap +package job import ( "context" "testing" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/agent/v3/redaction" "github.com/buildkite/agent/v3/tracetools" "github.com/opentracing/opentracing-go" @@ -83,7 +83,7 @@ func TestStartTracing_NoTracingBackend(t *testing.T) { var err error // When there's no tracing backend, the tracer should be a no-op. - b := New(Config{}) + b := NewExecutor(Config{}) oriCtx := context.Background() b.shell, err = shell.New() @@ -104,7 +104,7 @@ func TestStartTracing_Datadog(t *testing.T) { // With the Datadog tracing backend, the global tracer should be from Datadog. cfg := Config{TracingBackend: "datadog"} - b := New(cfg) + b := NewExecutor(cfg) oriCtx := context.Background() b.shell, err = shell.New() diff --git a/bootstrap/git.go b/job/git.go similarity index 99% rename from bootstrap/git.go rename to job/git.go index 6bfd9e6a63..93eb98d3c9 100644 --- a/bootstrap/git.go +++ b/job/git.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "bufio" @@ -11,7 +11,7 @@ import ( "regexp" "strings" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/shellwords" ) diff --git a/bootstrap/git_test.go b/job/git_test.go similarity index 99% rename from bootstrap/git_test.go rename to job/git_test.go index f862df09b5..5e12635415 100644 --- a/bootstrap/git_test.go +++ b/job/git_test.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "context" @@ -6,7 +6,7 @@ import ( "os" "testing" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/bootstrap/integration/artifact_integration_test.go b/job/integration/artifact_integration_test.go similarity index 90% rename from bootstrap/integration/artifact_integration_test.go rename to job/integration/artifact_integration_test.go index e03e0f4d70..f06624398b 100644 --- a/bootstrap/integration/artifact_integration_test.go +++ b/job/integration/artifact_integration_test.go @@ -11,9 +11,9 @@ import ( func TestArtifactsUploadAfterCommand(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -40,9 +40,9 @@ func TestArtifactsUploadAfterCommand(t *testing.T) { func TestArtifactsUploadAfterCommandFails(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -74,9 +74,9 @@ func TestArtifactsUploadAfterCommandFails(t *testing.T) { func TestArtifactsUploadAfterCommandHookFails(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() diff --git a/bootstrap/integration/checkout_integration_test.go b/job/integration/checkout_integration_test.go similarity index 92% rename from bootstrap/integration/checkout_integration_test.go rename to job/integration/checkout_integration_test.go index 96e916c941..2e454d5ac4 100644 --- a/bootstrap/integration/checkout_integration_test.go +++ b/job/integration/checkout_integration_test.go @@ -42,9 +42,9 @@ func TestCheckingOutGitHubPullRequestsWithGitMirrorsExperiment(t *testing.T) { // t.Parallel() cannot be used with experiments.Enable() defer experimentWithUndo(experiments.GitMirrors)() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -83,9 +83,9 @@ func TestWithResolvingCommitExperiment(t *testing.T) { // t.Parallel() cannot be used with experiments.Enable() defer experimentWithUndo(experiments.ResolveCommitAfterCheckout)() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -136,9 +136,9 @@ func TestWithResolvingCommitExperiment(t *testing.T) { func TestCheckingOutLocalGitProject(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -192,9 +192,9 @@ func TestCheckingOutLocalGitProjectWithSubmodules(t *testing.T) { t.Skip() } - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -277,9 +277,9 @@ func TestCheckingOutLocalGitProjectWithSubmodulesDisabled(t *testing.T) { t.Skip() } - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -346,9 +346,9 @@ func TestCheckingOutLocalGitProjectWithSubmodulesDisabled(t *testing.T) { func TestCheckingOutShallowCloneOfLocalGitProject(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -397,9 +397,9 @@ func TestCheckingOutShallowCloneOfLocalGitProject(t *testing.T) { func TestCheckingOutSetsCorrectGitMetadataAndSendsItToBuildkite(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -413,9 +413,9 @@ func TestCheckingOutSetsCorrectGitMetadataAndSendsItToBuildkite(t *testing.T) { func TestCheckingOutWithSSHKeyscan(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -446,9 +446,9 @@ func TestCheckingOutWithSSHKeyscan(t *testing.T) { func TestCheckingOutWithoutSSHKeyscan(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -467,9 +467,9 @@ func TestCheckingOutWithoutSSHKeyscan(t *testing.T) { func TestCheckingOutWithSSHKeyscanAndUnscannableRepo(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -501,9 +501,9 @@ func TestCleaningAnExistingCheckout(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -535,9 +535,9 @@ func TestCleaningAnExistingCheckout(t *testing.T) { } func TestForcingACleanCheckout(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -555,9 +555,9 @@ func TestForcingACleanCheckout(t *testing.T) { } func TestCheckoutOnAnExistingRepositoryWithoutAGitFolder(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -580,9 +580,9 @@ func TestCheckoutOnAnExistingRepositoryWithoutAGitFolder(t *testing.T) { } func TestCheckoutRetriesOnCleanFailure(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -606,9 +606,9 @@ func TestCheckoutRetriesOnCleanFailure(t *testing.T) { } func TestCheckoutRetriesOnCloneFailure(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -630,9 +630,9 @@ func TestCheckoutRetriesOnCloneFailure(t *testing.T) { } func TestCheckoutDoesNotRetryOnHookFailure(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -663,9 +663,9 @@ func TestRepositorylessCheckout(t *testing.T) { t.Skip("Not supported on windows") } - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -693,13 +693,13 @@ func TestGitMirrorEnv(t *testing.T) { // t.Parallel() cannot test experiment flags in parallel defer experimentWithUndo(experiments.GitMirrors)() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() - // assert the correct BUILDKITE_REPO_MIRROR _after_ the bootstrap has run + // assert the correct BUILDKITE_REPO_MIRROR _after_ the executor has run gitMirrorPath := "" tester.ExpectGlobalHook("pre-command").Once().AndCallFunc(func(c *bintest.Call) { gitMirrorPath = c.GetEnv("BUILDKITE_REPO_MIRROR") diff --git a/bootstrap/integration/command_integration_test.go b/job/integration/command_integration_test.go similarity index 90% rename from bootstrap/integration/command_integration_test.go rename to job/integration/command_integration_test.go index 8c84d3cd8f..5a8091a142 100644 --- a/bootstrap/integration/command_integration_test.go +++ b/job/integration/command_integration_test.go @@ -14,9 +14,9 @@ func TestMultilineCommandRunUnderBatch(t *testing.T) { t.Skip("batch test only applies to Windows") } - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -42,9 +42,9 @@ func TestMultilineCommandRunUnderBatch(t *testing.T) { } func TestPreExitHooksRunsAfterCommandFails(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() diff --git a/bootstrap/integration/doc.go b/job/integration/doc.go similarity index 90% rename from bootstrap/integration/doc.go rename to job/integration/doc.go index 019c673210..f6023927d4 100644 --- a/bootstrap/integration/doc.go +++ b/job/integration/doc.go @@ -1,4 +1,4 @@ -// Package integration defines a series of integration tests for the bootstrap. +// Package integration defines a series of integration tests for the executor. // // It is intended for internal use by buildkite-agent only. package integration diff --git a/bootstrap/integration/docker_integration_test.go b/job/integration/docker_integration_test.go similarity index 92% rename from bootstrap/integration/docker_integration_test.go rename to job/integration/docker_integration_test.go index bf2f9c5df5..4b7563245d 100644 --- a/bootstrap/integration/docker_integration_test.go +++ b/job/integration/docker_integration_test.go @@ -17,9 +17,9 @@ func argumentForCommand(cmd string) any { } func TestRunningCommandWithDocker(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -50,9 +50,9 @@ func TestRunningCommandWithDocker(t *testing.T) { } func TestRunningCommandWithDockerAndCustomDockerfile(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -84,9 +84,9 @@ func TestRunningCommandWithDockerAndCustomDockerfile(t *testing.T) { } func TestRunningFailingCommandWithDocker(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -123,9 +123,9 @@ func TestRunningFailingCommandWithDocker(t *testing.T) { } func TestRunningCommandWithDockerCompose(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -156,9 +156,9 @@ func TestRunningCommandWithDockerCompose(t *testing.T) { } func TestRunningFailingCommandWithDockerCompose(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -196,9 +196,9 @@ func TestRunningFailingCommandWithDockerCompose(t *testing.T) { } func TestRunningCommandWithDockerComposeAndExtraConfig(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -230,9 +230,9 @@ func TestRunningCommandWithDockerComposeAndExtraConfig(t *testing.T) { } func TestRunningCommandWithDockerComposeAndBuildAll(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -254,7 +254,7 @@ func TestRunningCommandWithDockerComposeAndBuildAll(t *testing.T) { tester.RunAndCheck(t, env...) } -func expectCommandHooks(exitStatus string, t *testing.T, tester *BootstrapTester) { +func expectCommandHooks(exitStatus string, t *testing.T, tester *ExecutorTester) { tester.ExpectGlobalHook("pre-command").Once() tester.ExpectLocalHook("pre-command").Once() tester.ExpectGlobalHook("post-command").Once() diff --git a/bootstrap/integration/bootstrap_tester.go b/job/integration/executor_tester.go similarity index 79% rename from bootstrap/integration/bootstrap_tester.go rename to job/integration/executor_tester.go index 68e5b9f719..8f47492cdd 100644 --- a/bootstrap/integration/bootstrap_tester.go +++ b/job/integration/executor_tester.go @@ -23,8 +23,8 @@ import ( "github.com/buildkite/bintest/v3" ) -// BootstrapTester invokes a buildkite-agent bootstrap script with a temporary environment -type BootstrapTester struct { +// ExecutorTester invokes a buildkite-agent executor script with a temporary environment +type ExecutorTester struct { Name string Args []string Env []string @@ -43,7 +43,7 @@ type BootstrapTester struct { mocks []*bintest.Mock } -func NewBootstrapTester() (*BootstrapTester, error) { +func NewExecutorTester() (*ExecutorTester, error) { // The job API experiment adds a unix domain socket to a directory in the home directory // UDS names are limited to 108 characters, so we need to use a shorter home directory // Who knows what's going on in windowsland @@ -57,24 +57,24 @@ func NewBootstrapTester() (*BootstrapTester, error) { return nil, fmt.Errorf("making home directory: %w", err) } - pathDir, err := os.MkdirTemp("", "bootstrap-path") + pathDir, err := os.MkdirTemp("", "executor-path") if err != nil { - return nil, fmt.Errorf("making bootstrap-path directory: %w", err) + return nil, fmt.Errorf("making executor-path directory: %w", err) } - buildDir, err := os.MkdirTemp("", "bootstrap-builds") + buildDir, err := os.MkdirTemp("", "executor-builds") if err != nil { - return nil, fmt.Errorf("making bootstrap-builds directory: %w", err) + return nil, fmt.Errorf("making executor-builds directory: %w", err) } - hooksDir, err := os.MkdirTemp("", "bootstrap-hooks") + hooksDir, err := os.MkdirTemp("", "executor-hooks") if err != nil { - return nil, fmt.Errorf("making bootstrap-hooks directory: %w", err) + return nil, fmt.Errorf("making executor-hooks directory: %w", err) } - pluginsDir, err := os.MkdirTemp("", "bootstrap-plugins") + pluginsDir, err := os.MkdirTemp("", "executor-plugins") if err != nil { - return nil, fmt.Errorf("making bootstrap-plugins directory: %w", err) + return nil, fmt.Errorf("making executor-plugins directory: %w", err) } repo, err := createTestGitRespository() @@ -82,9 +82,9 @@ func NewBootstrapTester() (*BootstrapTester, error) { return nil, fmt.Errorf("creating test git repo: %w", err) } - bt := &BootstrapTester{ + bt := &ExecutorTester{ Name: os.Args[0], - Args: []string{"bootstrap"}, + Args: []string{"job", "run"}, Repo: repo, Env: []string{ "HOME=" + homeDir, @@ -118,9 +118,9 @@ func NewBootstrapTester() (*BootstrapTester, error) { bt.Env = append(bt.Env, "BUILDKITE_AGENT_EXPERIMENT="+strings.Join(exp, ",")) if experiments.IsEnabled(experiments.GitMirrors) { - gitMirrorsDir, err := os.MkdirTemp("", "bootstrap-git-mirrors") + gitMirrorsDir, err := os.MkdirTemp("", "executor-git-mirrors") if err != nil { - return nil, fmt.Errorf("making bootstrap-git-mirrors directory: %w", err) + return nil, fmt.Errorf("making executor-git-mirrors directory: %w", err) } bt.GitMirrorsDir = gitMirrorsDir @@ -156,7 +156,7 @@ func NewBootstrapTester() (*BootstrapTester, error) { } // Mock creates a mock for a binary using bintest -func (b *BootstrapTester) Mock(name string) (*bintest.Mock, error) { +func (b *ExecutorTester) Mock(name string) (*bintest.Mock, error) { mock, err := bintest.NewMock(filepath.Join(b.PathDir, name)) if err != nil { return mock, err @@ -167,16 +167,16 @@ func (b *BootstrapTester) Mock(name string) (*bintest.Mock, error) { } // MustMock will fail the test if creating the mock fails -func (b *BootstrapTester) MustMock(t *testing.T, name string) *bintest.Mock { +func (b *ExecutorTester) MustMock(t *testing.T, name string) *bintest.Mock { mock, err := b.Mock(name) if err != nil { - t.Fatalf("BootstrapTester.Mock(%q) error = %v", name, err) + t.Fatalf("ExecutorTester.Mock(%q) error = %v", name, err) } return mock } // HasMock returns true if a mock has been created by that name -func (b *BootstrapTester) HasMock(name string) bool { +func (b *ExecutorTester) HasMock(name string) bool { for _, m := range b.mocks { if strings.TrimSuffix(m.Name, filepath.Ext(m.Name)) == name { return true @@ -186,7 +186,7 @@ func (b *BootstrapTester) HasMock(name string) bool { } // MockAgent creates a mock for the buildkite-agent binary -func (b *BootstrapTester) MockAgent(t *testing.T) *bintest.Mock { +func (b *ExecutorTester) MockAgent(t *testing.T) *bintest.Mock { agent := b.MustMock(t, "buildkite-agent") agent.Expect("env", "dump"). Min(0). @@ -197,7 +197,7 @@ func (b *BootstrapTester) MockAgent(t *testing.T) *bintest.Mock { } // writeHookScript generates a buildkite-agent hook script that calls a mock binary -func (b *BootstrapTester) writeHookScript(m *bintest.Mock, name string, dir string, args ...string) (string, error) { +func (b *ExecutorTester) writeHookScript(m *bintest.Mock, name string, dir string, args ...string) (string, error) { hookScript := filepath.Join(dir, name) body := "" @@ -217,7 +217,7 @@ func (b *BootstrapTester) writeHookScript(m *bintest.Mock, name string, dir stri // ExpectLocalHook creates a mock object and a script in the git repository's buildkite hooks dir // that proxies to the mock. This lets you set up expectations on a local hook -func (b *BootstrapTester) ExpectLocalHook(name string) *bintest.Expectation { +func (b *ExecutorTester) ExpectLocalHook(name string) *bintest.Expectation { hooksDir := filepath.Join(b.Repo.Path, ".buildkite", "hooks") if err := os.MkdirAll(hooksDir, 0700); err != nil { @@ -241,7 +241,7 @@ func (b *BootstrapTester) ExpectLocalHook(name string) *bintest.Expectation { // ExpectGlobalHook creates a mock object and a script in the global buildkite hooks dir // that proxies to the mock. This lets you set up expectations on a global hook -func (b *BootstrapTester) ExpectGlobalHook(name string) *bintest.Expectation { +func (b *ExecutorTester) ExpectGlobalHook(name string) *bintest.Expectation { _, err := b.writeHookScript(b.hookMock, name, b.HooksDir, "global", name) if err != nil { panic(err) @@ -250,8 +250,8 @@ func (b *BootstrapTester) ExpectGlobalHook(name string) *bintest.Expectation { return b.hookMock.Expect("global", name) } -// Run the bootstrap and return any errors -func (b *BootstrapTester) Run(t *testing.T, env ...string) error { +// Run the executor and return any errors +func (b *ExecutorTester) Run(t *testing.T, env ...string) error { // Mock out the meta-data calls to the agent after checkout if !b.HasMock("buildkite-agent") { agent := b.MockAgent(t) @@ -271,7 +271,7 @@ func (b *BootstrapTester) Run(t *testing.T, env ...string) error { buf := &buffer{} - if os.Getenv(`DEBUG_BOOTSTRAP`) == "1" { + if os.Getenv("DEBUG_JOB_EXEC") == "1" { w := newTestLogWriter(t) b.cmd.Stdout = io.MultiWriter(buf, w) b.cmd.Stderr = io.MultiWriter(buf, w) @@ -295,24 +295,24 @@ func (b *BootstrapTester) Run(t *testing.T, env ...string) error { return err } -func (b *BootstrapTester) Cancel() error { +func (b *ExecutorTester) Cancel() error { b.cmdLock.Lock() defer b.cmdLock.Unlock() log.Printf("Killing pid %d", b.cmd.Process.Pid) return b.cmd.Process.Signal(syscall.SIGINT) } -func (b *BootstrapTester) CheckMocks(t *testing.T) { +func (b *ExecutorTester) CheckMocks(t *testing.T) { for _, mock := range b.mocks { mock.Check(t) } } -func (b *BootstrapTester) CheckoutDir() string { +func (b *ExecutorTester) CheckoutDir() string { return filepath.Join(b.BuildDir, "test-agent", "test", "test-project") } -func (b *BootstrapTester) ReadEnvFromOutput(key string) (string, bool) { +func (b *ExecutorTester) ReadEnvFromOutput(key string) (string, bool) { re := regexp.MustCompile(key + "=(.+)\n") matches := re.FindStringSubmatch(b.Output) if len(matches) == 0 { @@ -321,20 +321,20 @@ func (b *BootstrapTester) ReadEnvFromOutput(key string) (string, bool) { return matches[1], true } -// Run the bootstrap and then check the mocks -func (b *BootstrapTester) RunAndCheck(t *testing.T, env ...string) { +// Run the executor and then check the mocks +func (b *ExecutorTester) RunAndCheck(t *testing.T, env ...string) { err := b.Run(t, env...) - t.Logf("Bootstrap output:\n%s", b.Output) + t.Logf("Executor output:\n%s", b.Output) if err != nil { - t.Fatalf("BootstrapTester.Run(%q) = %v", env, err) + t.Fatalf("ExecutorTester.Run(%q) = %v", env, err) } b.CheckMocks(t) } // Close the tester, delete all the directories and mocks -func (b *BootstrapTester) Close() error { +func (b *ExecutorTester) Close() error { for _, mock := range b.mocks { if err := mock.Close(); err != nil { return err @@ -368,11 +368,11 @@ func (b *BootstrapTester) Close() error { return nil } -func mockEnvAsJSONOnStdout(b *BootstrapTester) func(c *bintest.Call) { +func mockEnvAsJSONOnStdout(b *ExecutorTester) func(c *bintest.Call) { return func(c *bintest.Call) { envMap := map[string]string{} - for _, e := range b.Env { // The env from the bootstrap tester + for _, e := range b.Env { // The env from the executor tester k, v, _ := env.Split(e) envMap[k] = v } diff --git a/bootstrap/integration/git.go b/job/integration/git.go similarity index 78% rename from bootstrap/integration/git.go rename to job/integration/git.go index c14dd01144..a9dff4a712 100644 --- a/bootstrap/integration/git.go +++ b/job/integration/git.go @@ -10,47 +10,47 @@ import ( func createTestGitRespository() (*gitRepository, error) { repo, err := newGitRepository() if err != nil { - return nil, err + return nil, fmt.Errorf("creating git repo: %w", err) } if err = repo.CreateBranch("main"); err != nil { - return nil, err + return nil, fmt.Errorf("creating main branch: %w", err) } if err := os.WriteFile(filepath.Join(repo.Path, "test.txt"), []byte("This is a test"), 0600); err != nil { - return nil, err + return nil, fmt.Errorf("writing test.txt on branch: master: %w", err) } if err = repo.Add("test.txt"); err != nil { - return nil, err + return nil, fmt.Errorf("`git add`ing test.txt on branch: master : %w", err) } if err = repo.Commit("Initial Commit"); err != nil { - return nil, err + return nil, fmt.Errorf("committing on branch: master: %w", err) } if err = repo.CreateBranch("update-test-txt"); err != nil { - return nil, err + return nil, fmt.Errorf("creating update-test-txt branch: %w", err) } if err := os.WriteFile(filepath.Join(repo.Path, "test.txt"), []byte("This is a test pull request"), 0600); err != nil { - return nil, err + return nil, fmt.Errorf("writing test.txt on branch update-test-text: %w", err) } if err = repo.Add("test.txt"); err != nil { - return nil, err + return nil, fmt.Errorf("`git add`ing test.txt on branch update-test-txt: %w", err) } if err = repo.Commit("PR Commit"); err != nil { - return nil, err + return nil, fmt.Errorf("creating PR commit: %w", err) } if _, err = repo.Execute("update-ref", "refs/pull/123/head", "HEAD"); err != nil { - return nil, err + return nil, fmt.Errorf("updateing refs/pull/123/head to HEAD: %w", err) } if err = repo.CheckoutBranch("main"); err != nil { - return nil, err + return nil, fmt.Errorf("checking out main: %w", err) } return repo, nil @@ -63,7 +63,7 @@ type gitRepository struct { func newGitRepository() (*gitRepository, error) { tempDirRaw, err := os.MkdirTemp("", "git-repo") if err != nil { - return nil, fmt.Errorf("Error creating temp dir: %v", err) + return nil, fmt.Errorf("creating temp dir: %w", err) } // io.TempDir on Windows tilde-shortens (8.3 style?) long filenames in the path. @@ -84,7 +84,7 @@ func newGitRepository() (*gitRepository, error) { // EvalSymlinks seems best? Maybe there's a better way? tempDir, err := filepath.EvalSymlinks(tempDirRaw) if err != nil { - return nil, fmt.Errorf("EvalSymlinks for temp dir: %v", err) + return nil, fmt.Errorf("EvalSymlinks for temp dir: %w", err) } gr := &gitRepository{Path: tempDir} @@ -132,8 +132,9 @@ func (gr *gitRepository) Close() error { func (gr *gitRepository) Execute(args ...string) (string, error) { path, err := exec.LookPath("git") if err != nil { - return "", err + return "", fmt.Errorf("finding git executable on path: %w", err) } + cmd := exec.Command(path, args...) cmd.Dir = gr.Path // log.Printf("$ git %v", args) @@ -144,8 +145,8 @@ func (gr *gitRepository) Execute(args ...string) (string, error) { func (gr *gitRepository) ExecuteAll(argsSlice [][]string) error { for _, args := range argsSlice { - if _, err := gr.Execute(args...); err != nil { - return err + if out, err := gr.Execute(args...); err != nil { + return fmt.Errorf("executing git %v: %s (%v)", args, out, err) } } return nil diff --git a/bootstrap/integration/hooks_integration_test.go b/job/integration/hooks_integration_test.go similarity index 90% rename from bootstrap/integration/hooks_integration_test.go rename to job/integration/hooks_integration_test.go index 119aa872d4..fdf93b1ac0 100644 --- a/bootstrap/integration/hooks_integration_test.go +++ b/job/integration/hooks_integration_test.go @@ -10,16 +10,16 @@ import ( "testing" "time" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/bintest/v3" ) func TestEnvironmentVariablesPassBetweenHooks(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -64,9 +64,9 @@ func TestEnvironmentVariablesPassBetweenHooks(t *testing.T) { func TestHooksCanUnsetEnvironmentVariables(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -124,9 +124,9 @@ func TestHooksCanUnsetEnvironmentVariables(t *testing.T) { func TestDirectoryPassesBetweenHooks(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -158,9 +158,9 @@ func TestDirectoryPassesBetweenHooks(t *testing.T) { } func TestDirectoryPassesBetweenHooksIgnoredUnderExit(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -195,9 +195,9 @@ func TestDirectoryPassesBetweenHooksIgnoredUnderExit(t *testing.T) { func TestCheckingOutFiresCorrectHooks(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -225,9 +225,9 @@ func TestCheckingOutFiresCorrectHooks(t *testing.T) { func TestReplacingCheckoutHook(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -254,9 +254,9 @@ func TestReplacingCheckoutHook(t *testing.T) { func TestReplacingGlobalCommandHook(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -279,9 +279,9 @@ func TestReplacingGlobalCommandHook(t *testing.T) { func TestReplacingLocalCommandHook(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -305,9 +305,9 @@ func TestReplacingLocalCommandHook(t *testing.T) { func TestPreExitHooksFireAfterCommandFailures(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -346,9 +346,9 @@ func TestPreExitHooksFireAfterHookFailures(t *testing.T) { t.Run(tc.failingHook, func(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -396,9 +396,9 @@ func TestPreExitHooksFireAfterHookFailures(t *testing.T) { func TestNoLocalHooksCalledWhenConfigSet(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -430,9 +430,9 @@ func TestExitCodesPropagateOutFromGlobalHooks(t *testing.T) { // "post-artifact", } { t.Run(hook, func(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -460,9 +460,9 @@ func TestPreExitHooksFireAfterCancel(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() diff --git a/bootstrap/integration/job_api_integration_test.go b/job/integration/job_api_integration_test.go similarity index 97% rename from bootstrap/integration/job_api_integration_test.go rename to job/integration/job_api_integration_test.go index e419ffa728..70d8539657 100644 --- a/bootstrap/integration/job_api_integration_test.go +++ b/job/integration/job_api_integration_test.go @@ -18,9 +18,9 @@ import ( func TestBootstrapRunsJobAPI(t *testing.T) { defer experimentWithUndo(experiments.JobAPI)() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() diff --git a/bootstrap/integration/main_test.go b/job/integration/main_test.go similarity index 76% rename from bootstrap/integration/main_test.go rename to job/integration/main_test.go index b55c57b0e9..579e36bc0f 100644 --- a/bootstrap/integration/main_test.go +++ b/job/integration/main_test.go @@ -13,13 +13,13 @@ import ( ) func TestMain(m *testing.M) { - // If we are passed "bootstrap", execute like the bootstrap cli - if len(os.Args) > 1 && os.Args[1] == "bootstrap" { + // If we are passed "job run", execute like the job run cli + if sliceEq(os.Args, []string{"job", "run"}) { app := cli.NewApp() app.Name = "buildkite-agent" app.Version = version.Version() app.Commands = []cli.Command{ - clicommand.BootstrapCommand, + clicommand.JobRunCommand, } if err := app.Run(os.Args); err != nil { @@ -50,3 +50,17 @@ func TestMain(m *testing.M) { code := m.Run() os.Exit(code) } + +func sliceEq(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} diff --git a/bootstrap/integration/plugin_integration_test.go b/job/integration/plugin_integration_test.go similarity index 91% rename from bootstrap/integration/plugin_integration_test.go rename to job/integration/plugin_integration_test.go index 6c1e97c1f5..5400dc111c 100644 --- a/bootstrap/integration/plugin_integration_test.go +++ b/job/integration/plugin_integration_test.go @@ -10,16 +10,16 @@ import ( "strings" "testing" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/bintest/v3" ) func TestRunningPlugins(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -78,9 +78,9 @@ func TestRunningPlugins(t *testing.T) { func TestExitCodesPropagateOutFromPlugins(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -122,12 +122,12 @@ func TestExitCodesPropagateOutFromPlugins(t *testing.T) { tester.CheckMocks(t) } -func TestMalformedPluginNamesDontCrashBootstrap(t *testing.T) { +func TestMalformedPluginNamesDontCrashExecutor(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -149,9 +149,9 @@ func TestMalformedPluginNamesDontCrashBootstrap(t *testing.T) { func TestOverlappingPluginHooks(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -213,9 +213,9 @@ func TestPluginCloneRetried(t *testing.T) { t.Parallel() - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() @@ -278,23 +278,23 @@ func TestPluginCloneRetried(t *testing.T) { // that a plugin modified upstream is treated as expected. That is, by default, the updates won't // take effect, but with BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH set, they will. func TestModifiedPluginNoForcePull(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() - // Let's set a fixed location for plugins, otherwise NewBootstrapTester() gives us a random new + // Let's set a fixed location for plugins, otherwise NewExecutorTester() gives us a random new // tempdir every time, which defeats our test. Later we'll use this pluginsDir for the second // test run, too. - pluginsDir, err := os.MkdirTemp("", "bootstrap-plugins") + pluginsDir, err := os.MkdirTemp("", "executor-plugins") if err != nil { - t.Fatalf(`os.MkdirTemp("", "bootstrap-plugins") error = %v`, err) + t.Fatalf(`os.MkdirTemp("", "executor-plugins") error = %v`, err) } tester.PluginsDir = pluginsDir // There's a bit of machinery in replacePluginPathInEnv to modify only the - // BUILDKITE_PLUGINS_PATH, leaving the rest of the environment variables NewBootstrapTester() + // BUILDKITE_PLUGINS_PATH, leaving the rest of the environment variables NewExecutorTester() // gave us as-is. tester.Env = replacePluginPathInEnv(tester.Env, pluginsDir) @@ -343,9 +343,9 @@ func TestModifiedPluginNoForcePull(t *testing.T) { tester.RunAndCheck(t, env...) // Now, we want to "repeat" the test build, having modified the plugin's contents. - tester2, err := NewBootstrapTester() + tester2, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester2.Close() @@ -385,17 +385,17 @@ func TestModifiedPluginNoForcePull(t *testing.T) { // and after modifying a plugin's source, but this time with BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH // set to true. So, we expect the upstream plugin changes to take effect on our second build. func TestModifiedPluginWithForcePull(t *testing.T) { - tester, err := NewBootstrapTester() + tester, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester.Close() // Let's set a fixed location for plugins, otherwise it'll be a random new tempdir every time // which defeats our test. - pluginsDir, err := os.MkdirTemp("", "bootstrap-plugins") + pluginsDir, err := os.MkdirTemp("", "executor-plugins") if err != nil { - t.Fatalf(`os.MkdirTemp("", "bootstrap-plugins") error = %v`, err) + t.Fatalf(`os.MkdirTemp("", "executor-plugins") error = %v`, err) } // Same resetting code for BUILDKITE_PLUGINS_PATH as in the previous test @@ -442,9 +442,9 @@ func TestModifiedPluginWithForcePull(t *testing.T) { tester.RunAndCheck(t, env...) - tester2, err := NewBootstrapTester() + tester2, err := NewExecutorTester() if err != nil { - t.Fatalf("NewBootstrapTester() error = %v", err) + t.Fatalf("NewExecutorTester() error = %v", err) } defer tester2.Close() @@ -573,7 +573,7 @@ func (tp *testPlugin) MarshalJSON() ([]byte, error) { } // replacePluginPathInEnv is useful for modifying the Env blob of a tester created with -// NewBootstrapTester(). We need to do that because the tester relies on BUILDKITE_PLUGINS_PATH, +// NewExecutorTester(). We need to do that because the tester relies on BUILDKITE_PLUGINS_PATH, // not on the .PluginsDir field as one might expect. func replacePluginPathInEnv(originalEnv []string, pluginsDir string) (newEnv []string) { newEnv = []string{} diff --git a/bootstrap/knownhosts.go b/job/knownhosts.go similarity index 98% rename from bootstrap/knownhosts.go rename to job/knownhosts.go index 4455c0df75..5991c1087f 100644 --- a/bootstrap/knownhosts.go +++ b/job/knownhosts.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "bufio" @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" homedir "github.com/mitchellh/go-homedir" "golang.org/x/crypto/ssh/knownhosts" ) diff --git a/bootstrap/knownhosts_test.go b/job/knownhosts_test.go similarity index 96% rename from bootstrap/knownhosts_test.go rename to job/knownhosts_test.go index e1417351b5..aa57ff97d2 100644 --- a/bootstrap/knownhosts_test.go +++ b/job/knownhosts_test.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "context" @@ -7,7 +7,7 @@ import ( "os" "testing" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/gliderlabs/ssh" ) diff --git a/bootstrap/shell/batch.go b/job/shell/batch.go similarity index 100% rename from bootstrap/shell/batch.go rename to job/shell/batch.go diff --git a/bootstrap/shell/export_test.go b/job/shell/export_test.go similarity index 100% rename from bootstrap/shell/export_test.go rename to job/shell/export_test.go diff --git a/bootstrap/shell/logger.go b/job/shell/logger.go similarity index 98% rename from bootstrap/shell/logger.go rename to job/shell/logger.go index 45babc4f38..c8bbb8e0e4 100644 --- a/bootstrap/shell/logger.go +++ b/job/shell/logger.go @@ -26,7 +26,7 @@ type Logger interface { // Errorf shows a Buildkite formatted error expands the previous group Errorf(format string, v ...any) - // Warningf shows a buildkite bootstrap warning + // Warningf shows a buildkite executor warning Warningf(format string, v ...any) // Promptf prints a shell prompt diff --git a/bootstrap/shell/logger_test.go b/job/shell/logger_test.go similarity index 97% rename from bootstrap/shell/logger_test.go rename to job/shell/logger_test.go index f87ed1c719..7e8e6757f2 100644 --- a/bootstrap/shell/logger_test.go +++ b/job/shell/logger_test.go @@ -6,7 +6,7 @@ import ( "runtime" "testing" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/google/go-cmp/cmp" ) diff --git a/bootstrap/shell/lookpath.go b/job/shell/lookpath.go similarity index 100% rename from bootstrap/shell/lookpath.go rename to job/shell/lookpath.go diff --git a/bootstrap/shell/lookpath_windows.go b/job/shell/lookpath_windows.go similarity index 100% rename from bootstrap/shell/lookpath_windows.go rename to job/shell/lookpath_windows.go diff --git a/bootstrap/shell/shell.go b/job/shell/shell.go similarity index 100% rename from bootstrap/shell/shell.go rename to job/shell/shell.go diff --git a/bootstrap/shell/shell_test.go b/job/shell/shell_test.go similarity index 99% rename from bootstrap/shell/shell_test.go rename to job/shell/shell_test.go index f3339fcf54..0f2c99f343 100644 --- a/bootstrap/shell/shell_test.go +++ b/job/shell/shell_test.go @@ -14,8 +14,8 @@ import ( "testing" "time" - "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/experiments" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/bintest/v3" "github.com/google/go-cmp/cmp" ) diff --git a/bootstrap/shell/signal.go b/job/shell/signal.go similarity index 100% rename from bootstrap/shell/signal.go rename to job/shell/signal.go diff --git a/bootstrap/shell/signal_windows.go b/job/shell/signal_windows.go similarity index 100% rename from bootstrap/shell/signal_windows.go rename to job/shell/signal_windows.go diff --git a/bootstrap/shell/tempfile.go b/job/shell/tempfile.go similarity index 100% rename from bootstrap/shell/tempfile.go rename to job/shell/tempfile.go diff --git a/bootstrap/shell/test.go b/job/shell/test.go similarity index 100% rename from bootstrap/shell/test.go rename to job/shell/test.go diff --git a/bootstrap/ssh.go b/job/ssh.go similarity index 97% rename from bootstrap/ssh.go rename to job/ssh.go index 847cb09bbf..d633e6cd63 100644 --- a/bootstrap/ssh.go +++ b/job/ssh.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "context" @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/roko" ) diff --git a/bootstrap/ssh_test.go b/job/ssh_test.go similarity index 97% rename from bootstrap/ssh_test.go rename to job/ssh_test.go index 705db89b83..3abaf59d5e 100644 --- a/bootstrap/ssh_test.go +++ b/job/ssh_test.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "context" @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/bintest/v3" "github.com/stretchr/testify/assert" ) diff --git a/bootstrap/tracing.go b/job/tracing.go similarity index 92% rename from bootstrap/tracing.go rename to job/tracing.go index 732abc0a72..77e8b5af8f 100644 --- a/bootstrap/tracing.go +++ b/job/tracing.go @@ -1,4 +1,4 @@ -package bootstrap +package job import ( "context" @@ -36,7 +36,7 @@ type stopper func() func noopStopper() {} -func (b *Bootstrap) startTracing(ctx context.Context) (tracetools.Span, context.Context, stopper) { +func (b *Executor) startTracing(ctx context.Context) (tracetools.Span, context.Context, stopper) { switch b.Config.TracingBackend { case tracetools.BackendDatadog: // Newer versions of the tracing libs print out diagnostic info which spams the @@ -55,12 +55,12 @@ func (b *Bootstrap) startTracing(ctx context.Context) (tracetools.Span, context. default: b.shell.Commentf("An invalid tracing backend was provided: %q. Tracing will not occur.", b.Config.TracingBackend) - b.Config.TracingBackend = tracetools.BackendNone // Ensure that we don't do any tracing after this, some of the stuff in tracetools uses the bootstrap's tracking backend + b.Config.TracingBackend = tracetools.BackendNone // Ensure that we don't do any tracing after this, some of the stuff in tracetools uses the executor's tracking backend return &tracetools.NoopSpan{}, ctx, noopStopper } } -func (b *Bootstrap) ddResourceName() string { +func (b *Executor) ddResourceName() string { label, ok := b.shell.Env.Get("BUILDKITE_LABEL") if !ok { label = "job" @@ -71,7 +71,7 @@ func (b *Bootstrap) ddResourceName() string { // startTracingDatadog sets up tracing based on the config values. It uses opentracing as an // abstraction so the agent can support multiple libraries if needbe. -func (b *Bootstrap) startTracingDatadog(ctx context.Context) (tracetools.Span, context.Context, stopper) { +func (b *Executor) startTracingDatadog(ctx context.Context) (tracetools.Span, context.Context, stopper) { opts := []tracer.StartOption{ tracer.WithService(b.Config.TracingServiceName), tracer.WithSampler(tracer.NewAllSampler()), @@ -99,7 +99,7 @@ func (b *Bootstrap) startTracingDatadog(ctx context.Context) (tracetools.Span, c // extractTraceCtx pulls encoded distributed tracing information from the env vars. // Note: This should match the injectTraceCtx code in shell. -func (b *Bootstrap) extractDDTraceCtx() opentracing.SpanContext { +func (b *Executor) extractDDTraceCtx() opentracing.SpanContext { sctx, err := tracetools.DecodeTraceContext(b.shell.Env.Dump()) if err != nil { // Return nil so a new span will be created @@ -108,7 +108,7 @@ func (b *Bootstrap) extractDDTraceCtx() opentracing.SpanContext { return sctx } -func (b *Bootstrap) otRootSpanName() string { +func (b *Executor) otRootSpanName() string { base := b.OrganizationSlug + "/" + b.PipelineSlug + "/" key, ok := b.shell.Env.Get("BUILDKITE_STEP_KEY") if ok && key != "" { @@ -123,7 +123,7 @@ func (b *Bootstrap) otRootSpanName() string { return base + "job" } -func (b *Bootstrap) startTracingOpenTelemetry(ctx context.Context) (tracetools.Span, context.Context, stopper) { +func (b *Executor) startTracingOpenTelemetry(ctx context.Context) (tracetools.Span, context.Context, stopper) { client := otlptracegrpc.NewClient() exporter, err := otlptrace.New(ctx, client) if err != nil { @@ -182,7 +182,7 @@ func (b *Bootstrap) startTracingOpenTelemetry(ctx context.Context) (tracetools.S return tracetools.NewOpenTelemetrySpan(span), ctx, stop } -func GenericTracingExtras(b *Bootstrap, env *env.Environment) map[string]any { +func GenericTracingExtras(b *Executor, env *env.Environment) map[string]any { buildID, _ := env.Get("BUILDKITE_BUILD_ID") buildNumber, _ := env.Get("BUILDKITE_BUILD_NUMBER") buildURL, _ := env.Get("BUILDKITE_BUILD_URL") @@ -287,7 +287,7 @@ func toOpenTelemetryAttributes(extras map[string]any) ([]attribute.KeyValue, map return attrs, unknownAttrTypes } -func (b *Bootstrap) implementationSpecificSpanName(otelName, ddName string) string { +func (b *Executor) implementationSpecificSpanName(otelName, ddName string) string { switch b.TracingBackend { case tracetools.BackendDatadog: return ddName diff --git a/jobapi/middleware.go b/jobapi/middleware.go index 3368b793a3..0fc76a3b91 100644 --- a/jobapi/middleware.go +++ b/jobapi/middleware.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" ) func LoggerMiddleware(l shell.Logger) func(http.Handler) http.Handler { diff --git a/jobapi/server.go b/jobapi/server.go index 0882a132ad..946da5d88f 100644 --- a/jobapi/server.go +++ b/jobapi/server.go @@ -14,8 +14,8 @@ import ( "sync" "time" - "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/env" + "github.com/buildkite/agent/v3/job/shell" ) // Server is a Job API server. It provides an HTTP API with which to interact with the job currently running in the buildkite agent diff --git a/jobapi/server_test.go b/jobapi/server_test.go index e8acb42e3f..ddd11521ec 100644 --- a/jobapi/server_test.go +++ b/jobapi/server_test.go @@ -14,8 +14,8 @@ import ( "testing" "time" - "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/env" + "github.com/buildkite/agent/v3/job/shell" "github.com/buildkite/agent/v3/jobapi" "github.com/google/go-cmp/cmp" ) diff --git a/main.go b/main.go index 8987630ccd..6a0ee3ffa5 100644 --- a/main.go +++ b/main.go @@ -137,6 +137,13 @@ func main() { }, }, clicommand.BootstrapCommand, + { + Name: "job", + Usage: "Interact with Buildkite jobs", + Subcommands: []cli.Command{ + clicommand.JobRunCommand, + }, + }, } app.ErrWriter = os.Stderr diff --git a/packaging/docker/alpine-k8s/buildkite-agent.cfg b/packaging/docker/alpine-k8s/buildkite-agent.cfg index 4899c2c8de..2950159dac 100644 --- a/packaging/docker/alpine-k8s/buildkite-agent.cfg +++ b/packaging/docker/alpine-k8s/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent job run`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-run-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/docker/alpine/buildkite-agent.cfg b/packaging/docker/alpine/buildkite-agent.cfg index 4899c2c8de..2950159dac 100644 --- a/packaging/docker/alpine/buildkite-agent.cfg +++ b/packaging/docker/alpine/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent job run`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-run-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/docker/sidecar/buildkite-agent.cfg b/packaging/docker/sidecar/buildkite-agent.cfg index 4899c2c8de..2950159dac 100644 --- a/packaging/docker/sidecar/buildkite-agent.cfg +++ b/packaging/docker/sidecar/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent job run`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-run-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/docker/ubuntu-18.04/buildkite-agent.cfg b/packaging/docker/ubuntu-18.04/buildkite-agent.cfg index 4899c2c8de..2950159dac 100644 --- a/packaging/docker/ubuntu-18.04/buildkite-agent.cfg +++ b/packaging/docker/ubuntu-18.04/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent job run`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-run-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/docker/ubuntu-20.04/buildkite-agent.cfg b/packaging/docker/ubuntu-20.04/buildkite-agent.cfg index 4899c2c8de..2950159dac 100644 --- a/packaging/docker/ubuntu-20.04/buildkite-agent.cfg +++ b/packaging/docker/ubuntu-20.04/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent job run`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-run-script="" # Path to where the builds will run from build-path="/buildkite/builds" diff --git a/packaging/github/linux/buildkite-agent.cfg b/packaging/github/linux/buildkite-agent.cfg index affe7792fe..9a96bd837a 100644 --- a/packaging/github/linux/buildkite-agent.cfg +++ b/packaging/github/linux/buildkite-agent.cfg @@ -25,10 +25,10 @@ name="%hostname-%spawn" # Include the host's Google Cloud instance labels as tags # tags-from-gcp-labels=true -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent job run`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-run-script="" # Path to where the builds will run from build-path="$HOME/.buildkite-agent/builds" diff --git a/packaging/github/windows/buildkite-agent.cfg b/packaging/github/windows/buildkite-agent.cfg index f98f8fed4d..ed1c3a6449 100644 --- a/packaging/github/windows/buildkite-agent.cfg +++ b/packaging/github/windows/buildkite-agent.cfg @@ -13,10 +13,10 @@ name="%hostname-%spawn" # Tags for the agent (default is "queue=default") # tags="key1=val2,key2=val2" -# Path to a custom bootstrap command to run. By default this is `buildkite-agent bootstrap`. +# Path to a custom job execution command to run. By default this is `buildkite-agent job run`. # This allows you to override the entire execution of a job. Generally you should use hooks instead! # See https://buildkite.com/docs/agent/hooks -# bootstrap-script="" +# job-run-script="" # Path to where the builds will run from build-path="C:\buildkite-agent\builds" diff --git a/redaction/redactor.go b/redaction/redactor.go index 7a2b3bab28..fdab7e7601 100644 --- a/redaction/redactor.go +++ b/redaction/redactor.go @@ -8,7 +8,7 @@ import ( "io" "path" - "github.com/buildkite/agent/v3/bootstrap/shell" + "github.com/buildkite/agent/v3/job/shell" ) // RedactLengthMin is the shortest string length that will be considered a diff --git a/utils/path_test.go b/utils/path_test.go index 7d1742095f..3956bd8066 100644 --- a/utils/path_test.go +++ b/utils/path_test.go @@ -47,9 +47,9 @@ func TestNormalizingCommands(t *testing.T) { usr, err := user.Current() assert.NoError(t, err) - c, err := NormalizeCommand(filepath.Join("~/", "buildkite-agent", "bootstrap.sh")) + c, err := NormalizeCommand(filepath.Join("~/", "buildkite-agent", "job", "run")) assert.NoError(t, err) - assert.Equal(t, filepath.Join(usr.HomeDir, "buildkite-agent", "bootstrap.sh"), c) + assert.Equal(t, filepath.Join(usr.HomeDir, "buildkite-agent", "job", "run"), c) c, err = NormalizeCommand("cat test.log") assert.NoError(t, err)