Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use terraform-exec #268

Closed
wants to merge 4 commits into from
Closed

Use terraform-exec #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5ea8398
Migrate terraform execution to terraform-exec
appilon Sep 17, 2020
666470a
Refactor executor mock
appilon Sep 23, 2020
d96cbf1
bump terraform-exec
appilon Sep 23, 2020
1bade99
Update TODOs
appilon Sep 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,22 @@ require (
github.com/gammazero/workerpool v1.0.0
github.com/google/go-cmp v0.5.1
github.com/hashicorp/go-multierror v1.1.0
github.com/hashicorp/go-version v1.2.0
github.com/hashicorp/go-version v1.2.1
github.com/hashicorp/hcl/v2 v2.5.2-0.20200528183353-fa7c453538de
github.com/hashicorp/terraform-config-inspect v0.0.0-20200806211835-c481b8bfa41e
github.com/hashicorp/terraform-exec v0.11.0
github.com/hashicorp/terraform-json v0.5.0
github.com/hashicorp/terraform-svchost v0.0.0-20191119180714-d2e4933b9136
github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
github.com/mitchellh/cli v1.0.0
github.com/mitchellh/cli v1.1.1
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/mapstructure v1.3.2
github.com/pmezard/go-difflib v1.0.0
github.com/sourcegraph/go-lsp v0.0.0-20200117082640-b19bb38222e2
github.com/spf13/afero v1.3.2
github.com/zclconf/go-cty v1.2.1
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb
golang.org/x/net v0.0.0-20200301022130-244492dfa37a
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect
)

replace github.com/sourcegraph/go-lsp => github.com/radeksimko/go-lsp v0.1.0
186 changes: 184 additions & 2 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

235 changes: 51 additions & 184 deletions internal/terraform/exec/exec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,47 +3,32 @@ package exec
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"strings"
"time"

"github.com/hashicorp/go-version"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
"github.com/hashicorp/terraform-ls/logging"
)

var defaultExecTimeout = 30 * time.Second

// We pass through all variables, but longer term we'll need to reflect
// that some variables might be workspace/directory specific
// and passing through these may be dangerous once the LS
// starts to execute commands which can mutate the state
var passthroughEnvVars = os.Environ()

// cmdCtxFunc allows mocking of Terraform in tests while retaining
// ability to pass context for timeout/cancellation
type cmdCtxFunc func(context.Context, string, ...string) *exec.Cmd

// ExecutorFactory can be used in external consumers of exec pkg
// to enable easy swapping with MockExecutor
type ExecutorFactory func(path string) *Executor

type Executor struct {
tf *tfexec.Terraform

timeout time.Duration

execPath string
workDir string
logger *log.Logger
execLogPath string

cmdCtxFunc cmdCtxFunc
}

type command struct {
Expand All @@ -59,12 +44,31 @@ func NewExecutor(path string) *Executor {
timeout: defaultExecTimeout,
execPath: path,
logger: log.New(ioutil.Discard, "", 0),
cmdCtxFunc: func(ctx context.Context, path string, arg ...string) *exec.Cmd {
return exec.CommandContext(ctx, path, arg...)
},
}
}

// tfExec should be used to initialize and return an instance of tfexec.Terraform just in
// time, the instance will be cached. This is helpful because the language server sets the working
// directory after the creation of the Executor instance.
func (e *Executor) tfExec() *tfexec.Terraform {
if e.tf == nil {
tf, err := tfexec.NewTerraform(e.workDir, e.execPath)
if err != nil {
panic(err)
}
tf.SetLogger(e.logger)

// TODO: support log filename template upstream
// if e.execLogPath != "" {
// logPath, err := logging.ParseExecLogPath(cmd.Args, e.execLogPath)
// tf.SetLogPath(logPath)
// }

e.tf = tf
}
return e.tf
}

func (e *Executor) SetLogger(logger *log.Logger) {
e.logger = logger
}
Expand All @@ -85,115 +89,13 @@ func (e *Executor) GetExecPath() string {
return e.execPath
}

func (e *Executor) cmd(ctx context.Context, args ...string) (*command, error) {
if e.workDir == "" {
return nil, fmt.Errorf("no work directory set")
}

cancel := func() {}
if e.timeout > 0 {
ctx, cancel = context.WithTimeout(ctx, e.timeout)
}

var outBuf bytes.Buffer
var errBuf bytes.Buffer

cmd := e.cmdCtxFunc(ctx, e.execPath, args...)
cmd.Args = append([]string{"terraform"}, args...)
cmd.Dir = e.workDir
cmd.Stderr = &errBuf
cmd.Stdout = &outBuf

// We don't perform upgrade from the context of executor
// and don't report outdated version to users,
// so we don't need to ask checkpoint for upgrades.
cmd.Env = append(cmd.Env, "CHECKPOINT_DISABLE=1")

cmd.Env = append(cmd.Env, passthroughEnvVars...)

if e.execLogPath != "" {
logPath, err := logging.ParseExecLogPath(cmd.Args, e.execLogPath)
if err != nil {
return &command{
Cmd: cmd,
Context: ctx,
CancelFunc: cancel,
StdoutBuffer: &outBuf,
StderrBuffer: &errBuf,
}, fmt.Errorf("failed to parse log path: %w", err)
}
cmd.Env = append(cmd.Env, "TF_LOG=TRACE")
cmd.Env = append(cmd.Env, "TF_LOG_PATH="+logPath)

e.logger.Printf("Execution will be logged to %s", logPath)
}
return &command{
Cmd: cmd,
Context: ctx,
CancelFunc: cancel,
StdoutBuffer: &outBuf,
StderrBuffer: &errBuf,
}, nil
}

func (e *Executor) waitCmd(command *command) ([]byte, error) {
args := command.Cmd.Args
e.logger.Printf("Waiting for command to finish ...")
err := command.Cmd.Wait()
if err != nil {
if tErr, ok := err.(*exec.ExitError); ok {
exitErr := &ExitError{
Err: tErr,
Path: command.Cmd.Path,
Stdout: command.StdoutBuffer.String(),
Stderr: command.StderrBuffer.String(),
}

ctxErr := command.Context.Err()
if errors.Is(ctxErr, context.DeadlineExceeded) {
exitErr.CtxErr = ExecTimeoutError(args, e.timeout)
}
if errors.Is(ctxErr, context.Canceled) {
exitErr.CtxErr = ExecCanceledError(args)
}

return nil, exitErr
}

return nil, err
}

pc := command.Cmd.ProcessState
e.logger.Printf("terraform run (%s %q, in %q, pid %d) finished with exit code %d",
e.execPath, args, e.workDir, pc.Pid(), pc.ExitCode())

return command.StdoutBuffer.Bytes(), nil
}

func (e *Executor) runCmd(command *command) ([]byte, error) {
args := command.Cmd.Args
e.logger.Printf("Starting %s %q in %q...", e.execPath, args, e.workDir)
err := command.Cmd.Start()
if err != nil {
return nil, err
}

return e.waitCmd(command)
}

func (e *Executor) run(ctx context.Context, args ...string) ([]byte, error) {
cmd, err := e.cmd(ctx, args...)
e.logger.Printf("running with timeout %s", e.timeout)
defer cmd.CancelFunc()
if err != nil {
return nil, err
}
return e.runCmd(cmd)
}

type Formatter func(ctx context.Context, input []byte) ([]byte, error)

func (e *Executor) FormatterForVersion(v string) (Formatter, error) {
return formatterForVersion(v, e.Format)
}

func formatterForVersion(v string, f Formatter) (Formatter, error) {
if v == "" {
return nil, fmt.Errorf("unknown version - unable to provide formatter")
}
Expand All @@ -210,65 +112,30 @@ func (e *Executor) FormatterForVersion(v string) (Formatter, error) {
}

if ver.GreaterThanOrEqual(fmtCapableVersion) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You could move this version compatibility check to terraform-exec if you wanted.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is something @radeksimko upstreaming now. Depending on the outcome of the upstreaming work, this PR may be superseded.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hashicorp/terraform-exec#88

return e.Format, nil
return f, nil
}

return nil, fmt.Errorf("no formatter available for %s", v)
}

func (e *Executor) Format(ctx context.Context, input []byte) ([]byte, error) {
cmd, err := e.cmd(ctx, "fmt", "-")
if err != nil {
return nil, err
}
ctx, cancel := contextWithTimeout(ctx, e.timeout)
defer cancel()

stdin, err := cmd.Cmd.StdinPipe()
if err != nil {
return nil, err
}

err = cmd.Cmd.Start()
if err != nil {
return nil, err
}

_, err = writeAndClose(stdin, input)
if err != nil {
return nil, err
}

out, err := e.waitCmd(cmd)
if err != nil {
return nil, fmt.Errorf("failed to format: %w", err)
}

return out, nil
}

func writeAndClose(w io.WriteCloser, input []byte) (int, error) {
defer w.Close()

n, err := w.Write(input)
if err != nil {
return n, err
}

return n, nil
formatted, err := tfexec.FormatString(ctx, e.execPath, string(input))
return []byte(formatted), err
}

func (e *Executor) Version(ctx context.Context) (string, error) {
out, err := e.run(ctx, "version")
ctx, cancel := contextWithTimeout(ctx, e.timeout)
defer cancel()

v, _, err := e.tfExec().Version(ctx, true)
if err != nil {
return "", fmt.Errorf("failed to get version: %w", err)
}
outString := string(out)
lines := strings.Split(outString, "\n")
if len(lines) < 1 {
return "", fmt.Errorf("unexpected version output: %q", outString)
return "", err
}
version := strings.TrimPrefix(lines[0], "Terraform v")

return version, nil
// TODO: consider refactoring codebase to work directly with go-version.Version
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I noticed a fair bit of back and forth between go-version.Version and the stringified representation. I think given I saw go-version types bled onto a few structs throughout the codebade, and given terraform-exec works with it directly it may be worth changing the language server codebase to work directly in that type, in a follow up PR.

return v.String(), nil
}

func (e *Executor) VersionIsSupported(ctx context.Context, c version.Constraints) error {
Expand All @@ -290,16 +157,16 @@ func (e *Executor) VersionIsSupported(ctx context.Context, c version.Constraints
}

func (e *Executor) ProviderSchemas(ctx context.Context) (*tfjson.ProviderSchemas, error) {
outBytes, err := e.run(ctx, "providers", "schema", "-json")
if err != nil {
return nil, fmt.Errorf("failed to get schemas: %w", err)
}
ctx, cancel := contextWithTimeout(ctx, e.timeout)
defer cancel()

var schemas tfjson.ProviderSchemas
err = json.Unmarshal(outBytes, &schemas)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
}
return e.tfExec().ProvidersSchema(ctx)
}

return &schemas, nil
func contextWithTimeout(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
cancel := func() {}
if timeout > 0 {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should it just be a WithCancel if its zero?

ctx, cancel = context.WithTimeout(ctx, timeout)
}
return ctx, cancel
}
Loading