From e19192d2a19b27f52bb368bce54ccefa8fd8cd27 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Wed, 10 Nov 2021 09:08:03 -0600 Subject: [PATCH] Add support for go-getter syntax when calling opa.Eval (#1026) * Add support for go-getter syntax when calling opa.Eval * Update modules/opa/download_policy.go --- go.mod | 1 + go.sum | 17 +++++ modules/opa/download_policy.go | 73 ++++++++++++++++++++ modules/opa/download_policy_test.go | 102 ++++++++++++++++++++++++++++ modules/opa/eval.go | 21 ++++-- test/terraform_opa_example_test.go | 16 +++++ 6 files changed, 223 insertions(+), 7 deletions(-) create mode 100644 modules/opa/download_policy.go create mode 100644 modules/opa/download_policy_test.go diff --git a/go.mod b/go.mod index 6c5758099..67f8ed82c 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/google/go-containerregistry v0.6.0 github.com/google/uuid v1.2.0 github.com/gruntwork-io/go-commons v0.8.0 + github.com/hashicorp/go-getter v1.5.9 github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/go-version v1.3.0 github.com/hashicorp/hcl/v2 v2.9.1 diff --git a/go.sum b/go.sum index 535d47cec..1e87ac790 100644 --- a/go.sum +++ b/go.sum @@ -122,12 +122,15 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/aws/aws-lambda-go v1.13.3 h1:SuCy7H3NLyp+1Mrfp+m80jcbi9KYWAs9/BXwppwRDzY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/aws/aws-sdk-go v1.40.56 h1:FM2yjR0UUYFzDTMx+mH9Vyw1k1EUUxsAFzk+BjkzANA= github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= @@ -145,6 +148,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -474,6 +478,10 @@ github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FK github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-getter v1.5.9 h1:b7ahZW50iQiUek/at3CvZhPK1/jiV6CtKcsJiR6E4R0= +github.com/hashicorp/go-getter v1.5.9/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= @@ -481,10 +489,13 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= +github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw= github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -533,6 +544,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= @@ -569,6 +581,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= @@ -585,6 +598,7 @@ github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HK github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= @@ -774,6 +788,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ= github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -1261,6 +1277,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= diff --git a/modules/opa/download_policy.go b/modules/opa/download_policy.go new file mode 100644 index 000000000..4309e7edf --- /dev/null +++ b/modules/opa/download_policy.go @@ -0,0 +1,73 @@ +package opa + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + + getter "github.com/hashicorp/go-getter" + + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/testing" +) + +var ( + // A map that maps the go-getter base URL to the temporary directory where it is downloaded. + policyDirCache sync.Map +) + +// downloadPolicyE takes in a rule path written in go-getter syntax and downloads it to a temporary directory so that it +// can be passed to opa. The temporary directory that is used is cached based on the go-getter base path, and reused +// across calls. +// For example, if you call downloadPolicyE with the go-getter URL multiple times: +// git::https://github.com/gruntwork-io/terratest.git//policies/foo.rego?ref=master +// The first time the gruntwork-io/terratest repo will be downloaded to a new temp directory. All subsequent calls will +// reuse that first temporary dir where the repo was cloned. This is preserved even if a different subdir is requested +// later, e.g.: git::https://github.com/gruntwork-io/terratest.git//examples/bar.rego?ref=master. +// Note that the query parameters are always included in the base URL. This means that if you use a different ref (e.g., +// git::https://github.com/gruntwork-io/terratest.git//examples/bar.rego?ref=v0.39.3), then that will be cloned to a new +// temporary directory rather than the cached dir. +func downloadPolicyE(t testing.TestingT, rulePath string) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + detected, err := getter.Detect(rulePath, cwd, getter.Detectors) + if err != nil { + return "", err + } + + // File getters are assumed to be a local path reference, so pass through the original path. + if strings.HasPrefix(detected, "file") { + return rulePath, nil + } + + // At this point we assume the getter URL is a remote URL, so we start the process of downloading it to a temp dir. + + // First, check if we had already downloaded the source and it is in our cache. + baseDir, subDir := getter.SourceDirSubdir(rulePath) + downloadPath, hasDownloaded := policyDirCache.Load(baseDir) + if hasDownloaded { + logger.Logf(t, "Previously downloaded %s: returning cached path", baseDir) + return filepath.Join(downloadPath.(string), subDir), nil + } + + // Not downloaded, so use go-getter to download the remote source to a temp dir. + tempDir, err := ioutil.TempDir("", "terratest-opa-policy-*") + if err != nil { + return "", err + } + // go-getter doesn't work if you give it a directory that already exists, so we add an additional path in the + // tempDir to make sure we feed a directory that doesn't exist yet. + tempDir = filepath.Join(tempDir, "getter") + + logger.Logf(t, "Downloading %s to temp dir %s", rulePath, tempDir) + if err := getter.GetAny(tempDir, baseDir); err != nil { + return "", err + } + policyDirCache.Store(baseDir, tempDir) + return filepath.Join(tempDir, subDir), nil +} diff --git a/modules/opa/download_policy_test.go b/modules/opa/download_policy_test.go new file mode 100644 index 000000000..12fb349cd --- /dev/null +++ b/modules/opa/download_policy_test.go @@ -0,0 +1,102 @@ +package opa + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/gruntwork-io/terratest/modules/files" + "github.com/gruntwork-io/terratest/modules/git" +) + +// Test to make sure the downloadPolicyE function returns a local path without processing it. +func TestDownloadPolicyReturnsLocalPath(t *testing.T) { + t.Parallel() + + localPath := "../../examples/terraform-opa-example/policy/enforce_source.rego" + path, err := downloadPolicyE(t, localPath) + require.NoError(t, err) + assert.Equal(t, localPath, path) +} + +// Test to make sure the downloadPolicyE function returns a remote path to a temporary directory. +func TestDownloadPolicyDownloadsRemote(t *testing.T) { + t.Parallel() + + curRef := git.GetCurrentGitRef(t) + baseDir := fmt.Sprintf("git::https://github.com/gruntwork-io/terratest.git?ref=%s", curRef) + localPath := "../../examples/terraform-opa-example/policy/enforce_source.rego" + remotePath := fmt.Sprintf("git::https://github.com/gruntwork-io/terratest.git//examples/terraform-opa-example/policy/enforce_source.rego?ref=%s", curRef) + + // Make sure we clean up the downloaded file, while simultaneously asserting that the download dir was stored in the + // cache. + defer func() { + downloadPathRaw, inCache := policyDirCache.Load(baseDir) + require.True(t, inCache) + downloadPath := downloadPathRaw.(string) + if strings.HasSuffix(downloadPath, "/getter") { + downloadPath = filepath.Dir(downloadPath) + } + assert.NoError(t, os.RemoveAll(downloadPath)) + }() + + path, err := downloadPolicyE(t, remotePath) + require.NoError(t, err) + + absPath, err := filepath.Abs(localPath) + require.NoError(t, err) + assert.NotEqual(t, absPath, path) + + localContents, err := ioutil.ReadFile(localPath) + require.NoError(t, err) + remoteContents, err := ioutil.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, localContents, remoteContents) +} + +// Test to make sure the downloadPolicyE function uses the cache if it has already downloaded an existing base path. +func TestDownloadPolicyReusesCachedDir(t *testing.T) { + t.Parallel() + + baseDir := "git::https://github.com/gruntwork-io/terratest.git?ref=master" + remotePath := "git::https://github.com/gruntwork-io/terratest.git//examples/terraform-opa-example/policy/enforce_source.rego?ref=master" + remotePathAltSubPath := "git::https://github.com/gruntwork-io/terratest.git//modules/opa/eval.go?ref=master" + + // Make sure we clean up the downloaded file, while simultaneously asserting that the download dir was stored in the + // cache. + defer func() { + downloadPathRaw, inCache := policyDirCache.Load(baseDir) + require.True(t, inCache) + downloadPath := downloadPathRaw.(string) + + if strings.HasSuffix(downloadPath, "/getter") { + downloadPath = filepath.Dir(downloadPath) + } + assert.NoError(t, os.RemoveAll(downloadPath)) + }() + + path, err := downloadPolicyE(t, remotePath) + require.NoError(t, err) + files.FileExists(path) + + downloadPathRaw, inCache := policyDirCache.Load(baseDir) + require.True(t, inCache) + downloadPath := downloadPathRaw.(string) + + // make sure the second call is exactly equal to the first call + newPath, err := downloadPolicyE(t, remotePath) + require.NoError(t, err) + assert.Equal(t, path, newPath) + + // Also make sure the cache is reused for alternative sub dirs. + newAltPath, err := downloadPolicyE(t, remotePathAltSubPath) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(path, downloadPath)) + assert.True(t, strings.HasPrefix(newAltPath, downloadPath)) +} diff --git a/modules/opa/eval.go b/modules/opa/eval.go index 80ebc63c8..37f2f2c2a 100644 --- a/modules/opa/eval.go +++ b/modules/opa/eval.go @@ -18,7 +18,8 @@ type EvalOptions struct { // Whether OPA should run checks with failure. FailMode FailMode - // Path to rego file containing the OPA rules. + // Path to rego file containing the OPA rules. Can also be a remote path defined in go-getter syntax. Refer to + // https://github.com/hashicorp/go-getter#url-format for supported options. RulePath string // Set a logger that should be used. See the logger package for more info. @@ -57,6 +58,11 @@ func Eval(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resu // opa eval -i $JSONFile -d $RulePath $ResultQuery // This will asynchronously run OPA on each file concurrently using goroutines. func EvalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) error { + downloadedPolicyPath, err := downloadPolicyE(t, options.RulePath) + if err != nil { + return err + } + wg := new(sync.WaitGroup) wg.Add(len(jsonFilePaths)) errorsOccurred := new(multierror.Error) @@ -64,7 +70,7 @@ func EvalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, res for i, jsonFilePath := range jsonFilePaths { errChan := make(chan error, 1) errChans[i] = errChan - go asyncEval(t, wg, errChan, options, jsonFilePath, resultQuery) + go asyncEval(t, wg, errChan, options, downloadedPolicyPath, jsonFilePath, resultQuery) } wg.Wait() for _, errChan := range errChans { @@ -82,27 +88,28 @@ func asyncEval( wg *sync.WaitGroup, errChan chan error, options *EvalOptions, + downloadedPolicyPath string, jsonFilePath string, resultQuery string, ) { defer wg.Done() cmd := shell.Command{ Command: "opa", - Args: formatOPAEvalArgs(options, jsonFilePath, resultQuery), + Args: formatOPAEvalArgs(options, downloadedPolicyPath, jsonFilePath, resultQuery), // Do not log output from shell package so we can log the full json without breaking it up. This is ok, because // opa eval is typically very quick. Logger: logger.Discard, } err := runCommandWithFullLoggingE(t, options.Logger, cmd) - ruleBasePath := filepath.Base(options.RulePath) + ruleBasePath := filepath.Base(downloadedPolicyPath) if err == nil { options.Logger.Logf(t, "opa eval passed on file %s (policy %s; query %s)", jsonFilePath, ruleBasePath, resultQuery) } else { options.Logger.Logf(t, "Failed opa eval on file %s (policy %s; query %s)", jsonFilePath, ruleBasePath, resultQuery) if options.DebugDisableQueryDataOnError == false { options.Logger.Logf(t, "DEBUG: rerunning opa eval to query for full data.") - cmd.Args = formatOPAEvalArgs(options, jsonFilePath, "data") + cmd.Args = formatOPAEvalArgs(options, downloadedPolicyPath, jsonFilePath, "data") // We deliberately ignore the error here as we want to only return the original error. runCommandWithFullLoggingE(t, options.Logger, cmd) } @@ -111,7 +118,7 @@ func asyncEval( } // formatOPAEvalArgs formats the arguments for the `opa eval` command. -func formatOPAEvalArgs(options *EvalOptions, jsonFilePath string, resultQuery string) []string { +func formatOPAEvalArgs(options *EvalOptions, rulePath, jsonFilePath, resultQuery string) []string { args := []string{"eval"} switch options.FailMode { @@ -125,7 +132,7 @@ func formatOPAEvalArgs(options *EvalOptions, jsonFilePath string, resultQuery st args, []string{ "-i", jsonFilePath, - "-d", options.RulePath, + "-d", rulePath, resultQuery, }..., ) diff --git a/test/terraform_opa_example_test.go b/test/terraform_opa_example_test.go index 4d0843196..0b734ab2c 100644 --- a/test/terraform_opa_example_test.go +++ b/test/terraform_opa_example_test.go @@ -47,3 +47,19 @@ func TestOPAEvalTerraformModuleFailsCheck(t *testing.T) { // website::tag::6:: Here we expect the checks to fail, so we use `OPAEvalE` to check the error. Note that on the files that failed, this function will rerun `opa eval` with the query set to `data`, so you can see the values of all the variables in the policy. This is useful for debugging failures. require.Error(t, terraform.OPAEvalE(t, tfOpts, opaOpts, "data.enforce_source.allow")) } + +// An example of how to use Terratest to run OPA policy checks on Terraform source code using a remote OPA policy source +// file. This will check the module called `pass` against the rego policy `enforce_source` defined in the +// `terraform-opa-example` folder of the terratest repository. +func TestOPAEvalTerraformModuleRemotePolicy(t *testing.T) { + t.Parallel() + + tfOpts := &terraform.Options{ + TerraformDir: "../examples/terraform-opa-example/pass", + } + opaOpts := &opa.EvalOptions{ + RulePath: "git::https://github.com/gruntwork-io/terratest.git//examples/terraform-opa-example/policy/enforce_source.rego?ref=master", + FailMode: opa.FailUndefined, + } + terraform.OPAEval(t, tfOpts, opaOpts, "data.enforce_source.allow") +}