diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b17dd9c..9f306b4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -107,7 +107,7 @@ jobs:
curl -sL "https://github.com/goss-org/goss/releases/download/${{ env.GOSSVERSION }}/goss-${GOSS_ARCH}" -o goss
chmod +x goss
go build cmd/tests/test-features.go
- ./test-features -f ${{ matrix.flavour }} -i test-image-${{ matrix.flavour }}-${{ matrix.arch}}:local -p ${{ matrix.arch }} -t ${PWD}/.testbed -g ${PWD}/goss -l debug
+ ./test-features -c -f ${{ matrix.flavour }} -i test-image-${{ matrix.flavour }}-${{ matrix.arch }}:local -p ${{ matrix.arch }} -t ${PWD}/.testbed -g ${PWD}/goss -l debug
- name: Cleanup
run: |
rm -rf .testbed
diff --git a/README.md b/README.md
index 3a2bdf6..c9c43b6 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,6 @@ Currently supported cloud flavours are:
Following features and tools are supported:
* 🐟 Fish Shell
* 📷 AzCopy
-* 🔐 Bitwarden
* 🪪 Certificates
* ⚙️ Direnv
* ⛵️ Helm
@@ -52,7 +51,6 @@ Following features and tools are supported:
* [Features](#features)
* [Fish Shell](#_fish)
* [AzCopy](#azcopy)
- * [Bitwarden](#bitwarden)
* [Certificates](#certificates)
* [Direnv](#direnv)
* [Helm](#helm)
@@ -373,15 +371,6 @@ Installs [AzCopy](https://github.com/Azure/azure-storage-azcopy)
* USE_azcopy: Enable this feature
* DEBUG_azcopy: Debug this feature
-### Bitwarden
-
-Installs the [Bitwarden CLI](https://bitwarden.com/help/cli/)
-
-#### Configuration
-
-* USE_bitwarden: Enable this feature
-* DEBUG_bitwarden: Debug this feature
-
### Certificates
Adds specified trusted certificate authorities into the container
@@ -728,20 +717,19 @@ To build all flavours with the same tag, use
## Testing
To run the test suite for a specific flavour, you need to create a local directory that holds flavour-specific data
-(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables.
+(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables. This is called
+ a "testbed" directory.
First, you need to compile the test runner:
- cd tests
- docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features
+ docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features cmd/tests/test-features
After that, download the latest goss binary for the target architecture you will test (linux/amd64 or linux/arm64) from
the [Goss site](https://github.com/goss-org/goss) and put it somewhere local.
Once that is done, run the tests like following:
- cd tests
- ./test-features -f [flavour] -i [image:tag] -t [path to flavour-data] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary]
+ ./test-features -f [flavour] -i [image:tag] -t [path to testbed directory] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary]
This will run the tests of all features that supply a test suite one by one and, if all succeed, will test all
features together for integration testing. Check out `test-features --help` for other options.
@@ -755,12 +743,19 @@ When the testrunner encounters such file it will check if CloudControl fails to
You can add a regular expression pattern into `.will-fail` to test if the container or command output matches it.
+### Unstable tests
+
+As we're dealing with a lot of moving targets in the features, sometimes a test might not be reliable. For these
+situations we support a .might-fail file. Just add it as a text file into the test suite subdirectory and put some text
+into it describing the problem. Failed test won't fail the test suite then but instead the description will be shown.
+
### Test debugging
-To check why a test failed, run the test-runner using the -x bash parameter to see the different commands it issues.
+To check why a test failed, use the -l parameter to enable debug logging. Additionally, you can use the -n parameter
+to specify the specific feature to test and use the -x parameter to stop testing if one test fails.
-Then, take the failing command and instead of `dgoss run` execute `docker run` with the same arguments to analyze the
-tests locally.
+When a test fails, the test container will not be removed automatically (unless you specified the -c parameter), so
+you can inspect the failing container as well.
## Building documentation ##
diff --git a/README.md.gotmpl b/README.md.gotmpl
index 01b3b70..07fb2ee 100644
--- a/README.md.gotmpl
+++ b/README.md.gotmpl
@@ -384,20 +384,19 @@ To build all flavours with the same tag, use
## Testing
To run the test suite for a specific flavour, you need to create a local directory that holds flavour-specific data
-(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables.
+(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables. This is called
+ a "testbed" directory.
First, you need to compile the test runner:
- cd tests
- docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features
+ docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features cmd/tests/test-features
After that, download the latest goss binary for the target architecture you will test (linux/amd64 or linux/arm64) from
the [Goss site](https://github.com/goss-org/goss) and put it somewhere local.
Once that is done, run the tests like following:
- cd tests
- ./test-features -f [flavour] -i [image:tag] -t [path to flavour-data] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary]
+ ./test-features -f [flavour] -i [image:tag] -t [path to testbed directory] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary]
This will run the tests of all features that supply a test suite one by one and, if all succeed, will test all
features together for integration testing. Check out `test-features --help` for other options.
@@ -411,12 +410,19 @@ When the testrunner encounters such file it will check if CloudControl fails to
You can add a regular expression pattern into `.will-fail` to test if the container or command output matches it.
+### Unstable tests
+
+As we're dealing with a lot of moving targets in the features, sometimes a test might not be reliable. For these
+situations we support a .might-fail file. Just add it as a text file into the test suite subdirectory and put some text
+into it describing the problem. Failed test won't fail the test suite then but instead the description will be shown.
+
### Test debugging
-To check why a test failed, run the test-runner using the -x bash parameter to see the different commands it issues.
+To check why a test failed, use the -l parameter to enable debug logging. Additionally, you can use the -n parameter
+to specify the specific feature to test and use the -x parameter to stop testing if one test fails.
-Then, take the failing command and instead of `dgoss run` execute `docker run` with the same arguments to analyze the
-tests locally.
+When a test fails, the test container will not be removed automatically (unless you specified the -c parameter), so
+you can inspect the failing container as well.
## Building documentation ##
diff --git a/cmd/tests/test-features.go b/cmd/tests/test-features.go
index 2282ef3..7250e7a 100644
--- a/cmd/tests/test-features.go
+++ b/cmd/tests/test-features.go
@@ -138,8 +138,7 @@ func main() {
includeFeatures := parser.List("n", "include", &argparse.Options{Help: "Only include these features when testing"})
excludeFeatures := parser.List("e", "exclude", &argparse.Options{Help: "Exclude these features when testing"})
skipIntegrationTest := parser.Flag("s", "skip-integration", &argparse.Options{
- Help: "Skip integration test at the end",
- Default: false,
+ Help: "Skip integration test at the end",
})
maxWait := parser.Int("w", "wait", &argparse.Options{
Help: "Maximum number of seconds goss_wait should wait",
@@ -149,6 +148,12 @@ func main() {
Help: "Loglevel to use",
Default: "error",
})
+ failFast := parser.Flag("x", "failfast", &argparse.Options{
+ Help: "Whether to stop testing directly when one test failes",
+ })
+ cleanup := parser.Flag("c", "cleanup", &argparse.Options{
+ Help: "Clean up container in case of error",
+ })
err := parser.Parse(os.Args)
if err != nil {
@@ -272,7 +277,7 @@ func main() {
for _, feature := range featuresToTest {
var featureTimer = time.Now()
- if err := lib.TestFeature(feature, testBed, containerAdapter); err != nil {
+ if err := lib.TestFeature(feature, testBed, containerAdapter, *cleanup); err != nil {
logrus.Errorf(
"❌ %s (%s) ⏱ %ds :\n\n%s\n",
feature.Name,
@@ -285,6 +290,10 @@ func main() {
ErrorMessage: err.Error(),
ElapsedSeconds: int(math.Round(time.Since(featureTimer).Seconds())),
})
+ if *failFast {
+ logrus.Error("Failfast activated, so failing directly.")
+ os.Exit(1)
+ }
} else {
logrus.Infof(
"✅ %s (%s) ⏱ %ds",
diff --git a/feature/azcopy/goss/.might-fail b/feature/azcopy/goss/.might-fail
new file mode 100644
index 0000000..545ab2e
--- /dev/null
+++ b/feature/azcopy/goss/.might-fail
@@ -0,0 +1 @@
+Might fail because of https://github.com/Azure/azure-storage-azcopy/issues/2483
\ No newline at end of file
diff --git a/feature/azcopy/goss/goss.yaml b/feature/azcopy/goss/goss.yaml
index 1fa0616..cd84c9f 100644
--- a/feature/azcopy/goss/goss.yaml
+++ b/feature/azcopy/goss/goss.yaml
@@ -1,6 +1,6 @@
command:
azcopy:
- exec: "/home/cloudcontrol/bin/azcopy --version"
+ exec: "/home/cloudcontrol/bin/azcopy --help"
exit-status: 0
stdout:
- - azcopy version
+ - AzCopy
diff --git a/go.mod b/go.mod
index a432475..4c47b05 100644
--- a/go.mod
+++ b/go.mod
@@ -6,10 +6,10 @@ require (
github.com/MakeNowJust/heredoc v1.0.0
github.com/Masterminds/sprig/v3 v3.2.3
github.com/akamensky/argparse v1.4.0
- github.com/docker/docker v20.10.22+incompatible
- github.com/opencontainers/image-spec v1.0.2
- github.com/otiai10/copy v1.9.0
- github.com/sirupsen/logrus v1.9.0
+ github.com/docker/docker v24.0.7+incompatible
+ github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
+ github.com/otiai10/copy v1.14.0
+ github.com/sirupsen/logrus v1.9.3
github.com/thoas/go-funk v0.9.3
gopkg.in/yaml.v3 v3.0.1
)
@@ -17,14 +17,16 @@ require (
require (
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
- github.com/Microsoft/go-winio v0.6.0 // indirect
+ github.com/Microsoft/go-winio v0.6.1 // indirect
+ github.com/containerd/containerd v1.7.11 // indirect
+ github.com/containerd/log v0.1.0 // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
- github.com/google/uuid v1.1.1 // indirect
+ github.com/google/uuid v1.3.0 // indirect
github.com/huandu/xstrings v1.3.3 // indirect
- github.com/imdario/mergo v0.3.11 // indirect
+ github.com/imdario/mergo v0.3.13 // indirect
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/reflectwalk v1.0.0 // indirect
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
@@ -33,11 +35,12 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
- golang.org/x/crypto v0.3.0 // indirect
- golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
- golang.org/x/net v0.2.0 // indirect
- golang.org/x/sys v0.2.0 // indirect
+ golang.org/x/crypto v0.14.0 // indirect
+ golang.org/x/mod v0.11.0 // indirect
+ golang.org/x/net v0.17.0 // indirect
+ golang.org/x/sync v0.3.0 // indirect
+ golang.org/x/sys v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
- golang.org/x/tools v0.1.12 // indirect
+ golang.org/x/tools v0.10.0 // indirect
gotest.tools/v3 v3.4.0 // indirect
)
diff --git a/go.sum b/go.sum
index 9debe30..0c26192 100644
--- a/go.sum
+++ b/go.sum
@@ -9,8 +9,14 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj
github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg=
github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
+github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
+github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc=
github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA=
+github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw=
+github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -18,6 +24,8 @@ github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6
github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v20.10.22+incompatible h1:6jX4yB+NtcbldT90k7vBSaWJDB3i+zkVJT9BEK8kQkk=
github.com/docker/docker v20.10.22+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -28,10 +36,14 @@ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
@@ -46,13 +58,18 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8=
+github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4=
github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI=
+github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
+github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4=
github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI=
+github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -61,6 +78,8 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -80,10 +99,14 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
+golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -92,10 +115,14 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -107,6 +134,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
@@ -123,6 +152,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg=
+golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -133,6 +164,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
diff --git a/internal/tests/lib/container/docker.go b/internal/tests/lib/container/docker.go
index 46fe9da..0faec44 100644
--- a/internal/tests/lib/container/docker.go
+++ b/internal/tests/lib/container/docker.go
@@ -72,7 +72,7 @@ func (d DockerAdapter) FindImage(image string, platform string) error {
if len(localImages) == 0 {
remoteImages, err := dockerCli.ImageSearch(context.Background(), image, types.ImageSearchOptions{Limit: 1})
- if err != nil {
+ if err != nil && !strings.Contains(err.Error(), "404") {
panic(fmt.Sprintf("Can not search remote images: %s", err.Error()))
}
if len(remoteImages) == 0 {
@@ -137,6 +137,7 @@ func (d DockerAdapter) StopContainer(containerID string) error {
}
func (d DockerAdapter) RunCommand(containerID string, cmd []string) (string, error) {
+ logrus.Debugf("Running command %s on container %s", strings.Join(cmd, " "), containerID)
dockerCli := d.getClient()
var executeID string
if idResponse, err := dockerCli.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{
diff --git a/internal/tests/lib/testRunner.go b/internal/tests/lib/testRunner.go
index e18187b..739e2f6 100644
--- a/internal/tests/lib/testRunner.go
+++ b/internal/tests/lib/testRunner.go
@@ -26,7 +26,7 @@ file:
`
// TestFeature tests a specific feature.
-func TestFeature(feature Feature, testBed TestBed, containerAdapter container.AdapterBase) error {
+func TestFeature(feature Feature, testBed TestBed, containerAdapter container.AdapterBase, cleanup bool) error {
var testSuites []string
if testSuitesGlob, err := filepath.Glob(filepath.Join(feature.FullPath, "goss*")); err != nil {
return fmt.Errorf("error searching for testsuites in %s: %w", feature.FullPath, err)
@@ -47,8 +47,11 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad
containerErr := func() error {
var testDir string
+ var hasProblems = false
var willFail = false
var failPattern = ".*"
+ var mightFail = false
+ var mightFailDescription = ""
if _, err := os.Stat(filepath.Join(testSuite, ".will-fail")); err == nil {
willFail = true
@@ -60,6 +63,16 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad
}
}
+ if _, err := os.Stat(filepath.Join(testSuite, ".might-fail")); err == nil {
+ mightFail = true
+ logrus.Debug("Found .might-fail file, test might fail.")
+ if tmp, err := os.ReadFile(filepath.Join(testSuite, ".might-fail")); err != nil {
+ return fmt.Errorf("loading the contents of %s failed: %w", filepath.Join(testSuite, ".might-fail"), err)
+ } else {
+ mightFailDescription = strings.TrimSuffix(string(tmp), "\n")
+ }
+ }
+
if tempReturn, err := os.MkdirTemp("", "cctest-*"); err != nil {
return fmt.Errorf("error creating temporary directory for testbed: %w", err)
} else {
@@ -87,7 +100,7 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad
if _, err := fileutils.CopyFile(
filepath.Join(testSuite, "goss.yaml"),
filepath.Join(testDir, "goss.yaml"),
- ); err != nil && !willFail {
+ ); err != nil && !willFail && !mightFail {
return fmt.Errorf("can not copy goss.yaml: %w", err)
}
@@ -144,15 +157,24 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad
binds,
testBed.Platform,
); err != nil {
+ hasProblems = true
return fmt.Errorf("error starting container: %w", err)
} else {
+ logrus.Debugf("Started container %s", startedContainerId)
containerId = startedContainerId
}
// deferring the removal of the container
defer func() {
- if err := containerAdapter.StopContainer(containerId); err != nil {
- panic(fmt.Errorf("can not stop goss container: %w", err))
+ if cleanup || !hasProblems {
+ logrus.Debugf("Stopping container %s", containerId)
+ if err := containerAdapter.StopContainer(containerId); err != nil {
+ panic(fmt.Errorf("can not stop goss container: %w", err))
+ } else {
+ logrus.Debugf("Stopped container with id %s", containerId)
+ }
+ } else {
+ logrus.Debugf("Not stopping container because cleanup was disabled and problems exist.")
}
}()
@@ -168,11 +190,14 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad
var runCommandError *container.RunCommandError
switch {
case errors.As(err, &runCommandError):
- if !willFail {
+ if mightFail {
+ logrus.Warnf("Waiting for Goss in %s failed with %s, but it might fail because: \n%s", testSuite, err.Error(), mightFailDescription)
+ } else if !willFail {
+ hasProblems = true
return &GossError{
returnCode: runCommandError.ReturnCode,
logOutput: fmt.Sprintf(
- "Command output:\n%s\n\nContainer output:\n%s\n&s",
+ "Command output:\n%s\n\nContainer output:\n%s\n%s",
runCommandError.CommandOutput,
runCommandError.ContainerOutput,
runCommandError.Error(),
@@ -182,6 +207,7 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad
failPattern,
[]byte(fmt.Sprintf("%s%s", runCommandError.CommandOutput, runCommandError.ContainerOutput)),
); err != nil || !found {
+ hasProblems = true
return &GossError{
returnCode: 0,
logOutput: fmt.Sprintf(
@@ -194,9 +220,12 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad
}
}
default:
- return err
+ if !willFail && !mightFail {
+ return err
+ }
}
} else if err == nil && willFail {
+ hasProblems = true
return &GossError{
returnCode: 0,
logOutput: fmt.Sprintf("Container was created successfully when it shouldn't have been: %s", waitOutput),
@@ -216,7 +245,10 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad
"documentation",
},
); err != nil {
- if !willFail {
+ if mightFail {
+ logrus.Warnf("Testing %s failed with %s, but it might fail because: \n%s", testSuite, err.Error(), mightFailDescription)
+ } else if !willFail {
+ hasProblems = true
var runCommandError *container.RunCommandError
switch {
case errors.As(err, &runCommandError):
@@ -229,10 +261,13 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad
),
}
default:
- return err
+ if !willFail && !mightFail {
+ return err
+ }
}
}
} else if err == nil && willFail {
+ hasProblems = true
return &GossError{
returnCode: 0,
logOutput: "Container was created successfully when it shouldn't have been.",
@@ -296,6 +331,14 @@ func IntegrationTests(features []Feature, testBed TestBed, containerAdapter cont
)
continue
}
+ if _, err := os.Stat(filepath.Join(testSuite, ".might-fail")); err == nil {
+ logrus.Infof(
+ "Test %s for feature %s is ignored for integration testing, because it might fail",
+ filepath.Base(testSuite),
+ feature.Name,
+ )
+ continue
+ }
envs = append(envs, fmt.Sprintf("USE_%s=yes", feature.Name))
for _, envFile := range []string{