fix&test: discard error for NewFileCache & E2E test for CRL with cache

cmd/notation/verify.go

@@ -246,11 +246,13 @@ func getVerifier(ctx context.Context) (notation.Verifier, error) {
 	}
 	fileCache, err := crl.NewFileCache(cacheRoot)
 	if err != nil {
-		return nil, err
-	}
-	crlFetcher.Cache = &clicrl.CacheWithLog{
-		Cache:             fileCache,
-		DiscardCacheError: crlFetcher.DiscardCacheError,
+		// discard NewFileCache error as cache errors are not critical
+		fmt.Fprintf(os.Stderr, "Warning: %v\n", err)
+	} else {
+		crlFetcher.Cache = &clicrl.CacheWithLog{
+			Cache:             fileCache,
+			DiscardCacheError: crlFetcher.DiscardCacheError,
+		}
 	}
 	revocationCodeSigningValidator, err := revocation.NewWithOptions(revocation.Options{
 		OCSPHTTPClient:   ocspHttpClient,

cmd/notation/verify_test.go

@@ -19,7 +19,6 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"runtime"
 	"testing"
 
 	"github.com/notaryproject/notation-go/dir"
@@ -119,18 +118,6 @@ func TestGetVerifier(t *testing.T) {
 			t.Fatalf("expected %s, but got %s", expectedErrMsg, err)
 		}
 	})
-
-	t.Run("failed to new crl file cache", func(t *testing.T) {
-		if runtime.GOOS == "windows" {
-			t.Skip("skipping test on Windows")
-		}
-		dir.UserCacheDir = "/cache"
-		expectedErrMsg := "failed to create crl file cache: mkdir /cache: permission denied"
-		_, err := getVerifier(context.Background())
-		if err == nil || err.Error() != expectedErrMsg {
-			t.Fatalf("expected %s, but got %s", expectedErrMsg, err)
-		}
-	})
 }
 
 func dummyOCIPolicyDocument() trustpolicy.OCIDocument {

test/e2e/go.mod

@@ -4,7 +4,8 @@ go 1.23
 
 require (
 	github.com/notaryproject/notation-core-go v1.2.0-rc.1
-	github.com/onsi/ginkgo/v2 v2.20.2
+	github.com/notaryproject/notation-go v1.2.0-beta.1.0.20240926015724-84c2ec076201
+	github.com/onsi/ginkgo/v2 v2.21.0
 	github.com/onsi/gomega v1.34.2
 	github.com/opencontainers/image-spec v1.1.0
 	oras.land/oras-go/v2 v2.5.0
@@ -15,16 +16,16 @@ require (
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
-	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
+	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
 	github.com/notaryproject/tspclient-go v0.2.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/veraison/go-cose v1.1.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.24.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/tools v0.26.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

test/e2e/go.sum

@@ -8,14 +8,16 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA=
-github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/notaryproject/notation-core-go v1.2.0-rc.1 h1:VMFlG+9a1JoNAQ3M96g8iqCq0cDRtE7XBaiTD8Ouvqw=
 github.com/notaryproject/notation-core-go v1.2.0-rc.1/go.mod h1:b/70rA4OgOHlg0A7pb8zTWKJadFO6781zS3a37KHEJQ=
+github.com/notaryproject/notation-go v1.2.0-beta.1.0.20240926015724-84c2ec076201 h1:2QBYa9Df+vMwMiaHaFqPoUiwfx5vcPEgM7KbusivTpw=
+github.com/notaryproject/notation-go v1.2.0-beta.1.0.20240926015724-84c2ec076201/go.mod h1:F6zMQl3PhVdCsI1xlIjK66kCorUQhWkoMtlZdvJWxFI=
 github.com/notaryproject/tspclient-go v0.2.0 h1:g/KpQGmyk/h7j60irIRG1mfWnibNOzJ8WhLqAzuiQAQ=
 github.com/notaryproject/tspclient-go v0.2.0/go.mod h1:LGyA/6Kwd2FlM0uk8Vc5il3j0CddbWSHBj/4kxQDbjs=
-github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4=
-github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag=
+github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
+github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
 github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -30,18 +32,18 @@ github.com/veraison/go-cose v1.1.0 h1:AalPS4VGiKavpAzIlBjrn7bhqXiXi4jbMYY/2+UC+4
 github.com/veraison/go-cose v1.1.0/go.mod h1:7ziE85vSq4ScFTg6wyoMXjucIGOf4JkFEZi/an96Ct4=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
-golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
-golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
-golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
-golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
-golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
-golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
-golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
+golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
 google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
 google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

test/e2e/internal/notation/host.go

@@ -123,7 +123,7 @@ func Opts(options ...utils.HostOption) []utils.HostOption {
 func BaseOptions() []utils.HostOption {
 	return Opts(
 		AuthOption("", ""),
-		AddKeyOption("e2e.key", "e2e.crt"),
+		AddKeyOption(filepath.Join(NotationE2ELocalKeysDir, "e2e.key"), filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")),
 		AddTrustStoreOption("e2e", filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")),
 		AddTrustPolicyOption("trustpolicy.json"),
 	)
@@ -141,18 +141,27 @@ func TimestampOptions(verifyTimestamp string) []utils.HostOption {
 
 	return Opts(
 		AuthOption("", ""),
-		AddKeyOption("e2e.key", "e2e.crt"),
+		AddKeyOption(filepath.Join(NotationE2ELocalKeysDir, "e2e.key"), filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")),
 		AddTrustStoreOption("e2e", filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")),
 		AddTimestampTrustStoreOption("e2e", filepath.Join(NotationE2EConfigPath, "timestamp", "globalsignTSARoot.cer")),
 		AddTimestampTrustStoreOption("e2e", filepath.Join(NotationE2EConfigPath, "timestamp", "DigiCertTSARootSHA384.cer")),
 		trustPolicyOption,
 	)
 }
 
+func CRLOptions() []utils.HostOption {
+	return Opts(
+		AuthOption("", ""),
+		AddKeyOption(filepath.Join(NotationE2EConfigPath, "crl", "leaf.key"), filepath.Join(NotationE2EConfigPath, "crl", "certchain_with_crl.pem")),
+		AddTrustStoreOption("e2e", filepath.Join(NotationE2EConfigPath, "crl", "root.crt")),
+		AddTrustPolicyOption("trustpolicy.json"),
+	)
+}
+
 func BaseOptionsWithExperimental() []utils.HostOption {
 	return Opts(
 		AuthOption("", ""),
-		AddKeyOption("e2e.key", "e2e.crt"),
+		AddKeyOption(filepath.Join(NotationE2ELocalKeysDir, "e2e.key"), filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")),
 		AddTrustStoreOption("e2e", filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")),
 		AddTrustPolicyOption("trustpolicy.json"),
 		EnableExperimental(),
@@ -163,7 +172,7 @@ func BaseOptionsWithExperimental() []utils.HostOption {
 // testing environment.
 func TestLoginOptions() []utils.HostOption {
 	return Opts(
-		AddKeyOption("e2e.key", "e2e.crt"),
+		AddKeyOption(filepath.Join(NotationE2ELocalKeysDir, "e2e.key"), filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")),
 		AddTrustStoreOption("e2e", filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")),
 		AddTrustPolicyOption("trustpolicy.json"),
 		AddConfigJsonOption("pass_credential_helper_config.json"),
@@ -193,9 +202,9 @@ func AuthOption(username, password string) utils.HostOption {
 
 // AddKeyOption adds the test signingkeys.json, key and cert files to
 // the notation directory.
-func AddKeyOption(keyName, certName string) utils.HostOption {
+func AddKeyOption(keyPath, certPath string) utils.HostOption {
 	return func(vhost *utils.VirtualHost) error {
-		return AddKeyPairs(vhost.AbsolutePath(NotationDirName), keyName, certName)
+		return AddKeyPairs(vhost.AbsolutePath(NotationDirName), keyPath, certPath)
 	}
 }
 

test/e2e/internal/notation/key.go

@@ -53,35 +53,37 @@ type SigningKeys struct {
 
 // AddKeyPairs creates the signingkeys.json file and the localkeys directory
 // with e2e.key and e2e.crt
-func AddKeyPairs(dir, keyName, certName string) error {
+func AddKeyPairs(destNotationConfigDir, srcKeyPath, srcCertPath string) error {
+	keyName := filepath.Base(srcKeyPath)
+	certName := filepath.Base(srcCertPath)
 	// create signingkeys.json files
 	if err := saveJSON(
-		generateSigningKeys(dir),
-		filepath.Join(dir, SigningKeysFileName)); err != nil {
+		generateSigningKeys(destNotationConfigDir, keyName, certName),
+		filepath.Join(destNotationConfigDir, SigningKeysFileName)); err != nil {
 		return err
 	}
 
 	// create localkeys directory
-	localKeysDir := filepath.Join(dir, LocalKeysDirName)
+	localKeysDir := filepath.Join(destNotationConfigDir, LocalKeysDirName)
 	os.MkdirAll(localKeysDir, 0700)
 
 	// copy key and cert files
-	if err := copyFile(filepath.Join(NotationE2ELocalKeysDir, keyName), filepath.Join(localKeysDir, "e2e.key")); err != nil {
+	if err := copyFile(srcKeyPath, filepath.Join(localKeysDir, keyName)); err != nil {
 		return err
 	}
-	return copyFile(filepath.Join(NotationE2ELocalKeysDir, certName), filepath.Join(localKeysDir, "e2e.crt"))
+	return copyFile(srcCertPath, filepath.Join(localKeysDir, certName))
 }
 
 // generateSigningKeys generates the signingkeys.json for notation.
-func generateSigningKeys(dir string) *SigningKeys {
+func generateSigningKeys(dir, keyName, certName string) *SigningKeys {
 	return &SigningKeys{
 		Default: "e2e",
 		Keys: []KeySuite{
 			{
 				Name: "e2e",
 				X509KeyPair: &X509KeyPair{
-					KeyPath:         filepath.Join(dir, "localkeys", "e2e.key"),
-					CertificatePath: filepath.Join(dir, "localkeys", "e2e.crt"),
+					KeyPath:         filepath.Join(dir, "localkeys", keyName),
+					CertificatePath: filepath.Join(dir, "localkeys", certName),
 				},
 			},
 		},

test/e2e/internal/utils/crl.go

@@ -0,0 +1,79 @@
+// Copyright The Notary Project Authors.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"fmt"
+	"net/http"
+)
+
+// LeafCRLRevoke sends http post request to http://localhost:10086/leaf/revoke
+func LeafCRLRevoke() error {
+	url := "http://localhost:10086/leaf/revoke"
+	resp, err := http.Post(url, "application/json", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	fmt.Printf("CRL of leaf certificate revoked with status code: %d\n", resp.StatusCode)
+	return nil
+}
+
+// LeafCRLUnrevoke sends http post request to http://localhost:10086/leaf/unrevoke
+func LeafCRLUnrevoke() error {
+	url := "http://localhost:10086/leaf/unrevoke"
+	resp, err := http.Post(url, "application/json", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	fmt.Printf("CRL of leaf certificate unrevoked with status code: %d\n", resp.StatusCode)
+	return nil
+}
+
+// LeafCRLExpired sends http post request to http://localhost:10086/leaf/expired
+func LeafCRLExpired() error {
+	url := "http://localhost:10086/leaf/expired"
+	resp, err := http.Post(url, "application/json", nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	fmt.Printf("CRL of leaf certificate expired with status code: %d\n", resp.StatusCode)
+	return nil
+}
+
+// IntermediateCRLRevoke sends http post request to http://localhost:10086/intermediate/revoke
+func IntermediateCRLRevoke() error {
+	url := "http://localhost:10086/intermediate/revoke"
+	resp, err := http.Post(url, "application/json", nil)
+	if err != nil {
+		return nil
+	}
+	defer resp.Body.Close()
+	fmt.Printf("CRL of intermediate certificate revoked with status code: %d\n", resp.StatusCode)
+	return nil
+}
+
+// IntermediateCRLUnrevoke sends http post request to http://localhost:10086/intermediate/unrevoke
+func IntermediateCRLUnrevoke() error {
+	url := "http://localhost:10086/intermediate/unrevoke"
+	resp, err := http.Post(url, "application/json", nil)
+	if err != nil {
+		return nil
+	}
+	defer resp.Body.Close()
+	fmt.Printf("CRL of intermediate certificate unrevoked with status code: %d\n", resp.StatusCode)
+	return nil
+}

test/e2e/internal/utils/host.go

@@ -38,7 +38,7 @@ func NewVirtualHost(binPath string, options ...HostOption) (*VirtualHost, error)
 	vhost.userDir = ginkgo.GinkgoT().TempDir()
 
 	// set user dir environment variables
-	vhost.UpdateEnv(UserConfigEnv(vhost.userDir))
+	vhost.UpdateEnv(UserEnv(vhost.userDir))
 
 	// set options
 	vhost.SetOption(options...)
@@ -77,11 +77,13 @@ func (h *VirtualHost) SetOption(options ...HostOption) {
 // HostOption is a function to set the host configuration.
 type HostOption func(vhost *VirtualHost) error
 
-// UserConfigEnv creates environment variable for changing
-// user config dir (By setting $XDG_CONFIG_HOME).
-func UserConfigEnv(dir string) map[string]string {
+// UserEnv creates environment variable for changing
+// user config dir by setting $XDG_CONFIG_HOME and user cache dir by
+// setting $NOTATION_CACHE.
+func UserEnv(dir string) map[string]string {
 	// create and set user dir for linux
 	return map[string]string{
 		"XDG_CONFIG_HOME": dir,
+		"NOTATION_CACHE":  filepath.Join(dir, ".cache"),
 	}
 }

test/e2e/run.sh

@@ -69,7 +69,7 @@ if [ ! -f "$NOTATION_E2E_OLD_BINARY_PATH" ]; then
 fi
 
 # install dependency
-go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.11.0
+go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.21.0
 
 # build e2e plugin and tar.gz
 PLUGIN_NAME=notation-e2e-plugin
@@ -95,9 +95,16 @@ esac
 
 setup_registry
 
+# run the CRL server in the background
+python3 ./scripts/crl_server.py &
+CRL_SERVER_PID=$!
+
 # defer cleanup registry
 function cleanup {
+    echo "Cleaning up..."
     cleanup_registry
+    echo "Stopping CRL server..."
+    kill $CRL_SERVER_PID
 }
 trap cleanup EXIT
 
@@ -111,4 +118,4 @@ export NOTATION_E2E_PLUGIN_TAR_GZ_PATH=$CWD/plugin/bin/$PLUGIN_NAME.tar.gz
 export NOTATION_E2E_MALICIOUS_PLUGIN_ARCHIVE_PATH=$CWD/testdata/malicious-plugin
 
 # run tests
-ginkgo -r -p -v
+ginkgo -r -p -v