Merge pull request #7468 from justinsb/sha256_hashes

Look for sha256 and sha1 files for artifacts

pkg/assets/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
         "//util/pkg/hashing:go_default_library",
         "//util/pkg/vfs:go_default_library",
         "//vendor/github.com/blang/semver:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],
 )

pkg/assets/builder.go

@@ -24,10 +24,11 @@ import (
 	"path"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/blang/semver"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog"
-
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/featureflag"
@@ -290,19 +291,31 @@ func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {
 		return nil, fmt.Errorf("file url is not defined")
 	}
 
-	for _, ext := range []string{".sha1"} {
-		hashURL := u.String() + ext
-		b, err := vfs.Context.ReadFile(hashURL)
-		if err != nil {
-			klog.Infof("error reading hash file %q: %v", hashURL, err)
-			continue
+	// We now prefer sha256 hashes
+	for backoffSteps := 1; backoffSteps <= 3; backoffSteps++ {
+		// We try first with a short backoff, so we don't
+		// waste too much time looking for files that don't
+		// exist before trying the next one
+		backoff := wait.Backoff{
+			Duration: 500 * time.Millisecond,
+			Factor:   2,
+			Steps:    backoffSteps,
 		}
-		hashString := strings.TrimSpace(string(b))
-		klog.V(2).Infof("Found hash %q for %q", hashString, u)
 
-		// Accept a hash string that is `<hash> <filename>`
-		fields := strings.Fields(hashString)
-		return hashing.FromString(fields[0])
+		for _, ext := range []string{".sha256", ".sha1"} {
+			hashURL := u.String() + ext
+			b, err := vfs.Context.ReadFile(hashURL, vfs.WithBackoff(backoff))
+			if err != nil {
+				klog.Infof("error reading hash file %q: %v", hashURL, err)
+				continue
+			}
+			hashString := strings.TrimSpace(string(b))
+			klog.V(2).Infof("Found hash %q for %q", hashString, u)
+
+			// Accept a hash string that is `<hash> <filename>`
+			fields := strings.Fields(hashString)
+			return hashing.FromString(fields[0])
+		}
 	}
 
 	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {

upup/pkg/fi/assettasks/copyfile.go

@@ -48,10 +48,30 @@ func (e *CopyFile) CompareWithID() *string {
 	return e.Name
 }
 
+// fileExtensionForSHA returns the expected extension for the given hash
+// If the hash length is not recognized, it returns an error.
+func fileExtensionForSHA(sha string) (string, error) {
+	switch len(sha) {
+	case 40:
+		return ".sha1", nil
+	case 64:
+		return ".sha256", nil
+	default:
+		return "", fmt.Errorf("unhandled sha length for %q", sha)
+	}
+}
+
 // Find attempts to find a file.
 func (e *CopyFile) Find(c *fi.Context) (*CopyFile, error) {
+	expectedSHA := strings.TrimSpace(fi.StringValue(e.SHA))
+
+	shaExtension, err := fileExtensionForSHA(expectedSHA)
+	if err != nil {
+		return nil, err
+	}
+
+	targetSHAFile := fi.StringValue(e.TargetFile) + shaExtension
 
-	targetSHAFile := fi.StringValue(e.TargetFile) + ".sha1"
 	targetSHABytes, err := vfs.Context.ReadFile(targetSHAFile)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -64,9 +84,9 @@ func (e *CopyFile) Find(c *fi.Context) (*CopyFile, error) {
 			return nil, nil
 		}
 	}
-
 	targetSHA := string(targetSHABytes)
-	if strings.TrimSpace(targetSHA) == strings.TrimSpace(fi.StringValue(e.SHA)) {
+
+	if strings.TrimSpace(targetSHA) == expectedSHA {
 		actual := &CopyFile{
 			Name:       e.Name,
 			TargetFile: e.TargetFile,
@@ -142,7 +162,12 @@ func transferFile(c *fi.Context, source string, target string, sha string) error
 		return fmt.Errorf("error building path %q: %v", objectStore, err)
 	}
 
-	shaTarget := objectStore + ".sha1"
+	shaExtension, err := fileExtensionForSHA(sha)
+	if err != nil {
+		return err
+	}
+
+	shaTarget := objectStore + shaExtension
 	shaVFS, err := vfs.Context.BuildVfsPath(shaTarget)
 	if err != nil {
 		return fmt.Errorf("error building path %q: %v", shaTarget, err)

upup/pkg/fi/cloudup/networking.go

@@ -129,8 +129,9 @@ const (
 	defaultCNIAssetHashStringK8s1_9 = "d595d3ded6499a64e8dac02466e2f5f2ce257c9f"
 
 	// defaultCNIAssetK8s1_11 is the CNI tarball for k8s >= 1.11
-	defaultCNIAssetK8s1_11           = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz"
-	defaultCNIAssetHashStringK8s1_11 = "52e9d2de8a5f927307d9397308735658ee44ab8d"
+	defaultCNIAssetK8s1_11             = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz"
+	defaultCNIAssetSHA1StringK8s1_11   = "52e9d2de8a5f927307d9397308735658ee44ab8d"
+	defaultCNIAssetSHA256StringK8s1_11 = "3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64"
 
 	// Environment variable for overriding CNI url
 	ENV_VAR_CNI_VERSION_URL       = "CNI_VERSION_URL"
@@ -167,9 +168,14 @@ func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL,
 	}
 
 	var cniAsset, cniAssetHash string
-	if util.IsKubernetesGTE("1.11", *sv) {
+	if util.IsKubernetesGTE("1.15", *sv) {
+		// We're still on the same asset, but we use sha256
 		cniAsset = defaultCNIAssetK8s1_11
-		cniAssetHash = defaultCNIAssetHashStringK8s1_11
+		cniAssetHash = defaultCNIAssetSHA256StringK8s1_11
+		klog.V(2).Infof("Adding default CNI asset for k8s >= 1.11: %s", defaultCNIAssetK8s1_9)
+	} else if util.IsKubernetesGTE("1.11", *sv) {
+		cniAsset = defaultCNIAssetK8s1_11
+		cniAssetHash = defaultCNIAssetSHA1StringK8s1_11
 		klog.V(2).Infof("Adding default CNI asset for k8s >= 1.11: %s", defaultCNIAssetK8s1_9)
 	} else if util.IsKubernetesGTE("1.9", *sv) {
 		cniAsset = defaultCNIAssetK8s1_9

util/pkg/vfs/context.go

@@ -56,11 +56,36 @@ var Context = VFSContext{
 	k8sContext: NewKubernetesContext(),
 }
 
+type vfsOptions struct {
+	backoff wait.Backoff
+}
+
+type VFSOption func(options *vfsOptions)
+
+// WithBackoff specifies a custom VFS backoff policy
+func WithBackoff(backoff wait.Backoff) VFSOption {
+	return func(options *vfsOptions) {
+		options.backoff = backoff
+	}
+}
+
 // ReadLocation reads a file from a vfs URL
 // It supports additional schemes which don't (yet) have full VFS implementations:
 //   metadata: reads from instance metadata on GCE/AWS
 //   http / https: reads from HTTP
-func (c *VFSContext) ReadFile(location string) ([]byte, error) {
+func (c *VFSContext) ReadFile(location string, options ...VFSOption) ([]byte, error) {
+	var opts vfsOptions
+	// Exponential backoff, starting with 500 milliseconds, doubling each time, 5 steps
+	opts.backoff = wait.Backoff{
+		Duration: 500 * time.Millisecond,
+		Factor:   2,
+		Steps:    5,
+	}
+
+	for _, option := range options {
+		option(&opts)
+	}
+
 	if strings.Contains(location, "://") && !strings.HasPrefix(location, "file://") {
 		// Handle our special case schemas
 		u, err := url.Parse(location)
@@ -75,20 +100,20 @@ func (c *VFSContext) ReadFile(location string) ([]byte, error) {
 				httpURL := "http://169.254.169.254/computeMetadata/v1/instance/attributes/" + u.Path
 				httpHeaders := make(map[string]string)
 				httpHeaders["Metadata-Flavor"] = "Google"
-				return c.readHttpLocation(httpURL, httpHeaders)
+				return c.readHttpLocation(httpURL, httpHeaders, opts)
 			case "aws":
 				httpURL := "http://169.254.169.254/latest/" + u.Path
-				return c.readHttpLocation(httpURL, nil)
+				return c.readHttpLocation(httpURL, nil, opts)
 			case "digitalocean":
 				httpURL := "http://169.254.169.254/metadata/v1" + u.Path
-				return c.readHttpLocation(httpURL, nil)
+				return c.readHttpLocation(httpURL, nil, opts)
 
 			default:
 				return nil, fmt.Errorf("unknown metadata type: %q in %q", u.Host, location)
 			}
 
 		case "http", "https":
-			return c.readHttpLocation(location, nil)
+			return c.readHttpLocation(location, nil, opts)
 		}
 	}
 
@@ -145,17 +170,10 @@ func (c *VFSContext) BuildVfsPath(p string) (Path, error) {
 // readHttpLocation reads an http (or https) url.
 // It returns the contents, or an error on any non-200 response.  On a 404, it will return os.ErrNotExist
 // It will retry a few times on a 500 class error
-func (c *VFSContext) readHttpLocation(httpURL string, httpHeaders map[string]string) ([]byte, error) {
-	// Exponential backoff, starting with 500 milliseconds, doubling each time, 5 steps
-	backoff := wait.Backoff{
-		Duration: 500 * time.Millisecond,
-		Factor:   2,
-		Steps:    5,
-	}
-
+func (c *VFSContext) readHttpLocation(httpURL string, httpHeaders map[string]string, opts vfsOptions) ([]byte, error) {
 	var body []byte
 
-	done, err := RetryWithBackoff(backoff, func() (bool, error) {
+	done, err := RetryWithBackoff(opts.backoff, func() (bool, error) {
 		klog.V(4).Infof("Performing HTTP request: GET %s", httpURL)
 		req, err := http.NewRequest("GET", httpURL, nil)
 		if err != nil {