Use embedded hashes for our well-known assets

Rather than downloading the hash every time, we can record the hashes
for our well-known assets and bake them into the kOps binary.  If the
hash is not baked in, we will continue to fall-back to downloading it,
this is important for new k8s versions, or where the user specifies a
version of one of our well-known assets (such as containerd).

cmd/kops/get_assets.go

@@ -132,7 +132,7 @@ func RunGetAssets(ctx context.Context, f *util.Factory, out io.Writer, options *
 		file := File{
 			Canonical: fileAsset.CanonicalURL.String(),
 			Download:  fileAsset.DownloadURL.String(),
-			SHA:       fileAsset.SHAValue,
+			SHA:       fileAsset.SHAValue.Hex(),
 		}
 		if !seen[file.Canonical] {
 			result.Files = append(result.Files, &file)

hack/generate-asset-hashes.sh

@@ -35,7 +35,7 @@ filestores:
 files:
 EOF
 
-  for ((patch = 0 ; patch < max_patch ; patch++ )); do
+  for ((patch = 0 ; patch <= max_patch ; patch++ )); do
     version="${prefix}.${patch}"
     echo "k8s ${version}"
 
@@ -62,12 +62,12 @@ function generate_runc_hashes() {
 # This file is generated by generate-asset-hashes.sh
 
 filestores:
-- base: https://dl.k8s.io/release/
+- base: https://github.com/opencontainers/runc/releases/download/
 
 files:
 EOF
 
-  for ((patch = 0 ; patch < max_patch ; patch++ )); do
+  for ((patch = 0 ; patch <= max_patch ; patch++ )); do
     version="${prefix}.${patch}"
     echo "runc ${version}"
 
@@ -83,9 +83,10 @@ EOF
 generate_k8s_hashes 1.24 17
 generate_k8s_hashes 1.25 16
 generate_k8s_hashes 1.26 15
-generate_k8s_hashes 1.27 12
-generate_k8s_hashes 1.28 8
-generate_k8s_hashes 1.29 3
+generate_k8s_hashes 1.27 13
+generate_k8s_hashes 1.28 9
+generate_k8s_hashes 1.29 4
+generate_k8s_hashes 1.30 0
 
 
 generate_runc_hashes 1.1 12

pkg/assets/assetdata/data_test.go

@@ -26,7 +26,22 @@ func TestGetHash(t *testing.T) {
 		Name string
 		Hash string
 	}{
-		{Name: "https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubelet", Hash: "sha256:b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5"},
+		{
+			Name: "https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubelet",
+			Hash: "b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5",
+		},
+		{
+			Name: "https://github.com/opencontainers/runc/releases/download/v1.1.0/runc.amd64",
+			Hash: "ab1c67fbcbdddbe481e48a55cf0ef9a86b38b166b5079e0010737fd87d7454bb",
+		},
+		{
+			Name: "https://github.com/opencontainers/runc/releases/download/v1.1.0/runc.arm64",
+			Hash: "9ec8e68feabc4e7083a4cfa45ebe4d529467391e0b03ee7de7ddda5770b05e68",
+		},
+		{
+			Name: "https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64",
+			Hash: "aadeef400b8f05645768c1476d1023f7875b78f52c7ff1967a6dbce236b8cbd8",
+		},
 	}
 
 	for _, g := range grid {
@@ -41,7 +56,7 @@ func TestGetHash(t *testing.T) {
 		if !found {
 			t.Fatalf("hash for %q was not found", g.Name)
 		}
-		got := h.String()
+		got := h.Hex()
 		want := g.Hash
 		if got != g.Hash {
 			t.Errorf("unexpected hash for %q; got %q, want %q", g.Name, got, want)

pkg/assets/assetdata/runc-1.1.yaml

@@ -1,7 +1,7 @@
 # This file is generated by generate-asset-hashes.sh
 
 filestores:
-- base: https://dl.k8s.io/release/
+- base: https://github.com/opencontainers/runc/releases/download/
 
 files:
 # runc 1.1.0

pkg/assets/builder.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/util"
+	"k8s.io/kops/pkg/assets/assetdata"
 	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/values"
@@ -97,7 +98,7 @@ type FileAsset struct {
 	// CanonicalURL is the canonical location of the asset, for example as distributed by the kops project
 	CanonicalURL *url.URL
 	// SHAValue is the SHA hash of the FileAsset.
-	SHAValue string
+	SHAValue *hashing.Hash
 }
 
 // NewAssetBuilder creates a new AssetBuilder.
@@ -235,73 +236,49 @@ func (a *AssetBuilder) RemapImage(image string) (string, error) {
 	return image + "@" + digest, nil
 }
 
-// RemapFileAndSHA returns a remapped URL for the file, if AssetsLocation is defined.
-// It also returns the SHA hash of the file.
-func (a *AssetBuilder) RemapFileAndSHA(fileURL *url.URL) (*url.URL, *hashing.Hash, error) {
-	if fileURL == nil {
-		return nil, nil, fmt.Errorf("unable to remap a nil URL")
+// RemapFile returns a remapped URL for the file, if AssetsLocation is defined.
+// It is returns in a FileAsset, alongside the SHA hash of the file.
+// The SHA hash is is knownHash is provided, and otherwise will be found first by
+// checking the canonical URL against our well-known hashes, and failing that via download.
+func (a *AssetBuilder) RemapFile(canonicalURL *url.URL, knownHash *hashing.Hash) (*FileAsset, error) {
+	if canonicalURL == nil {
+		return nil, fmt.Errorf("unable to remap a nil URL")
 	}
 
 	fileAsset := &FileAsset{
-		DownloadURL:  fileURL,
-		CanonicalURL: fileURL,
+		DownloadURL:  canonicalURL,
+		CanonicalURL: canonicalURL,
 	}
 
 	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
-
-		normalizedFile, err := a.remapURL(fileURL)
+		normalizedFile, err := a.remapURL(canonicalURL)
 		if err != nil {
-			return nil, nil, err
+			return nil, err
 		}
 
-		if fileURL.Host != normalizedFile.Host {
+		if canonicalURL.Host != normalizedFile.Host {
 			fileAsset.DownloadURL = normalizedFile
 			klog.V(4).Infof("adding remapped file: %q", fileAsset.DownloadURL.String())
 		}
 	}
 
-	h, err := a.findHash(fileAsset)
-	if err != nil {
-		return nil, nil, err
-	}
-	fileAsset.SHAValue = h.Hex()
-
-	klog.V(8).Infof("adding file: %+v", fileAsset)
-	a.FileAssets = append(a.FileAssets, fileAsset)
-
-	return fileAsset.DownloadURL, h, nil
-}
-
-// RemapFileAndSHAValue returns a remapped URL for the file without a SHA file in object storage, if AssetsLocation is defined.
-func (a *AssetBuilder) RemapFileAndSHAValue(fileURL *url.URL, shaValue string) (*url.URL, error) {
-	if fileURL == nil {
-		return nil, fmt.Errorf("unable to remap a nil URL")
-	}
-
-	fileAsset := &FileAsset{
-		DownloadURL:  fileURL,
-		CanonicalURL: fileURL,
-		SHAValue:     shaValue,
-	}
-
-	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
-		normalizedFile, err := a.remapURL(fileURL)
+	if knownHash == nil {
+		h, err := a.findHash(fileAsset)
 		if err != nil {
 			return nil, err
 		}
-		if fileURL.Host != normalizedFile.Host {
-			fileAsset.DownloadURL = normalizedFile
-			klog.V(4).Infof("adding remapped file: %q", fileAsset.DownloadURL.String())
-		}
+		knownHash = h
 	}
 
+	fileAsset.SHAValue = knownHash
+
 	klog.V(8).Infof("adding file: %+v", fileAsset)
 	a.FileAssets = append(a.FileAssets, fileAsset)
 
-	return fileAsset.DownloadURL, nil
+	return fileAsset, nil
 }
 
-// FindHash returns the hash value of a FileAsset.
+// findHash returns the hash value of a FileAsset.
 func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {
 	// If the phase is "assets" we use the CanonicalFileURL,
 	// but during other phases we use the hash from the FileRepository or the base kops path.
@@ -324,6 +301,16 @@ func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {
 		return nil, fmt.Errorf("file url is not defined")
 	}
 
+	knownHash, found, err := assetdata.GetHash(file.CanonicalURL)
+	if err != nil {
+		return nil, err
+	}
+	if found {
+		return knownHash, nil
+	}
+
+	klog.Infof("asset %q is not well-known, downloading hash", file.CanonicalURL)
+
 	// We now prefer sha256 hashes
 	for backoffSteps := 1; backoffSteps <= 3; backoffSteps++ {
 		// We try first with a short backoff, so we don't
@@ -338,7 +325,7 @@ func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {
 		for _, ext := range []string{".sha256", ".sha256sum"} {
 			for _, mirror := range FindURLMirrors(u.String()) {
 				hashURL := mirror + ext
-				klog.V(3).Infof("Trying to read hash fie: %q", hashURL)
+				klog.V(3).Infof("Trying to read hash file: %q", hashURL)
 				b, err := a.vfsContext.ReadFile(hashURL, vfs.WithBackoff(backoff))
 				if err != nil {
 					// Try to log without being too alarming - issue #7550

pkg/assets/copy.go

@@ -56,7 +56,7 @@ func Copy(imageAssets []*ImageAsset, fileAssets []*FileAsset, vfsContext *vfs.VF
 				Name:       fileAsset.CanonicalURL.String(),
 				TargetFile: fileAsset.DownloadURL.String(),
 				SourceFile: fileAsset.CanonicalURL.String(),
-				SHA:        fileAsset.SHAValue,
+				SHA:        fileAsset.SHAValue.Hex(),
 				VFSContext: vfsContext,
 				Cluster:    cluster,
 			}

pkg/assets/mirrored_asset.go

@@ -17,7 +17,6 @@ limitations under the License.
 package assets
 
 import (
-	"net/url"
 	"strings"
 
 	"k8s.io/klog/v2"
@@ -30,12 +29,14 @@ type MirroredAsset struct {
 }
 
 // BuildMirroredAsset checks to see if this is a file under the standard base location, and if so constructs some mirror locations
-func BuildMirroredAsset(u *url.URL, hash *hashing.Hash) *MirroredAsset {
+func BuildMirroredAsset(asset *FileAsset) *MirroredAsset {
+	u := asset.DownloadURL
+
 	a := &MirroredAsset{
-		Hash: hash,
+		Hash: asset.SHAValue,
 	}
 
-	if hash == nil {
+	if asset.SHAValue == nil {
 		klog.Warningf("not using mirrors for asset %s as it does not have a known hash", u)
 		a.Locations = []string{u.String()}
 	} else {

pkg/nodemodel/fileassets.go

@@ -75,11 +75,11 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
 			}
 			k.Path = path.Join(k.Path, an)
 
-			u, hash, err := assetBuilder.RemapFileAndSHA(k)
+			asset, err := assetBuilder.RemapFile(k, nil)
 			if err != nil {
 				return err
 			}
-			c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(u, hash))
+			c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(asset))
 		}
 
 		kubernetesVersion, _ := util.ParseKubernetesVersion(c.Cluster.Spec.KubernetesVersion)
@@ -98,6 +98,7 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
 					return err
 				}
 
+				// TODO: Move these hashes to assetdata
 				hashes := map[architectures.Architecture]string{
 					"amd64": "827d558953d861b81a35c3b599191a73f53c1f63bce42c61e7a3fee21a717a89",
 					"arm64": "f1617c0ef77f3718e12a3efc6f650375d5b5e96eebdbcbad3e465e89e781bdfa",
@@ -106,70 +107,69 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
 				if err != nil {
 					return fmt.Errorf("unable to parse auth-provider-gcp binary asset hash %q: %v", hashes[arch], err)
 				}
-				u, err := assetBuilder.RemapFileAndSHAValue(k, hashes[arch])
+				asset, err := assetBuilder.RemapFile(k, hash)
 				if err != nil {
 					return err
 				}
 
-				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(u, hash))
+				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(asset))
 			case kops.CloudProviderAWS:
 				binaryLocation := c.Cluster.Spec.CloudProvider.AWS.BinariesLocation
 				if binaryLocation == nil {
 					binaryLocation = fi.PtrTo("https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1")
 				}
 
-				k, err := url.Parse(fmt.Sprintf("%s/linux/%s/ecr-credential-provider-linux-%s", *binaryLocation, arch, arch))
+				u, err := url.Parse(fmt.Sprintf("%s/linux/%s/ecr-credential-provider-linux-%s", *binaryLocation, arch, arch))
 				if err != nil {
 					return err
 				}
-				u, hash, err := assetBuilder.RemapFileAndSHA(k)
+				asset, err := assetBuilder.RemapFile(u, nil)
 				if err != nil {
 					return err
 				}
-
-				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(u, hash))
+				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(asset))
 			}
 		}
 
 		{
-			cniAsset, cniAssetHash, err := wellknownassets.FindCNIAssets(c.Cluster, assetBuilder, arch)
+			cniAsset, err := wellknownassets.FindCNIAssets(c.Cluster, assetBuilder, arch)
 			if err != nil {
 				return err
 			}
-			c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(cniAsset, cniAssetHash))
+			c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(cniAsset))
 		}
 
 		if c.Cluster.Spec.Containerd == nil || !c.Cluster.Spec.Containerd.SkipInstall {
-			containerdAssetUrl, containerdAssetHash, err := wellknownassets.FindContainerdAsset(c.Cluster, assetBuilder, arch)
+			containerdAsset, err := wellknownassets.FindContainerdAsset(c.Cluster, assetBuilder, arch)
 			if err != nil {
 				return err
 			}
-			if containerdAssetUrl != nil && containerdAssetHash != nil {
-				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(containerdAssetUrl, containerdAssetHash))
+			if containerdAsset != nil {
+				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(containerdAsset))
 			}
 
-			runcAssetUrl, runcAssetHash, err := wellknownassets.FindRuncAsset(c.Cluster, assetBuilder, arch)
+			runcAsset, err := wellknownassets.FindRuncAsset(c.Cluster, assetBuilder, arch)
 			if err != nil {
 				return err
 			}
-			if runcAssetUrl != nil && runcAssetHash != nil {
-				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(runcAssetUrl, runcAssetHash))
+			if runcAsset != nil {
+				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(runcAsset))
 			}
-			nerdctlAssetUrl, nerdctlAssetHash, err := wellknownassets.FindNerdctlAsset(c.Cluster, assetBuilder, arch)
+			nerdctlAsset, err := wellknownassets.FindNerdctlAsset(c.Cluster, assetBuilder, arch)
 			if err != nil {
 				return err
 			}
-			if nerdctlAssetUrl != nil && nerdctlAssetHash != nil {
-				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(nerdctlAssetUrl, nerdctlAssetHash))
+			if nerdctlAsset != nil {
+				c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(nerdctlAsset))
 			}
 		}
 
-		crictlAssetUrl, crictlAssetHash, err := wellknownassets.FindCrictlAsset(c.Cluster, assetBuilder, arch)
+		crictlAsset, err := wellknownassets.FindCrictlAsset(c.Cluster, assetBuilder, arch)
 		if err != nil {
 			return err
 		}
-		if crictlAssetUrl != nil && crictlAssetHash != nil {
-			c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(crictlAssetUrl, crictlAssetHash))
+		if crictlAsset != nil {
+			c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(crictlAsset))
 		}
 
 		asset, err := wellknownassets.NodeUpAsset(assetBuilder, arch)