Create assetdata

We create some yaml files that record the hashes for well-known file assets,
and then we will be able to look them up by their canonical URL.

This is not yet actually used, that can be done in a future commit.

hack/generate-asset-hashes.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+cd "${REPO_ROOT}"
+
+
+function generate_k8s_hashes() {
+  prefix=$1
+  max_patch=$2
+
+  cat > "${REPO_ROOT}/pkg/assets/assetdata/k8s-${prefix}.yaml" <<EOF
+# This file is generated by generate-asset-hashes.sh
+
+filestores:
+- base: https://dl.k8s.io/release/
+
+files:
+EOF
+
+  for ((patch = 0 ; patch < max_patch ; patch++ )); do
+    version="${prefix}.${patch}"
+    echo "k8s ${version}"
+
+    # We exclude some files that we don't currently need, to keep the size down
+    go run ./pkg/assets/assetdata/tools/cmd/generatefileassets \
+      --base https://dl.k8s.io/release/ \
+      --prefix "v${version}/" \
+      --exclude "**/arm/**" \
+      --exclude "**/s390x/**" \
+      --exclude "**/ppc64le/**" \
+      --exclude "**/windows/**" \
+      --exclude "**/*-s390x.tar.gz" \
+      --exclude "**/*-ppc64le.tar.gz" \
+      --exclude "**/*.tar.gz" \
+      | sed "s@files:@# kubernetes ${version}@g" >> "${REPO_ROOT}/pkg/assets/assetdata/k8s-${prefix}.yaml"
+  done
+}
+
+function generate_runc_hashes() {
+  prefix=$1
+  max_patch=$2
+
+  cat > "${REPO_ROOT}/pkg/assets/assetdata/runc-${prefix}.yaml" <<EOF
+# This file is generated by generate-asset-hashes.sh
+
+filestores:
+- base: https://dl.k8s.io/release/
+
+files:
+EOF
+
+  for ((patch = 0 ; patch < max_patch ; patch++ )); do
+    version="${prefix}.${patch}"
+    echo "runc ${version}"
+
+    # We exclude some files that we don't currently need, to keep the size down
+    go run ./pkg/assets/assetdata/tools/cmd/generatefileassets \
+      --base https://github.com/opencontainers/runc/releases/download/ \
+      --prefix "v${version}/" \
+      --sums "https://github.com/opencontainers/runc/releases/download/v${version}/runc.sha256sum" \
+      | sed "s@files:@# runc ${version}@g" >> "${REPO_ROOT}/pkg/assets/assetdata/runc-${prefix}.yaml"
+  done
+}
+
+generate_k8s_hashes 1.24 17
+generate_k8s_hashes 1.25 16
+generate_k8s_hashes 1.26 15
+generate_k8s_hashes 1.27 12
+generate_k8s_hashes 1.28 8
+generate_k8s_hashes 1.29 3
+
+
+generate_runc_hashes 1.1 12

pkg/assets/assetdata/README.md

@@ -0,0 +1,11 @@
+This directory contains hashes for well-known dependencies of kubernetes / a kOps cluster.
+
+We store the hashes here, rather than downloading them every time - it is a little more secure,
+and it is more efficient.
+
+The yaml file structure is intended to mirror the asset structure used by the k8s project,
+e.g. by [kpromo](https://github.com/kubernetes-sigs/promo-tools/blob/main/docs/file-promotion.md).
+However, this should be treated as an implementation detail.
+
+Currently many hash files are manually curated.  Some of them can be automatically generated,
+and we have scripts named `generate-<foo>.sh` to generate them.

pkg/assets/assetdata/cni.yaml

@@ -0,0 +1,14 @@
+filestores:
+- base: https://storage.googleapis.com/k8s-artifacts-cni/release/
+
+files:
+- name: v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
+  sha256: 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7
+- name: v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
+  sha256: ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0
+
+- name: v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
+  sha256: f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37
+- name: v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz
+  sha256: 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57
+

pkg/assets/assetdata/containerd.yaml

@@ -0,0 +1,16 @@
+filestores:
+- base: https://github.com/containerd/containerd/releases/download/
+
+# Manually constructed; there is a .sha256sum file available
+
+files:
+- name: v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
+  sha256: bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2
+- name: v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
+  sha256: c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce
+
+- name: v1.7.13/containerd-1.7.13-linux-amd64.tar.gz
+  sha256: c2371c009dd8b7738663333d91e5ab50d204f8bcae24201f45d59060d12c3a23
+- name: v1.7.13/containerd-1.7.13-linux-arm64.tar.gz
+  sha256: 118759e398f35337109592b4d237538872dc12a207d38832b9d04515d0acbc4d
+

pkg/assets/assetdata/crictl.yaml

@@ -0,0 +1,9 @@
+filestores:
+- base: https://github.com/kubernetes-sigs/cri-tools/releases/download/
+
+# Manually constructed, though we could quickly get the hash by downloading the .sha256 file for each asset
+files:
+- name: v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz
+  sha256: d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb
+- name: v1.29.0/crictl-v1.29.0-linux-arm64.tar.gz
+  sha256: 0b615cfa00c331fb9c4524f3d4058a61cc487b33a3436d1269e7832cf283f925

pkg/assets/assetdata/data.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package assetdata
+
+import (
+	"embed"
+	"fmt"
+	"io/fs"
+	"net/url"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kops/util/pkg/hashing"
+	"sigs.k8s.io/yaml"
+)
+
+//go:embed *.yaml
+var embeddedDataFS embed.FS
+
+// GetHash returns the stored hash for the well-known asset, looking it up by the canonicalURL.
+// If found, it returns (hash, true, nil)
+// If not found, it returns (nil, false, nil)
+func GetHash(canonicalURL *url.URL) (*hashing.Hash, bool, error) {
+	var allMatches []*file
+
+	if err := fs.WalkDir(embeddedDataFS, ".", func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		b, err := fs.ReadFile(embeddedDataFS, p)
+		if err != nil {
+			return fmt.Errorf("reading embedded file %q: %w", p, err)
+		}
+
+		manifest, err := parseManifestFile(b)
+		if err != nil {
+			return fmt.Errorf("parsing embedded file %q: %w", p, err)
+		}
+
+		matches := manifest.Matches(canonicalURL.String())
+		allMatches = append(allMatches, matches...)
+		return nil
+	}); err != nil {
+		return nil, false, fmt.Errorf("walking embedded data: %w", err)
+	}
+
+	hashes := sets.New[string]()
+	for _, match := range allMatches {
+		hashes.Insert(match.SHA256)
+	}
+	if len(hashes) == 0 {
+		return nil, false, nil
+	}
+	if len(hashes) > 1 {
+		return nil, false, fmt.Errorf("found multiple matches for asset %q", canonicalURL)
+	}
+	h, err := hashing.FromString(hashes.UnsortedList()[0])
+	if err != nil {
+		return nil, false, err
+	}
+	return h, true, nil
+}
+
+type file struct {
+	Name   string `json:"name,omitempty"`
+	SHA256 string `json:"sha256,omitempty"`
+}
+
+type fileStore struct {
+	Base string `json:"base,omitempty"`
+}
+
+type manifest struct {
+	FileStores []fileStore `json:"filestores,omitempty"`
+	Files      []file      `json:"files,omitempty"`
+}
+
+func parseManifestFile(b []byte) (*manifest, error) {
+	m := &manifest{}
+	if err := yaml.Unmarshal(b, m); err != nil {
+		return nil, fmt.Errorf("parsing yaml: %w", err)
+	}
+	return m, nil
+}
+
+func (m *manifest) Matches(canonicalURL string) []*file {
+	var matches []*file
+	for _, fileStore := range m.FileStores {
+		if !strings.HasPrefix(canonicalURL, fileStore.Base) {
+			continue
+		}
+		relativePath := strings.TrimPrefix(canonicalURL, fileStore.Base)
+		for i := range m.Files {
+			f := &m.Files[i]
+			if f.Name == relativePath {
+				matches = append(matches, f)
+			}
+		}
+	}
+	return matches
+}

pkg/assets/assetdata/data_test.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package assetdata
+
+import (
+	"net/url"
+	"testing"
+)
+
+func TestGetHash(t *testing.T) {
+	grid := []struct {
+		Name string
+		Hash string
+	}{
+		{Name: "https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubelet", Hash: "sha256:b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5"},
+	}
+
+	for _, g := range grid {
+		u, err := url.Parse(g.Name)
+		if err != nil {
+			t.Fatalf("parsing url %q: %v", g.Name, err)
+		}
+		h, found, err := GetHash(u)
+		if err != nil {
+			t.Fatalf("getting hash for %q: %v", g.Name, err)
+		}
+		if !found {
+			t.Fatalf("hash for %q was not found", g.Name)
+		}
+		got := h.String()
+		want := g.Hash
+		if got != g.Hash {
+			t.Errorf("unexpected hash for %q; got %q, want %q", g.Name, got, want)
+		}
+	}
+}

pkg/assets/assetdata/ecr.yaml

@@ -0,0 +1,10 @@
+filestores:
+- base: https://artifacts.k8s.io/binaries/cloud-provider-aws/
+
+# Manually constructed; there is a .sha256 file available for each asset
+
+files:
+- name: v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
+  sha256: 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a
+- name: v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
+  sha256: b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481