Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Look for sha256 and sha1 files for artifacts #7468

Merged
merged 1 commit into from
Aug 24, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/assets/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ go_library(
"//util/pkg/hashing:go_default_library",
"//util/pkg/vfs:go_default_library",
"//vendor/github.com/blang/semver:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)
Expand Down
37 changes: 25 additions & 12 deletions pkg/assets/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,11 @@ import (
"path"
"regexp"
"strings"
"time"

"github.com/blang/semver"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog"

"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/kops/util"
"k8s.io/kops/pkg/featureflag"
Expand Down Expand Up @@ -290,19 +291,31 @@ func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {
return nil, fmt.Errorf("file url is not defined")
}

for _, ext := range []string{".sha1"} {
hashURL := u.String() + ext
b, err := vfs.Context.ReadFile(hashURL)
if err != nil {
klog.Infof("error reading hash file %q: %v", hashURL, err)
continue
// We now prefer sha256 hashes
for backoffSteps := 1; backoffSteps <= 3; backoffSteps++ {
// We try first with a short backoff, so we don't
// waste too much time looking for files that don't
// exist before trying the next one
backoff := wait.Backoff{
Duration: 500 * time.Millisecond,
Factor: 2,
Steps: backoffSteps,
}
hashString := strings.TrimSpace(string(b))
klog.V(2).Infof("Found hash %q for %q", hashString, u)

// Accept a hash string that is `<hash> <filename>`
fields := strings.Fields(hashString)
return hashing.FromString(fields[0])
for _, ext := range []string{".sha256", ".sha1"} {
hashURL := u.String() + ext
b, err := vfs.Context.ReadFile(hashURL, vfs.WithBackoff(backoff))
if err != nil {
klog.Infof("error reading hash file %q: %v", hashURL, err)
continue
}
hashString := strings.TrimSpace(string(b))
klog.V(2).Infof("Found hash %q for %q", hashString, u)

// Accept a hash string that is `<hash> <filename>`
fields := strings.Fields(hashString)
return hashing.FromString(fields[0])
}
}

if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
Expand Down
33 changes: 29 additions & 4 deletions upup/pkg/fi/assettasks/copyfile.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,30 @@ func (e *CopyFile) CompareWithID() *string {
return e.Name
}

// fileExtensionForSHA returns the expected extension for the given hash
// If the hash length is not recognized, it returns an error.
func fileExtensionForSHA(sha string) (string, error) {
switch len(sha) {
case 40:
return ".sha1", nil
case 64:
return ".sha256", nil
default:
return "", fmt.Errorf("unhandled sha length for %q", sha)
}
}

// Find attempts to find a file.
func (e *CopyFile) Find(c *fi.Context) (*CopyFile, error) {
expectedSHA := strings.TrimSpace(fi.StringValue(e.SHA))

shaExtension, err := fileExtensionForSHA(expectedSHA)
if err != nil {
return nil, err
}

targetSHAFile := fi.StringValue(e.TargetFile) + shaExtension

targetSHAFile := fi.StringValue(e.TargetFile) + ".sha1"
targetSHABytes, err := vfs.Context.ReadFile(targetSHAFile)
if err != nil {
if os.IsNotExist(err) {
Expand All @@ -64,9 +84,9 @@ func (e *CopyFile) Find(c *fi.Context) (*CopyFile, error) {
return nil, nil
}
}

targetSHA := string(targetSHABytes)
if strings.TrimSpace(targetSHA) == strings.TrimSpace(fi.StringValue(e.SHA)) {

if strings.TrimSpace(targetSHA) == expectedSHA {
actual := &CopyFile{
Name: e.Name,
TargetFile: e.TargetFile,
Expand Down Expand Up @@ -142,7 +162,12 @@ func transferFile(c *fi.Context, source string, target string, sha string) error
return fmt.Errorf("error building path %q: %v", objectStore, err)
}

shaTarget := objectStore + ".sha1"
shaExtension, err := fileExtensionForSHA(sha)
if err != nil {
return err
}

shaTarget := objectStore + shaExtension
shaVFS, err := vfs.Context.BuildVfsPath(shaTarget)
if err != nil {
return fmt.Errorf("error building path %q: %v", shaTarget, err)
Expand Down
14 changes: 10 additions & 4 deletions upup/pkg/fi/cloudup/networking.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,9 @@ const (
defaultCNIAssetHashStringK8s1_9 = "d595d3ded6499a64e8dac02466e2f5f2ce257c9f"

// defaultCNIAssetK8s1_11 is the CNI tarball for k8s >= 1.11
defaultCNIAssetK8s1_11 = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz"
defaultCNIAssetHashStringK8s1_11 = "52e9d2de8a5f927307d9397308735658ee44ab8d"
defaultCNIAssetK8s1_11 = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz"
defaultCNIAssetSHA1StringK8s1_11 = "52e9d2de8a5f927307d9397308735658ee44ab8d"
defaultCNIAssetSHA256StringK8s1_11 = "3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64"

// Environment variable for overriding CNI url
ENV_VAR_CNI_VERSION_URL = "CNI_VERSION_URL"
Expand Down Expand Up @@ -167,9 +168,14 @@ func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL,
}

var cniAsset, cniAssetHash string
if util.IsKubernetesGTE("1.11", *sv) {
if util.IsKubernetesGTE("1.15", *sv) {
// We're still on the same asset, but we use sha256
cniAsset = defaultCNIAssetK8s1_11
cniAssetHash = defaultCNIAssetHashStringK8s1_11
cniAssetHash = defaultCNIAssetSHA256StringK8s1_11
klog.V(2).Infof("Adding default CNI asset for k8s >= 1.11: %s", defaultCNIAssetK8s1_9)
} else if util.IsKubernetesGTE("1.11", *sv) {
cniAsset = defaultCNIAssetK8s1_11
cniAssetHash = defaultCNIAssetSHA1StringK8s1_11
klog.V(2).Infof("Adding default CNI asset for k8s >= 1.11: %s", defaultCNIAssetK8s1_9)
} else if util.IsKubernetesGTE("1.9", *sv) {
cniAsset = defaultCNIAssetK8s1_9
Expand Down
46 changes: 32 additions & 14 deletions util/pkg/vfs/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,36 @@ var Context = VFSContext{
k8sContext: NewKubernetesContext(),
}

type vfsOptions struct {
backoff wait.Backoff
}

type VFSOption func(options *vfsOptions)

// WithBackoff specifies a custom VFS backoff policy
func WithBackoff(backoff wait.Backoff) VFSOption {
return func(options *vfsOptions) {
options.backoff = backoff
}
}

// ReadLocation reads a file from a vfs URL
// It supports additional schemes which don't (yet) have full VFS implementations:
// metadata: reads from instance metadata on GCE/AWS
// http / https: reads from HTTP
func (c *VFSContext) ReadFile(location string) ([]byte, error) {
func (c *VFSContext) ReadFile(location string, options ...VFSOption) ([]byte, error) {
var opts vfsOptions
// Exponential backoff, starting with 500 milliseconds, doubling each time, 5 steps
opts.backoff = wait.Backoff{
Duration: 500 * time.Millisecond,
Factor: 2,
Steps: 5,
}

for _, option := range options {
option(&opts)
}

if strings.Contains(location, "://") && !strings.HasPrefix(location, "file://") {
// Handle our special case schemas
u, err := url.Parse(location)
Expand All @@ -75,20 +100,20 @@ func (c *VFSContext) ReadFile(location string) ([]byte, error) {
httpURL := "http://169.254.169.254/computeMetadata/v1/instance/attributes/" + u.Path
httpHeaders := make(map[string]string)
httpHeaders["Metadata-Flavor"] = "Google"
return c.readHttpLocation(httpURL, httpHeaders)
return c.readHttpLocation(httpURL, httpHeaders, opts)
case "aws":
httpURL := "http://169.254.169.254/latest/" + u.Path
return c.readHttpLocation(httpURL, nil)
return c.readHttpLocation(httpURL, nil, opts)
case "digitalocean":
httpURL := "http://169.254.169.254/metadata/v1" + u.Path
return c.readHttpLocation(httpURL, nil)
return c.readHttpLocation(httpURL, nil, opts)

default:
return nil, fmt.Errorf("unknown metadata type: %q in %q", u.Host, location)
}

case "http", "https":
return c.readHttpLocation(location, nil)
return c.readHttpLocation(location, nil, opts)
}
}

Expand Down Expand Up @@ -145,17 +170,10 @@ func (c *VFSContext) BuildVfsPath(p string) (Path, error) {
// readHttpLocation reads an http (or https) url.
// It returns the contents, or an error on any non-200 response. On a 404, it will return os.ErrNotExist
// It will retry a few times on a 500 class error
func (c *VFSContext) readHttpLocation(httpURL string, httpHeaders map[string]string) ([]byte, error) {
// Exponential backoff, starting with 500 milliseconds, doubling each time, 5 steps
backoff := wait.Backoff{
Duration: 500 * time.Millisecond,
Factor: 2,
Steps: 5,
}

func (c *VFSContext) readHttpLocation(httpURL string, httpHeaders map[string]string, opts vfsOptions) ([]byte, error) {
var body []byte

done, err := RetryWithBackoff(backoff, func() (bool, error) {
done, err := RetryWithBackoff(opts.backoff, func() (bool, error) {
klog.V(4).Infof("Performing HTTP request: GET %s", httpURL)
req, err := http.NewRequest("GET", httpURL, nil)
if err != nil {
Expand Down