From 36932e1128991ce251d6afed4791c6611cbd2073 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Mon, 19 Oct 2015 14:51:25 -0500 Subject: [PATCH 1/4] add basic checksum --- client.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/client.go b/client.go index 518e67f99..f9e7acb73 100644 --- a/client.go +++ b/client.go @@ -1,8 +1,12 @@ package getter import ( + "crypto/sha256" + "encoding/hex" "fmt" + "io" "io/ioutil" + "log" "os" "path/filepath" @@ -71,10 +75,25 @@ func (c *Client) Get() error { "download not supported for scheme '%s'", force) } + var sum string + sumRaw := u.Query()["checksum"] + if len(sumRaw) == 1 { + sum = sumRaw[0] + } + // If we're not downloading a directory, then just download the file // and return. if !c.Dir { - return g.GetFile(dst, u) + if sum == "" { + return g.GetFile(dst, u) + } + + err := g.GetFile(dst, u) + if err != nil { + return err + } + + return checksum(dst, sum) } // We're downloading a directory, which might require a bit more work @@ -97,5 +116,33 @@ func (c *Client) Get() error { return copyDir(realDst, filepath.Join(dst, subDir), false) } + return checksum(realDst, sum) +} + +// checksum is a simple method to compute the SHA256 checksum of a source (file +// or dir) and compare it to a given sum. +func checksum(source, sum string) error { + if sum == "" { + return nil + } + // compute and check checksum + log.Printf("[DEBUG] Running checksum on (%s)", source) + hasher := sha256.New() + file, err := os.Open(source) + if err != nil { + return fmt.Errorf("Failed to open file for checksum: %s", err) + } + + defer file.Close() + io.Copy(hasher, file) + + computed := hex.EncodeToString(hasher.Sum(nil)) + if sum != computed { + return fmt.Errorf( + "Checksums did not match.\nExpected (%s), got (%s)", + sum, + computed) + } + return nil } From b402d96f1d0fd089a31a0db6184bd0f61066f918 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 24 Oct 2015 18:37:34 -0700 Subject: [PATCH 2/4] support all major checksum types and add tests --- client.go | 92 +++++++++++++++++++++++++++++++++++++---------------- get_test.go | 79 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 27 deletions(-) diff --git a/client.go b/client.go index f9e7acb73..8e90f5965 100644 --- a/client.go +++ b/client.go @@ -1,14 +1,19 @@ package getter import ( + "bytes" + "crypto/md5" + "crypto/sha1" "crypto/sha256" + "crypto/sha512" "encoding/hex" "fmt" + "hash" "io" "io/ioutil" - "log" "os" "path/filepath" + "strings" urlhelper "github.com/hashicorp/terraform/helper/url" ) @@ -75,25 +80,63 @@ func (c *Client) Get() error { "download not supported for scheme '%s'", force) } - var sum string - sumRaw := u.Query()["checksum"] - if len(sumRaw) == 1 { - sum = sumRaw[0] + // Determine if we have a checksum + var checksumHash hash.Hash + var checksumValue []byte + if v := u.Query().Get("checksum"); v != "" { + // Delete the query parameter if we have it. + u.Query().Del("checksum") + + // If we're getting a directory, then this is an error. You cannot + // checksum a directory. TODO: test + if c.Dir { + return fmt.Errorf( + "checksum cannot be specified for directory download") + } + + // Determine the checksum hash type + checksumType := "" + idx := strings.Index(v, ":") + if idx > -1 { + checksumType = v[:idx] + } + switch checksumType { + case "md5": + checksumHash = md5.New() + case "sha1": + checksumHash = sha1.New() + case "sha256": + checksumHash = sha256.New() + case "sha512": + checksumHash = sha512.New() + default: + return fmt.Errorf( + "unsupported checksum type: %s", checksumType) + } + + // Get the remainder of the value and parse it into bytes + b, err := hex.DecodeString(v[idx+1:]) + if err != nil { + return fmt.Errorf("invalid checksum: %s", err) + } + + // Set our value + checksumValue = b } // If we're not downloading a directory, then just download the file // and return. if !c.Dir { - if sum == "" { - return g.GetFile(dst, u) - } - err := g.GetFile(dst, u) if err != nil { return err } - return checksum(dst, sum) + if checksumHash != nil { + return checksum(dst, checksumHash, checksumValue) + } + + return nil } // We're downloading a directory, which might require a bit more work @@ -116,32 +159,27 @@ func (c *Client) Get() error { return copyDir(realDst, filepath.Join(dst, subDir), false) } - return checksum(realDst, sum) + return nil } -// checksum is a simple method to compute the SHA256 checksum of a source (file -// or dir) and compare it to a given sum. -func checksum(source, sum string) error { - if sum == "" { - return nil - } - // compute and check checksum - log.Printf("[DEBUG] Running checksum on (%s)", source) - hasher := sha256.New() - file, err := os.Open(source) +// checksum is a simple method to compute the checksum of a source file +// and compare it to the given expected value. +func checksum(source string, h hash.Hash, v []byte) error { + f, err := os.Open(source) if err != nil { return fmt.Errorf("Failed to open file for checksum: %s", err) } + defer f.Close() - defer file.Close() - io.Copy(hasher, file) + if _, err := io.Copy(h, f); err != nil { + return fmt.Errorf("Failed to hash: %s", err) + } - computed := hex.EncodeToString(hasher.Sum(nil)) - if sum != computed { + if actual := h.Sum(nil); !bytes.Equal(actual, v) { return fmt.Errorf( "Checksums did not match.\nExpected (%s), got (%s)", - sum, - computed) + hex.EncodeToString(v), + hex.EncodeToString(actual)) } return nil diff --git a/get_test.go b/get_test.go index 8c3b07493..94042c5c7 100644 --- a/get_test.go +++ b/get_test.go @@ -59,3 +59,82 @@ func TestGet_fileSubdir(t *testing.T) { t.Fatalf("err: %s", err) } } + +func TestGetFile(t *testing.T) { + dst := tempFile(t) + u := testModule("basic-file/foo.txt") + + if err := GetFile(dst, u); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + assertContents(t, dst, "Hello\n") +} + +func TestGetFile_checksum(t *testing.T) { + cases := []struct { + Append string + Err bool + }{ + { + "", + false, + }, + + // MD5 + { + "?checksum=md5:09f7e02f1290be211da707a266f153b3", + false, + }, + { + "?checksum=md5:09f7e02f1290be211da707a266f153b4", + true, + }, + + // SHA1 + { + "?checksum=sha1:1d229271928d3f9e2bb0375bd6ce5db6c6d348d9", + false, + }, + { + "?checksum=sha1:1d229271928d3f9e2bb0375bd6ce5db6c6d348d0", + true, + }, + + // SHA256 + { + "?checksum=sha256:66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18", + false, + }, + { + "?checksum=sha256:66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f19", + true, + }, + + // SHA512 + { + "?checksum=sha512:c2bad2223811194582af4d1508ac02cd69eeeeedeeb98d54fcae4dcefb13cc882e7640328206603d3fb9cd5f949a9be0db054dd34fbfa190c498a5fe09750cef", + false, + }, + { + "?checksum=sha512:c2bad2223811194582af4d1508ac02cd69eeeeedeeb98d54fcae4dcefb13cc882e7640328206603d3fb9cd5f949a9be0db054dd34fbfa190c498a5fe09750ced", + true, + }, + } + + for _, tc := range cases { + u := testModule("basic-file/foo.txt") + tc.Append + + func() { + dst := tempFile(t) + defer os.Remove(dst) + if err := GetFile(dst, u); (err != nil) != tc.Err { + t.Fatalf("append: %s\n\nerr: %s", tc.Append, err) + } + + // Verify the main file exists + assertContents(t, dst, "Hello\n") + }() + } +} From 35916edf47a81580990dce0b51010084805db83d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 24 Oct 2015 18:47:34 -0700 Subject: [PATCH 3/4] test that checksum query param is removed --- client.go | 6 ++++-- get_mock.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ get_test.go | 23 +++++++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 get_mock.go diff --git a/client.go b/client.go index 8e90f5965..8a956509b 100644 --- a/client.go +++ b/client.go @@ -83,9 +83,11 @@ func (c *Client) Get() error { // Determine if we have a checksum var checksumHash hash.Hash var checksumValue []byte - if v := u.Query().Get("checksum"); v != "" { + q := u.Query() + if v := q.Get("checksum"); v != "" { // Delete the query parameter if we have it. - u.Query().Del("checksum") + q.Del("checksum") + u.RawQuery = q.Encode() // If we're getting a directory, then this is an error. You cannot // checksum a directory. TODO: test diff --git a/get_mock.go b/get_mock.go new file mode 100644 index 000000000..a7d3d3052 --- /dev/null +++ b/get_mock.go @@ -0,0 +1,45 @@ +package getter + +import ( + "net/url" +) + +// MockGetter is an implementation of Getter that can be used for tests. +type MockGetter struct { + // Proxy, if set, will be called after recording the calls below. + // If it isn't set, then the *Err values will be returned. + Proxy Getter + + GetCalled bool + GetDst string + GetURL *url.URL + GetErr error + + GetFileCalled bool + GetFileDst string + GetFileURL *url.URL + GetFileErr error +} + +func (g *MockGetter) Get(dst string, u *url.URL) error { + g.GetCalled = true + g.GetDst = dst + g.GetURL = u + + if g.Proxy != nil { + return g.Proxy.Get(dst, u) + } + + return g.GetErr +} + +func (g *MockGetter) GetFile(dst string, u *url.URL) error { + g.GetFileCalled = true + g.GetFileDst = dst + g.GetFileURL = u + + if g.Proxy != nil { + return g.Proxy.GetFile(dst, u) + } + return g.GetFileErr +} diff --git a/get_test.go b/get_test.go index 94042c5c7..47b506cf7 100644 --- a/get_test.go +++ b/get_test.go @@ -138,3 +138,26 @@ func TestGetFile_checksum(t *testing.T) { }() } } + +func TestGetFile_checksumURL(t *testing.T) { + dst := tempFile(t) + u := testModule("basic-file/foo.txt") + "?checksum=md5:09f7e02f1290be211da707a266f153b3" + + getter := &MockGetter{Proxy: new(FileGetter)} + client := &Client{ + Src: u, + Dst: dst, + Dir: false, + Getters: map[string]Getter{ + "file": getter, + }, + } + + if err := client.Get(); err != nil { + t.Fatalf("err: %s", err) + } + + if v := getter.GetFileURL.Query().Get("checksum"); v != "" { + t.Fatalf("bad: %s", v) + } +} From 9e68b212bbf65d00c91fed876f2d97e00bd334f6 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 24 Oct 2015 18:48:40 -0700 Subject: [PATCH 4/4] update error message --- client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.go b/client.go index 8a956509b..fb87235c6 100644 --- a/client.go +++ b/client.go @@ -179,7 +179,7 @@ func checksum(source string, h hash.Hash, v []byte) error { if actual := h.Sum(nil); !bytes.Equal(actual, v) { return fmt.Errorf( - "Checksums did not match.\nExpected (%s), got (%s)", + "Checksums did not match.\nExpected: %s\nGot: %s", hex.EncodeToString(v), hex.EncodeToString(actual)) }