Add lz4 to compressutil (#5403)

helper/compressutil/compress.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/golang/snappy"
 	"github.com/hashicorp/errwrap"
+	"github.com/pierrec/lz4"
 )
 
 const (
@@ -17,41 +18,39 @@ const (
 	// The value of this constant should not be a first character of any
 	// valid JSON string.
 
-	// Byte value used as canary when using Gzip format
+	CompressionTypeGzip        = "gzip"
 	CompressionCanaryGzip byte = 'G'
 
-	// Byte value used as canary when using Lzw format
-	CompressionCanaryLzw byte = 'L'
+	CompressionTypeLZW        = "lzw"
+	CompressionCanaryLZW byte = 'L'
 
-	// Byte value used as canary when using Snappy format
+	CompressionTypeSnappy        = "snappy"
 	CompressionCanarySnappy byte = 'S'
 
-	CompressionTypeLzw = "lzw"
-
-	CompressionTypeGzip = "gzip"
-
-	CompressionTypeSnappy = "snappy"
+	CompressionTypeLZ4        = "lz4"
+	CompressionCanaryLZ4 byte = '4'
 )
 
 // SnappyReadCloser embeds the snappy reader which implements the io.Reader
 // interface. The decompress procedure in this utility expects an
 // io.ReadCloser. This type implements the io.Closer interface to retain the
 // generic way of decompression.
-type SnappyReadCloser struct {
-	*snappy.Reader
+type CompressUtilReadCloser struct {
+	io.Reader
 }
 
 // Close is a noop method implemented only to satisfy the io.Closer interface
-func (s *SnappyReadCloser) Close() error {
+func (c *CompressUtilReadCloser) Close() error {
 	return nil
 }
 
 // CompressionConfig is used to select a compression type to be performed by
 // Compress and Decompress utilities.
 // Supported types are:
-// * CompressionTypeLzw
+// * CompressionTypeLZW
 // * CompressionTypeGzip
 // * CompressionTypeSnappy
+// * CompressionTypeLZ4
 //
 // When using CompressionTypeGzip, the compression levels can also be chosen:
 // * gzip.DefaultCompression
@@ -82,10 +81,10 @@ func Compress(data []byte, config *CompressionConfig) ([]byte, error) {
 	// Write the canary into the buffer and create writer to compress the
 	// input data based on the configured type
 	switch config.Type {
-	case CompressionTypeLzw:
-		buf.Write([]byte{CompressionCanaryLzw})
-
+	case CompressionTypeLZW:
+		buf.Write([]byte{CompressionCanaryLZW})
 		writer = lzw.NewWriter(&buf, lzw.LSB, 8)
+
 	case CompressionTypeGzip:
 		buf.Write([]byte{CompressionCanaryGzip})
 
@@ -100,9 +99,15 @@ func Compress(data []byte, config *CompressionConfig) ([]byte, error) {
 			config.GzipCompressionLevel = gzip.DefaultCompression
 		}
 		writer, err = gzip.NewWriterLevel(&buf, config.GzipCompressionLevel)
+
 	case CompressionTypeSnappy:
 		buf.Write([]byte{CompressionCanarySnappy})
 		writer = snappy.NewBufferedWriter(&buf)
+
+	case CompressionTypeLZ4:
+		buf.Write([]byte{CompressionCanaryLZ4})
+		writer = lz4.NewWriter(&buf)
+
 	default:
 		return nil, fmt.Errorf("unsupported compression type")
 	}
@@ -142,30 +147,40 @@ func Decompress(data []byte) ([]byte, bool, error) {
 		return nil, false, fmt.Errorf("'data' being decompressed is empty")
 	}
 
-	switch {
+	canary := data[0]
+	cData := data[1:]
+
+	switch canary {
 	// If the first byte matches the canary byte, remove the canary
 	// byte and try to decompress the data that is after the canary.
-	case data[0] == CompressionCanaryGzip:
+	case CompressionCanaryGzip:
 		if len(data) < 2 {
 			return nil, false, fmt.Errorf("invalid 'data' after the canary")
 		}
-		data = data[1:]
-		reader, err = gzip.NewReader(bytes.NewReader(data))
-	case data[0] == CompressionCanaryLzw:
+		reader, err = gzip.NewReader(bytes.NewReader(cData))
+
+	case CompressionCanaryLZW:
 		if len(data) < 2 {
 			return nil, false, fmt.Errorf("invalid 'data' after the canary")
 		}
-		data = data[1:]
-		reader = lzw.NewReader(bytes.NewReader(data), lzw.LSB, 8)
+		reader = lzw.NewReader(bytes.NewReader(cData), lzw.LSB, 8)
 
-	case data[0] == CompressionCanarySnappy:
+	case CompressionCanarySnappy:
 		if len(data) < 2 {
 			return nil, false, fmt.Errorf("invalid 'data' after the canary")
 		}
-		data = data[1:]
-		reader = &SnappyReadCloser{
-			Reader: snappy.NewReader(bytes.NewReader(data)),
+		reader = &CompressUtilReadCloser{
+			Reader: snappy.NewReader(bytes.NewReader(cData)),
 		}
+
+	case CompressionCanaryLZ4:
+		if len(data) < 2 {
+			return nil, false, fmt.Errorf("invalid 'data' after the canary")
+		}
+		reader = &CompressUtilReadCloser{
+			Reader: lz4.NewReader(bytes.NewReader(cData)),
+		}
+
 	default:
 		// If the first byte doesn't match the canary byte, it means
 		// that the content was not compressed at all. Indicate the

helper/compressutil/compress_test.go

@@ -74,7 +74,7 @@ func TestCompressUtil_CompressDecompress(t *testing.T) {
 
 	// Compress input using lzw format
 	compressedJSONBytes, err := Compress(inputJSONBytes, &CompressionConfig{
-		Type: CompressionTypeLzw,
+		Type: CompressionTypeLZW,
 	})
 	if err != nil {
 		t.Fatal("expected an error")
@@ -83,8 +83,8 @@ func TestCompressUtil_CompressDecompress(t *testing.T) {
 		t.Fatal("failed to compress data in lzw format")
 	}
 	// Check the presence of the canary
-	if compressedJSONBytes[0] != CompressionCanaryLzw {
-		t.Fatalf("bad: compression canary: expected: %d actual: %d", CompressionCanaryLzw, compressedJSONBytes[0])
+	if compressedJSONBytes[0] != CompressionCanaryLZW {
+		t.Fatalf("bad: compression canary: expected: %d actual: %d", CompressionCanaryLZW, compressedJSONBytes[0])
 	}
 
 	// Decompress the input and check the output

helper/forwarding/util.go

@@ -39,7 +39,7 @@ func GenerateForwardedHTTPRequest(req *http.Request, addr string) (*http.Request
 		newBody, err = jsonutil.EncodeJSON(fq)
 	case "json_compress":
 		newBody, err = jsonutil.EncodeJSONAndCompress(fq, &compressutil.CompressionConfig{
-			Type: compressutil.CompressionTypeLzw,
+			Type: compressutil.CompressionTypeLZW,
 		})
 	case "proto3":
 		fallthrough